diff options
520 files changed, 18116 insertions, 8663 deletions
diff --git a/Android.mk b/Android.mk index e4f4e74cb2c..47bcaac3439 100644 --- a/Android.mk +++ b/Android.mk @@ -298,7 +298,7 @@ test-art-target-jit$(ART_PHONY_TEST_TARGET_SUFFIX): test-art-target-run-test-jit $(hide) $(call ART_TEST_PREREQ_FINISHED,$@) # Secondary target architecture variants: -ifdef TARGET_2ND_ARCH +ifdef 2ND_ART_PHONY_TEST_TARGET_SUFFIX .PHONY: test-art-target$(2ND_ART_PHONY_TEST_TARGET_SUFFIX) test-art-target$(2ND_ART_PHONY_TEST_TARGET_SUFFIX): test-art-target-gtest$(2ND_ART_PHONY_TEST_TARGET_SUFFIX) \ test-art-target-run-test$(2ND_ART_PHONY_TEST_TARGET_SUFFIX) diff --git a/build/Android.common.mk b/build/Android.common.mk index b0fa124e483..a6a9f0fc479 100644 --- a/build/Android.common.mk +++ b/build/Android.common.mk @@ -59,8 +59,6 @@ ifdef TARGET_2ND_ARCH ART_PHONY_TEST_TARGET_SUFFIX := 64 2ND_ART_PHONY_TEST_TARGET_SUFFIX := 32 else - # TODO: ??? - $(warning Do not know what to do with this multi-target configuration!) ART_PHONY_TEST_TARGET_SUFFIX := 32 2ND_ART_PHONY_TEST_TARGET_SUFFIX := endif diff --git a/build/Android.gtest.mk b/build/Android.gtest.mk index b483e5f6f29..a4a72f4ccc0 100644 --- a/build/Android.gtest.mk +++ b/build/Android.gtest.mk @@ -89,43 +89,59 @@ ART_TEST_TARGET_GTEST_EmptyUncompressed_DEX := $(basename $(ART_TEST_TARGET_GTES ART_TEST_HOST_GTEST_MultiDexUncompressed_DEX := $(basename $(ART_TEST_HOST_GTEST_MultiDex_DEX))Uncompressed$(suffix $(ART_TEST_HOST_GTEST_MultiDex_DEX)) ART_TEST_TARGET_GTEST_MultiDexUncompressed_DEX := $(basename $(ART_TEST_TARGET_GTEST_MultiDex_DEX))Uncompressed$(suffix $(ART_TEST_TARGET_GTEST_MultiDex_DEX)) +ifdef ART_TEST_HOST_GTEST_Main_DEX $(ART_TEST_HOST_GTEST_MainStripped_DEX): $(ART_TEST_HOST_GTEST_Main_DEX) cp $< $@ $(call dexpreopt-remove-classes.dex,$@) +endif +ifdef ART_TEST_TARGET_GTEST_Main_DEX $(ART_TEST_TARGET_GTEST_MainStripped_DEX): $(ART_TEST_TARGET_GTEST_Main_DEX) cp $< $@ $(call dexpreopt-remove-classes.dex,$@) +endif +ifdef ART_TEST_HOST_GTEST_Main_DEX $(ART_TEST_HOST_GTEST_MainUncompressed_DEX): $(ART_TEST_HOST_GTEST_Main_DEX) $(ZIPALIGN) cp $< $@ $(call uncompress-dexs, $@) $(call align-package, $@) +endif +ifdef ART_TEST_TARGET_GTEST_Main_DEX $(ART_TEST_TARGET_GTEST_MainUncompressed_DEX): $(ART_TEST_TARGET_GTEST_Main_DEX) $(ZIPALIGN) cp $< $@ $(call uncompress-dexs, $@) $(call align-package, $@) +endif +ifdef ART_TEST_HOST_GTEST_Main_DEX $(ART_TEST_HOST_GTEST_EmptyUncompressed_DEX): $(ZIPALIGN) touch $(dir $@)classes.dex zip -j -qD -X -0 $@ $(dir $@)classes.dex rm $(dir $@)classes.dex +endif +ifdef ART_TEST_TARGET_GTEST_Main_DEX $(ART_TEST_TARGET_GTEST_EmptyUncompressed_DEX): $(ZIPALIGN) touch $(dir $@)classes.dex zip -j -qD -X -0 $@ $(dir $@)classes.dex rm $(dir $@)classes.dex +endif +ifdef ART_TEST_HOST_GTEST_MultiDex_DEX $(ART_TEST_HOST_GTEST_MultiDexUncompressed_DEX): $(ART_TEST_HOST_GTEST_MultiDex_DEX) $(ZIPALIGN) cp $< $@ $(call uncompress-dexs, $@) $(call align-package, $@) +endif +ifdef ART_TEST_TARGET_GTEST_MultiDex_DEX $(ART_TEST_TARGET_GTEST_MultiDexUncompressed_DEX): $(ART_TEST_TARGET_GTEST_MultiDex_DEX) $(ZIPALIGN) cp $< $@ $(call uncompress-dexs, $@) $(call align-package, $@) +endif ART_TEST_GTEST_VerifierDeps_SRC := $(abspath $(wildcard $(LOCAL_PATH)/VerifierDeps/*.smali)) ART_TEST_GTEST_VerifierDepsMulti_SRC := $(abspath $(wildcard $(LOCAL_PATH)/VerifierDepsMulti/*.smali)) @@ -156,6 +172,7 @@ ART_GTEST_class_loader_context_test_DEX_DEPS := Main MultiDex MyClass ForClassLo ART_GTEST_class_table_test_DEX_DEPS := XandY ART_GTEST_compiler_driver_test_DEX_DEPS := AbstractMethod StaticLeafMethods ProfileTestMultiDex ART_GTEST_dex_cache_test_DEX_DEPS := Main Packages MethodTypes +ART_GTEST_dexanalyze_test_DEX_DEPS := MultiDex ART_GTEST_dexlayout_test_DEX_DEPS := ManyMethods ART_GTEST_dex2oat_test_DEX_DEPS := $(ART_GTEST_dex2oat_environment_tests_DEX_DEPS) ManyMethods Statics VerifierDeps MainUncompressed EmptyUncompressed ART_GTEST_dex2oat_image_test_DEX_DEPS := $(ART_GTEST_dex2oat_environment_tests_DEX_DEPS) Statics VerifierDeps @@ -295,6 +312,12 @@ ART_GTEST_imgdiag_test_TARGET_DEPS := \ $(TARGET_CORE_IMAGE_DEFAULT_32) \ imgdiagd-target +# Dex analyze test requires dexanalyze. +ART_GTEST_dexanalyze_test_HOST_DEPS := \ + dexanalyze-host +ART_GTEST_dexanalyze_test_TARGET_DEPS := \ + dexanalyze-target + # Oatdump test requires an image and oatfile to dump. ART_GTEST_oatdump_test_HOST_DEPS := \ $(HOST_CORE_IMAGE_DEFAULT_64) \ @@ -337,6 +360,7 @@ ART_TEST_MODULES := \ art_compiler_tests \ art_compiler_host_tests \ art_dex2oat_tests \ + art_dexanalyze_tests \ art_dexdiag_tests \ art_dexdump_tests \ art_dexlayout_tests \ @@ -672,7 +696,7 @@ endef # define-art-gtest-host-both ifeq ($(ART_BUILD_TARGET),true) $(foreach file,$(ART_TARGET_GTEST_FILES), $(eval $(call define-art-gtest-target,$(file),))) - ifdef TARGET_2ND_ARCH + ifdef 2ND_ART_PHONY_TEST_TARGET_SUFFIX $(foreach file,$(2ND_ART_TARGET_GTEST_FILES), $(eval $(call define-art-gtest-target,$(file),2ND_))) endif # Rules to run the different architecture versions of the gtest. @@ -689,7 +713,7 @@ endif # Used outside the art project to get a list of the current tests RUNTIME_TARGET_GTEST_MAKE_TARGETS := -$(foreach file, $(ART_TARGET_GTEST_FILES), $(eval RUNTIME_TARGET_GTEST_MAKE_TARGETS += $$(notdir $$(basename $$(file))))) +$(foreach file, $(ART_TARGET_GTEST_FILES), $(eval RUNTIME_TARGET_GTEST_MAKE_TARGETS += $$(notdir $$(patsubst %/,%,$$(dir $$(file))))_$$(notdir $$(basename $$(file))))) COMPILER_TARGET_GTEST_MAKE_TARGETS := # Define all the combinations of host/target, valgrind and suffix such as: @@ -732,7 +756,7 @@ $(eval $(call define-test-art-gtest-combination,target,TARGET,,)) $(eval $(call define-test-art-gtest-combination,target,TARGET,valgrind-,)) $(eval $(call define-test-art-gtest-combination,target,TARGET,,$(ART_PHONY_TEST_TARGET_SUFFIX))) $(eval $(call define-test-art-gtest-combination,target,TARGET,valgrind-,$(ART_PHONY_TEST_TARGET_SUFFIX))) -ifdef TARGET_2ND_ARCH +ifdef 2ND_ART_PHONY_TEST_TARGET_SUFFIX $(eval $(call define-test-art-gtest-combination,target,TARGET,,$(2ND_ART_PHONY_TEST_TARGET_SUFFIX))) $(eval $(call define-test-art-gtest-combination,target,TARGET,valgrind-,$(2ND_ART_PHONY_TEST_TARGET_SUFFIX))) endif @@ -782,6 +806,7 @@ ART_GTEST_jni_internal_test_DEX_DEPS := ART_GTEST_oat_file_assistant_test_DEX_DEPS := ART_GTEST_oat_file_assistant_test_HOST_DEPS := ART_GTEST_oat_file_assistant_test_TARGET_DEPS := +ART_GTEST_dexanalyze_test_DEX_DEPS := ART_GTEST_dexoptanalyzer_test_DEX_DEPS := ART_GTEST_dexoptanalyzer_test_HOST_DEPS := ART_GTEST_dexoptanalyzer_test_TARGET_DEPS := diff --git a/cmdline/cmdline_types.h b/cmdline/cmdline_types.h index c8be69d9227..5a25a6ceb68 100644 --- a/cmdline/cmdline_types.h +++ b/cmdline/cmdline_types.h @@ -416,8 +416,6 @@ static gc::CollectorType ParseCollectorType(const std::string& option) { return gc::kCollectorTypeGSS; } else if (option == "CC") { return gc::kCollectorTypeCC; - } else if (option == "MC") { - return gc::kCollectorTypeMC; } else { return gc::kCollectorTypeNone; } diff --git a/compiler/Android.bp b/compiler/Android.bp index e42261c5561..cde64b058cb 100644 --- a/compiler/Android.bp +++ b/compiler/Android.bp @@ -39,7 +39,6 @@ art_cc_defaults { "linker/file_output_stream.cc", "linker/output_stream.cc", "linker/vector_output_stream.cc", - "linker/relative_patcher.cc", "jit/jit_compiler.cc", "jit/jit_logger.cc", "jni/quick/calling_convention.cc", @@ -70,6 +69,7 @@ art_cc_defaults { "optimizing/load_store_analysis.cc", "optimizing/load_store_elimination.cc", "optimizing/locations.cc", + "optimizing/loop_analysis.cc", "optimizing/loop_optimization.cc", "optimizing/nodes.cc", "optimizing/optimization.cc", @@ -101,8 +101,6 @@ art_cc_defaults { arm: { srcs: [ "jni/quick/arm/calling_convention_arm.cc", - "linker/arm/relative_patcher_arm_base.cc", - "linker/arm/relative_patcher_thumb2.cc", "optimizing/code_generator_arm_vixl.cc", "optimizing/code_generator_vector_arm_vixl.cc", "optimizing/instruction_simplifier_arm.cc", @@ -119,7 +117,6 @@ art_cc_defaults { arm64: { srcs: [ "jni/quick/arm64/calling_convention_arm64.cc", - "linker/arm64/relative_patcher_arm64.cc", "optimizing/code_generator_arm64.cc", "optimizing/code_generator_vector_arm64.cc", "optimizing/scheduler_arm64.cc", @@ -133,7 +130,6 @@ art_cc_defaults { mips: { srcs: [ "jni/quick/mips/calling_convention_mips.cc", - "linker/mips/relative_patcher_mips.cc", "optimizing/code_generator_mips.cc", "optimizing/code_generator_vector_mips.cc", "optimizing/instruction_simplifier_mips.cc", @@ -146,7 +142,6 @@ art_cc_defaults { mips64: { srcs: [ "jni/quick/mips64/calling_convention_mips64.cc", - "linker/mips64/relative_patcher_mips64.cc", "optimizing/code_generator_mips64.cc", "optimizing/code_generator_vector_mips64.cc", "optimizing/intrinsics_mips64.cc", @@ -157,8 +152,6 @@ art_cc_defaults { x86: { srcs: [ "jni/quick/x86/calling_convention_x86.cc", - "linker/x86/relative_patcher_x86.cc", - "linker/x86/relative_patcher_x86_base.cc", "optimizing/code_generator_x86.cc", "optimizing/code_generator_vector_x86.cc", "optimizing/intrinsics_x86.cc", @@ -172,7 +165,6 @@ art_cc_defaults { x86_64: { srcs: [ "jni/quick/x86_64/calling_convention_x86_64.cc", - "linker/x86_64/relative_patcher_x86_64.cc", "optimizing/intrinsics_x86_64.cc", "optimizing/code_generator_x86_64.cc", "optimizing/code_generator_vector_x86_64.cc", @@ -372,31 +364,25 @@ art_cc_test { codegen: { arm: { srcs: [ - "linker/arm/relative_patcher_thumb2_test.cc", "utils/arm/managed_register_arm_test.cc", ], }, arm64: { srcs: [ - "linker/arm64/relative_patcher_arm64_test.cc", "utils/arm64/managed_register_arm64_test.cc", ], }, mips: { srcs: [ - "linker/mips/relative_patcher_mips_test.cc", - "linker/mips/relative_patcher_mips32r6_test.cc", ], }, mips64: { srcs: [ - "linker/mips64/relative_patcher_mips64_test.cc", "utils/mips64/managed_register_mips64_test.cc", ], }, x86: { srcs: [ - "linker/x86/relative_patcher_x86_test.cc", "utils/x86/managed_register_x86_test.cc", // These tests are testing architecture-independent @@ -412,7 +398,8 @@ art_cc_test { }, x86_64: { srcs: [ - "linker/x86_64/relative_patcher_x86_64_test.cc", + // Is this test a bit-rotten copy of the x86 test? b/77951326 + // "utils/x86_64/managed_register_x86_64_test.cc", ], }, }, diff --git a/compiler/cfi_test.h b/compiler/cfi_test.h index 29ff235cea7..581edaa773c 100644 --- a/compiler/cfi_test.h +++ b/compiler/cfi_test.h @@ -37,8 +37,8 @@ constexpr dwarf::CFIFormat kCFIFormat = dwarf::DW_DEBUG_FRAME_FORMAT; class CFITest : public dwarf::DwarfTest { public: void GenerateExpected(FILE* f, InstructionSet isa, const char* isa_str, - const std::vector<uint8_t>& actual_asm, - const std::vector<uint8_t>& actual_cfi) { + ArrayRef<const uint8_t> actual_asm, + ArrayRef<const uint8_t> actual_cfi) { std::vector<std::string> lines; // Print the raw bytes. fprintf(f, "static constexpr uint8_t expected_asm_%s[] = {", isa_str); @@ -50,11 +50,18 @@ class CFITest : public dwarf::DwarfTest { // Pretty-print CFI opcodes. constexpr bool is64bit = false; dwarf::DebugFrameOpCodeWriter<> initial_opcodes; - dwarf::WriteCIE(is64bit, dwarf::Reg(8), - initial_opcodes, kCFIFormat, &debug_frame_data_); + dwarf::WriteCIE(is64bit, dwarf::Reg(8), initial_opcodes, kCFIFormat, &debug_frame_data_); std::vector<uintptr_t> debug_frame_patches; - dwarf::WriteFDE(is64bit, 0, 0, 0, actual_asm.size(), ArrayRef<const uint8_t>(actual_cfi), - kCFIFormat, 0, &debug_frame_data_, &debug_frame_patches); + dwarf::WriteFDE(is64bit, + /* section_address */ 0, + /* cie_address */ 0, + /* code_address */ 0, + actual_asm.size(), + actual_cfi, + kCFIFormat, + /* buffer_address */ 0, + &debug_frame_data_, + &debug_frame_patches); ReformatCfi(Objdump(false, "-W"), &lines); // Pretty-print assembly. const uint8_t* asm_base = actual_asm.data(); @@ -142,7 +149,7 @@ class CFITest : public dwarf::DwarfTest { } // Pretty-print byte array. 12 bytes per line. - static void HexDump(FILE* f, const std::vector<uint8_t>& data) { + static void HexDump(FILE* f, ArrayRef<const uint8_t> data) { for (size_t i = 0; i < data.size(); i++) { fprintf(f, i % 12 == 0 ? "\n " : " "); // Whitespace. fprintf(f, "0x%02X,", data[i]); diff --git a/compiler/common_compiler_test.cc b/compiler/common_compiler_test.cc index d3e3a51f7a5..96a0c1be4db 100644 --- a/compiler/common_compiler_test.cc +++ b/compiler/common_compiler_test.cc @@ -138,20 +138,6 @@ std::unordered_set<std::string>* CommonCompilerTest::GetImageClasses() { return new std::unordered_set<std::string>(); } -// Get the set of compiled classes given to the compiler-driver in SetUp. Note: the compiler -// driver assumes ownership of the set, so the test should properly release the set. -std::unordered_set<std::string>* CommonCompilerTest::GetCompiledClasses() { - // Null, no selection of compiled-classes. - return nullptr; -} - -// Get the set of compiled methods given to the compiler-driver in SetUp. Note: the compiler -// driver assumes ownership of the set, so the test should properly release the set. -std::unordered_set<std::string>* CommonCompilerTest::GetCompiledMethods() { - // Null, no selection of compiled-methods. - return nullptr; -} - // Get ProfileCompilationInfo that should be passed to the driver. ProfileCompilationInfo* CommonCompilerTest::GetProfileCompilationInfo() { // Null, profile information will not be taken into account. @@ -190,8 +176,6 @@ void CommonCompilerTest::CreateCompilerDriver(Compiler::Kind kind, isa, instruction_set_features_.get(), GetImageClasses(), - GetCompiledClasses(), - GetCompiledMethods(), number_of_threads, /* swap_fd */ -1, GetProfileCompilationInfo())); diff --git a/compiler/common_compiler_test.h b/compiler/common_compiler_test.h index 8af29d44f0c..39c8bd817bb 100644 --- a/compiler/common_compiler_test.h +++ b/compiler/common_compiler_test.h @@ -67,14 +67,6 @@ class CommonCompilerTest : public CommonRuntimeTest { // driver assumes ownership of the set, so the test should properly release the set. virtual std::unordered_set<std::string>* GetImageClasses(); - // Get the set of compiled classes given to the compiler-driver in SetUp. Note: the compiler - // driver assumes ownership of the set, so the test should properly release the set. - virtual std::unordered_set<std::string>* GetCompiledClasses(); - - // Get the set of compiled methods given to the compiler-driver in SetUp. Note: the compiler - // driver assumes ownership of the set, so the test should properly release the set. - virtual std::unordered_set<std::string>* GetCompiledMethods(); - virtual ProfileCompilationInfo* GetProfileCompilationInfo(); virtual CompilerFilter::Filter GetCompilerFilter() const { diff --git a/compiler/driver/compiled_method_storage.cc b/compiler/driver/compiled_method_storage.cc index a26a985ff9b..aa8277edb4d 100644 --- a/compiler/driver/compiled_method_storage.cc +++ b/compiler/driver/compiled_method_storage.cc @@ -161,6 +161,46 @@ class CompiledMethodStorage::LengthPrefixedArrayAlloc { SwapSpace* const swap_space_; }; +class CompiledMethodStorage::ThunkMapKey { + public: + ThunkMapKey(linker::LinkerPatch::Type type, uint32_t custom_value1, uint32_t custom_value2) + : type_(type), custom_value1_(custom_value1), custom_value2_(custom_value2) {} + + bool operator<(const ThunkMapKey& other) const { + if (custom_value1_ != other.custom_value1_) { + return custom_value1_ < other.custom_value1_; + } + if (custom_value2_ != other.custom_value2_) { + return custom_value2_ < other.custom_value2_; + } + return type_ < other.type_; + } + + private: + linker::LinkerPatch::Type type_; + uint32_t custom_value1_; + uint32_t custom_value2_; +}; + +class CompiledMethodStorage::ThunkMapValue { + public: + ThunkMapValue(std::vector<uint8_t, SwapAllocator<uint8_t>>&& code, + const std::string& debug_name) + : code_(std::move(code)), debug_name_(debug_name) {} + + ArrayRef<const uint8_t> GetCode() const { + return ArrayRef<const uint8_t>(code_); + } + + const std::string& GetDebugName() const { + return debug_name_; + } + + private: + std::vector<uint8_t, SwapAllocator<uint8_t>> code_; + std::string debug_name_; +}; + CompiledMethodStorage::CompiledMethodStorage(int swap_fd) : swap_space_(swap_fd == -1 ? nullptr : new SwapSpace(swap_fd, 10 * MB)), dedupe_enabled_(true), @@ -171,7 +211,9 @@ CompiledMethodStorage::CompiledMethodStorage(int swap_fd) LengthPrefixedArrayAlloc<uint8_t>(swap_space_.get())), dedupe_cfi_info_("dedupe cfi info", LengthPrefixedArrayAlloc<uint8_t>(swap_space_.get())), dedupe_linker_patches_("dedupe cfi info", - LengthPrefixedArrayAlloc<linker::LinkerPatch>(swap_space_.get())) { + LengthPrefixedArrayAlloc<linker::LinkerPatch>(swap_space_.get())), + thunk_map_lock_("thunk_map_lock"), + thunk_map_(std::less<ThunkMapKey>(), SwapAllocator<ThunkMapValueType>(swap_space_.get())) { } CompiledMethodStorage::~CompiledMethodStorage() { @@ -237,4 +279,55 @@ void CompiledMethodStorage::ReleaseLinkerPatches( ReleaseArrayIfNotDeduplicated(linker_patches); } +CompiledMethodStorage::ThunkMapKey CompiledMethodStorage::GetThunkMapKey( + const linker::LinkerPatch& linker_patch) { + uint32_t custom_value1 = 0u; + uint32_t custom_value2 = 0u; + switch (linker_patch.GetType()) { + case linker::LinkerPatch::Type::kBakerReadBarrierBranch: + custom_value1 = linker_patch.GetBakerCustomValue1(); + custom_value2 = linker_patch.GetBakerCustomValue2(); + break; + case linker::LinkerPatch::Type::kCallRelative: + // No custom values. + break; + default: + LOG(FATAL) << "Unexpected patch type: " << linker_patch.GetType(); + UNREACHABLE(); + } + return ThunkMapKey(linker_patch.GetType(), custom_value1, custom_value2); +} + +ArrayRef<const uint8_t> CompiledMethodStorage::GetThunkCode(const linker::LinkerPatch& linker_patch, + /*out*/ std::string* debug_name) { + ThunkMapKey key = GetThunkMapKey(linker_patch); + MutexLock lock(Thread::Current(), thunk_map_lock_); + auto it = thunk_map_.find(key); + if (it != thunk_map_.end()) { + const ThunkMapValue& value = it->second; + if (debug_name != nullptr) { + *debug_name = value.GetDebugName(); + } + return value.GetCode(); + } else { + if (debug_name != nullptr) { + *debug_name = std::string(); + } + return ArrayRef<const uint8_t>(); + } +} + +void CompiledMethodStorage::SetThunkCode(const linker::LinkerPatch& linker_patch, + ArrayRef<const uint8_t> code, + const std::string& debug_name) { + DCHECK(!code.empty()); + ThunkMapKey key = GetThunkMapKey(linker_patch); + std::vector<uint8_t, SwapAllocator<uint8_t>> code_copy( + code.begin(), code.end(), SwapAllocator<uint8_t>(swap_space_.get())); + ThunkMapValue value(std::move(code_copy), debug_name); + MutexLock lock(Thread::Current(), thunk_map_lock_); + // Note: Multiple threads can try and compile the same thunk, so this may not create a new entry. + thunk_map_.emplace(key, std::move(value)); +} + } // namespace art diff --git a/compiler/driver/compiled_method_storage.h b/compiler/driver/compiled_method_storage.h index 249f06c20f3..1634facb7ca 100644 --- a/compiler/driver/compiled_method_storage.h +++ b/compiler/driver/compiled_method_storage.h @@ -18,6 +18,7 @@ #define ART_COMPILER_DRIVER_COMPILED_METHOD_STORAGE_H_ #include <iosfwd> +#include <map> #include <memory> #include "base/array_ref.h" @@ -67,7 +68,29 @@ class CompiledMethodStorage { const ArrayRef<const linker::LinkerPatch>& linker_patches); void ReleaseLinkerPatches(const LengthPrefixedArray<linker::LinkerPatch>* linker_patches); + // Returns the code associated with the given patch. + // If the code has not been set, returns empty data. + // If `debug_name` is not null, stores the associated debug name in `*debug_name`. + ArrayRef<const uint8_t> GetThunkCode(const linker::LinkerPatch& linker_patch, + /*out*/ std::string* debug_name = nullptr); + + // Sets the code and debug name associated with the given patch. + void SetThunkCode(const linker::LinkerPatch& linker_patch, + ArrayRef<const uint8_t> code, + const std::string& debug_name); + private: + class ThunkMapKey; + class ThunkMapValue; + using ThunkMapValueType = std::pair<const ThunkMapKey, ThunkMapValue>; + using ThunkMap = std::map<ThunkMapKey, + ThunkMapValue, + std::less<ThunkMapKey>, + SwapAllocator<ThunkMapValueType>>; + static_assert(std::is_same<ThunkMapValueType, ThunkMap::value_type>::value, "Value type check."); + + static ThunkMapKey GetThunkMapKey(const linker::LinkerPatch& linker_patch); + template <typename T, typename DedupeSetType> const LengthPrefixedArray<T>* AllocateOrDeduplicateArray(const ArrayRef<const T>& data, DedupeSetType* dedupe_set); @@ -102,6 +125,9 @@ class CompiledMethodStorage { ArrayDedupeSet<uint8_t> dedupe_cfi_info_; ArrayDedupeSet<linker::LinkerPatch> dedupe_linker_patches_; + Mutex thunk_map_lock_; + ThunkMap thunk_map_ GUARDED_BY(thunk_map_lock_); + DISALLOW_COPY_AND_ASSIGN(CompiledMethodStorage); }; diff --git a/compiler/driver/compiled_method_storage_test.cc b/compiler/driver/compiled_method_storage_test.cc index 0769561d0ed..42fbba5109e 100644 --- a/compiler/driver/compiled_method_storage_test.cc +++ b/compiler/driver/compiled_method_storage_test.cc @@ -34,8 +34,6 @@ TEST(CompiledMethodStorage, Deduplicate) { /* instruction_set_ */ InstructionSet::kNone, /* instruction_set_features */ nullptr, /* image_classes */ nullptr, - /* compiled_classes */ nullptr, - /* compiled_methods */ nullptr, /* thread_count */ 1u, /* swap_fd */ -1, /* profile_compilation_info */ nullptr); diff --git a/compiler/driver/compiler_driver.cc b/compiler/driver/compiler_driver.cc index 53604761d12..41b7e7be47f 100644 --- a/compiler/driver/compiler_driver.cc +++ b/compiler/driver/compiler_driver.cc @@ -264,8 +264,6 @@ CompilerDriver::CompilerDriver( InstructionSet instruction_set, const InstructionSetFeatures* instruction_set_features, std::unordered_set<std::string>* image_classes, - std::unordered_set<std::string>* compiled_classes, - std::unordered_set<std::string>* compiled_methods, size_t thread_count, int swap_fd, const ProfileCompilationInfo* profile_compilation_info) @@ -279,8 +277,6 @@ CompilerDriver::CompilerDriver( requires_constructor_barrier_lock_("constructor barrier lock"), non_relative_linker_patch_count_(0u), image_classes_(image_classes), - classes_to_compile_(compiled_classes), - methods_to_compile_(compiled_methods), number_of_soft_verifier_failures_(0), had_hard_verifier_failure_(false), parallel_thread_count_(thread_count), @@ -638,7 +634,6 @@ static void CompileMethodQuick( (verified_method->GetEncounteredVerificationFailures() & (verifier::VERIFY_ERROR_FORCE_INTERPRETER | verifier::VERIFY_ERROR_LOCKING)) == 0 && // Is eligable for compilation by methods-to-compile filter. - driver->IsMethodToCompile(method_ref) && driver->ShouldCompileBasedOnProfile(method_ref); if (compile) { @@ -781,7 +776,8 @@ void CompilerDriver::Resolve(jobject class_loader, // TODO: Collect the relevant string indices in parallel, then allocate them sequentially in a // stable order. -static void ResolveConstStrings(Handle<mirror::DexCache> dex_cache, +static void ResolveConstStrings(ClassLinker* class_linker, + Handle<mirror::DexCache> dex_cache, const DexFile& dex_file, const DexFile::CodeItem* code_item) REQUIRES_SHARED(Locks::mutator_lock_) { @@ -790,7 +786,6 @@ static void ResolveConstStrings(Handle<mirror::DexCache> dex_cache, return; } - ClassLinker* const class_linker = Runtime::Current()->GetClassLinker(); for (const DexInstructionPcPair& inst : CodeItemInstructionAccessor(dex_file, code_item)) { switch (inst->Opcode()) { case Instruction::CONST_STRING: @@ -838,22 +833,105 @@ static void ResolveConstStrings(CompilerDriver* driver, dex_file->StringByTypeIdx(class_def.class_idx_)); if (!compilation_enabled) { // Compilation is skipped, do not resolve const-string in code of this class. - // TODO: Make sure that inlining honors this. + // FIXME: Make sure that inlining honors this. b/26687569 continue; } // Direct and virtual methods. - int64_t previous_method_idx = -1; while (it.HasNextMethod()) { - uint32_t method_idx = it.GetMemberIndex(); - if (method_idx == previous_method_idx) { - // smali can create dex files with two encoded_methods sharing the same method_idx - // http://code.google.com/p/smali/issues/detail?id=119 - it.Next(); - continue; + ResolveConstStrings(class_linker, dex_cache, *dex_file, it.GetMethodCodeItem()); + it.Next(); + } + DCHECK(!it.HasNext()); + } + } +} + +// Initialize type check bit strings for check-cast and instance-of in the code. Done to have +// deterministic allocation behavior. Right now this is single-threaded for simplicity. +// TODO: Collect the relevant type indices in parallel, then process them sequentially in a +// stable order. + +static void InitializeTypeCheckBitstrings(CompilerDriver* driver, + ClassLinker* class_linker, + Handle<mirror::DexCache> dex_cache, + const DexFile& dex_file, + const DexFile::CodeItem* code_item) + REQUIRES_SHARED(Locks::mutator_lock_) { + if (code_item == nullptr) { + // Abstract or native method. + return; + } + + for (const DexInstructionPcPair& inst : CodeItemInstructionAccessor(dex_file, code_item)) { + switch (inst->Opcode()) { + case Instruction::CHECK_CAST: + case Instruction::INSTANCE_OF: { + dex::TypeIndex type_index( + (inst->Opcode() == Instruction::CHECK_CAST) ? inst->VRegB_21c() : inst->VRegC_22c()); + const char* descriptor = dex_file.StringByTypeIdx(type_index); + // We currently do not use the bitstring type check for array or final (including + // primitive) classes. We may reconsider this in future if it's deemed to be beneficial. + // And we cannot use it for classes outside the boot image as we do not know the runtime + // value of their bitstring when compiling (it may not even get assigned at runtime). + if (descriptor[0] == 'L' && driver->IsImageClass(descriptor)) { + ObjPtr<mirror::Class> klass = + class_linker->LookupResolvedType(type_index, + dex_cache.Get(), + /* class_loader */ nullptr); + CHECK(klass != nullptr) << descriptor << " should have been previously resolved."; + // Now assign the bitstring if the class is not final. Keep this in sync with sharpening. + if (!klass->IsFinal()) { + MutexLock subtype_check_lock(Thread::Current(), *Locks::subtype_check_lock_); + SubtypeCheck<ObjPtr<mirror::Class>>::EnsureAssigned(klass); + } } - previous_method_idx = method_idx; - ResolveConstStrings(dex_cache, *dex_file, it.GetMethodCodeItem()); + break; + } + + default: + break; + } + } +} + +static void InitializeTypeCheckBitstrings(CompilerDriver* driver, + const std::vector<const DexFile*>& dex_files, + TimingLogger* timings) { + ScopedObjectAccess soa(Thread::Current()); + StackHandleScope<1> hs(soa.Self()); + ClassLinker* const class_linker = Runtime::Current()->GetClassLinker(); + MutableHandle<mirror::DexCache> dex_cache(hs.NewHandle<mirror::DexCache>(nullptr)); + + for (const DexFile* dex_file : dex_files) { + dex_cache.Assign(class_linker->FindDexCache(soa.Self(), *dex_file)); + TimingLogger::ScopedTiming t("Initialize type check bitstrings", timings); + + size_t class_def_count = dex_file->NumClassDefs(); + for (size_t class_def_index = 0; class_def_index < class_def_count; ++class_def_index) { + const DexFile::ClassDef& class_def = dex_file->GetClassDef(class_def_index); + + const uint8_t* class_data = dex_file->GetClassData(class_def); + if (class_data == nullptr) { + // empty class, probably a marker interface + continue; + } + + ClassDataItemIterator it(*dex_file, class_data); + it.SkipAllFields(); + + bool compilation_enabled = driver->IsClassToCompile( + dex_file->StringByTypeIdx(class_def.class_idx_)); + if (!compilation_enabled) { + // Compilation is skipped, do not look for type checks in code of this class. + // FIXME: Make sure that inlining honors this. b/26687569 + continue; + } + + // Direct and virtual methods. + while (it.HasNextMethod()) { + InitializeTypeCheckBitstrings( + driver, class_linker, dex_cache, *dex_file, it.GetMethodCodeItem()); it.Next(); } DCHECK(!it.HasNext()); @@ -955,6 +1033,14 @@ void CompilerDriver::PreCompile(jobject class_loader, UpdateImageClasses(timings); VLOG(compiler) << "UpdateImageClasses: " << GetMemoryUsageString(false); + + if (kBitstringSubtypeCheckEnabled && + GetCompilerOptions().IsForceDeterminism() && GetCompilerOptions().IsBootImage()) { + // Initialize type check bit string used by check-cast and instanceof. + // Do this now to have a deterministic image. + // Note: This is done after UpdateImageClasses() at it relies on the image classes to be final. + InitializeTypeCheckBitstrings(this, dex_files, timings); + } } bool CompilerDriver::IsImageClass(const char* descriptor) const { @@ -974,15 +1060,6 @@ bool CompilerDriver::IsClassToCompile(const char* descriptor) const { return classes_to_compile_->find(descriptor) != classes_to_compile_->end(); } -bool CompilerDriver::IsMethodToCompile(const MethodReference& method_ref) const { - if (methods_to_compile_ == nullptr) { - return true; - } - - std::string tmp = method_ref.PrettyMethod(); - return methods_to_compile_->find(tmp.c_str()) != methods_to_compile_->end(); -} - bool CompilerDriver::ShouldCompileBasedOnProfile(const MethodReference& method_ref) const { // Profile compilation info may be null if no profile is passed. if (!CompilerFilter::DependsOnProfile(compiler_options_->GetCompilerFilter())) { @@ -1555,7 +1632,7 @@ class ParallelCompilationManager { self->AssertNoPendingException(); CHECK_GT(work_units, 0U); - index_.StoreRelaxed(begin); + index_.store(begin, std::memory_order_relaxed); for (size_t i = 0; i < work_units; ++i) { thread_pool_->AddTask(self, new ForAllClosureLambda<Fn>(this, end, fn)); } @@ -1573,7 +1650,7 @@ class ParallelCompilationManager { } size_t NextIndex() { - return index_.FetchAndAddSequentiallyConsistent(1); + return index_.fetch_add(1, std::memory_order_seq_cst); } private: @@ -2838,7 +2915,8 @@ void CompilerDriver::AddCompiledMethod(const MethodReference& method_ref, /*expected*/ nullptr, compiled_method); CHECK(result == MethodTable::kInsertResultSuccess); - non_relative_linker_patch_count_.FetchAndAddRelaxed(non_relative_linker_patch_count); + non_relative_linker_patch_count_.fetch_add(non_relative_linker_patch_count, + std::memory_order_relaxed); DCHECK(GetCompiledMethod(method_ref) != nullptr) << method_ref.PrettyMethod(); } @@ -2949,7 +3027,7 @@ bool CompilerDriver::IsMethodVerifiedWithoutFailures(uint32_t method_idx, } size_t CompilerDriver::GetNonRelativeLinkerPatchCount() const { - return non_relative_linker_patch_count_.LoadRelaxed(); + return non_relative_linker_patch_count_.load(std::memory_order_relaxed); } void CompilerDriver::SetRequiresConstructorBarrier(Thread* self, diff --git a/compiler/driver/compiler_driver.h b/compiler/driver/compiler_driver.h index a5462eefe2a..55f3561e3a8 100644 --- a/compiler/driver/compiler_driver.h +++ b/compiler/driver/compiler_driver.h @@ -100,8 +100,6 @@ class CompilerDriver { InstructionSet instruction_set, const InstructionSetFeatures* instruction_set_features, std::unordered_set<std::string>* image_classes, - std::unordered_set<std::string>* compiled_classes, - std::unordered_set<std::string>* compiled_methods, size_t thread_count, int swap_fd, const ProfileCompilationInfo* profile_compilation_info); @@ -316,9 +314,6 @@ class CompilerDriver { // Checks whether the provided class should be compiled, i.e., is in classes_to_compile_. bool IsClassToCompile(const char* descriptor) const; - // Checks whether the provided method should be compiled, i.e., is in method_to_compile_. - bool IsMethodToCompile(const MethodReference& method_ref) const; - // Checks whether profile guided compilation is enabled and if the method should be compiled // according to the profile file. bool ShouldCompileBasedOnProfile(const MethodReference& method_ref) const; @@ -505,12 +500,8 @@ class CompilerDriver { // This option may be restricted to the boot image, depending on a flag in the implementation. std::unique_ptr<std::unordered_set<std::string>> classes_to_compile_; - // Specifies the methods that will be compiled. Note that if methods_to_compile_ is null, - // all methods are eligible for compilation (compilation filters etc. will still apply). - // This option may be restricted to the boot image, depending on a flag in the implementation. - std::unique_ptr<std::unordered_set<std::string>> methods_to_compile_; - std::atomic<uint32_t> number_of_soft_verifier_failures_; + bool had_hard_verifier_failure_; // A thread pool that can (potentially) run tasks in parallel. diff --git a/compiler/driver/compiler_driver_test.cc b/compiler/driver/compiler_driver_test.cc index 162904c0e73..1332280d20c 100644 --- a/compiler/driver/compiler_driver_test.cc +++ b/compiler/driver/compiler_driver_test.cc @@ -184,59 +184,6 @@ TEST_F(CompilerDriverTest, AbstractMethodErrorStub) { } } -class CompilerDriverMethodsTest : public CompilerDriverTest { - protected: - std::unordered_set<std::string>* GetCompiledMethods() OVERRIDE { - return new std::unordered_set<std::string>({ - "byte StaticLeafMethods.identity(byte)", - "int StaticLeafMethods.sum(int, int, int)", - "double StaticLeafMethods.sum(double, double, double, double)" - }); - } -}; - -TEST_F(CompilerDriverMethodsTest, Selection) { - Thread* self = Thread::Current(); - jobject class_loader; - { - ScopedObjectAccess soa(self); - class_loader = LoadDex("StaticLeafMethods"); - } - ASSERT_NE(class_loader, nullptr); - - // Need to enable dex-file writability. Methods rejected to be compiled will run through the - // dex-to-dex compiler. - for (const DexFile* dex_file : GetDexFiles(class_loader)) { - ASSERT_TRUE(dex_file->EnableWrite()); - } - - CompileAll(class_loader); - - ClassLinker* class_linker = Runtime::Current()->GetClassLinker(); - ScopedObjectAccess soa(self); - StackHandleScope<1> hs(self); - Handle<mirror::ClassLoader> h_loader( - hs.NewHandle(soa.Decode<mirror::ClassLoader>(class_loader))); - mirror::Class* klass = class_linker->FindClass(self, "LStaticLeafMethods;", h_loader); - ASSERT_NE(klass, nullptr); - - std::unique_ptr<std::unordered_set<std::string>> expected(GetCompiledMethods()); - - const auto pointer_size = class_linker->GetImagePointerSize(); - for (auto& m : klass->GetDirectMethods(pointer_size)) { - std::string name = m.PrettyMethod(true); - const void* code = m.GetEntryPointFromQuickCompiledCodePtrSize(pointer_size); - ASSERT_NE(code, nullptr); - if (expected->find(name) != expected->end()) { - expected->erase(name); - EXPECT_FALSE(class_linker->IsQuickToInterpreterBridge(code)); - } else { - EXPECT_TRUE(class_linker->IsQuickToInterpreterBridge(code)); - } - } - EXPECT_TRUE(expected->empty()); -} - class CompilerDriverProfileTest : public CompilerDriverTest { protected: ProfileCompilationInfo* GetProfileCompilationInfo() OVERRIDE { diff --git a/compiler/exception_test.cc b/compiler/exception_test.cc index f582341b180..c139fcf1d8a 100644 --- a/compiler/exception_test.cc +++ b/compiler/exception_test.cc @@ -20,6 +20,7 @@ #include "base/callee_save_type.h" #include "base/enums.h" #include "base/leb128.h" +#include "base/malloc_arena_pool.h" #include "class_linker.h" #include "common_runtime_test.h" #include "dex/code_item_accessors-inl.h" @@ -67,7 +68,7 @@ class ExceptionTest : public CommonRuntimeTest { fake_code_.push_back(0x70 | i); } - ArenaPool pool; + MallocArenaPool pool; ArenaStack arena_stack(&pool); ScopedArenaAllocator allocator(&arena_stack); StackMapStream stack_maps(&allocator, kRuntimeISA); diff --git a/compiler/jit/jit_compiler.cc b/compiler/jit/jit_compiler.cc index ac5c6fb01f8..0de00a82fa4 100644 --- a/compiler/jit/jit_compiler.cc +++ b/compiler/jit/jit_compiler.cc @@ -142,8 +142,6 @@ JitCompiler::JitCompiler() { instruction_set, instruction_set_features_.get(), /* image_classes */ nullptr, - /* compiled_classes */ nullptr, - /* compiled_methods */ nullptr, /* thread_count */ 1, /* swap_fd */ -1, /* profile_compilation_info */ nullptr)); diff --git a/compiler/jni/jni_cfi_test.cc b/compiler/jni/jni_cfi_test.cc index 236b5c0c2e3..920a3a8da63 100644 --- a/compiler/jni/jni_cfi_test.cc +++ b/compiler/jni/jni_cfi_test.cc @@ -20,6 +20,7 @@ #include "arch/instruction_set.h" #include "base/arena_allocator.h" #include "base/enums.h" +#include "base/malloc_arena_pool.h" #include "cfi_test.h" #include "gtest/gtest.h" #include "jni/quick/calling_convention.h" @@ -61,7 +62,7 @@ class JNICFITest : public CFITest { const bool is_synchronized = false; const char* shorty = "IIFII"; - ArenaPool pool; + MallocArenaPool pool; ArenaAllocator allocator(&pool); std::unique_ptr<JniCallingConvention> jni_conv( @@ -94,7 +95,11 @@ class JNICFITest : public CFITest { const std::vector<uint8_t>& actual_cfi = *(jni_asm->cfi().data()); if (kGenerateExpected) { - GenerateExpected(stdout, isa, isa_str, actual_asm, actual_cfi); + GenerateExpected(stdout, + isa, + isa_str, + ArrayRef<const uint8_t>(actual_asm), + ArrayRef<const uint8_t>(actual_cfi)); } else { EXPECT_EQ(expected_asm, actual_asm); EXPECT_EQ(expected_cfi, actual_cfi); diff --git a/compiler/jni/jni_compiler_test.cc b/compiler/jni/jni_compiler_test.cc index 451a9099651..730a1a63e8e 100644 --- a/compiler/jni/jni_compiler_test.cc +++ b/compiler/jni/jni_compiler_test.cc @@ -21,6 +21,7 @@ #include "art_method-inl.h" #include "base/bit_utils.h" +#include "base/mem_map.h" #include "class_linker.h" #include "common_compiler_test.h" #include "compiler.h" @@ -29,7 +30,6 @@ #include "indirect_reference_table.h" #include "java_vm_ext.h" #include "jni_internal.h" -#include "mem_map.h" #include "mirror/class-inl.h" #include "mirror/class_loader.h" #include "mirror/object-inl.h" diff --git a/compiler/jni/quick/jni_compiler.cc b/compiler/jni/quick/jni_compiler.cc index d001cfe4fc5..8cb1998f7f6 100644 --- a/compiler/jni/quick/jni_compiler.cc +++ b/compiler/jni/quick/jni_compiler.cc @@ -27,6 +27,8 @@ #include "base/enums.h" #include "base/logging.h" // For VLOG. #include "base/macros.h" +#include "base/malloc_arena_pool.h" +#include "base/memory_region.h" #include "base/utils.h" #include "calling_convention.h" #include "class_linker.h" @@ -36,7 +38,6 @@ #include "driver/compiler_options.h" #include "entrypoints/quick/quick_entrypoints.h" #include "jni_env_ext.h" -#include "memory_region.h" #include "thread.h" #include "utils/arm/managed_register_arm.h" #include "utils/arm64/managed_register_arm64.h" @@ -174,7 +175,7 @@ static JniCompiledMethod ArtJniCompileMethodInternal(CompilerDriver* driver, } } - ArenaPool pool; + MallocArenaPool pool; ArenaAllocator allocator(&pool); // Calling conventions used to iterate over parameters to method diff --git a/compiler/linker/arm/relative_patcher_thumb2.cc b/compiler/linker/arm/relative_patcher_thumb2.cc deleted file mode 100644 index 78755176e43..00000000000 --- a/compiler/linker/arm/relative_patcher_thumb2.cc +++ /dev/null @@ -1,484 +0,0 @@ -/* - * Copyright (C) 2015 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "linker/arm/relative_patcher_thumb2.h" - -#include <sstream> - -#include "arch/arm/asm_support_arm.h" -#include "art_method.h" -#include "base/bit_utils.h" -#include "compiled_method.h" -#include "entrypoints/quick/quick_entrypoints_enum.h" -#include "linker/linker_patch.h" -#include "lock_word.h" -#include "mirror/array-inl.h" -#include "mirror/object.h" -#include "read_barrier.h" -#include "utils/arm/assembler_arm_vixl.h" - -namespace art { -namespace linker { - -// PC displacement from patch location; Thumb2 PC is always at instruction address + 4. -static constexpr int32_t kPcDisplacement = 4; - -// Maximum positive and negative displacement for method call measured from the patch location. -// (Signed 25 bit displacement with the last bit 0 has range [-2^24, 2^24-2] measured from -// the Thumb2 PC pointing right after the BL, i.e. 4 bytes later than the patch location.) -constexpr uint32_t kMaxMethodCallPositiveDisplacement = (1u << 24) - 2 + kPcDisplacement; -constexpr uint32_t kMaxMethodCallNegativeDisplacement = (1u << 24) - kPcDisplacement; - -// Maximum positive and negative displacement for a conditional branch measured from the patch -// location. (Signed 21 bit displacement with the last bit 0 has range [-2^20, 2^20-2] measured -// from the Thumb2 PC pointing right after the B.cond, i.e. 4 bytes later than the patch location.) -constexpr uint32_t kMaxBcondPositiveDisplacement = (1u << 20) - 2u + kPcDisplacement; -constexpr uint32_t kMaxBcondNegativeDisplacement = (1u << 20) - kPcDisplacement; - -Thumb2RelativePatcher::Thumb2RelativePatcher(RelativePatcherTargetProvider* provider) - : ArmBaseRelativePatcher(provider, InstructionSet::kThumb2) { -} - -void Thumb2RelativePatcher::PatchCall(std::vector<uint8_t>* code, - uint32_t literal_offset, - uint32_t patch_offset, - uint32_t target_offset) { - DCHECK_LE(literal_offset + 4u, code->size()); - DCHECK_EQ(literal_offset & 1u, 0u); - DCHECK_EQ(patch_offset & 1u, 0u); - DCHECK_EQ(target_offset & 1u, 1u); // Thumb2 mode bit. - uint32_t displacement = CalculateMethodCallDisplacement(patch_offset, target_offset & ~1u); - displacement -= kPcDisplacement; // The base PC is at the end of the 4-byte patch. - DCHECK_EQ(displacement & 1u, 0u); - DCHECK((displacement >> 24) == 0u || (displacement >> 24) == 255u); // 25-bit signed. - uint32_t signbit = (displacement >> 31) & 0x1; - uint32_t i1 = (displacement >> 23) & 0x1; - uint32_t i2 = (displacement >> 22) & 0x1; - uint32_t imm10 = (displacement >> 12) & 0x03ff; - uint32_t imm11 = (displacement >> 1) & 0x07ff; - uint32_t j1 = i1 ^ (signbit ^ 1); - uint32_t j2 = i2 ^ (signbit ^ 1); - uint32_t value = (signbit << 26) | (j1 << 13) | (j2 << 11) | (imm10 << 16) | imm11; - value |= 0xf000d000; // BL - - // Check that we're just overwriting an existing BL. - DCHECK_EQ(GetInsn32(code, literal_offset) & 0xf800d000, 0xf000d000); - // Write the new BL. - SetInsn32(code, literal_offset, value); -} - -void Thumb2RelativePatcher::PatchPcRelativeReference(std::vector<uint8_t>* code, - const LinkerPatch& patch, - uint32_t patch_offset, - uint32_t target_offset) { - uint32_t literal_offset = patch.LiteralOffset(); - uint32_t pc_literal_offset = patch.PcInsnOffset(); - uint32_t pc_base = patch_offset + (pc_literal_offset - literal_offset) + 4u /* PC adjustment */; - uint32_t diff = target_offset - pc_base; - - uint32_t insn = GetInsn32(code, literal_offset); - DCHECK_EQ(insn & 0xff7ff0ffu, 0xf2400000u); // MOVW/MOVT, unpatched (imm16 == 0). - uint32_t diff16 = ((insn & 0x00800000u) != 0u) ? (diff >> 16) : (diff & 0xffffu); - uint32_t imm4 = (diff16 >> 12) & 0xfu; - uint32_t imm = (diff16 >> 11) & 0x1u; - uint32_t imm3 = (diff16 >> 8) & 0x7u; - uint32_t imm8 = diff16 & 0xffu; - insn = (insn & 0xfbf08f00u) | (imm << 26) | (imm4 << 16) | (imm3 << 12) | imm8; - SetInsn32(code, literal_offset, insn); -} - -void Thumb2RelativePatcher::PatchBakerReadBarrierBranch(std::vector<uint8_t>* code, - const LinkerPatch& patch, - uint32_t patch_offset) { - DCHECK_ALIGNED(patch_offset, 2u); - uint32_t literal_offset = patch.LiteralOffset(); - DCHECK_ALIGNED(literal_offset, 2u); - DCHECK_LT(literal_offset, code->size()); - uint32_t insn = GetInsn32(code, literal_offset); - DCHECK_EQ(insn, 0xf0408000); // BNE +0 (unpatched) - ThunkKey key = GetBakerThunkKey(patch); - if (kIsDebugBuild) { - const uint32_t encoded_data = key.GetCustomValue1(); - BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data); - // Check that the next instruction matches the expected LDR. - switch (kind) { - case BakerReadBarrierKind::kField: { - BakerReadBarrierWidth width = BakerReadBarrierWidthField::Decode(encoded_data); - if (width == BakerReadBarrierWidth::kWide) { - DCHECK_GE(code->size() - literal_offset, 8u); - uint32_t next_insn = GetInsn32(code, literal_offset + 4u); - // LDR (immediate), encoding T3, with correct base_reg. - CheckValidReg((next_insn >> 12) & 0xfu); // Check destination register. - const uint32_t base_reg = BakerReadBarrierFirstRegField::Decode(encoded_data); - CHECK_EQ(next_insn & 0xffff0000u, 0xf8d00000u | (base_reg << 16)); - } else { - DCHECK_GE(code->size() - literal_offset, 6u); - uint32_t next_insn = GetInsn16(code, literal_offset + 4u); - // LDR (immediate), encoding T1, with correct base_reg. - CheckValidReg(next_insn & 0x7u); // Check destination register. - const uint32_t base_reg = BakerReadBarrierFirstRegField::Decode(encoded_data); - CHECK_EQ(next_insn & 0xf838u, 0x6800u | (base_reg << 3)); - } - break; - } - case BakerReadBarrierKind::kArray: { - DCHECK_GE(code->size() - literal_offset, 8u); - uint32_t next_insn = GetInsn32(code, literal_offset + 4u); - // LDR (register) with correct base_reg, S=1 and option=011 (LDR Wt, [Xn, Xm, LSL #2]). - CheckValidReg((next_insn >> 12) & 0xfu); // Check destination register. - const uint32_t base_reg = BakerReadBarrierFirstRegField::Decode(encoded_data); - CHECK_EQ(next_insn & 0xffff0ff0u, 0xf8500020u | (base_reg << 16)); - CheckValidReg(next_insn & 0xf); // Check index register - break; - } - case BakerReadBarrierKind::kGcRoot: { - BakerReadBarrierWidth width = BakerReadBarrierWidthField::Decode(encoded_data); - if (width == BakerReadBarrierWidth::kWide) { - DCHECK_GE(literal_offset, 4u); - uint32_t prev_insn = GetInsn32(code, literal_offset - 4u); - // LDR (immediate), encoding T3, with correct root_reg. - const uint32_t root_reg = BakerReadBarrierFirstRegField::Decode(encoded_data); - CHECK_EQ(prev_insn & 0xfff0f000u, 0xf8d00000u | (root_reg << 12)); - } else { - DCHECK_GE(literal_offset, 2u); - uint32_t prev_insn = GetInsn16(code, literal_offset - 2u); - // LDR (immediate), encoding T1, with correct root_reg. - const uint32_t root_reg = BakerReadBarrierFirstRegField::Decode(encoded_data); - CHECK_EQ(prev_insn & 0xf807u, 0x6800u | root_reg); - } - break; - } - default: - LOG(FATAL) << "Unexpected type: " << static_cast<uint32_t>(key.GetType()); - UNREACHABLE(); - } - } - uint32_t target_offset = GetThunkTargetOffset(key, patch_offset); - DCHECK_ALIGNED(target_offset, 4u); - uint32_t disp = target_offset - (patch_offset + kPcDisplacement); - DCHECK((disp >> 20) == 0u || (disp >> 20) == 0xfffu); // 21-bit signed. - insn |= ((disp << (26 - 20)) & 0x04000000u) | // Shift bit 20 to 26, "S". - ((disp >> (19 - 11)) & 0x00000800u) | // Shift bit 19 to 13, "J1". - ((disp >> (18 - 13)) & 0x00002000u) | // Shift bit 18 to 11, "J2". - ((disp << (16 - 12)) & 0x003f0000u) | // Shift bits 12-17 to 16-25, "imm6". - ((disp >> (1 - 0)) & 0x000007ffu); // Shift bits 1-12 to 0-11, "imm11". - SetInsn32(code, literal_offset, insn); -} - -#define __ assembler.GetVIXLAssembler()-> - -static void EmitGrayCheckAndFastPath(arm::ArmVIXLAssembler& assembler, - vixl::aarch32::Register base_reg, - vixl::aarch32::MemOperand& lock_word, - vixl::aarch32::Label* slow_path, - int32_t raw_ldr_offset) { - using namespace vixl::aarch32; // NOLINT(build/namespaces) - // Load the lock word containing the rb_state. - __ Ldr(ip, lock_word); - // Given the numeric representation, it's enough to check the low bit of the rb_state. - static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); - static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); - __ Tst(ip, Operand(LockWord::kReadBarrierStateMaskShifted)); - __ B(ne, slow_path, /* is_far_target */ false); - __ Add(lr, lr, raw_ldr_offset); - // Introduce a dependency on the lock_word including rb_state, - // to prevent load-load reordering, and without using - // a memory barrier (which would be more expensive). - __ Add(base_reg, base_reg, Operand(ip, LSR, 32)); - __ Bx(lr); // And return back to the function. - // Note: The fake dependency is unnecessary for the slow path. -} - -// Load the read barrier introspection entrypoint in register `entrypoint` -static void LoadReadBarrierMarkIntrospectionEntrypoint(arm::ArmVIXLAssembler& assembler, - vixl::aarch32::Register entrypoint) { - using vixl::aarch32::MemOperand; - using vixl::aarch32::ip; - // Thread Register. - const vixl::aarch32::Register tr = vixl::aarch32::r9; - - // The register where the read barrier introspection entrypoint is loaded - // is fixed: `Thumb2RelativePatcher::kBakerCcEntrypointRegister` (R4). - DCHECK_EQ(entrypoint.GetCode(), Thumb2RelativePatcher::kBakerCcEntrypointRegister); - // entrypoint = Thread::Current()->pReadBarrierMarkReg12, i.e. pReadBarrierMarkIntrospection. - DCHECK_EQ(ip.GetCode(), 12u); - const int32_t entry_point_offset = - Thread::ReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ip.GetCode()); - __ Ldr(entrypoint, MemOperand(tr, entry_point_offset)); -} - -void Thumb2RelativePatcher::CompileBakerReadBarrierThunk(arm::ArmVIXLAssembler& assembler, - uint32_t encoded_data) { - using namespace vixl::aarch32; // NOLINT(build/namespaces) - BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data); - switch (kind) { - case BakerReadBarrierKind::kField: { - // Check if the holder is gray and, if not, add fake dependency to the base register - // and return to the LDR instruction to load the reference. Otherwise, use introspection - // to load the reference and call the entrypoint (in kBakerCcEntrypointRegister) - // that performs further checks on the reference and marks it if needed. - Register base_reg(BakerReadBarrierFirstRegField::Decode(encoded_data)); - CheckValidReg(base_reg.GetCode()); - Register holder_reg(BakerReadBarrierSecondRegField::Decode(encoded_data)); - CheckValidReg(holder_reg.GetCode()); - BakerReadBarrierWidth width = BakerReadBarrierWidthField::Decode(encoded_data); - UseScratchRegisterScope temps(assembler.GetVIXLAssembler()); - temps.Exclude(ip); - // If base_reg differs from holder_reg, the offset was too large and we must have - // emitted an explicit null check before the load. Otherwise, we need to null-check - // the holder as we do not necessarily do that check before going to the thunk. - vixl::aarch32::Label throw_npe; - if (holder_reg.Is(base_reg)) { - __ CompareAndBranchIfZero(holder_reg, &throw_npe, /* is_far_target */ false); - } - vixl::aarch32::Label slow_path; - MemOperand lock_word(holder_reg, mirror::Object::MonitorOffset().Int32Value()); - const int32_t raw_ldr_offset = (width == BakerReadBarrierWidth::kWide) - ? BAKER_MARK_INTROSPECTION_FIELD_LDR_WIDE_OFFSET - : BAKER_MARK_INTROSPECTION_FIELD_LDR_NARROW_OFFSET; - EmitGrayCheckAndFastPath(assembler, base_reg, lock_word, &slow_path, raw_ldr_offset); - __ Bind(&slow_path); - const int32_t ldr_offset = /* Thumb state adjustment (LR contains Thumb state). */ -1 + - raw_ldr_offset; - Register ep_reg(kBakerCcEntrypointRegister); - LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ep_reg); - if (width == BakerReadBarrierWidth::kWide) { - MemOperand ldr_half_address(lr, ldr_offset + 2); - __ Ldrh(ip, ldr_half_address); // Load the LDR immediate half-word with "Rt | imm12". - __ Ubfx(ip, ip, 0, 12); // Extract the offset imm12. - __ Ldr(ip, MemOperand(base_reg, ip)); // Load the reference. - } else { - MemOperand ldr_address(lr, ldr_offset); - __ Ldrh(ip, ldr_address); // Load the LDR immediate, encoding T1. - __ Add(ep_reg, // Adjust the entrypoint address to the entrypoint - ep_reg, // for narrow LDR. - Operand(BAKER_MARK_INTROSPECTION_FIELD_LDR_NARROW_ENTRYPOINT_OFFSET)); - __ Ubfx(ip, ip, 6, 5); // Extract the imm5, i.e. offset / 4. - __ Ldr(ip, MemOperand(base_reg, ip, LSL, 2)); // Load the reference. - } - // Do not unpoison. With heap poisoning enabled, the entrypoint expects a poisoned reference. - __ Bx(ep_reg); // Jump to the entrypoint. - if (holder_reg.Is(base_reg)) { - // Add null check slow path. The stack map is at the address pointed to by LR. - __ Bind(&throw_npe); - int32_t offset = GetThreadOffset<kArmPointerSize>(kQuickThrowNullPointer).Int32Value(); - __ Ldr(ip, MemOperand(/* Thread* */ vixl::aarch32::r9, offset)); - __ Bx(ip); - } - break; - } - case BakerReadBarrierKind::kArray: { - Register base_reg(BakerReadBarrierFirstRegField::Decode(encoded_data)); - CheckValidReg(base_reg.GetCode()); - DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data)); - DCHECK(BakerReadBarrierWidthField::Decode(encoded_data) == BakerReadBarrierWidth::kWide); - UseScratchRegisterScope temps(assembler.GetVIXLAssembler()); - temps.Exclude(ip); - vixl::aarch32::Label slow_path; - int32_t data_offset = - mirror::Array::DataOffset(Primitive::ComponentSize(Primitive::kPrimNot)).Int32Value(); - MemOperand lock_word(base_reg, mirror::Object::MonitorOffset().Int32Value() - data_offset); - DCHECK_LT(lock_word.GetOffsetImmediate(), 0); - const int32_t raw_ldr_offset = BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET; - EmitGrayCheckAndFastPath(assembler, base_reg, lock_word, &slow_path, raw_ldr_offset); - __ Bind(&slow_path); - const int32_t ldr_offset = /* Thumb state adjustment (LR contains Thumb state). */ -1 + - raw_ldr_offset; - MemOperand ldr_address(lr, ldr_offset + 2); - __ Ldrb(ip, ldr_address); // Load the LDR (register) byte with "00 | imm2 | Rm", - // i.e. Rm+32 because the scale in imm2 is 2. - Register ep_reg(kBakerCcEntrypointRegister); - LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ep_reg); - __ Bfi(ep_reg, ip, 3, 6); // Insert ip to the entrypoint address to create - // a switch case target based on the index register. - __ Mov(ip, base_reg); // Move the base register to ip0. - __ Bx(ep_reg); // Jump to the entrypoint's array switch case. - break; - } - case BakerReadBarrierKind::kGcRoot: { - // Check if the reference needs to be marked and if so (i.e. not null, not marked yet - // and it does not have a forwarding address), call the correct introspection entrypoint; - // otherwise return the reference (or the extracted forwarding address). - // There is no gray bit check for GC roots. - Register root_reg(BakerReadBarrierFirstRegField::Decode(encoded_data)); - CheckValidReg(root_reg.GetCode()); - DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data)); - BakerReadBarrierWidth width = BakerReadBarrierWidthField::Decode(encoded_data); - UseScratchRegisterScope temps(assembler.GetVIXLAssembler()); - temps.Exclude(ip); - vixl::aarch32::Label return_label, not_marked, forwarding_address; - __ CompareAndBranchIfZero(root_reg, &return_label, /* is_far_target */ false); - MemOperand lock_word(root_reg, mirror::Object::MonitorOffset().Int32Value()); - __ Ldr(ip, lock_word); - __ Tst(ip, LockWord::kMarkBitStateMaskShifted); - __ B(eq, ¬_marked); - __ Bind(&return_label); - __ Bx(lr); - __ Bind(¬_marked); - static_assert(LockWord::kStateShift == 30 && LockWord::kStateForwardingAddress == 3, - "To use 'CMP ip, #modified-immediate; BHS', we need the lock word state in " - " the highest bits and the 'forwarding address' state to have all bits set"); - __ Cmp(ip, Operand(0xc0000000)); - __ B(hs, &forwarding_address); - Register ep_reg(kBakerCcEntrypointRegister); - LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ep_reg); - // Adjust the art_quick_read_barrier_mark_introspection address in kBakerCcEntrypointRegister - // to art_quick_read_barrier_mark_introspection_gc_roots. - int32_t entrypoint_offset = (width == BakerReadBarrierWidth::kWide) - ? BAKER_MARK_INTROSPECTION_GC_ROOT_LDR_WIDE_ENTRYPOINT_OFFSET - : BAKER_MARK_INTROSPECTION_GC_ROOT_LDR_NARROW_ENTRYPOINT_OFFSET; - __ Add(ep_reg, ep_reg, Operand(entrypoint_offset)); - __ Mov(ip, root_reg); - __ Bx(ep_reg); - __ Bind(&forwarding_address); - __ Lsl(root_reg, ip, LockWord::kForwardingAddressShift); - __ Bx(lr); - break; - } - default: - LOG(FATAL) << "Unexpected kind: " << static_cast<uint32_t>(kind); - UNREACHABLE(); - } -} - -std::vector<uint8_t> Thumb2RelativePatcher::CompileThunk(const ThunkKey& key) { - ArenaPool pool; - ArenaAllocator allocator(&pool); - arm::ArmVIXLAssembler assembler(&allocator); - - switch (key.GetType()) { - case ThunkType::kMethodCall: - // The thunk just uses the entry point in the ArtMethod. This works even for calls - // to the generic JNI and interpreter trampolines. - assembler.LoadFromOffset( - arm::kLoadWord, - vixl::aarch32::pc, - vixl::aarch32::r0, - ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value()); - __ Bkpt(0); - break; - case ThunkType::kBakerReadBarrier: - CompileBakerReadBarrierThunk(assembler, key.GetCustomValue1()); - break; - } - - assembler.FinalizeCode(); - std::vector<uint8_t> thunk_code(assembler.CodeSize()); - MemoryRegion code(thunk_code.data(), thunk_code.size()); - assembler.FinalizeInstructions(code); - return thunk_code; -} - -std::string Thumb2RelativePatcher::GetThunkDebugName(const ThunkKey& key) { - switch (key.GetType()) { - case ThunkType::kMethodCall: - return "MethodCallThunk"; - - case ThunkType::kBakerReadBarrier: { - uint32_t encoded_data = key.GetCustomValue1(); - BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data); - std::ostringstream oss; - oss << "BakerReadBarrierThunk"; - switch (kind) { - case BakerReadBarrierKind::kField: - oss << "Field"; - if (BakerReadBarrierWidthField::Decode(encoded_data) == BakerReadBarrierWidth::kWide) { - oss << "Wide"; - } - oss << "_r" << BakerReadBarrierFirstRegField::Decode(encoded_data) - << "_r" << BakerReadBarrierSecondRegField::Decode(encoded_data); - break; - case BakerReadBarrierKind::kArray: - oss << "Array_r" << BakerReadBarrierFirstRegField::Decode(encoded_data); - DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data)); - DCHECK(BakerReadBarrierWidthField::Decode(encoded_data) == BakerReadBarrierWidth::kWide); - break; - case BakerReadBarrierKind::kGcRoot: - oss << "GcRoot"; - if (BakerReadBarrierWidthField::Decode(encoded_data) == BakerReadBarrierWidth::kWide) { - oss << "Wide"; - } - oss << "_r" << BakerReadBarrierFirstRegField::Decode(encoded_data); - DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data)); - break; - } - return oss.str(); - } - } -} - -#undef __ - -uint32_t Thumb2RelativePatcher::MaxPositiveDisplacement(const ThunkKey& key) { - switch (key.GetType()) { - case ThunkType::kMethodCall: - return kMaxMethodCallPositiveDisplacement; - case ThunkType::kBakerReadBarrier: - return kMaxBcondPositiveDisplacement; - } -} - -uint32_t Thumb2RelativePatcher::MaxNegativeDisplacement(const ThunkKey& key) { - switch (key.GetType()) { - case ThunkType::kMethodCall: - return kMaxMethodCallNegativeDisplacement; - case ThunkType::kBakerReadBarrier: - return kMaxBcondNegativeDisplacement; - } -} - -void Thumb2RelativePatcher::SetInsn32(std::vector<uint8_t>* code, uint32_t offset, uint32_t value) { - DCHECK_LE(offset + 4u, code->size()); - DCHECK_ALIGNED(offset, 2u); - uint8_t* addr = &(*code)[offset]; - addr[0] = (value >> 16) & 0xff; - addr[1] = (value >> 24) & 0xff; - addr[2] = (value >> 0) & 0xff; - addr[3] = (value >> 8) & 0xff; -} - -uint32_t Thumb2RelativePatcher::GetInsn32(ArrayRef<const uint8_t> code, uint32_t offset) { - DCHECK_LE(offset + 4u, code.size()); - DCHECK_ALIGNED(offset, 2u); - const uint8_t* addr = &code[offset]; - return - (static_cast<uint32_t>(addr[0]) << 16) + - (static_cast<uint32_t>(addr[1]) << 24) + - (static_cast<uint32_t>(addr[2]) << 0)+ - (static_cast<uint32_t>(addr[3]) << 8); -} - -template <typename Vector> -uint32_t Thumb2RelativePatcher::GetInsn32(Vector* code, uint32_t offset) { - static_assert(std::is_same<typename Vector::value_type, uint8_t>::value, "Invalid value type"); - return GetInsn32(ArrayRef<const uint8_t>(*code), offset); -} - -uint32_t Thumb2RelativePatcher::GetInsn16(ArrayRef<const uint8_t> code, uint32_t offset) { - DCHECK_LE(offset + 2u, code.size()); - DCHECK_ALIGNED(offset, 2u); - const uint8_t* addr = &code[offset]; - return (static_cast<uint32_t>(addr[0]) << 0) + (static_cast<uint32_t>(addr[1]) << 8); -} - -template <typename Vector> -uint32_t Thumb2RelativePatcher::GetInsn16(Vector* code, uint32_t offset) { - static_assert(std::is_same<typename Vector::value_type, uint8_t>::value, "Invalid value type"); - return GetInsn16(ArrayRef<const uint8_t>(*code), offset); -} - -} // namespace linker -} // namespace art diff --git a/compiler/linker/arm/relative_patcher_thumb2.h b/compiler/linker/arm/relative_patcher_thumb2.h deleted file mode 100644 index 68386c00f4a..00000000000 --- a/compiler/linker/arm/relative_patcher_thumb2.h +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Copyright (C) 2015 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef ART_COMPILER_LINKER_ARM_RELATIVE_PATCHER_THUMB2_H_ -#define ART_COMPILER_LINKER_ARM_RELATIVE_PATCHER_THUMB2_H_ - -#include "arch/arm/registers_arm.h" -#include "base/array_ref.h" -#include "base/bit_field.h" -#include "base/bit_utils.h" -#include "linker/arm/relative_patcher_arm_base.h" - -namespace art { - -namespace arm { -class ArmVIXLAssembler; -} // namespace arm - -namespace linker { - -class Thumb2RelativePatcher FINAL : public ArmBaseRelativePatcher { - public: - static constexpr uint32_t kBakerCcEntrypointRegister = 4u; - - static uint32_t EncodeBakerReadBarrierFieldData(uint32_t base_reg, - uint32_t holder_reg, - bool narrow) { - CheckValidReg(base_reg); - CheckValidReg(holder_reg); - DCHECK(!narrow || base_reg < 8u) << base_reg; - BakerReadBarrierWidth width = - narrow ? BakerReadBarrierWidth::kNarrow : BakerReadBarrierWidth::kWide; - return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kField) | - BakerReadBarrierFirstRegField::Encode(base_reg) | - BakerReadBarrierSecondRegField::Encode(holder_reg) | - BakerReadBarrierWidthField::Encode(width); - } - - static uint32_t EncodeBakerReadBarrierArrayData(uint32_t base_reg) { - CheckValidReg(base_reg); - return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kArray) | - BakerReadBarrierFirstRegField::Encode(base_reg) | - BakerReadBarrierSecondRegField::Encode(kInvalidEncodedReg) | - BakerReadBarrierWidthField::Encode(BakerReadBarrierWidth::kWide); - } - - static uint32_t EncodeBakerReadBarrierGcRootData(uint32_t root_reg, bool narrow) { - CheckValidReg(root_reg); - DCHECK(!narrow || root_reg < 8u) << root_reg; - BakerReadBarrierWidth width = - narrow ? BakerReadBarrierWidth::kNarrow : BakerReadBarrierWidth::kWide; - return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kGcRoot) | - BakerReadBarrierFirstRegField::Encode(root_reg) | - BakerReadBarrierSecondRegField::Encode(kInvalidEncodedReg) | - BakerReadBarrierWidthField::Encode(width); - } - - explicit Thumb2RelativePatcher(RelativePatcherTargetProvider* provider); - - void PatchCall(std::vector<uint8_t>* code, - uint32_t literal_offset, - uint32_t patch_offset, - uint32_t target_offset) OVERRIDE; - void PatchPcRelativeReference(std::vector<uint8_t>* code, - const LinkerPatch& patch, - uint32_t patch_offset, - uint32_t target_offset) OVERRIDE; - void PatchBakerReadBarrierBranch(std::vector<uint8_t>* code, - const LinkerPatch& patch, - uint32_t patch_offset) OVERRIDE; - - protected: - std::vector<uint8_t> CompileThunk(const ThunkKey& key) OVERRIDE; - std::string GetThunkDebugName(const ThunkKey& key) OVERRIDE; - uint32_t MaxPositiveDisplacement(const ThunkKey& key) OVERRIDE; - uint32_t MaxNegativeDisplacement(const ThunkKey& key) OVERRIDE; - - private: - static constexpr uint32_t kInvalidEncodedReg = /* pc is invalid */ 15u; - - enum class BakerReadBarrierKind : uint8_t { - kField, // Field get or array get with constant offset (i.e. constant index). - kArray, // Array get with index in register. - kGcRoot, // GC root load. - kLast = kGcRoot - }; - - enum class BakerReadBarrierWidth : uint8_t { - kWide, // 32-bit LDR (and 32-bit NEG if heap poisoning is enabled). - kNarrow, // 16-bit LDR (and 16-bit NEG if heap poisoning is enabled). - kLast = kNarrow - }; - - static constexpr size_t kBitsForBakerReadBarrierKind = - MinimumBitsToStore(static_cast<size_t>(BakerReadBarrierKind::kLast)); - static constexpr size_t kBitsForRegister = 4u; - using BakerReadBarrierKindField = - BitField<BakerReadBarrierKind, 0, kBitsForBakerReadBarrierKind>; - using BakerReadBarrierFirstRegField = - BitField<uint32_t, kBitsForBakerReadBarrierKind, kBitsForRegister>; - using BakerReadBarrierSecondRegField = - BitField<uint32_t, kBitsForBakerReadBarrierKind + kBitsForRegister, kBitsForRegister>; - static constexpr size_t kBitsForBakerReadBarrierWidth = - MinimumBitsToStore(static_cast<size_t>(BakerReadBarrierWidth::kLast)); - using BakerReadBarrierWidthField = BitField<BakerReadBarrierWidth, - kBitsForBakerReadBarrierKind + 2 * kBitsForRegister, - kBitsForBakerReadBarrierWidth>; - - static void CheckValidReg(uint32_t reg) { - DCHECK(reg < 12u && reg != kBakerCcEntrypointRegister) << reg; - } - - void CompileBakerReadBarrierThunk(arm::ArmVIXLAssembler& assembler, uint32_t encoded_data); - - void SetInsn32(std::vector<uint8_t>* code, uint32_t offset, uint32_t value); - static uint32_t GetInsn32(ArrayRef<const uint8_t> code, uint32_t offset); - - template <typename Vector> - static uint32_t GetInsn32(Vector* code, uint32_t offset); - - static uint32_t GetInsn16(ArrayRef<const uint8_t> code, uint32_t offset); - - template <typename Vector> - static uint32_t GetInsn16(Vector* code, uint32_t offset); - - friend class Thumb2RelativePatcherTest; - - DISALLOW_COPY_AND_ASSIGN(Thumb2RelativePatcher); -}; - -} // namespace linker -} // namespace art - -#endif // ART_COMPILER_LINKER_ARM_RELATIVE_PATCHER_THUMB2_H_ diff --git a/compiler/linker/elf_builder.h b/compiler/linker/elf_builder.h index a5f60992cae..3da7a437627 100644 --- a/compiler/linker/elf_builder.h +++ b/compiler/linker/elf_builder.h @@ -529,6 +529,8 @@ class ElfBuilder FINAL { stream_(output), rodata_(this, ".rodata", SHT_PROGBITS, SHF_ALLOC, nullptr, 0, kPageSize, 0), text_(this, ".text", SHT_PROGBITS, SHF_ALLOC | SHF_EXECINSTR, nullptr, 0, kPageSize, 0), + data_bimg_rel_ro_( + this, ".data.bimg.rel.ro", SHT_PROGBITS, SHF_ALLOC, nullptr, 0, kPageSize, 0), bss_(this, ".bss", SHT_NOBITS, SHF_ALLOC, nullptr, 0, kPageSize, 0), dex_(this, ".dex", SHT_NOBITS, SHF_ALLOC, nullptr, 0, kPageSize, 0), dynstr_(this, ".dynstr", SHF_ALLOC, kPageSize), @@ -552,6 +554,7 @@ class ElfBuilder FINAL { loaded_size_(0u), virtual_address_(0) { text_.phdr_flags_ = PF_R | PF_X; + data_bimg_rel_ro_.phdr_flags_ = PF_R | PF_W; // Shall be made read-only at run time. bss_.phdr_flags_ = PF_R | PF_W; dex_.phdr_flags_ = PF_R; dynamic_.phdr_flags_ = PF_R | PF_W; @@ -566,6 +569,7 @@ class ElfBuilder FINAL { BuildIdSection* GetBuildId() { return &build_id_; } Section* GetRoData() { return &rodata_; } Section* GetText() { return &text_; } + Section* GetDataBimgRelRo() { return &data_bimg_rel_ro_; } Section* GetBss() { return &bss_; } Section* GetDex() { return &dex_; } StringSection* GetStrTab() { return &strtab_; } @@ -694,6 +698,7 @@ class ElfBuilder FINAL { void PrepareDynamicSection(const std::string& elf_file_path, Elf_Word rodata_size, Elf_Word text_size, + Elf_Word data_bimg_rel_ro_size, Elf_Word bss_size, Elf_Word bss_methods_offset, Elf_Word bss_roots_offset, @@ -707,6 +712,9 @@ class ElfBuilder FINAL { // Allocate all pre-dynamic sections. rodata_.AllocateVirtualMemory(rodata_size); text_.AllocateVirtualMemory(text_size); + if (data_bimg_rel_ro_size != 0) { + data_bimg_rel_ro_.AllocateVirtualMemory(data_bimg_rel_ro_size); + } if (bss_size != 0) { bss_.AllocateVirtualMemory(bss_size); } @@ -735,6 +743,24 @@ class ElfBuilder FINAL { Elf_Word oatlastword_address = rodata_.GetAddress() + rodata_size - 4; dynsym_.Add(oatlastword, &rodata_, oatlastword_address, 4, STB_GLOBAL, STT_OBJECT); } + if (data_bimg_rel_ro_size != 0u) { + Elf_Word oatdatabimgrelro = dynstr_.Add("oatdatabimgrelro"); + dynsym_.Add(oatdatabimgrelro, + &data_bimg_rel_ro_, + data_bimg_rel_ro_.GetAddress(), + data_bimg_rel_ro_size, + STB_GLOBAL, + STT_OBJECT); + Elf_Word oatdatabimgrelrolastword = dynstr_.Add("oatdatabimgrelrolastword"); + Elf_Word oatdatabimgrelrolastword_address = + data_bimg_rel_ro_.GetAddress() + data_bimg_rel_ro_size - 4; + dynsym_.Add(oatdatabimgrelrolastword, + &data_bimg_rel_ro_, + oatdatabimgrelrolastword_address, + 4, + STB_GLOBAL, + STT_OBJECT); + } DCHECK_LE(bss_roots_offset, bss_size); if (bss_size != 0u) { Elf_Word oatbss = dynstr_.Add("oatbss"); @@ -1010,6 +1036,7 @@ class ElfBuilder FINAL { Section rodata_; Section text_; + Section data_bimg_rel_ro_; Section bss_; Section dex_; CachedStringSection dynstr_; diff --git a/compiler/linker/linker_patch.h b/compiler/linker/linker_patch.h index 77d689d4dbb..7b35fd9b0c3 100644 --- a/compiler/linker/linker_patch.h +++ b/compiler/linker/linker_patch.h @@ -41,19 +41,27 @@ class LinkerPatch { // choose to squeeze the Type into fewer than 8 bits, we'll have to declare // patch_type_ as an uintN_t and do explicit static_cast<>s. enum class Type : uint8_t { + kDataBimgRelRo, // NOTE: Actual patching is instruction_set-dependent. kMethodRelative, // NOTE: Actual patching is instruction_set-dependent. kMethodBssEntry, // NOTE: Actual patching is instruction_set-dependent. kCall, kCallRelative, // NOTE: Actual patching is instruction_set-dependent. kTypeRelative, // NOTE: Actual patching is instruction_set-dependent. - kTypeClassTable, // NOTE: Actual patching is instruction_set-dependent. kTypeBssEntry, // NOTE: Actual patching is instruction_set-dependent. kStringRelative, // NOTE: Actual patching is instruction_set-dependent. - kStringInternTable, // NOTE: Actual patching is instruction_set-dependent. kStringBssEntry, // NOTE: Actual patching is instruction_set-dependent. kBakerReadBarrierBranch, // NOTE: Actual patching is instruction_set-dependent. }; + static LinkerPatch DataBimgRelRoPatch(size_t literal_offset, + uint32_t pc_insn_offset, + uint32_t boot_image_offset) { + LinkerPatch patch(literal_offset, Type::kDataBimgRelRo, /* target_dex_file */ nullptr); + patch.boot_image_offset_ = boot_image_offset; + patch.pc_insn_offset_ = pc_insn_offset; + return patch; + } + static LinkerPatch RelativeMethodPatch(size_t literal_offset, const DexFile* target_dex_file, uint32_t pc_insn_offset, @@ -100,16 +108,6 @@ class LinkerPatch { return patch; } - static LinkerPatch TypeClassTablePatch(size_t literal_offset, - const DexFile* target_dex_file, - uint32_t pc_insn_offset, - uint32_t target_type_idx) { - LinkerPatch patch(literal_offset, Type::kTypeClassTable, target_dex_file); - patch.type_idx_ = target_type_idx; - patch.pc_insn_offset_ = pc_insn_offset; - return patch; - } - static LinkerPatch TypeBssEntryPatch(size_t literal_offset, const DexFile* target_dex_file, uint32_t pc_insn_offset, @@ -130,16 +128,6 @@ class LinkerPatch { return patch; } - static LinkerPatch StringInternTablePatch(size_t literal_offset, - const DexFile* target_dex_file, - uint32_t pc_insn_offset, - uint32_t target_string_idx) { - LinkerPatch patch(literal_offset, Type::kStringInternTable, target_dex_file); - patch.string_idx_ = target_string_idx; - patch.pc_insn_offset_ = pc_insn_offset; - return patch; - } - static LinkerPatch StringBssEntryPatch(size_t literal_offset, const DexFile* target_dex_file, uint32_t pc_insn_offset, @@ -153,7 +141,7 @@ class LinkerPatch { static LinkerPatch BakerReadBarrierBranchPatch(size_t literal_offset, uint32_t custom_value1 = 0u, uint32_t custom_value2 = 0u) { - LinkerPatch patch(literal_offset, Type::kBakerReadBarrierBranch, nullptr); + LinkerPatch patch(literal_offset, Type::kBakerReadBarrierBranch, /* target_dex_file */ nullptr); patch.baker_custom_value1_ = custom_value1; patch.baker_custom_value2_ = custom_value2; return patch; @@ -172,14 +160,13 @@ class LinkerPatch { bool IsPcRelative() const { switch (GetType()) { + case Type::kDataBimgRelRo: case Type::kMethodRelative: case Type::kMethodBssEntry: case Type::kCallRelative: case Type::kTypeRelative: - case Type::kTypeClassTable: case Type::kTypeBssEntry: case Type::kStringRelative: - case Type::kStringInternTable: case Type::kStringBssEntry: case Type::kBakerReadBarrierBranch: return true; @@ -188,6 +175,11 @@ class LinkerPatch { } } + uint32_t BootImageOffset() const { + DCHECK(patch_type_ == Type::kDataBimgRelRo); + return boot_image_offset_; + } + MethodReference TargetMethod() const { DCHECK(patch_type_ == Type::kMethodRelative || patch_type_ == Type::kMethodBssEntry || @@ -198,40 +190,35 @@ class LinkerPatch { const DexFile* TargetTypeDexFile() const { DCHECK(patch_type_ == Type::kTypeRelative || - patch_type_ == Type::kTypeClassTable || patch_type_ == Type::kTypeBssEntry); return target_dex_file_; } dex::TypeIndex TargetTypeIndex() const { DCHECK(patch_type_ == Type::kTypeRelative || - patch_type_ == Type::kTypeClassTable || patch_type_ == Type::kTypeBssEntry); return dex::TypeIndex(type_idx_); } const DexFile* TargetStringDexFile() const { DCHECK(patch_type_ == Type::kStringRelative || - patch_type_ == Type::kStringInternTable || patch_type_ == Type::kStringBssEntry); return target_dex_file_; } dex::StringIndex TargetStringIndex() const { DCHECK(patch_type_ == Type::kStringRelative || - patch_type_ == Type::kStringInternTable || patch_type_ == Type::kStringBssEntry); return dex::StringIndex(string_idx_); } uint32_t PcInsnOffset() const { - DCHECK(patch_type_ == Type::kMethodRelative || + DCHECK(patch_type_ == Type::kDataBimgRelRo || + patch_type_ == Type::kMethodRelative || patch_type_ == Type::kMethodBssEntry || patch_type_ == Type::kTypeRelative || - patch_type_ == Type::kTypeClassTable || patch_type_ == Type::kTypeBssEntry || patch_type_ == Type::kStringRelative || - patch_type_ == Type::kStringInternTable || patch_type_ == Type::kStringBssEntry); return pc_insn_offset_; } @@ -263,10 +250,11 @@ class LinkerPatch { uint32_t literal_offset_ : 24; // Method code size up to 16MiB. Type patch_type_ : 8; union { - uint32_t cmp1_; // Used for relational operators. - uint32_t method_idx_; // Method index for Call/Method patches. - uint32_t type_idx_; // Type index for Type patches. - uint32_t string_idx_; // String index for String patches. + uint32_t cmp1_; // Used for relational operators. + uint32_t boot_image_offset_; // Data to write to the .data.bimg.rel.ro entry. + uint32_t method_idx_; // Method index for Call/Method patches. + uint32_t type_idx_; // Type index for Type patches. + uint32_t string_idx_; // String index for String patches. uint32_t baker_custom_value1_; static_assert(sizeof(method_idx_) == sizeof(cmp1_), "needed by relational operators"); static_assert(sizeof(type_idx_) == sizeof(cmp1_), "needed by relational operators"); diff --git a/compiler/optimizing/code_generator.cc b/compiler/optimizing/code_generator.cc index 6abda9b3026..231017f55e6 100644 --- a/compiler/optimizing/code_generator.cc +++ b/compiler/optimizing/code_generator.cc @@ -51,6 +51,8 @@ #include "dex/verified_method.h" #include "driver/compiler_driver.h" #include "graph_visualizer.h" +#include "image.h" +#include "gc/space/image_space.h" #include "intern_table.h" #include "intrinsics.h" #include "mirror/array-inl.h" @@ -447,6 +449,18 @@ void CodeGenerator::EmitLinkerPatches( // No linker patches by default. } +bool CodeGenerator::NeedsThunkCode(const linker::LinkerPatch& patch ATTRIBUTE_UNUSED) const { + // Code generators that create patches requiring thunk compilation should override this function. + return false; +} + +void CodeGenerator::EmitThunkCode(const linker::LinkerPatch& patch ATTRIBUTE_UNUSED, + /*out*/ ArenaVector<uint8_t>* code ATTRIBUTE_UNUSED, + /*out*/ std::string* debug_name ATTRIBUTE_UNUSED) { + // Code generators that create patches requiring thunk compilation should override this function. + LOG(FATAL) << "Unexpected call to EmitThunkCode()."; +} + void CodeGenerator::InitializeCodeGeneration(size_t number_of_spill_slots, size_t maximum_safepoint_spill_size, size_t number_of_out_slots, @@ -722,6 +736,47 @@ void CodeGenerator::GenerateLoadClassRuntimeCall(HLoadClass* cls) { } } +static uint32_t GetBootImageOffsetImpl(const void* object, ImageHeader::ImageSections section) { + Runtime* runtime = Runtime::Current(); + DCHECK(runtime->IsAotCompiler()); + const std::vector<gc::space::ImageSpace*>& boot_image_spaces = + runtime->GetHeap()->GetBootImageSpaces(); + // Check that the `object` is in the expected section of one of the boot image files. + DCHECK(std::any_of(boot_image_spaces.begin(), + boot_image_spaces.end(), + [object, section](gc::space::ImageSpace* space) { + uintptr_t begin = reinterpret_cast<uintptr_t>(space->Begin()); + uintptr_t offset = reinterpret_cast<uintptr_t>(object) - begin; + return space->GetImageHeader().GetImageSection(section).Contains(offset); + })); + uintptr_t begin = reinterpret_cast<uintptr_t>(boot_image_spaces.front()->Begin()); + uintptr_t offset = reinterpret_cast<uintptr_t>(object) - begin; + return dchecked_integral_cast<uint32_t>(offset); +} + +// NO_THREAD_SAFETY_ANALYSIS: Avoid taking the mutator lock, boot image classes are non-moveable. +uint32_t CodeGenerator::GetBootImageOffset(HLoadClass* load_class) NO_THREAD_SAFETY_ANALYSIS { + DCHECK_EQ(load_class->GetLoadKind(), HLoadClass::LoadKind::kBootImageRelRo); + ObjPtr<mirror::Class> klass = load_class->GetClass().Get(); + DCHECK(klass != nullptr); + return GetBootImageOffsetImpl(klass.Ptr(), ImageHeader::kSectionObjects); +} + +// NO_THREAD_SAFETY_ANALYSIS: Avoid taking the mutator lock, boot image strings are non-moveable. +uint32_t CodeGenerator::GetBootImageOffset(HLoadString* load_string) NO_THREAD_SAFETY_ANALYSIS { + DCHECK_EQ(load_string->GetLoadKind(), HLoadString::LoadKind::kBootImageRelRo); + ObjPtr<mirror::String> string = load_string->GetString().Get(); + DCHECK(string != nullptr); + return GetBootImageOffsetImpl(string.Ptr(), ImageHeader::kSectionObjects); +} + +uint32_t CodeGenerator::GetBootImageOffset(HInvokeStaticOrDirect* invoke) { + DCHECK_EQ(invoke->GetMethodLoadKind(), HInvokeStaticOrDirect::MethodLoadKind::kBootImageRelRo); + ArtMethod* method = invoke->GetResolvedMethod(); + DCHECK(method != nullptr); + return GetBootImageOffsetImpl(method, ImageHeader::kSectionArtMethods); +} + void CodeGenerator::BlockIfInRegister(Location location, bool is_out) const { // The DCHECKS below check that a register is not specified twice in // the summary. The out location can overlap with an input, so we need diff --git a/compiler/optimizing/code_generator.h b/compiler/optimizing/code_generator.h index f784a1a8573..62cacebaa1e 100644 --- a/compiler/optimizing/code_generator.h +++ b/compiler/optimizing/code_generator.h @@ -21,15 +21,16 @@ #include "arch/instruction_set_features.h" #include "base/arena_containers.h" #include "base/arena_object.h" +#include "base/array_ref.h" #include "base/bit_field.h" #include "base/bit_utils.h" #include "base/enums.h" +#include "base/memory_region.h" #include "dex/string_reference.h" #include "dex/type_reference.h" #include "globals.h" #include "graph_visualizer.h" #include "locations.h" -#include "memory_region.h" #include "nodes.h" #include "optimizing_compiler_stats.h" #include "read_barrier_option.h" @@ -74,6 +75,7 @@ class CodeAllocator { virtual ~CodeAllocator() {} virtual uint8_t* Allocate(size_t size) = 0; + virtual ArrayRef<const uint8_t> GetMemory() const = 0; private: DISALLOW_COPY_AND_ASSIGN(CodeAllocator); @@ -210,6 +212,10 @@ class CodeGenerator : public DeletableArenaObject<kArenaAllocCodeGenerator> { virtual void Initialize() = 0; virtual void Finalize(CodeAllocator* allocator); virtual void EmitLinkerPatches(ArenaVector<linker::LinkerPatch>* linker_patches); + virtual bool NeedsThunkCode(const linker::LinkerPatch& patch) const; + virtual void EmitThunkCode(const linker::LinkerPatch& patch, + /*out*/ ArenaVector<uint8_t>* code, + /*out*/ std::string* debug_name); virtual void GenerateFrameEntry() = 0; virtual void GenerateFrameExit() = 0; virtual void Bind(HBasicBlock* block) = 0; @@ -438,6 +444,8 @@ class CodeGenerator : public DeletableArenaObject<kArenaAllocCodeGenerator> { case TypeCheckKind::kArrayCheck: case TypeCheckKind::kUnresolvedCheck: return false; + case TypeCheckKind::kBitstringCheck: + return true; } LOG(FATAL) << "Unreachable"; UNREACHABLE(); @@ -556,6 +564,10 @@ class CodeGenerator : public DeletableArenaObject<kArenaAllocCodeGenerator> { Location runtime_return_location); void GenerateLoadClassRuntimeCall(HLoadClass* cls); + uint32_t GetBootImageOffset(HLoadClass* load_class); + uint32_t GetBootImageOffset(HLoadString* load_string); + uint32_t GetBootImageOffset(HInvokeStaticOrDirect* invoke); + static void CreateSystemArrayCopyLocationSummary(HInvoke* invoke); void SetDisassemblyInformation(DisassemblyInformation* info) { disasm_info_ = info; } diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc index 60f8f98757d..d4cfab82de3 100644 --- a/compiler/optimizing/code_generator_arm64.cc +++ b/compiler/optimizing/code_generator_arm64.cc @@ -30,7 +30,6 @@ #include "heap_poisoning.h" #include "intrinsics.h" #include "intrinsics_arm64.h" -#include "linker/arm64/relative_patcher_arm64.h" #include "linker/linker_patch.h" #include "lock_word.h" #include "mirror/array-inl.h" @@ -78,6 +77,7 @@ using helpers::OutputFPRegister; using helpers::OutputRegister; using helpers::QRegisterFrom; using helpers::RegisterFrom; +using helpers::SRegisterFrom; using helpers::StackOperandFrom; using helpers::VIXLRegCodeFromART; using helpers::WRegisterFrom; @@ -1424,6 +1424,62 @@ void CodeGeneratorARM64::Finalize(CodeAllocator* allocator) { __ FinalizeCode(); CodeGenerator::Finalize(allocator); + + // Verify Baker read barrier linker patches. + if (kIsDebugBuild) { + ArrayRef<const uint8_t> code = allocator->GetMemory(); + for (const BakerReadBarrierPatchInfo& info : baker_read_barrier_patches_) { + DCHECK(info.label.IsBound()); + uint32_t literal_offset = info.label.GetLocation(); + DCHECK_ALIGNED(literal_offset, 4u); + + auto GetInsn = [&code](uint32_t offset) { + DCHECK_ALIGNED(offset, 4u); + return + (static_cast<uint32_t>(code[offset + 0]) << 0) + + (static_cast<uint32_t>(code[offset + 1]) << 8) + + (static_cast<uint32_t>(code[offset + 2]) << 16)+ + (static_cast<uint32_t>(code[offset + 3]) << 24); + }; + + const uint32_t encoded_data = info.custom_data; + BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data); + // Check that the next instruction matches the expected LDR. + switch (kind) { + case BakerReadBarrierKind::kField: { + DCHECK_GE(code.size() - literal_offset, 8u); + uint32_t next_insn = GetInsn(literal_offset + 4u); + // LDR (immediate) with correct base_reg. + CheckValidReg(next_insn & 0x1fu); // Check destination register. + const uint32_t base_reg = BakerReadBarrierFirstRegField::Decode(encoded_data); + CHECK_EQ(next_insn & 0xffc003e0u, 0xb9400000u | (base_reg << 5)); + break; + } + case BakerReadBarrierKind::kArray: { + DCHECK_GE(code.size() - literal_offset, 8u); + uint32_t next_insn = GetInsn(literal_offset + 4u); + // LDR (register) with the correct base_reg, size=10 (32-bit), option=011 (extend = LSL), + // and S=1 (shift amount = 2 for 32-bit version), i.e. LDR Wt, [Xn, Xm, LSL #2]. + CheckValidReg(next_insn & 0x1fu); // Check destination register. + const uint32_t base_reg = BakerReadBarrierFirstRegField::Decode(encoded_data); + CHECK_EQ(next_insn & 0xffe0ffe0u, 0xb8607800u | (base_reg << 5)); + CheckValidReg((next_insn >> 16) & 0x1f); // Check index register + break; + } + case BakerReadBarrierKind::kGcRoot: { + DCHECK_GE(literal_offset, 4u); + uint32_t prev_insn = GetInsn(literal_offset - 4u); + // LDR (immediate) with correct root_reg. + const uint32_t root_reg = BakerReadBarrierFirstRegField::Decode(encoded_data); + CHECK_EQ(prev_insn & 0xffc0001fu, 0xb9400000u | root_reg); + break; + } + default: + LOG(FATAL) << "Unexpected kind: " << static_cast<uint32_t>(kind); + UNREACHABLE(); + } + } + } } void ParallelMoveResolverARM64::PrepareForEmitNativeCode() { @@ -2128,6 +2184,26 @@ void InstructionCodeGeneratorARM64::GenerateClassInitializationCheck(SlowPathCod __ Bind(slow_path->GetExitLabel()); } +void InstructionCodeGeneratorARM64::GenerateBitstringTypeCheckCompare( + HTypeCheckInstruction* check, vixl::aarch64::Register temp) { + uint32_t path_to_root = check->GetBitstringPathToRoot(); + uint32_t mask = check->GetBitstringMask(); + DCHECK(IsPowerOfTwo(mask + 1)); + size_t mask_bits = WhichPowerOf2(mask + 1); + + if (mask_bits == 16u) { + // Load only the bitstring part of the status word. + __ Ldrh(temp, HeapOperand(temp, mirror::Class::StatusOffset())); + } else { + // /* uint32_t */ temp = temp->status_ + __ Ldr(temp, HeapOperand(temp, mirror::Class::StatusOffset())); + // Extract the bitstring bits. + __ Ubfx(temp, temp, 0, mask_bits); + } + // Compare the bitstring bits to `path_to_root`. + __ Cmp(temp, path_to_root); +} + void CodeGeneratorARM64::GenerateMemoryBarrier(MemBarrierKind kind) { BarrierType type = BarrierAll; @@ -3865,6 +3941,8 @@ void LocationsBuilderARM64::VisitInstanceOf(HInstanceOf* instruction) { case TypeCheckKind::kInterfaceCheck: call_kind = LocationSummary::kCallOnSlowPath; break; + case TypeCheckKind::kBitstringCheck: + break; } LocationSummary* locations = @@ -3873,7 +3951,13 @@ void LocationsBuilderARM64::VisitInstanceOf(HInstanceOf* instruction) { locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. } locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); + if (type_check_kind == TypeCheckKind::kBitstringCheck) { + locations->SetInAt(1, Location::ConstantLocation(instruction->InputAt(1)->AsConstant())); + locations->SetInAt(2, Location::ConstantLocation(instruction->InputAt(2)->AsConstant())); + locations->SetInAt(3, Location::ConstantLocation(instruction->InputAt(3)->AsConstant())); + } else { + locations->SetInAt(1, Location::RequiresRegister()); + } // The "out" register is used as a temporary, so it overlaps with the inputs. // Note that TypeCheckSlowPathARM64 uses this register too. locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); @@ -3886,7 +3970,9 @@ void InstructionCodeGeneratorARM64::VisitInstanceOf(HInstanceOf* instruction) { LocationSummary* locations = instruction->GetLocations(); Location obj_loc = locations->InAt(0); Register obj = InputRegisterAt(instruction, 0); - Register cls = InputRegisterAt(instruction, 1); + Register cls = (type_check_kind == TypeCheckKind::kBitstringCheck) + ? Register() + : InputRegisterAt(instruction, 1); Location out_loc = locations->Out(); Register out = OutputRegister(instruction); const size_t num_temps = NumberOfInstanceOfTemps(type_check_kind); @@ -4072,6 +4158,23 @@ void InstructionCodeGeneratorARM64::VisitInstanceOf(HInstanceOf* instruction) { } break; } + + case TypeCheckKind::kBitstringCheck: { + // /* HeapReference<Class> */ temp = obj->klass_ + GenerateReferenceLoadTwoRegisters(instruction, + out_loc, + obj_loc, + class_offset, + maybe_temp_loc, + kWithoutReadBarrier); + + GenerateBitstringTypeCheckCompare(instruction, out); + __ Cset(out, eq); + if (zero.IsLinked()) { + __ B(&done); + } + break; + } } if (zero.IsLinked()) { @@ -4094,7 +4197,13 @@ void LocationsBuilderARM64::VisitCheckCast(HCheckCast* instruction) { LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction, call_kind); locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); + if (type_check_kind == TypeCheckKind::kBitstringCheck) { + locations->SetInAt(1, Location::ConstantLocation(instruction->InputAt(1)->AsConstant())); + locations->SetInAt(2, Location::ConstantLocation(instruction->InputAt(2)->AsConstant())); + locations->SetInAt(3, Location::ConstantLocation(instruction->InputAt(3)->AsConstant())); + } else { + locations->SetInAt(1, Location::RequiresRegister()); + } // Add temps for read barriers and other uses. One is used by TypeCheckSlowPathARM64. locations->AddRegisterTemps(NumberOfCheckCastTemps(type_check_kind)); } @@ -4104,7 +4213,9 @@ void InstructionCodeGeneratorARM64::VisitCheckCast(HCheckCast* instruction) { LocationSummary* locations = instruction->GetLocations(); Location obj_loc = locations->InAt(0); Register obj = InputRegisterAt(instruction, 0); - Register cls = InputRegisterAt(instruction, 1); + Register cls = (type_check_kind == TypeCheckKind::kBitstringCheck) + ? Register() + : InputRegisterAt(instruction, 1); const size_t num_temps = NumberOfCheckCastTemps(type_check_kind); DCHECK_GE(num_temps, 1u); DCHECK_LE(num_temps, 3u); @@ -4285,6 +4396,20 @@ void InstructionCodeGeneratorARM64::VisitCheckCast(HCheckCast* instruction) { __ B(ne, &start_loop); break; } + + case TypeCheckKind::kBitstringCheck: { + // /* HeapReference<Class> */ temp = obj->klass_ + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + maybe_temp2_loc, + kWithoutReadBarrier); + + GenerateBitstringTypeCheckCompare(instruction, temp); + __ B(ne, type_check_slow_path->GetEntryLabel()); + break; + } } __ Bind(&done); @@ -4459,12 +4584,23 @@ void CodeGeneratorARM64::GenerateStaticOrDirectCall( // Load method address from literal pool. __ Ldr(XRegisterFrom(temp), DeduplicateUint64Literal(invoke->GetMethodAddress())); break; + case HInvokeStaticOrDirect::MethodLoadKind::kBootImageRelRo: { + // Add ADRP with its PC-relative .data.bimg.rel.ro patch. + uint32_t boot_image_offset = GetBootImageOffset(invoke); + vixl::aarch64::Label* adrp_label = NewBootImageRelRoPatch(boot_image_offset); + EmitAdrpPlaceholder(adrp_label, XRegisterFrom(temp)); + // Add LDR with its PC-relative .data.bimg.rel.ro patch. + vixl::aarch64::Label* ldr_label = NewBootImageRelRoPatch(boot_image_offset, adrp_label); + // Note: Boot image is in the low 4GiB and the entry is 32-bit, so emit a 32-bit load. + EmitLdrOffsetPlaceholder(ldr_label, WRegisterFrom(temp), XRegisterFrom(temp)); + break; + } case HInvokeStaticOrDirect::MethodLoadKind::kBssEntry: { - // Add ADRP with its PC-relative DexCache access patch. + // Add ADRP with its PC-relative .bss entry patch. MethodReference target_method(&GetGraph()->GetDexFile(), invoke->GetDexMethodIndex()); vixl::aarch64::Label* adrp_label = NewMethodBssEntryPatch(target_method); EmitAdrpPlaceholder(adrp_label, XRegisterFrom(temp)); - // Add LDR with its PC-relative DexCache access patch. + // Add LDR with its PC-relative .bss entry patch. vixl::aarch64::Label* ldr_label = NewMethodBssEntryPatch(target_method, adrp_label); EmitLdrOffsetPlaceholder(ldr_label, XRegisterFrom(temp), XRegisterFrom(temp)); @@ -4559,6 +4695,13 @@ void InstructionCodeGeneratorARM64::VisitInvokePolymorphic(HInvokePolymorphic* i codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__); } +vixl::aarch64::Label* CodeGeneratorARM64::NewBootImageRelRoPatch( + uint32_t boot_image_offset, + vixl::aarch64::Label* adrp_label) { + return NewPcRelativePatch( + /* dex_file */ nullptr, boot_image_offset, adrp_label, &boot_image_method_patches_); +} + vixl::aarch64::Label* CodeGeneratorARM64::NewBootImageMethodPatch( MethodReference target_method, vixl::aarch64::Label* adrp_label) { @@ -4681,6 +4824,14 @@ inline void CodeGeneratorARM64::EmitPcRelativeLinkerPatches( } } +linker::LinkerPatch DataBimgRelRoPatchAdapter(size_t literal_offset, + const DexFile* target_dex_file, + uint32_t pc_insn_offset, + uint32_t boot_image_offset) { + DCHECK(target_dex_file == nullptr); // Unused for DataBimgRelRoPatch(), should be null. + return linker::LinkerPatch::DataBimgRelRoPatch(literal_offset, pc_insn_offset, boot_image_offset); +} + void CodeGeneratorARM64::EmitLinkerPatches(ArenaVector<linker::LinkerPatch>* linker_patches) { DCHECK(linker_patches->empty()); size_t size = @@ -4700,11 +4851,10 @@ void CodeGeneratorARM64::EmitLinkerPatches(ArenaVector<linker::LinkerPatch>* lin EmitPcRelativeLinkerPatches<linker::LinkerPatch::RelativeStringPatch>( boot_image_string_patches_, linker_patches); } else { - DCHECK(boot_image_method_patches_.empty()); - EmitPcRelativeLinkerPatches<linker::LinkerPatch::TypeClassTablePatch>( - boot_image_type_patches_, linker_patches); - EmitPcRelativeLinkerPatches<linker::LinkerPatch::StringInternTablePatch>( - boot_image_string_patches_, linker_patches); + EmitPcRelativeLinkerPatches<DataBimgRelRoPatchAdapter>( + boot_image_method_patches_, linker_patches); + DCHECK(boot_image_type_patches_.empty()); + DCHECK(boot_image_string_patches_.empty()); } EmitPcRelativeLinkerPatches<linker::LinkerPatch::MethodBssEntryPatch>( method_bss_entry_patches_, linker_patches); @@ -4719,6 +4869,44 @@ void CodeGeneratorARM64::EmitLinkerPatches(ArenaVector<linker::LinkerPatch>* lin DCHECK_EQ(size, linker_patches->size()); } +bool CodeGeneratorARM64::NeedsThunkCode(const linker::LinkerPatch& patch) const { + return patch.GetType() == linker::LinkerPatch::Type::kBakerReadBarrierBranch || + patch.GetType() == linker::LinkerPatch::Type::kCallRelative; +} + +void CodeGeneratorARM64::EmitThunkCode(const linker::LinkerPatch& patch, + /*out*/ ArenaVector<uint8_t>* code, + /*out*/ std::string* debug_name) { + Arm64Assembler assembler(GetGraph()->GetAllocator()); + switch (patch.GetType()) { + case linker::LinkerPatch::Type::kCallRelative: { + // The thunk just uses the entry point in the ArtMethod. This works even for calls + // to the generic JNI and interpreter trampolines. + Offset offset(ArtMethod::EntryPointFromQuickCompiledCodeOffset( + kArm64PointerSize).Int32Value()); + assembler.JumpTo(ManagedRegister(arm64::X0), offset, ManagedRegister(arm64::IP0)); + if (GetCompilerOptions().GenerateAnyDebugInfo()) { + *debug_name = "MethodCallThunk"; + } + break; + } + case linker::LinkerPatch::Type::kBakerReadBarrierBranch: { + DCHECK_EQ(patch.GetBakerCustomValue2(), 0u); + CompileBakerReadBarrierThunk(assembler, patch.GetBakerCustomValue1(), debug_name); + break; + } + default: + LOG(FATAL) << "Unexpected patch type " << patch.GetType(); + UNREACHABLE(); + } + + // Ensure we emit the literal pool if any. + assembler.FinalizeCode(); + code->resize(assembler.CodeSize()); + MemoryRegion code_region(code->data(), code->size()); + assembler.FinalizeInstructions(code_region); +} + vixl::aarch64::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateUint32Literal(uint32_t value) { return uint32_literals_.GetOrCreate( value, @@ -4779,7 +4967,7 @@ HLoadClass::LoadKind CodeGeneratorARM64::GetSupportedLoadClassKind( case HLoadClass::LoadKind::kReferrersClass: break; case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: - case HLoadClass::LoadKind::kBootImageClassTable: + case HLoadClass::LoadKind::kBootImageRelRo: case HLoadClass::LoadKind::kBssEntry: DCHECK(!Runtime::Current()->UseJitCompilation()); break; @@ -4859,12 +5047,12 @@ void InstructionCodeGeneratorARM64::VisitLoadClass(HLoadClass* cls) NO_THREAD_SA DCHECK(!cls->MustGenerateClinitCheck()); // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ Register current_method = InputRegisterAt(cls, 0); - GenerateGcRootFieldLoad(cls, - out_loc, - current_method, - ArtMethod::DeclaringClassOffset().Int32Value(), - /* fixup_label */ nullptr, - read_barrier_option); + codegen_->GenerateGcRootFieldLoad(cls, + out_loc, + current_method, + ArtMethod::DeclaringClassOffset().Int32Value(), + /* fixup_label */ nullptr, + read_barrier_option); break; } case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: { @@ -4888,23 +5076,16 @@ void InstructionCodeGeneratorARM64::VisitLoadClass(HLoadClass* cls) NO_THREAD_SA __ Ldr(out.W(), codegen_->DeduplicateBootImageAddressLiteral(address)); break; } - case HLoadClass::LoadKind::kBootImageClassTable: { + case HLoadClass::LoadKind::kBootImageRelRo: { DCHECK(!codegen_->GetCompilerOptions().IsBootImage()); - // Add ADRP with its PC-relative type patch. - const DexFile& dex_file = cls->GetDexFile(); - dex::TypeIndex type_index = cls->GetTypeIndex(); - vixl::aarch64::Label* adrp_label = codegen_->NewBootImageTypePatch(dex_file, type_index); + uint32_t boot_image_offset = codegen_->GetBootImageOffset(cls); + // Add ADRP with its PC-relative .data.bimg.rel.ro patch. + vixl::aarch64::Label* adrp_label = codegen_->NewBootImageRelRoPatch(boot_image_offset); codegen_->EmitAdrpPlaceholder(adrp_label, out.X()); - // Add LDR with its PC-relative type patch. + // Add LDR with its PC-relative .data.bimg.rel.ro patch. vixl::aarch64::Label* ldr_label = - codegen_->NewBootImageTypePatch(dex_file, type_index, adrp_label); + codegen_->NewBootImageRelRoPatch(boot_image_offset, adrp_label); codegen_->EmitLdrOffsetPlaceholder(ldr_label, out.W(), out.X()); - // Extract the reference from the slot data, i.e. clear the hash bits. - int32_t masked_hash = ClassTable::TableSlot::MaskHash( - ComputeModifiedUtf8Hash(dex_file.StringByTypeIdx(type_index))); - if (masked_hash != 0) { - __ Sub(out.W(), out.W(), Operand(masked_hash)); - } break; } case HLoadClass::LoadKind::kBssEntry: { @@ -4914,16 +5095,16 @@ void InstructionCodeGeneratorARM64::VisitLoadClass(HLoadClass* cls) NO_THREAD_SA vixl::aarch64::Register temp = XRegisterFrom(out_loc); vixl::aarch64::Label* adrp_label = codegen_->NewBssEntryTypePatch(dex_file, type_index); codegen_->EmitAdrpPlaceholder(adrp_label, temp); - // Add LDR with its PC-relative Class patch. + // Add LDR with its PC-relative Class .bss entry patch. vixl::aarch64::Label* ldr_label = codegen_->NewBssEntryTypePatch(dex_file, type_index, adrp_label); // /* GcRoot<mirror::Class> */ out = *(base_address + offset) /* PC-relative */ - GenerateGcRootFieldLoad(cls, - out_loc, - temp, - /* offset placeholder */ 0u, - ldr_label, - read_barrier_option); + codegen_->GenerateGcRootFieldLoad(cls, + out_loc, + temp, + /* offset placeholder */ 0u, + ldr_label, + read_barrier_option); generate_null_check = true; break; } @@ -4931,12 +5112,12 @@ void InstructionCodeGeneratorARM64::VisitLoadClass(HLoadClass* cls) NO_THREAD_SA __ Ldr(out, codegen_->DeduplicateJitClassLiteral(cls->GetDexFile(), cls->GetTypeIndex(), cls->GetClass())); - GenerateGcRootFieldLoad(cls, - out_loc, - out.X(), - /* offset */ 0, - /* fixup_label */ nullptr, - read_barrier_option); + codegen_->GenerateGcRootFieldLoad(cls, + out_loc, + out.X(), + /* offset */ 0, + /* fixup_label */ nullptr, + read_barrier_option); break; } case HLoadClass::LoadKind::kRuntimeCall: @@ -4989,7 +5170,7 @@ HLoadString::LoadKind CodeGeneratorARM64::GetSupportedLoadStringKind( HLoadString::LoadKind desired_string_load_kind) { switch (desired_string_load_kind) { case HLoadString::LoadKind::kBootImageLinkTimePcRelative: - case HLoadString::LoadKind::kBootImageInternTable: + case HLoadString::LoadKind::kBootImageRelRo: case HLoadString::LoadKind::kBssEntry: DCHECK(!Runtime::Current()->UseJitCompilation()); break; @@ -5055,16 +5236,15 @@ void InstructionCodeGeneratorARM64::VisitLoadString(HLoadString* load) NO_THREAD __ Ldr(out.W(), codegen_->DeduplicateBootImageAddressLiteral(address)); return; } - case HLoadString::LoadKind::kBootImageInternTable: { + case HLoadString::LoadKind::kBootImageRelRo: { DCHECK(!codegen_->GetCompilerOptions().IsBootImage()); - // Add ADRP with its PC-relative String patch. - const DexFile& dex_file = load->GetDexFile(); - const dex::StringIndex string_index = load->GetStringIndex(); - vixl::aarch64::Label* adrp_label = codegen_->NewBootImageStringPatch(dex_file, string_index); + // Add ADRP with its PC-relative .data.bimg.rel.ro patch. + uint32_t boot_image_offset = codegen_->GetBootImageOffset(load); + vixl::aarch64::Label* adrp_label = codegen_->NewBootImageRelRoPatch(boot_image_offset); codegen_->EmitAdrpPlaceholder(adrp_label, out.X()); - // Add LDR with its PC-relative String patch. + // Add LDR with its PC-relative .data.bimg.rel.ro patch. vixl::aarch64::Label* ldr_label = - codegen_->NewBootImageStringPatch(dex_file, string_index, adrp_label); + codegen_->NewBootImageRelRoPatch(boot_image_offset, adrp_label); codegen_->EmitLdrOffsetPlaceholder(ldr_label, out.W(), out.X()); return; } @@ -5076,16 +5256,16 @@ void InstructionCodeGeneratorARM64::VisitLoadString(HLoadString* load) NO_THREAD Register temp = XRegisterFrom(out_loc); vixl::aarch64::Label* adrp_label = codegen_->NewStringBssEntryPatch(dex_file, string_index); codegen_->EmitAdrpPlaceholder(adrp_label, temp); - // Add LDR with its .bss entry String patch. + // Add LDR with its PC-relative String .bss entry patch. vixl::aarch64::Label* ldr_label = codegen_->NewStringBssEntryPatch(dex_file, string_index, adrp_label); // /* GcRoot<mirror::String> */ out = *(base_address + offset) /* PC-relative */ - GenerateGcRootFieldLoad(load, - out_loc, - temp, - /* offset placeholder */ 0u, - ldr_label, - kCompilerReadBarrierOption); + codegen_->GenerateGcRootFieldLoad(load, + out_loc, + temp, + /* offset placeholder */ 0u, + ldr_label, + kCompilerReadBarrierOption); SlowPathCodeARM64* slow_path = new (codegen_->GetScopedAllocator()) LoadStringSlowPathARM64(load); codegen_->AddSlowPath(slow_path); @@ -5098,12 +5278,12 @@ void InstructionCodeGeneratorARM64::VisitLoadString(HLoadString* load) NO_THREAD __ Ldr(out, codegen_->DeduplicateJitStringLiteral(load->GetDexFile(), load->GetStringIndex(), load->GetString())); - GenerateGcRootFieldLoad(load, - out_loc, - out.X(), - /* offset */ 0, - /* fixup_label */ nullptr, - kCompilerReadBarrierOption); + codegen_->GenerateGcRootFieldLoad(load, + out_loc, + out.X(), + /* offset */ 0, + /* fixup_label */ nullptr, + kCompilerReadBarrierOption); return; } default: @@ -5462,6 +5642,153 @@ void InstructionCodeGeneratorARM64::VisitRem(HRem* rem) { } } +// TODO: integrate with HandleBinaryOp? +static void CreateMinMaxLocations(ArenaAllocator* allocator, HBinaryOperation* minmax) { + LocationSummary* locations = new (allocator) LocationSummary(minmax); + switch (minmax->GetResultType()) { + case DataType::Type::kInt32: + case DataType::Type::kInt64: + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); + break; + case DataType::Type::kFloat32: + case DataType::Type::kFloat64: + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetInAt(1, Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); + break; + default: + LOG(FATAL) << "Unexpected type for HMinMax " << minmax->GetResultType(); + } +} + +void InstructionCodeGeneratorARM64::GenerateMinMaxInt(LocationSummary* locations, + bool is_min, + DataType::Type type) { + Location op1 = locations->InAt(0); + Location op2 = locations->InAt(1); + Location out = locations->Out(); + + Register op1_reg; + Register op2_reg; + Register out_reg; + if (type == DataType::Type::kInt64) { + op1_reg = XRegisterFrom(op1); + op2_reg = XRegisterFrom(op2); + out_reg = XRegisterFrom(out); + } else { + DCHECK_EQ(type, DataType::Type::kInt32); + op1_reg = WRegisterFrom(op1); + op2_reg = WRegisterFrom(op2); + out_reg = WRegisterFrom(out); + } + + __ Cmp(op1_reg, op2_reg); + __ Csel(out_reg, op1_reg, op2_reg, is_min ? lt : gt); +} + +void InstructionCodeGeneratorARM64::GenerateMinMaxFP(LocationSummary* locations, + bool is_min, + DataType::Type type) { + Location op1 = locations->InAt(0); + Location op2 = locations->InAt(1); + Location out = locations->Out(); + + FPRegister op1_reg; + FPRegister op2_reg; + FPRegister out_reg; + if (type == DataType::Type::kFloat64) { + op1_reg = DRegisterFrom(op1); + op2_reg = DRegisterFrom(op2); + out_reg = DRegisterFrom(out); + } else { + DCHECK_EQ(type, DataType::Type::kFloat32); + op1_reg = SRegisterFrom(op1); + op2_reg = SRegisterFrom(op2); + out_reg = SRegisterFrom(out); + } + + if (is_min) { + __ Fmin(out_reg, op1_reg, op2_reg); + } else { + __ Fmax(out_reg, op1_reg, op2_reg); + } +} + +// TODO: integrate with HandleBinaryOp? +void InstructionCodeGeneratorARM64::GenerateMinMax(HBinaryOperation* minmax, bool is_min) { + DataType::Type type = minmax->GetResultType(); + switch (type) { + case DataType::Type::kInt32: + case DataType::Type::kInt64: + GenerateMinMaxInt(minmax->GetLocations(), is_min, type); + break; + case DataType::Type::kFloat32: + case DataType::Type::kFloat64: + GenerateMinMaxFP(minmax->GetLocations(), is_min, type); + break; + default: + LOG(FATAL) << "Unexpected type for HMinMax " << type; + } +} + +void LocationsBuilderARM64::VisitMin(HMin* min) { + CreateMinMaxLocations(GetGraph()->GetAllocator(), min); +} + +void InstructionCodeGeneratorARM64::VisitMin(HMin* min) { + GenerateMinMax(min, /*is_min*/ true); +} + +void LocationsBuilderARM64::VisitMax(HMax* max) { + CreateMinMaxLocations(GetGraph()->GetAllocator(), max); +} + +void InstructionCodeGeneratorARM64::VisitMax(HMax* max) { + GenerateMinMax(max, /*is_min*/ false); +} + +void LocationsBuilderARM64::VisitAbs(HAbs* abs) { + LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(abs); + switch (abs->GetResultType()) { + case DataType::Type::kInt32: + case DataType::Type::kInt64: + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); + break; + case DataType::Type::kFloat32: + case DataType::Type::kFloat64: + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); + break; + default: + LOG(FATAL) << "Unexpected type for abs operation " << abs->GetResultType(); + } +} + +void InstructionCodeGeneratorARM64::VisitAbs(HAbs* abs) { + switch (abs->GetResultType()) { + case DataType::Type::kInt32: + case DataType::Type::kInt64: { + Register in_reg = InputRegisterAt(abs, 0); + Register out_reg = OutputRegister(abs); + __ Cmp(in_reg, Operand(0)); + __ Cneg(out_reg, in_reg, lt); + break; + } + case DataType::Type::kFloat32: + case DataType::Type::kFloat64: { + FPRegister in_reg = InputFPRegisterAt(abs, 0); + FPRegister out_reg = OutputFPRegister(abs); + __ Fabs(out_reg, in_reg); + break; + } + default: + LOG(FATAL) << "Unexpected type for abs operation " << abs->GetResultType(); + } +} + void LocationsBuilderARM64::VisitConstructorFence(HConstructorFence* constructor_fence) { constructor_fence->SetLocations(nullptr); } @@ -5905,7 +6232,7 @@ void InstructionCodeGeneratorARM64::GenerateReferenceLoadTwoRegisters( } } -void InstructionCodeGeneratorARM64::GenerateGcRootFieldLoad( +void CodeGeneratorARM64::GenerateGcRootFieldLoad( HInstruction* instruction, Location root, Register obj, @@ -5939,9 +6266,8 @@ void InstructionCodeGeneratorARM64::GenerateGcRootFieldLoad( DCHECK(temps.IsAvailable(ip0)); DCHECK(temps.IsAvailable(ip1)); temps.Exclude(ip0, ip1); - uint32_t custom_data = - linker::Arm64RelativePatcher::EncodeBakerReadBarrierGcRootData(root_reg.GetCode()); - vixl::aarch64::Label* cbnz_label = codegen_->NewBakerReadBarrierPatch(custom_data); + uint32_t custom_data = EncodeBakerReadBarrierGcRootData(root_reg.GetCode()); + vixl::aarch64::Label* cbnz_label = NewBakerReadBarrierPatch(custom_data); EmissionCheckScope guard(GetVIXLAssembler(), 3 * vixl::aarch64::kInstructionSize); vixl::aarch64::Label return_address; @@ -5970,14 +6296,14 @@ void InstructionCodeGeneratorARM64::GenerateGcRootFieldLoad( // Slow path marking the GC root `root`. The entrypoint will // be loaded by the slow path code. SlowPathCodeARM64* slow_path = - new (codegen_->GetScopedAllocator()) ReadBarrierMarkSlowPathARM64(instruction, root); - codegen_->AddSlowPath(slow_path); + new (GetScopedAllocator()) ReadBarrierMarkSlowPathARM64(instruction, root); + AddSlowPath(slow_path); // /* GcRoot<mirror::Object> */ root = *(obj + offset) if (fixup_label == nullptr) { __ Ldr(root_reg, MemOperand(obj, offset)); } else { - codegen_->EmitLdrOffsetPlaceholder(fixup_label, root_reg, obj); + EmitLdrOffsetPlaceholder(fixup_label, root_reg, obj); } static_assert( sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>), @@ -5997,10 +6323,10 @@ void InstructionCodeGeneratorARM64::GenerateGcRootFieldLoad( if (fixup_label == nullptr) { __ Add(root_reg.X(), obj.X(), offset); } else { - codegen_->EmitAddPlaceholder(fixup_label, root_reg.X(), obj.X()); + EmitAddPlaceholder(fixup_label, root_reg.X(), obj.X()); } // /* mirror::Object* */ root = root->Read() - codegen_->GenerateReadBarrierForRootSlow(instruction, root, root); + GenerateReadBarrierForRootSlow(instruction, root, root); } } else { // Plain GC root load with no read barrier. @@ -6008,12 +6334,12 @@ void InstructionCodeGeneratorARM64::GenerateGcRootFieldLoad( if (fixup_label == nullptr) { __ Ldr(root_reg, MemOperand(obj, offset)); } else { - codegen_->EmitLdrOffsetPlaceholder(fixup_label, root_reg, obj.X()); + EmitLdrOffsetPlaceholder(fixup_label, root_reg, obj.X()); } // Note that GC roots are not affected by heap poisoning, thus we // do not have to unpoison `root_reg` here. } - codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__); + MaybeGenerateMarkingRegisterCheck(/* code */ __LINE__); } void CodeGeneratorARM64::GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction, @@ -6062,9 +6388,7 @@ void CodeGeneratorARM64::GenerateFieldLoadWithBakerReadBarrier(HInstruction* ins DCHECK(temps.IsAvailable(ip0)); DCHECK(temps.IsAvailable(ip1)); temps.Exclude(ip0, ip1); - uint32_t custom_data = linker::Arm64RelativePatcher::EncodeBakerReadBarrierFieldData( - base.GetCode(), - obj.GetCode()); + uint32_t custom_data = EncodeBakerReadBarrierFieldData(base.GetCode(), obj.GetCode()); vixl::aarch64::Label* cbnz_label = NewBakerReadBarrierPatch(custom_data); { @@ -6149,8 +6473,7 @@ void CodeGeneratorARM64::GenerateArrayLoadWithBakerReadBarrier(HInstruction* ins DCHECK(temps.IsAvailable(ip0)); DCHECK(temps.IsAvailable(ip1)); temps.Exclude(ip0, ip1); - uint32_t custom_data = - linker::Arm64RelativePatcher::EncodeBakerReadBarrierArrayData(temp.GetCode()); + uint32_t custom_data = EncodeBakerReadBarrierArrayData(temp.GetCode()); vixl::aarch64::Label* cbnz_label = NewBakerReadBarrierPatch(custom_data); __ Add(temp.X(), obj.X(), Operand(data_offset)); @@ -6510,5 +6833,176 @@ void CodeGeneratorARM64::EmitJitRootPatches(uint8_t* code, const uint8_t* roots_ #undef __ #undef QUICK_ENTRY_POINT +#define __ assembler.GetVIXLAssembler()-> + +static void EmitGrayCheckAndFastPath(arm64::Arm64Assembler& assembler, + vixl::aarch64::Register base_reg, + vixl::aarch64::MemOperand& lock_word, + vixl::aarch64::Label* slow_path, + vixl::aarch64::Label* throw_npe = nullptr) { + // Load the lock word containing the rb_state. + __ Ldr(ip0.W(), lock_word); + // Given the numeric representation, it's enough to check the low bit of the rb_state. + static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); + static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); + __ Tbnz(ip0.W(), LockWord::kReadBarrierStateShift, slow_path); + static_assert( + BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET == BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET, + "Field and array LDR offsets must be the same to reuse the same code."); + // To throw NPE, we return to the fast path; the artificial dependence below does not matter. + if (throw_npe != nullptr) { + __ Bind(throw_npe); + } + // Adjust the return address back to the LDR (1 instruction; 2 for heap poisoning). + static_assert(BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET == (kPoisonHeapReferences ? -8 : -4), + "Field LDR must be 1 instruction (4B) before the return address label; " + " 2 instructions (8B) for heap poisoning."); + __ Add(lr, lr, BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET); + // Introduce a dependency on the lock_word including rb_state, + // to prevent load-load reordering, and without using + // a memory barrier (which would be more expensive). + __ Add(base_reg, base_reg, Operand(ip0, LSR, 32)); + __ Br(lr); // And return back to the function. + // Note: The fake dependency is unnecessary for the slow path. +} + +// Load the read barrier introspection entrypoint in register `entrypoint`. +static void LoadReadBarrierMarkIntrospectionEntrypoint(arm64::Arm64Assembler& assembler, + vixl::aarch64::Register entrypoint) { + // entrypoint = Thread::Current()->pReadBarrierMarkReg16, i.e. pReadBarrierMarkIntrospection. + DCHECK_EQ(ip0.GetCode(), 16u); + const int32_t entry_point_offset = + Thread::ReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(ip0.GetCode()); + __ Ldr(entrypoint, MemOperand(tr, entry_point_offset)); +} + +void CodeGeneratorARM64::CompileBakerReadBarrierThunk(Arm64Assembler& assembler, + uint32_t encoded_data, + /*out*/ std::string* debug_name) { + BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data); + switch (kind) { + case BakerReadBarrierKind::kField: { + auto base_reg = + Register::GetXRegFromCode(BakerReadBarrierFirstRegField::Decode(encoded_data)); + CheckValidReg(base_reg.GetCode()); + auto holder_reg = + Register::GetXRegFromCode(BakerReadBarrierSecondRegField::Decode(encoded_data)); + CheckValidReg(holder_reg.GetCode()); + UseScratchRegisterScope temps(assembler.GetVIXLAssembler()); + temps.Exclude(ip0, ip1); + // If base_reg differs from holder_reg, the offset was too large and we must have emitted + // an explicit null check before the load. Otherwise, for implicit null checks, we need to + // null-check the holder as we do not necessarily do that check before going to the thunk. + vixl::aarch64::Label throw_npe_label; + vixl::aarch64::Label* throw_npe = nullptr; + if (GetCompilerOptions().GetImplicitNullChecks() && holder_reg.Is(base_reg)) { + throw_npe = &throw_npe_label; + __ Cbz(holder_reg.W(), throw_npe); + } + // Check if the holder is gray and, if not, add fake dependency to the base register + // and return to the LDR instruction to load the reference. Otherwise, use introspection + // to load the reference and call the entrypoint that performs further checks on the + // reference and marks it if needed. + vixl::aarch64::Label slow_path; + MemOperand lock_word(holder_reg, mirror::Object::MonitorOffset().Int32Value()); + EmitGrayCheckAndFastPath(assembler, base_reg, lock_word, &slow_path, throw_npe); + __ Bind(&slow_path); + MemOperand ldr_address(lr, BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET); + __ Ldr(ip0.W(), ldr_address); // Load the LDR (immediate) unsigned offset. + LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ip1); + __ Ubfx(ip0.W(), ip0.W(), 10, 12); // Extract the offset. + __ Ldr(ip0.W(), MemOperand(base_reg, ip0, LSL, 2)); // Load the reference. + // Do not unpoison. With heap poisoning enabled, the entrypoint expects a poisoned reference. + __ Br(ip1); // Jump to the entrypoint. + break; + } + case BakerReadBarrierKind::kArray: { + auto base_reg = + Register::GetXRegFromCode(BakerReadBarrierFirstRegField::Decode(encoded_data)); + CheckValidReg(base_reg.GetCode()); + DCHECK_EQ(kBakerReadBarrierInvalidEncodedReg, + BakerReadBarrierSecondRegField::Decode(encoded_data)); + UseScratchRegisterScope temps(assembler.GetVIXLAssembler()); + temps.Exclude(ip0, ip1); + vixl::aarch64::Label slow_path; + int32_t data_offset = + mirror::Array::DataOffset(Primitive::ComponentSize(Primitive::kPrimNot)).Int32Value(); + MemOperand lock_word(base_reg, mirror::Object::MonitorOffset().Int32Value() - data_offset); + DCHECK_LT(lock_word.GetOffset(), 0); + EmitGrayCheckAndFastPath(assembler, base_reg, lock_word, &slow_path); + __ Bind(&slow_path); + MemOperand ldr_address(lr, BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET); + __ Ldr(ip0.W(), ldr_address); // Load the LDR (register) unsigned offset. + LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ip1); + __ Ubfx(ip0, ip0, 16, 6); // Extract the index register, plus 32 (bit 21 is set). + __ Bfi(ip1, ip0, 3, 6); // Insert ip0 to the entrypoint address to create + // a switch case target based on the index register. + __ Mov(ip0, base_reg); // Move the base register to ip0. + __ Br(ip1); // Jump to the entrypoint's array switch case. + break; + } + case BakerReadBarrierKind::kGcRoot: { + // Check if the reference needs to be marked and if so (i.e. not null, not marked yet + // and it does not have a forwarding address), call the correct introspection entrypoint; + // otherwise return the reference (or the extracted forwarding address). + // There is no gray bit check for GC roots. + auto root_reg = + Register::GetWRegFromCode(BakerReadBarrierFirstRegField::Decode(encoded_data)); + CheckValidReg(root_reg.GetCode()); + DCHECK_EQ(kBakerReadBarrierInvalidEncodedReg, + BakerReadBarrierSecondRegField::Decode(encoded_data)); + UseScratchRegisterScope temps(assembler.GetVIXLAssembler()); + temps.Exclude(ip0, ip1); + vixl::aarch64::Label return_label, not_marked, forwarding_address; + __ Cbz(root_reg, &return_label); + MemOperand lock_word(root_reg.X(), mirror::Object::MonitorOffset().Int32Value()); + __ Ldr(ip0.W(), lock_word); + __ Tbz(ip0.W(), LockWord::kMarkBitStateShift, ¬_marked); + __ Bind(&return_label); + __ Br(lr); + __ Bind(¬_marked); + __ Tst(ip0.W(), Operand(ip0.W(), LSL, 1)); + __ B(&forwarding_address, mi); + LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ip1); + // Adjust the art_quick_read_barrier_mark_introspection address in IP1 to + // art_quick_read_barrier_mark_introspection_gc_roots. + __ Add(ip1, ip1, Operand(BAKER_MARK_INTROSPECTION_GC_ROOT_ENTRYPOINT_OFFSET)); + __ Mov(ip0.W(), root_reg); + __ Br(ip1); + __ Bind(&forwarding_address); + __ Lsl(root_reg, ip0.W(), LockWord::kForwardingAddressShift); + __ Br(lr); + break; + } + default: + LOG(FATAL) << "Unexpected kind: " << static_cast<uint32_t>(kind); + UNREACHABLE(); + } + + if (GetCompilerOptions().GenerateAnyDebugInfo()) { + std::ostringstream oss; + oss << "BakerReadBarrierThunk"; + switch (kind) { + case BakerReadBarrierKind::kField: + oss << "Field_r" << BakerReadBarrierFirstRegField::Decode(encoded_data) + << "_r" << BakerReadBarrierSecondRegField::Decode(encoded_data); + break; + case BakerReadBarrierKind::kArray: + oss << "Array_r" << BakerReadBarrierFirstRegField::Decode(encoded_data); + DCHECK_EQ(kBakerReadBarrierInvalidEncodedReg, + BakerReadBarrierSecondRegField::Decode(encoded_data)); + break; + case BakerReadBarrierKind::kGcRoot: + oss << "GcRoot_r" << BakerReadBarrierFirstRegField::Decode(encoded_data); + DCHECK_EQ(kBakerReadBarrierInvalidEncodedReg, + BakerReadBarrierSecondRegField::Decode(encoded_data)); + break; + } + *debug_name = oss.str(); + } +} + +#undef __ + } // namespace arm64 } // namespace art diff --git a/compiler/optimizing/code_generator_arm64.h b/compiler/optimizing/code_generator_arm64.h index 0654046de5d..aa343b1185d 100644 --- a/compiler/optimizing/code_generator_arm64.h +++ b/compiler/optimizing/code_generator_arm64.h @@ -18,6 +18,7 @@ #define ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM64_H_ #include "arch/arm64/quick_method_frame_info_arm64.h" +#include "base/bit_field.h" #include "code_generator.h" #include "common_arm64.h" #include "dex/dex_file_types.h" @@ -36,6 +37,11 @@ #pragma GCC diagnostic pop namespace art { + +namespace linker { +class Arm64RelativePatcherTest; +} // namespace linker + namespace arm64 { class CodeGeneratorARM64; @@ -264,6 +270,8 @@ class InstructionCodeGeneratorARM64 : public InstructionCodeGenerator { private: void GenerateClassInitializationCheck(SlowPathCodeARM64* slow_path, vixl::aarch64::Register class_reg); + void GenerateBitstringTypeCheckCompare(HTypeCheckInstruction* check, + vixl::aarch64::Register temp); void GenerateSuspendCheck(HSuspendCheck* instruction, HBasicBlock* successor); void HandleBinaryOp(HBinaryOperation* instr); @@ -273,6 +281,10 @@ class InstructionCodeGeneratorARM64 : public InstructionCodeGenerator { void HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info); void HandleCondition(HCondition* instruction); + void GenerateMinMaxInt(LocationSummary* locations, bool is_min, DataType::Type type); + void GenerateMinMaxFP(LocationSummary* locations, bool is_min, DataType::Type type); + void GenerateMinMax(HBinaryOperation* minmax, bool is_min); + // Generate a heap reference load using one register `out`: // // out <- *(out + offset) @@ -303,17 +315,6 @@ class InstructionCodeGeneratorARM64 : public InstructionCodeGenerator { uint32_t offset, Location maybe_temp, ReadBarrierOption read_barrier_option); - // Generate a GC root reference load: - // - // root <- *(obj + offset) - // - // while honoring read barriers based on read_barrier_option. - void GenerateGcRootFieldLoad(HInstruction* instruction, - Location root, - vixl::aarch64::Register obj, - uint32_t offset, - vixl::aarch64::Label* fixup_label, - ReadBarrierOption read_barrier_option); // Generate a floating-point comparison. void GenerateFcmp(HInstruction* instruction); @@ -561,7 +562,14 @@ class CodeGeneratorARM64 : public CodeGenerator { UNIMPLEMENTED(FATAL); } - // Add a new PC-relative method patch for an instruction and return the label + // Add a new boot image relocation patch for an instruction and return the label + // to be bound before the instruction. The instruction will be either the + // ADRP (pass `adrp_label = null`) or the LDR (pass `adrp_label` pointing + // to the associated ADRP patch label). + vixl::aarch64::Label* NewBootImageRelRoPatch(uint32_t boot_image_offset, + vixl::aarch64::Label* adrp_label = nullptr); + + // Add a new boot image method patch for an instruction and return the label // to be bound before the instruction. The instruction will be either the // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing // to the associated ADRP patch label). @@ -575,7 +583,7 @@ class CodeGeneratorARM64 : public CodeGenerator { vixl::aarch64::Label* NewMethodBssEntryPatch(MethodReference target_method, vixl::aarch64::Label* adrp_label = nullptr); - // Add a new PC-relative type patch for an instruction and return the label + // Add a new boot image type patch for an instruction and return the label // to be bound before the instruction. The instruction will be either the // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing // to the associated ADRP patch label). @@ -591,7 +599,7 @@ class CodeGeneratorARM64 : public CodeGenerator { dex::TypeIndex type_index, vixl::aarch64::Label* adrp_label = nullptr); - // Add a new PC-relative string patch for an instruction and return the label + // Add a new boot image string patch for an instruction and return the label // to be bound before the instruction. The instruction will be either the // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing // to the associated ADRP patch label). @@ -628,9 +636,24 @@ class CodeGeneratorARM64 : public CodeGenerator { vixl::aarch64::Register base); void EmitLinkerPatches(ArenaVector<linker::LinkerPatch>* linker_patches) OVERRIDE; + bool NeedsThunkCode(const linker::LinkerPatch& patch) const OVERRIDE; + void EmitThunkCode(const linker::LinkerPatch& patch, + /*out*/ ArenaVector<uint8_t>* code, + /*out*/ std::string* debug_name) OVERRIDE; void EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) OVERRIDE; + // Generate a GC root reference load: + // + // root <- *(obj + offset) + // + // while honoring read barriers based on read_barrier_option. + void GenerateGcRootFieldLoad(HInstruction* instruction, + Location root, + vixl::aarch64::Register obj, + uint32_t offset, + vixl::aarch64::Label* fixup_label, + ReadBarrierOption read_barrier_option); // Fast path implementation of ReadBarrier::Barrier for a heap // reference field load when Baker's read barriers are used. void GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction, @@ -765,6 +788,62 @@ class CodeGeneratorARM64 : public CodeGenerator { void GenerateExplicitNullCheck(HNullCheck* instruction) OVERRIDE; private: + // Encoding of thunk type and data for link-time generated thunks for Baker read barriers. + + enum class BakerReadBarrierKind : uint8_t { + kField, // Field get or array get with constant offset (i.e. constant index). + kArray, // Array get with index in register. + kGcRoot, // GC root load. + kLast = kGcRoot + }; + + static constexpr uint32_t kBakerReadBarrierInvalidEncodedReg = /* sp/zr is invalid */ 31u; + + static constexpr size_t kBitsForBakerReadBarrierKind = + MinimumBitsToStore(static_cast<size_t>(BakerReadBarrierKind::kLast)); + static constexpr size_t kBakerReadBarrierBitsForRegister = + MinimumBitsToStore(kBakerReadBarrierInvalidEncodedReg); + using BakerReadBarrierKindField = + BitField<BakerReadBarrierKind, 0, kBitsForBakerReadBarrierKind>; + using BakerReadBarrierFirstRegField = + BitField<uint32_t, kBitsForBakerReadBarrierKind, kBakerReadBarrierBitsForRegister>; + using BakerReadBarrierSecondRegField = + BitField<uint32_t, + kBitsForBakerReadBarrierKind + kBakerReadBarrierBitsForRegister, + kBakerReadBarrierBitsForRegister>; + + static void CheckValidReg(uint32_t reg) { + DCHECK(reg < vixl::aarch64::lr.GetCode() && + reg != vixl::aarch64::ip0.GetCode() && + reg != vixl::aarch64::ip1.GetCode()) << reg; + } + + static inline uint32_t EncodeBakerReadBarrierFieldData(uint32_t base_reg, uint32_t holder_reg) { + CheckValidReg(base_reg); + CheckValidReg(holder_reg); + return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kField) | + BakerReadBarrierFirstRegField::Encode(base_reg) | + BakerReadBarrierSecondRegField::Encode(holder_reg); + } + + static inline uint32_t EncodeBakerReadBarrierArrayData(uint32_t base_reg) { + CheckValidReg(base_reg); + return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kArray) | + BakerReadBarrierFirstRegField::Encode(base_reg) | + BakerReadBarrierSecondRegField::Encode(kBakerReadBarrierInvalidEncodedReg); + } + + static inline uint32_t EncodeBakerReadBarrierGcRootData(uint32_t root_reg) { + CheckValidReg(root_reg); + return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kGcRoot) | + BakerReadBarrierFirstRegField::Encode(root_reg) | + BakerReadBarrierSecondRegField::Encode(kBakerReadBarrierInvalidEncodedReg); + } + + void CompileBakerReadBarrierThunk(Arm64Assembler& assembler, + uint32_t encoded_data, + /*out*/ std::string* debug_name); + using Uint64ToLiteralMap = ArenaSafeMap<uint64_t, vixl::aarch64::Literal<uint64_t>*>; using Uint32ToLiteralMap = ArenaSafeMap<uint32_t, vixl::aarch64::Literal<uint32_t>*>; using StringToLiteralMap = ArenaSafeMap<StringReference, @@ -820,7 +899,8 @@ class CodeGeneratorARM64 : public CodeGenerator { Uint32ToLiteralMap uint32_literals_; // Deduplication map for 64-bit literals, used for non-patchable method address or method code. Uint64ToLiteralMap uint64_literals_; - // PC-relative method patch info for kBootImageLinkTimePcRelative. + // PC-relative method patch info for kBootImageLinkTimePcRelative/BootImageRelRo. + // Also used for type/string patches for kBootImageRelRo (same linker patch as for methods). ArenaDeque<PcRelativePatchInfo> boot_image_method_patches_; // PC-relative method patch info for kBssEntry. ArenaDeque<PcRelativePatchInfo> method_bss_entry_patches_; @@ -828,7 +908,7 @@ class CodeGeneratorARM64 : public CodeGenerator { ArenaDeque<PcRelativePatchInfo> boot_image_type_patches_; // PC-relative type patch info for kBssEntry. ArenaDeque<PcRelativePatchInfo> type_bss_entry_patches_; - // PC-relative String patch info; type depends on configuration (intern table or boot image PIC). + // PC-relative String patch info for kBootImageLinkTimePcRelative. ArenaDeque<PcRelativePatchInfo> boot_image_string_patches_; // PC-relative String patch info for kBssEntry. ArenaDeque<PcRelativePatchInfo> string_bss_entry_patches_; @@ -840,6 +920,7 @@ class CodeGeneratorARM64 : public CodeGenerator { // Patches for class literals in JIT compiled code. TypeToLiteralMap jit_class_patches_; + friend class linker::Arm64RelativePatcherTest; DISALLOW_COPY_AND_ASSIGN(CodeGeneratorARM64); }; diff --git a/compiler/optimizing/code_generator_arm_vixl.cc b/compiler/optimizing/code_generator_arm_vixl.cc index 2f495fc15fd..7350b146f95 100644 --- a/compiler/optimizing/code_generator_arm_vixl.cc +++ b/compiler/optimizing/code_generator_arm_vixl.cc @@ -29,7 +29,6 @@ #include "gc/accounting/card_table.h" #include "heap_poisoning.h" #include "intrinsics_arm_vixl.h" -#include "linker/arm/relative_patcher_thumb2.h" #include "linker/linker_patch.h" #include "mirror/array-inl.h" #include "mirror/class-inl.h" @@ -94,9 +93,6 @@ constexpr bool kBakerReadBarrierLinkTimeThunksEnableForFields = true; constexpr bool kBakerReadBarrierLinkTimeThunksEnableForArrays = true; constexpr bool kBakerReadBarrierLinkTimeThunksEnableForGcRoots = true; -// The reserved entrypoint register for link-time generated thunks. -const vixl32::Register kBakerCcEntrypointRegister = r4; - // Using a base helps identify when we hit Marking Register check breakpoints. constexpr int kMarkingRegisterCheckBreakCodeBaseCode = 0x10; @@ -116,8 +112,6 @@ static inline void ExcludeIPAndBakerCcEntrypointRegister(UseScratchRegisterScope DCHECK(temps->IsAvailable(ip)); temps->Exclude(ip); DCHECK(!temps->IsAvailable(kBakerCcEntrypointRegister)); - DCHECK_EQ(kBakerCcEntrypointRegister.GetCode(), - linker::Thumb2RelativePatcher::kBakerCcEntrypointRegister); DCHECK_NE(instruction->GetLocations()->GetTempCount(), 0u); DCHECK(RegisterFrom(instruction->GetLocations()->GetTemp( instruction->GetLocations()->GetTempCount() - 1u)).Is(kBakerCcEntrypointRegister)); @@ -2422,6 +2416,80 @@ void CodeGeneratorARMVIXL::Finalize(CodeAllocator* allocator) { FixJumpTables(); GetAssembler()->FinalizeCode(); CodeGenerator::Finalize(allocator); + + // Verify Baker read barrier linker patches. + if (kIsDebugBuild) { + ArrayRef<const uint8_t> code = allocator->GetMemory(); + for (const BakerReadBarrierPatchInfo& info : baker_read_barrier_patches_) { + DCHECK(info.label.IsBound()); + uint32_t literal_offset = info.label.GetLocation(); + DCHECK_ALIGNED(literal_offset, 2u); + + auto GetInsn16 = [&code](uint32_t offset) { + DCHECK_ALIGNED(offset, 2u); + return (static_cast<uint32_t>(code[offset + 0]) << 0) + + (static_cast<uint32_t>(code[offset + 1]) << 8); + }; + auto GetInsn32 = [=](uint32_t offset) { + return (GetInsn16(offset) << 16) + (GetInsn16(offset + 2u) << 0); + }; + + uint32_t encoded_data = info.custom_data; + BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data); + // Check that the next instruction matches the expected LDR. + switch (kind) { + case BakerReadBarrierKind::kField: { + BakerReadBarrierWidth width = BakerReadBarrierWidthField::Decode(encoded_data); + if (width == BakerReadBarrierWidth::kWide) { + DCHECK_GE(code.size() - literal_offset, 8u); + uint32_t next_insn = GetInsn32(literal_offset + 4u); + // LDR (immediate), encoding T3, with correct base_reg. + CheckValidReg((next_insn >> 12) & 0xfu); // Check destination register. + const uint32_t base_reg = BakerReadBarrierFirstRegField::Decode(encoded_data); + CHECK_EQ(next_insn & 0xffff0000u, 0xf8d00000u | (base_reg << 16)); + } else { + DCHECK_GE(code.size() - literal_offset, 6u); + uint32_t next_insn = GetInsn16(literal_offset + 4u); + // LDR (immediate), encoding T1, with correct base_reg. + CheckValidReg(next_insn & 0x7u); // Check destination register. + const uint32_t base_reg = BakerReadBarrierFirstRegField::Decode(encoded_data); + CHECK_EQ(next_insn & 0xf838u, 0x6800u | (base_reg << 3)); + } + break; + } + case BakerReadBarrierKind::kArray: { + DCHECK_GE(code.size() - literal_offset, 8u); + uint32_t next_insn = GetInsn32(literal_offset + 4u); + // LDR (register) with correct base_reg, S=1 and option=011 (LDR Wt, [Xn, Xm, LSL #2]). + CheckValidReg((next_insn >> 12) & 0xfu); // Check destination register. + const uint32_t base_reg = BakerReadBarrierFirstRegField::Decode(encoded_data); + CHECK_EQ(next_insn & 0xffff0ff0u, 0xf8500020u | (base_reg << 16)); + CheckValidReg(next_insn & 0xf); // Check index register + break; + } + case BakerReadBarrierKind::kGcRoot: { + BakerReadBarrierWidth width = BakerReadBarrierWidthField::Decode(encoded_data); + if (width == BakerReadBarrierWidth::kWide) { + DCHECK_GE(literal_offset, 4u); + uint32_t prev_insn = GetInsn32(literal_offset - 4u); + // LDR (immediate), encoding T3, with correct root_reg. + const uint32_t root_reg = BakerReadBarrierFirstRegField::Decode(encoded_data); + CHECK_EQ(prev_insn & 0xfff0f000u, 0xf8d00000u | (root_reg << 12)); + } else { + DCHECK_GE(literal_offset, 2u); + uint32_t prev_insn = GetInsn16(literal_offset - 2u); + // LDR (immediate), encoding T1, with correct root_reg. + const uint32_t root_reg = BakerReadBarrierFirstRegField::Decode(encoded_data); + CHECK_EQ(prev_insn & 0xf807u, 0x6800u | root_reg); + } + break; + } + default: + LOG(FATAL) << "Unexpected kind: " << static_cast<uint32_t>(kind); + UNREACHABLE(); + } + } + } } void CodeGeneratorARMVIXL::SetupBlockedRegisters() const { @@ -4690,6 +4758,299 @@ void InstructionCodeGeneratorARMVIXL::VisitRem(HRem* rem) { } } +static void CreateMinMaxLocations(ArenaAllocator* allocator, HBinaryOperation* minmax) { + LocationSummary* locations = new (allocator) LocationSummary(minmax); + switch (minmax->GetResultType()) { + case DataType::Type::kInt32: + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); + break; + case DataType::Type::kInt64: + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RequiresRegister()); + locations->SetOut(Location::SameAsFirstInput()); + break; + case DataType::Type::kFloat32: + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetInAt(1, Location::RequiresFpuRegister()); + locations->SetOut(Location::SameAsFirstInput()); + locations->AddTemp(Location::RequiresRegister()); + break; + case DataType::Type::kFloat64: + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetInAt(1, Location::RequiresFpuRegister()); + locations->SetOut(Location::SameAsFirstInput()); + break; + default: + LOG(FATAL) << "Unexpected type for HMinMax " << minmax->GetResultType(); + } +} + +void InstructionCodeGeneratorARMVIXL::GenerateMinMaxInt(LocationSummary* locations, bool is_min) { + Location op1_loc = locations->InAt(0); + Location op2_loc = locations->InAt(1); + Location out_loc = locations->Out(); + + vixl32::Register op1 = RegisterFrom(op1_loc); + vixl32::Register op2 = RegisterFrom(op2_loc); + vixl32::Register out = RegisterFrom(out_loc); + + __ Cmp(op1, op2); + + { + ExactAssemblyScope aas(GetVIXLAssembler(), + 3 * kMaxInstructionSizeInBytes, + CodeBufferCheckScope::kMaximumSize); + + __ ite(is_min ? lt : gt); + __ mov(is_min ? lt : gt, out, op1); + __ mov(is_min ? ge : le, out, op2); + } +} + +void InstructionCodeGeneratorARMVIXL::GenerateMinMaxLong(LocationSummary* locations, bool is_min) { + Location op1_loc = locations->InAt(0); + Location op2_loc = locations->InAt(1); + Location out_loc = locations->Out(); + + // Optimization: don't generate any code if inputs are the same. + if (op1_loc.Equals(op2_loc)) { + DCHECK(out_loc.Equals(op1_loc)); // out_loc is set as SameAsFirstInput() in location builder. + return; + } + + vixl32::Register op1_lo = LowRegisterFrom(op1_loc); + vixl32::Register op1_hi = HighRegisterFrom(op1_loc); + vixl32::Register op2_lo = LowRegisterFrom(op2_loc); + vixl32::Register op2_hi = HighRegisterFrom(op2_loc); + vixl32::Register out_lo = LowRegisterFrom(out_loc); + vixl32::Register out_hi = HighRegisterFrom(out_loc); + UseScratchRegisterScope temps(GetVIXLAssembler()); + const vixl32::Register temp = temps.Acquire(); + + DCHECK(op1_lo.Is(out_lo)); + DCHECK(op1_hi.Is(out_hi)); + + // Compare op1 >= op2, or op1 < op2. + __ Cmp(out_lo, op2_lo); + __ Sbcs(temp, out_hi, op2_hi); + + // Now GE/LT condition code is correct for the long comparison. + { + vixl32::ConditionType cond = is_min ? ge : lt; + ExactAssemblyScope it_scope(GetVIXLAssembler(), + 3 * kMaxInstructionSizeInBytes, + CodeBufferCheckScope::kMaximumSize); + __ itt(cond); + __ mov(cond, out_lo, op2_lo); + __ mov(cond, out_hi, op2_hi); + } +} + +void InstructionCodeGeneratorARMVIXL::GenerateMinMaxFloat(HInstruction* minmax, bool is_min) { + LocationSummary* locations = minmax->GetLocations(); + Location op1_loc = locations->InAt(0); + Location op2_loc = locations->InAt(1); + Location out_loc = locations->Out(); + + // Optimization: don't generate any code if inputs are the same. + if (op1_loc.Equals(op2_loc)) { + DCHECK(out_loc.Equals(op1_loc)); // out_loc is set as SameAsFirstInput() in location builder. + return; + } + + vixl32::SRegister op1 = SRegisterFrom(op1_loc); + vixl32::SRegister op2 = SRegisterFrom(op2_loc); + vixl32::SRegister out = SRegisterFrom(out_loc); + + UseScratchRegisterScope temps(GetVIXLAssembler()); + const vixl32::Register temp1 = temps.Acquire(); + vixl32::Register temp2 = RegisterFrom(locations->GetTemp(0)); + vixl32::Label nan, done; + vixl32::Label* final_label = codegen_->GetFinalLabel(minmax, &done); + + DCHECK(op1.Is(out)); + + __ Vcmp(op1, op2); + __ Vmrs(RegisterOrAPSR_nzcv(kPcCode), FPSCR); + __ B(vs, &nan, /* far_target */ false); // if un-ordered, go to NaN handling. + + // op1 <> op2 + vixl32::ConditionType cond = is_min ? gt : lt; + { + ExactAssemblyScope it_scope(GetVIXLAssembler(), + 2 * kMaxInstructionSizeInBytes, + CodeBufferCheckScope::kMaximumSize); + __ it(cond); + __ vmov(cond, F32, out, op2); + } + // for <>(not equal), we've done min/max calculation. + __ B(ne, final_label, /* far_target */ false); + + // handle op1 == op2, max(+0.0,-0.0), min(+0.0,-0.0). + __ Vmov(temp1, op1); + __ Vmov(temp2, op2); + if (is_min) { + __ Orr(temp1, temp1, temp2); + } else { + __ And(temp1, temp1, temp2); + } + __ Vmov(out, temp1); + __ B(final_label); + + // handle NaN input. + __ Bind(&nan); + __ Movt(temp1, High16Bits(kNanFloat)); // 0x7FC0xxxx is a NaN. + __ Vmov(out, temp1); + + if (done.IsReferenced()) { + __ Bind(&done); + } +} + +void InstructionCodeGeneratorARMVIXL::GenerateMinMaxDouble(HInstruction* minmax, bool is_min) { + LocationSummary* locations = minmax->GetLocations(); + Location op1_loc = locations->InAt(0); + Location op2_loc = locations->InAt(1); + Location out_loc = locations->Out(); + + // Optimization: don't generate any code if inputs are the same. + if (op1_loc.Equals(op2_loc)) { + DCHECK(out_loc.Equals(op1_loc)); // out_loc is set as SameAsFirstInput() in. + return; + } + + vixl32::DRegister op1 = DRegisterFrom(op1_loc); + vixl32::DRegister op2 = DRegisterFrom(op2_loc); + vixl32::DRegister out = DRegisterFrom(out_loc); + vixl32::Label handle_nan_eq, done; + vixl32::Label* final_label = codegen_->GetFinalLabel(minmax, &done); + + DCHECK(op1.Is(out)); + + __ Vcmp(op1, op2); + __ Vmrs(RegisterOrAPSR_nzcv(kPcCode), FPSCR); + __ B(vs, &handle_nan_eq, /* far_target */ false); // if un-ordered, go to NaN handling. + + // op1 <> op2 + vixl32::ConditionType cond = is_min ? gt : lt; + { + ExactAssemblyScope it_scope(GetVIXLAssembler(), + 2 * kMaxInstructionSizeInBytes, + CodeBufferCheckScope::kMaximumSize); + __ it(cond); + __ vmov(cond, F64, out, op2); + } + // for <>(not equal), we've done min/max calculation. + __ B(ne, final_label, /* far_target */ false); + + // handle op1 == op2, max(+0.0,-0.0). + if (!is_min) { + __ Vand(F64, out, op1, op2); + __ B(final_label); + } + + // handle op1 == op2, min(+0.0,-0.0), NaN input. + __ Bind(&handle_nan_eq); + __ Vorr(F64, out, op1, op2); // assemble op1/-0.0/NaN. + + if (done.IsReferenced()) { + __ Bind(&done); + } +} + +void InstructionCodeGeneratorARMVIXL::GenerateMinMax(HBinaryOperation* minmax, bool is_min) { + DataType::Type type = minmax->GetResultType(); + switch (type) { + case DataType::Type::kInt32: + GenerateMinMaxInt(minmax->GetLocations(), is_min); + break; + case DataType::Type::kInt64: + GenerateMinMaxLong(minmax->GetLocations(), is_min); + break; + case DataType::Type::kFloat32: + GenerateMinMaxFloat(minmax, is_min); + break; + case DataType::Type::kFloat64: + GenerateMinMaxDouble(minmax, is_min); + break; + default: + LOG(FATAL) << "Unexpected type for HMinMax " << type; + } +} + +void LocationsBuilderARMVIXL::VisitMin(HMin* min) { + CreateMinMaxLocations(GetGraph()->GetAllocator(), min); +} + +void InstructionCodeGeneratorARMVIXL::VisitMin(HMin* min) { + GenerateMinMax(min, /*is_min*/ true); +} + +void LocationsBuilderARMVIXL::VisitMax(HMax* max) { + CreateMinMaxLocations(GetGraph()->GetAllocator(), max); +} + +void InstructionCodeGeneratorARMVIXL::VisitMax(HMax* max) { + GenerateMinMax(max, /*is_min*/ false); +} + +void LocationsBuilderARMVIXL::VisitAbs(HAbs* abs) { + LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(abs); + switch (abs->GetResultType()) { + case DataType::Type::kInt32: + case DataType::Type::kInt64: + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); + locations->AddTemp(Location::RequiresRegister()); + break; + case DataType::Type::kFloat32: + case DataType::Type::kFloat64: + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); + break; + default: + LOG(FATAL) << "Unexpected type for abs operation " << abs->GetResultType(); + } +} + +void InstructionCodeGeneratorARMVIXL::VisitAbs(HAbs* abs) { + LocationSummary* locations = abs->GetLocations(); + switch (abs->GetResultType()) { + case DataType::Type::kInt32: { + vixl32::Register in_reg = RegisterFrom(locations->InAt(0)); + vixl32::Register out_reg = RegisterFrom(locations->Out()); + vixl32::Register mask = RegisterFrom(locations->GetTemp(0)); + __ Asr(mask, in_reg, 31); + __ Add(out_reg, in_reg, mask); + __ Eor(out_reg, out_reg, mask); + break; + } + case DataType::Type::kInt64: { + Location in = locations->InAt(0); + vixl32::Register in_reg_lo = LowRegisterFrom(in); + vixl32::Register in_reg_hi = HighRegisterFrom(in); + Location output = locations->Out(); + vixl32::Register out_reg_lo = LowRegisterFrom(output); + vixl32::Register out_reg_hi = HighRegisterFrom(output); + DCHECK(!out_reg_lo.Is(in_reg_hi)) << "Diagonal overlap unexpected."; + vixl32::Register mask = RegisterFrom(locations->GetTemp(0)); + __ Asr(mask, in_reg_hi, 31); + __ Adds(out_reg_lo, in_reg_lo, mask); + __ Adc(out_reg_hi, in_reg_hi, mask); + __ Eor(out_reg_lo, out_reg_lo, mask); + __ Eor(out_reg_hi, out_reg_hi, mask); + break; + } + case DataType::Type::kFloat32: + case DataType::Type::kFloat64: + __ Vabs(OutputVRegister(abs), InputVRegisterAt(abs, 0)); + break; + default: + LOG(FATAL) << "Unexpected type for abs operation " << abs->GetResultType(); + } +} void LocationsBuilderARMVIXL::VisitDivZeroCheck(HDivZeroCheck* instruction) { LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction); @@ -7033,7 +7394,7 @@ HLoadClass::LoadKind CodeGeneratorARMVIXL::GetSupportedLoadClassKind( case HLoadClass::LoadKind::kReferrersClass: break; case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: - case HLoadClass::LoadKind::kBootImageClassTable: + case HLoadClass::LoadKind::kBootImageRelRo: case HLoadClass::LoadKind::kBssEntry: DCHECK(!Runtime::Current()->UseJitCompilation()); break; @@ -7120,11 +7481,11 @@ void InstructionCodeGeneratorARMVIXL::VisitLoadClass(HLoadClass* cls) NO_THREAD_ DCHECK(!cls->MustGenerateClinitCheck()); // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ vixl32::Register current_method = InputRegisterAt(cls, 0); - GenerateGcRootFieldLoad(cls, - out_loc, - current_method, - ArtMethod::DeclaringClassOffset().Int32Value(), - read_barrier_option); + codegen_->GenerateGcRootFieldLoad(cls, + out_loc, + current_method, + ArtMethod::DeclaringClassOffset().Int32Value(), + read_barrier_option); break; } case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: { @@ -7143,25 +7504,19 @@ void InstructionCodeGeneratorARMVIXL::VisitLoadClass(HLoadClass* cls) NO_THREAD_ __ Ldr(out, codegen_->DeduplicateBootImageAddressLiteral(address)); break; } - case HLoadClass::LoadKind::kBootImageClassTable: { + case HLoadClass::LoadKind::kBootImageRelRo: { DCHECK(!codegen_->GetCompilerOptions().IsBootImage()); CodeGeneratorARMVIXL::PcRelativePatchInfo* labels = - codegen_->NewBootImageTypePatch(cls->GetDexFile(), cls->GetTypeIndex()); + codegen_->NewBootImageRelRoPatch(codegen_->GetBootImageOffset(cls)); codegen_->EmitMovwMovtPlaceholder(labels, out); __ Ldr(out, MemOperand(out, /* offset */ 0)); - // Extract the reference from the slot data, i.e. clear the hash bits. - int32_t masked_hash = ClassTable::TableSlot::MaskHash( - ComputeModifiedUtf8Hash(cls->GetDexFile().StringByTypeIdx(cls->GetTypeIndex()))); - if (masked_hash != 0) { - __ Sub(out, out, Operand(masked_hash)); - } break; } case HLoadClass::LoadKind::kBssEntry: { CodeGeneratorARMVIXL::PcRelativePatchInfo* labels = codegen_->NewTypeBssEntryPatch(cls->GetDexFile(), cls->GetTypeIndex()); codegen_->EmitMovwMovtPlaceholder(labels, out); - GenerateGcRootFieldLoad(cls, out_loc, out, /* offset */ 0, read_barrier_option); + codegen_->GenerateGcRootFieldLoad(cls, out_loc, out, /* offset */ 0, read_barrier_option); generate_null_check = true; break; } @@ -7170,7 +7525,7 @@ void InstructionCodeGeneratorARMVIXL::VisitLoadClass(HLoadClass* cls) NO_THREAD_ cls->GetTypeIndex(), cls->GetClass())); // /* GcRoot<mirror::Class> */ out = *out - GenerateGcRootFieldLoad(cls, out_loc, out, /* offset */ 0, read_barrier_option); + codegen_->GenerateGcRootFieldLoad(cls, out_loc, out, /* offset */ 0, read_barrier_option); break; } case HLoadClass::LoadKind::kRuntimeCall: @@ -7236,11 +7591,72 @@ void InstructionCodeGeneratorARMVIXL::GenerateClassInitializationCheck( __ Bind(slow_path->GetExitLabel()); } +void InstructionCodeGeneratorARMVIXL::GenerateBitstringTypeCheckCompare( + HTypeCheckInstruction* check, + vixl32::Register temp, + vixl32::FlagsUpdate flags_update) { + uint32_t path_to_root = check->GetBitstringPathToRoot(); + uint32_t mask = check->GetBitstringMask(); + DCHECK(IsPowerOfTwo(mask + 1)); + size_t mask_bits = WhichPowerOf2(mask + 1); + + // Note that HInstanceOf shall check for zero value in `temp` but HCheckCast needs + // the Z flag for BNE. This is indicated by the `flags_update` parameter. + if (mask_bits == 16u) { + // Load only the bitstring part of the status word. + __ Ldrh(temp, MemOperand(temp, mirror::Class::StatusOffset().Int32Value())); + // Check if the bitstring bits are equal to `path_to_root`. + if (flags_update == SetFlags) { + __ Cmp(temp, path_to_root); + } else { + __ Sub(temp, temp, path_to_root); + } + } else { + // /* uint32_t */ temp = temp->status_ + __ Ldr(temp, MemOperand(temp, mirror::Class::StatusOffset().Int32Value())); + if (GetAssembler()->ShifterOperandCanHold(SUB, path_to_root)) { + // Compare the bitstring bits using SUB. + __ Sub(temp, temp, path_to_root); + // Shift out bits that do not contribute to the comparison. + __ Lsl(flags_update, temp, temp, dchecked_integral_cast<uint32_t>(32u - mask_bits)); + } else if (IsUint<16>(path_to_root)) { + if (temp.IsLow()) { + // Note: Optimized for size but contains one more dependent instruction than necessary. + // MOVW+SUB(register) would be 8 bytes unless we find a low-reg temporary but the + // macro assembler would use the high reg IP for the constant by default. + // Compare the bitstring bits using SUB. + __ Sub(temp, temp, path_to_root & 0x00ffu); // 16-bit SUB (immediate) T2 + __ Sub(temp, temp, path_to_root & 0xff00u); // 32-bit SUB (immediate) T3 + // Shift out bits that do not contribute to the comparison. + __ Lsl(flags_update, temp, temp, dchecked_integral_cast<uint32_t>(32u - mask_bits)); + } else { + // Extract the bitstring bits. + __ Ubfx(temp, temp, 0, mask_bits); + // Check if the bitstring bits are equal to `path_to_root`. + if (flags_update == SetFlags) { + __ Cmp(temp, path_to_root); + } else { + __ Sub(temp, temp, path_to_root); + } + } + } else { + // Shift out bits that do not contribute to the comparison. + __ Lsl(temp, temp, dchecked_integral_cast<uint32_t>(32u - mask_bits)); + // Check if the shifted bitstring bits are equal to `path_to_root << (32u - mask_bits)`. + if (flags_update == SetFlags) { + __ Cmp(temp, path_to_root << (32u - mask_bits)); + } else { + __ Sub(temp, temp, path_to_root << (32u - mask_bits)); + } + } + } +} + HLoadString::LoadKind CodeGeneratorARMVIXL::GetSupportedLoadStringKind( HLoadString::LoadKind desired_string_load_kind) { switch (desired_string_load_kind) { case HLoadString::LoadKind::kBootImageLinkTimePcRelative: - case HLoadString::LoadKind::kBootImageInternTable: + case HLoadString::LoadKind::kBootImageRelRo: case HLoadString::LoadKind::kBssEntry: DCHECK(!Runtime::Current()->UseJitCompilation()); break; @@ -7304,10 +7720,10 @@ void InstructionCodeGeneratorARMVIXL::VisitLoadString(HLoadString* load) NO_THRE __ Ldr(out, codegen_->DeduplicateBootImageAddressLiteral(address)); return; } - case HLoadString::LoadKind::kBootImageInternTable: { + case HLoadString::LoadKind::kBootImageRelRo: { DCHECK(!codegen_->GetCompilerOptions().IsBootImage()); CodeGeneratorARMVIXL::PcRelativePatchInfo* labels = - codegen_->NewBootImageStringPatch(load->GetDexFile(), load->GetStringIndex()); + codegen_->NewBootImageRelRoPatch(codegen_->GetBootImageOffset(load)); codegen_->EmitMovwMovtPlaceholder(labels, out); __ Ldr(out, MemOperand(out, /* offset */ 0)); return; @@ -7317,7 +7733,8 @@ void InstructionCodeGeneratorARMVIXL::VisitLoadString(HLoadString* load) NO_THRE CodeGeneratorARMVIXL::PcRelativePatchInfo* labels = codegen_->NewStringBssEntryPatch(load->GetDexFile(), load->GetStringIndex()); codegen_->EmitMovwMovtPlaceholder(labels, out); - GenerateGcRootFieldLoad(load, out_loc, out, /* offset */ 0, kCompilerReadBarrierOption); + codegen_->GenerateGcRootFieldLoad( + load, out_loc, out, /* offset */ 0, kCompilerReadBarrierOption); LoadStringSlowPathARMVIXL* slow_path = new (codegen_->GetScopedAllocator()) LoadStringSlowPathARMVIXL(load); codegen_->AddSlowPath(slow_path); @@ -7331,7 +7748,8 @@ void InstructionCodeGeneratorARMVIXL::VisitLoadString(HLoadString* load) NO_THRE load->GetStringIndex(), load->GetString())); // /* GcRoot<mirror::String> */ out = *out - GenerateGcRootFieldLoad(load, out_loc, out, /* offset */ 0, kCompilerReadBarrierOption); + codegen_->GenerateGcRootFieldLoad( + load, out_loc, out, /* offset */ 0, kCompilerReadBarrierOption); return; } default: @@ -7427,6 +7845,8 @@ void LocationsBuilderARMVIXL::VisitInstanceOf(HInstanceOf* instruction) { case TypeCheckKind::kInterfaceCheck: call_kind = LocationSummary::kCallOnSlowPath; break; + case TypeCheckKind::kBitstringCheck: + break; } LocationSummary* locations = @@ -7435,7 +7855,13 @@ void LocationsBuilderARMVIXL::VisitInstanceOf(HInstanceOf* instruction) { locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. } locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); + if (type_check_kind == TypeCheckKind::kBitstringCheck) { + locations->SetInAt(1, Location::ConstantLocation(instruction->InputAt(1)->AsConstant())); + locations->SetInAt(2, Location::ConstantLocation(instruction->InputAt(2)->AsConstant())); + locations->SetInAt(3, Location::ConstantLocation(instruction->InputAt(3)->AsConstant())); + } else { + locations->SetInAt(1, Location::RequiresRegister()); + } // The "out" register is used as a temporary, so it overlaps with the inputs. // Note that TypeCheckSlowPathARM uses this register too. locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); @@ -7450,7 +7876,9 @@ void InstructionCodeGeneratorARMVIXL::VisitInstanceOf(HInstanceOf* instruction) LocationSummary* locations = instruction->GetLocations(); Location obj_loc = locations->InAt(0); vixl32::Register obj = InputRegisterAt(instruction, 0); - vixl32::Register cls = InputRegisterAt(instruction, 1); + vixl32::Register cls = (type_check_kind == TypeCheckKind::kBitstringCheck) + ? vixl32::Register() + : InputRegisterAt(instruction, 1); Location out_loc = locations->Out(); vixl32::Register out = OutputRegister(instruction); const size_t num_temps = NumberOfInstanceOfTemps(type_check_kind); @@ -7690,6 +8118,26 @@ void InstructionCodeGeneratorARMVIXL::VisitInstanceOf(HInstanceOf* instruction) __ B(slow_path->GetEntryLabel()); break; } + + case TypeCheckKind::kBitstringCheck: { + // /* HeapReference<Class> */ temp = obj->klass_ + GenerateReferenceLoadTwoRegisters(instruction, + out_loc, + obj_loc, + class_offset, + maybe_temp_loc, + kWithoutReadBarrier); + + GenerateBitstringTypeCheckCompare(instruction, out, DontCare); + // If `out` is a low reg and we would have another low reg temp, we could + // optimize this as RSBS+ADC, see GenerateConditionWithZero(). + // + // Also, in some cases when `out` is a low reg and we're loading a constant to IP + // it would make sense to use CMP+MOV+IT+MOV instead of SUB+CLZ+LSR as the code size + // would be the same and we would have fewer direct data dependencies. + codegen_->GenerateConditionWithZero(kCondEQ, out, out); // CLZ+LSR + break; + } } if (done.IsReferenced()) { @@ -7707,7 +8155,13 @@ void LocationsBuilderARMVIXL::VisitCheckCast(HCheckCast* instruction) { LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction, call_kind); locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); + if (type_check_kind == TypeCheckKind::kBitstringCheck) { + locations->SetInAt(1, Location::ConstantLocation(instruction->InputAt(1)->AsConstant())); + locations->SetInAt(2, Location::ConstantLocation(instruction->InputAt(2)->AsConstant())); + locations->SetInAt(3, Location::ConstantLocation(instruction->InputAt(3)->AsConstant())); + } else { + locations->SetInAt(1, Location::RequiresRegister()); + } locations->AddRegisterTemps(NumberOfCheckCastTemps(type_check_kind)); } @@ -7716,7 +8170,9 @@ void InstructionCodeGeneratorARMVIXL::VisitCheckCast(HCheckCast* instruction) { LocationSummary* locations = instruction->GetLocations(); Location obj_loc = locations->InAt(0); vixl32::Register obj = InputRegisterAt(instruction, 0); - vixl32::Register cls = InputRegisterAt(instruction, 1); + vixl32::Register cls = (type_check_kind == TypeCheckKind::kBitstringCheck) + ? vixl32::Register() + : InputRegisterAt(instruction, 1); Location temp_loc = locations->GetTemp(0); vixl32::Register temp = RegisterFrom(temp_loc); const size_t num_temps = NumberOfCheckCastTemps(type_check_kind); @@ -7901,6 +8357,20 @@ void InstructionCodeGeneratorARMVIXL::VisitCheckCast(HCheckCast* instruction) { __ B(ne, &start_loop, /* far_target */ false); break; } + + case TypeCheckKind::kBitstringCheck: { + // /* HeapReference<Class> */ temp = obj->klass_ + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + maybe_temp2_loc, + kWithoutReadBarrier); + + GenerateBitstringTypeCheckCompare(instruction, temp, SetFlags); + __ B(ne, type_check_slow_path->GetEntryLabel()); + break; + } } if (done.IsReferenced()) { __ Bind(&done); @@ -8330,7 +8800,7 @@ void InstructionCodeGeneratorARMVIXL::GenerateReferenceLoadTwoRegisters( } } -void InstructionCodeGeneratorARMVIXL::GenerateGcRootFieldLoad( +void CodeGeneratorARMVIXL::GenerateGcRootFieldLoad( HInstruction* instruction, Location root, vixl32::Register obj, @@ -8361,9 +8831,8 @@ void InstructionCodeGeneratorARMVIXL::GenerateGcRootFieldLoad( UseScratchRegisterScope temps(GetVIXLAssembler()); ExcludeIPAndBakerCcEntrypointRegister(&temps, instruction); bool narrow = CanEmitNarrowLdr(root_reg, obj, offset); - uint32_t custom_data = linker::Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData( - root_reg.GetCode(), narrow); - vixl32::Label* bne_label = codegen_->NewBakerReadBarrierPatch(custom_data); + uint32_t custom_data = EncodeBakerReadBarrierGcRootData(root_reg.GetCode(), narrow); + vixl32::Label* bne_label = NewBakerReadBarrierPatch(custom_data); vixl::EmissionCheckScope guard(GetVIXLAssembler(), 4 * vixl32::kMaxInstructionSizeInBytes); vixl32::Label return_address; @@ -8374,7 +8843,7 @@ void InstructionCodeGeneratorARMVIXL::GenerateGcRootFieldLoad( DCHECK_LT(offset, kReferenceLoadMinFarOffset); ptrdiff_t old_offset = GetVIXLAssembler()->GetBuffer()->GetCursorOffset(); __ ldr(EncodingSize(narrow ? Narrow : Wide), root_reg, MemOperand(obj, offset)); - EmitPlaceholderBne(codegen_, bne_label); + EmitPlaceholderBne(this, bne_label); __ Bind(&return_address); DCHECK_EQ(old_offset - GetVIXLAssembler()->GetBuffer()->GetCursorOffset(), narrow ? BAKER_MARK_INTROSPECTION_GC_ROOT_LDR_NARROW_OFFSET @@ -8394,8 +8863,8 @@ void InstructionCodeGeneratorARMVIXL::GenerateGcRootFieldLoad( // Slow path marking the GC root `root`. The entrypoint will // be loaded by the slow path code. SlowPathCodeARMVIXL* slow_path = - new (codegen_->GetScopedAllocator()) ReadBarrierMarkSlowPathARMVIXL(instruction, root); - codegen_->AddSlowPath(slow_path); + new (GetScopedAllocator()) ReadBarrierMarkSlowPathARMVIXL(instruction, root); + AddSlowPath(slow_path); // /* GcRoot<mirror::Object> */ root = *(obj + offset) GetAssembler()->LoadFromOffset(kLoadWord, root_reg, obj, offset); @@ -8416,7 +8885,7 @@ void InstructionCodeGeneratorARMVIXL::GenerateGcRootFieldLoad( // /* GcRoot<mirror::Object>* */ root = obj + offset __ Add(root_reg, obj, offset); // /* mirror::Object* */ root = root->Read() - codegen_->GenerateReadBarrierForRootSlow(instruction, root, root); + GenerateReadBarrierForRootSlow(instruction, root, root); } } else { // Plain GC root load with no read barrier. @@ -8425,7 +8894,7 @@ void InstructionCodeGeneratorARMVIXL::GenerateGcRootFieldLoad( // Note that GC roots are not affected by heap poisoning, thus we // do not have to unpoison `root_reg` here. } - codegen_->MaybeGenerateMarkingRegisterCheck(/* code */ 18); + MaybeGenerateMarkingRegisterCheck(/* code */ 18); } void CodeGeneratorARMVIXL::MaybeAddBakerCcEntrypointTempForFields(LocationSummary* locations) { @@ -8486,8 +8955,7 @@ void CodeGeneratorARMVIXL::GenerateFieldLoadWithBakerReadBarrier(HInstruction* i } UseScratchRegisterScope temps(GetVIXLAssembler()); ExcludeIPAndBakerCcEntrypointRegister(&temps, instruction); - uint32_t custom_data = linker::Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData( - base.GetCode(), obj.GetCode(), narrow); + uint32_t custom_data = EncodeBakerReadBarrierFieldData(base.GetCode(), obj.GetCode(), narrow); vixl32::Label* bne_label = NewBakerReadBarrierPatch(custom_data); { @@ -8573,8 +9041,7 @@ void CodeGeneratorARMVIXL::GenerateArrayLoadWithBakerReadBarrier(HInstruction* i UseScratchRegisterScope temps(GetVIXLAssembler()); ExcludeIPAndBakerCcEntrypointRegister(&temps, instruction); - uint32_t custom_data = - linker::Thumb2RelativePatcher::EncodeBakerReadBarrierArrayData(data_reg.GetCode()); + uint32_t custom_data = EncodeBakerReadBarrierArrayData(data_reg.GetCode()); vixl32::Label* bne_label = NewBakerReadBarrierPatch(custom_data); __ Add(data_reg, obj, Operand(data_offset)); @@ -8711,7 +9178,7 @@ void CodeGeneratorARMVIXL::UpdateReferenceFieldWithBakerReadBarrier(HInstruction void CodeGeneratorARMVIXL::GenerateRawReferenceLoad(HInstruction* instruction, Location ref, - vixl::aarch32::Register obj, + vixl32::Register obj, uint32_t offset, Location index, ScaleFactor scale_factor, @@ -8901,6 +9368,14 @@ void CodeGeneratorARMVIXL::GenerateStaticOrDirectCall( case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress: __ Mov(RegisterFrom(temp), Operand::From(invoke->GetMethodAddress())); break; + case HInvokeStaticOrDirect::MethodLoadKind::kBootImageRelRo: { + uint32_t boot_image_offset = GetBootImageOffset(invoke); + PcRelativePatchInfo* labels = NewBootImageRelRoPatch(boot_image_offset); + vixl32::Register temp_reg = RegisterFrom(temp); + EmitMovwMovtPlaceholder(labels, temp_reg); + GetAssembler()->LoadFromOffset(kLoadWord, temp_reg, temp_reg, /* offset*/ 0); + break; + } case HInvokeStaticOrDirect::MethodLoadKind::kBssEntry: { PcRelativePatchInfo* labels = NewMethodBssEntryPatch( MethodReference(&GetGraph()->GetDexFile(), invoke->GetDexMethodIndex())); @@ -8998,6 +9473,13 @@ void CodeGeneratorARMVIXL::GenerateVirtualCall( } } +CodeGeneratorARMVIXL::PcRelativePatchInfo* CodeGeneratorARMVIXL::NewBootImageRelRoPatch( + uint32_t boot_image_offset) { + return NewPcRelativePatch(/* dex_file */ nullptr, + boot_image_offset, + &boot_image_method_patches_); +} + CodeGeneratorARMVIXL::PcRelativePatchInfo* CodeGeneratorARMVIXL::NewBootImageMethodPatch( MethodReference target_method) { return NewPcRelativePatch( @@ -9036,7 +9518,7 @@ CodeGeneratorARMVIXL::PcRelativePatchInfo* CodeGeneratorARMVIXL::NewPcRelativePa return &patches->back(); } -vixl::aarch32::Label* CodeGeneratorARMVIXL::NewBakerReadBarrierPatch(uint32_t custom_data) { +vixl32::Label* CodeGeneratorARMVIXL::NewBakerReadBarrierPatch(uint32_t custom_data) { baker_read_barrier_patches_.emplace_back(custom_data); return &baker_read_barrier_patches_.back().label; } @@ -9088,6 +9570,14 @@ inline void CodeGeneratorARMVIXL::EmitPcRelativeLinkerPatches( } } +linker::LinkerPatch DataBimgRelRoPatchAdapter(size_t literal_offset, + const DexFile* target_dex_file, + uint32_t pc_insn_offset, + uint32_t boot_image_offset) { + DCHECK(target_dex_file == nullptr); // Unused for DataBimgRelRoPatch(), should be null. + return linker::LinkerPatch::DataBimgRelRoPatch(literal_offset, pc_insn_offset, boot_image_offset); +} + void CodeGeneratorARMVIXL::EmitLinkerPatches(ArenaVector<linker::LinkerPatch>* linker_patches) { DCHECK(linker_patches->empty()); size_t size = @@ -9107,11 +9597,10 @@ void CodeGeneratorARMVIXL::EmitLinkerPatches(ArenaVector<linker::LinkerPatch>* l EmitPcRelativeLinkerPatches<linker::LinkerPatch::RelativeStringPatch>( boot_image_string_patches_, linker_patches); } else { - DCHECK(boot_image_method_patches_.empty()); - EmitPcRelativeLinkerPatches<linker::LinkerPatch::TypeClassTablePatch>( - boot_image_type_patches_, linker_patches); - EmitPcRelativeLinkerPatches<linker::LinkerPatch::StringInternTablePatch>( - boot_image_string_patches_, linker_patches); + EmitPcRelativeLinkerPatches<DataBimgRelRoPatchAdapter>( + boot_image_method_patches_, linker_patches); + DCHECK(boot_image_type_patches_.empty()); + DCHECK(boot_image_string_patches_.empty()); } EmitPcRelativeLinkerPatches<linker::LinkerPatch::MethodBssEntryPatch>( method_bss_entry_patches_, linker_patches); @@ -9126,6 +9615,45 @@ void CodeGeneratorARMVIXL::EmitLinkerPatches(ArenaVector<linker::LinkerPatch>* l DCHECK_EQ(size, linker_patches->size()); } +bool CodeGeneratorARMVIXL::NeedsThunkCode(const linker::LinkerPatch& patch) const { + return patch.GetType() == linker::LinkerPatch::Type::kBakerReadBarrierBranch || + patch.GetType() == linker::LinkerPatch::Type::kCallRelative; +} + +void CodeGeneratorARMVIXL::EmitThunkCode(const linker::LinkerPatch& patch, + /*out*/ ArenaVector<uint8_t>* code, + /*out*/ std::string* debug_name) { + arm::ArmVIXLAssembler assembler(GetGraph()->GetAllocator()); + switch (patch.GetType()) { + case linker::LinkerPatch::Type::kCallRelative: + // The thunk just uses the entry point in the ArtMethod. This works even for calls + // to the generic JNI and interpreter trampolines. + assembler.LoadFromOffset( + arm::kLoadWord, + vixl32::pc, + vixl32::r0, + ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value()); + assembler.GetVIXLAssembler()->Bkpt(0); + if (GetCompilerOptions().GenerateAnyDebugInfo()) { + *debug_name = "MethodCallThunk"; + } + break; + case linker::LinkerPatch::Type::kBakerReadBarrierBranch: + DCHECK_EQ(patch.GetBakerCustomValue2(), 0u); + CompileBakerReadBarrierThunk(assembler, patch.GetBakerCustomValue1(), debug_name); + break; + default: + LOG(FATAL) << "Unexpected patch type " << patch.GetType(); + UNREACHABLE(); + } + + // Ensure we emit the literal pool if any. + assembler.FinalizeCode(); + code->resize(assembler.CodeSize()); + MemoryRegion code_region(code->data(), code->size()); + assembler.FinalizeInstructions(code_region); +} + VIXLUInt32Literal* CodeGeneratorARMVIXL::DeduplicateUint32Literal( uint32_t value, Uint32ToLiteralMap* map) { @@ -9370,5 +9898,211 @@ void CodeGeneratorARMVIXL::EmitMovwMovtPlaceholder( #undef QUICK_ENTRY_POINT #undef TODO_VIXL32 +#define __ assembler.GetVIXLAssembler()-> + +static void EmitGrayCheckAndFastPath(ArmVIXLAssembler& assembler, + vixl32::Register base_reg, + vixl32::MemOperand& lock_word, + vixl32::Label* slow_path, + int32_t raw_ldr_offset, + vixl32::Label* throw_npe = nullptr) { + // Load the lock word containing the rb_state. + __ Ldr(ip, lock_word); + // Given the numeric representation, it's enough to check the low bit of the rb_state. + static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); + static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); + __ Tst(ip, Operand(LockWord::kReadBarrierStateMaskShifted)); + __ B(ne, slow_path, /* is_far_target */ false); + // To throw NPE, we return to the fast path; the artificial dependence below does not matter. + if (throw_npe != nullptr) { + __ Bind(throw_npe); + } + __ Add(lr, lr, raw_ldr_offset); + // Introduce a dependency on the lock_word including rb_state, + // to prevent load-load reordering, and without using + // a memory barrier (which would be more expensive). + __ Add(base_reg, base_reg, Operand(ip, LSR, 32)); + __ Bx(lr); // And return back to the function. + // Note: The fake dependency is unnecessary for the slow path. +} + +// Load the read barrier introspection entrypoint in register `entrypoint` +static void LoadReadBarrierMarkIntrospectionEntrypoint(ArmVIXLAssembler& assembler, + vixl32::Register entrypoint) { + // The register where the read barrier introspection entrypoint is loaded + // is fixed: `kBakerCcEntrypointRegister` (R4). + DCHECK(entrypoint.Is(kBakerCcEntrypointRegister)); + // entrypoint = Thread::Current()->pReadBarrierMarkReg12, i.e. pReadBarrierMarkIntrospection. + DCHECK_EQ(ip.GetCode(), 12u); + const int32_t entry_point_offset = + Thread::ReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ip.GetCode()); + __ Ldr(entrypoint, MemOperand(tr, entry_point_offset)); +} + +void CodeGeneratorARMVIXL::CompileBakerReadBarrierThunk(ArmVIXLAssembler& assembler, + uint32_t encoded_data, + /*out*/ std::string* debug_name) { + BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data); + switch (kind) { + case BakerReadBarrierKind::kField: { + vixl32::Register base_reg(BakerReadBarrierFirstRegField::Decode(encoded_data)); + CheckValidReg(base_reg.GetCode()); + vixl32::Register holder_reg(BakerReadBarrierSecondRegField::Decode(encoded_data)); + CheckValidReg(holder_reg.GetCode()); + BakerReadBarrierWidth width = BakerReadBarrierWidthField::Decode(encoded_data); + UseScratchRegisterScope temps(assembler.GetVIXLAssembler()); + temps.Exclude(ip); + // If base_reg differs from holder_reg, the offset was too large and we must have emitted + // an explicit null check before the load. Otherwise, for implicit null checks, we need to + // null-check the holder as we do not necessarily do that check before going to the thunk. + vixl32::Label throw_npe_label; + vixl32::Label* throw_npe = nullptr; + if (GetCompilerOptions().GetImplicitNullChecks() && holder_reg.Is(base_reg)) { + throw_npe = &throw_npe_label; + __ CompareAndBranchIfZero(holder_reg, throw_npe, /* is_far_target */ false); + } + // Check if the holder is gray and, if not, add fake dependency to the base register + // and return to the LDR instruction to load the reference. Otherwise, use introspection + // to load the reference and call the entrypoint that performs further checks on the + // reference and marks it if needed. + vixl32::Label slow_path; + MemOperand lock_word(holder_reg, mirror::Object::MonitorOffset().Int32Value()); + const int32_t raw_ldr_offset = (width == BakerReadBarrierWidth::kWide) + ? BAKER_MARK_INTROSPECTION_FIELD_LDR_WIDE_OFFSET + : BAKER_MARK_INTROSPECTION_FIELD_LDR_NARROW_OFFSET; + EmitGrayCheckAndFastPath( + assembler, base_reg, lock_word, &slow_path, raw_ldr_offset, throw_npe); + __ Bind(&slow_path); + const int32_t ldr_offset = /* Thumb state adjustment (LR contains Thumb state). */ -1 + + raw_ldr_offset; + vixl32::Register ep_reg(kBakerCcEntrypointRegister); + LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ep_reg); + if (width == BakerReadBarrierWidth::kWide) { + MemOperand ldr_half_address(lr, ldr_offset + 2); + __ Ldrh(ip, ldr_half_address); // Load the LDR immediate half-word with "Rt | imm12". + __ Ubfx(ip, ip, 0, 12); // Extract the offset imm12. + __ Ldr(ip, MemOperand(base_reg, ip)); // Load the reference. + } else { + MemOperand ldr_address(lr, ldr_offset); + __ Ldrh(ip, ldr_address); // Load the LDR immediate, encoding T1. + __ Add(ep_reg, // Adjust the entrypoint address to the entrypoint + ep_reg, // for narrow LDR. + Operand(BAKER_MARK_INTROSPECTION_FIELD_LDR_NARROW_ENTRYPOINT_OFFSET)); + __ Ubfx(ip, ip, 6, 5); // Extract the imm5, i.e. offset / 4. + __ Ldr(ip, MemOperand(base_reg, ip, LSL, 2)); // Load the reference. + } + // Do not unpoison. With heap poisoning enabled, the entrypoint expects a poisoned reference. + __ Bx(ep_reg); // Jump to the entrypoint. + break; + } + case BakerReadBarrierKind::kArray: { + vixl32::Register base_reg(BakerReadBarrierFirstRegField::Decode(encoded_data)); + CheckValidReg(base_reg.GetCode()); + DCHECK_EQ(kBakerReadBarrierInvalidEncodedReg, + BakerReadBarrierSecondRegField::Decode(encoded_data)); + DCHECK(BakerReadBarrierWidthField::Decode(encoded_data) == BakerReadBarrierWidth::kWide); + UseScratchRegisterScope temps(assembler.GetVIXLAssembler()); + temps.Exclude(ip); + vixl32::Label slow_path; + int32_t data_offset = + mirror::Array::DataOffset(Primitive::ComponentSize(Primitive::kPrimNot)).Int32Value(); + MemOperand lock_word(base_reg, mirror::Object::MonitorOffset().Int32Value() - data_offset); + DCHECK_LT(lock_word.GetOffsetImmediate(), 0); + const int32_t raw_ldr_offset = BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET; + EmitGrayCheckAndFastPath(assembler, base_reg, lock_word, &slow_path, raw_ldr_offset); + __ Bind(&slow_path); + const int32_t ldr_offset = /* Thumb state adjustment (LR contains Thumb state). */ -1 + + raw_ldr_offset; + MemOperand ldr_address(lr, ldr_offset + 2); + __ Ldrb(ip, ldr_address); // Load the LDR (register) byte with "00 | imm2 | Rm", + // i.e. Rm+32 because the scale in imm2 is 2. + vixl32::Register ep_reg(kBakerCcEntrypointRegister); + LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ep_reg); + __ Bfi(ep_reg, ip, 3, 6); // Insert ip to the entrypoint address to create + // a switch case target based on the index register. + __ Mov(ip, base_reg); // Move the base register to ip0. + __ Bx(ep_reg); // Jump to the entrypoint's array switch case. + break; + } + case BakerReadBarrierKind::kGcRoot: { + // Check if the reference needs to be marked and if so (i.e. not null, not marked yet + // and it does not have a forwarding address), call the correct introspection entrypoint; + // otherwise return the reference (or the extracted forwarding address). + // There is no gray bit check for GC roots. + vixl32::Register root_reg(BakerReadBarrierFirstRegField::Decode(encoded_data)); + CheckValidReg(root_reg.GetCode()); + DCHECK_EQ(kBakerReadBarrierInvalidEncodedReg, + BakerReadBarrierSecondRegField::Decode(encoded_data)); + BakerReadBarrierWidth width = BakerReadBarrierWidthField::Decode(encoded_data); + UseScratchRegisterScope temps(assembler.GetVIXLAssembler()); + temps.Exclude(ip); + vixl32::Label return_label, not_marked, forwarding_address; + __ CompareAndBranchIfZero(root_reg, &return_label, /* is_far_target */ false); + MemOperand lock_word(root_reg, mirror::Object::MonitorOffset().Int32Value()); + __ Ldr(ip, lock_word); + __ Tst(ip, LockWord::kMarkBitStateMaskShifted); + __ B(eq, ¬_marked); + __ Bind(&return_label); + __ Bx(lr); + __ Bind(¬_marked); + static_assert(LockWord::kStateShift == 30 && LockWord::kStateForwardingAddress == 3, + "To use 'CMP ip, #modified-immediate; BHS', we need the lock word state in " + " the highest bits and the 'forwarding address' state to have all bits set"); + __ Cmp(ip, Operand(0xc0000000)); + __ B(hs, &forwarding_address); + vixl32::Register ep_reg(kBakerCcEntrypointRegister); + LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ep_reg); + // Adjust the art_quick_read_barrier_mark_introspection address in kBakerCcEntrypointRegister + // to art_quick_read_barrier_mark_introspection_gc_roots. + int32_t entrypoint_offset = (width == BakerReadBarrierWidth::kWide) + ? BAKER_MARK_INTROSPECTION_GC_ROOT_LDR_WIDE_ENTRYPOINT_OFFSET + : BAKER_MARK_INTROSPECTION_GC_ROOT_LDR_NARROW_ENTRYPOINT_OFFSET; + __ Add(ep_reg, ep_reg, Operand(entrypoint_offset)); + __ Mov(ip, root_reg); + __ Bx(ep_reg); + __ Bind(&forwarding_address); + __ Lsl(root_reg, ip, LockWord::kForwardingAddressShift); + __ Bx(lr); + break; + } + default: + LOG(FATAL) << "Unexpected kind: " << static_cast<uint32_t>(kind); + UNREACHABLE(); + } + + if (GetCompilerOptions().GenerateAnyDebugInfo()) { + std::ostringstream oss; + oss << "BakerReadBarrierThunk"; + switch (kind) { + case BakerReadBarrierKind::kField: + oss << "Field"; + if (BakerReadBarrierWidthField::Decode(encoded_data) == BakerReadBarrierWidth::kWide) { + oss << "Wide"; + } + oss << "_r" << BakerReadBarrierFirstRegField::Decode(encoded_data) + << "_r" << BakerReadBarrierSecondRegField::Decode(encoded_data); + break; + case BakerReadBarrierKind::kArray: + oss << "Array_r" << BakerReadBarrierFirstRegField::Decode(encoded_data); + DCHECK_EQ(kBakerReadBarrierInvalidEncodedReg, + BakerReadBarrierSecondRegField::Decode(encoded_data)); + DCHECK(BakerReadBarrierWidthField::Decode(encoded_data) == BakerReadBarrierWidth::kWide); + break; + case BakerReadBarrierKind::kGcRoot: + oss << "GcRoot"; + if (BakerReadBarrierWidthField::Decode(encoded_data) == BakerReadBarrierWidth::kWide) { + oss << "Wide"; + } + oss << "_r" << BakerReadBarrierFirstRegField::Decode(encoded_data); + DCHECK_EQ(kBakerReadBarrierInvalidEncodedReg, + BakerReadBarrierSecondRegField::Decode(encoded_data)); + break; + } + *debug_name = oss.str(); + } +} + +#undef __ + } // namespace arm } // namespace art diff --git a/compiler/optimizing/code_generator_arm_vixl.h b/compiler/optimizing/code_generator_arm_vixl.h index 536da41d07f..6b9919ab15e 100644 --- a/compiler/optimizing/code_generator_arm_vixl.h +++ b/compiler/optimizing/code_generator_arm_vixl.h @@ -36,6 +36,11 @@ #pragma GCC diagnostic pop namespace art { + +namespace linker { +class Thumb2RelativePatcherTest; +} // namespace linker + namespace arm { // This constant is used as an approximate margin when emission of veneer and literal pools @@ -108,6 +113,9 @@ static const vixl::aarch32::SRegister kRuntimeParameterFpuRegistersVIXL[] = { static const size_t kRuntimeParameterFpuRegistersLengthVIXL = arraysize(kRuntimeParameterFpuRegistersVIXL); +// The reserved entrypoint register for link-time generated thunks. +const vixl::aarch32::Register kBakerCcEntrypointRegister = vixl32::r4; + class LoadClassSlowPathARMVIXL; class CodeGeneratorARMVIXL; @@ -322,6 +330,9 @@ class InstructionCodeGeneratorARMVIXL : public InstructionCodeGenerator { void GenerateSuspendCheck(HSuspendCheck* instruction, HBasicBlock* successor); void GenerateClassInitializationCheck(LoadClassSlowPathARMVIXL* slow_path, vixl32::Register class_reg); + void GenerateBitstringTypeCheckCompare(HTypeCheckInstruction* check, + vixl::aarch32::Register temp, + vixl::aarch32::FlagsUpdate flags_update); void GenerateAndConst(vixl::aarch32::Register out, vixl::aarch32::Register first, uint32_t value); void GenerateOrrConst(vixl::aarch32::Register out, vixl::aarch32::Register first, uint32_t value); void GenerateEorConst(vixl::aarch32::Register out, vixl::aarch32::Register first, uint32_t value); @@ -349,6 +360,12 @@ class InstructionCodeGeneratorARMVIXL : public InstructionCodeGenerator { bool value_can_be_null); void HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info); + void GenerateMinMaxInt(LocationSummary* locations, bool is_min); + void GenerateMinMaxLong(LocationSummary* locations, bool is_min); + void GenerateMinMaxFloat(HInstruction* minmax, bool is_min); + void GenerateMinMaxDouble(HInstruction* minmax, bool is_min); + void GenerateMinMax(HBinaryOperation* minmax, bool is_min); + // Generate a heap reference load using one register `out`: // // out <- *(out + offset) @@ -379,16 +396,6 @@ class InstructionCodeGeneratorARMVIXL : public InstructionCodeGenerator { uint32_t offset, Location maybe_temp, ReadBarrierOption read_barrier_option); - // Generate a GC root reference load: - // - // root <- *(obj + offset) - // - // while honoring read barriers based on read_barrier_option. - void GenerateGcRootFieldLoad(HInstruction* instruction, - Location root, - vixl::aarch32::Register obj, - uint32_t offset, - ReadBarrierOption read_barrier_option); void GenerateTestAndBranch(HInstruction* instruction, size_t condition_input_index, vixl::aarch32::Label* true_target, @@ -574,6 +581,7 @@ class CodeGeneratorARMVIXL : public CodeGenerator { vixl::aarch32::Label add_pc_label; }; + PcRelativePatchInfo* NewBootImageRelRoPatch(uint32_t boot_image_offset); PcRelativePatchInfo* NewBootImageMethodPatch(MethodReference target_method); PcRelativePatchInfo* NewMethodBssEntryPatch(MethodReference target_method); PcRelativePatchInfo* NewBootImageTypePatch(const DexFile& dex_file, dex::TypeIndex type_index); @@ -596,6 +604,10 @@ class CodeGeneratorARMVIXL : public CodeGenerator { Handle<mirror::Class> handle); void EmitLinkerPatches(ArenaVector<linker::LinkerPatch>* linker_patches) OVERRIDE; + bool NeedsThunkCode(const linker::LinkerPatch& patch) const OVERRIDE; + void EmitThunkCode(const linker::LinkerPatch& patch, + /*out*/ ArenaVector<uint8_t>* code, + /*out*/ std::string* debug_name) OVERRIDE; void EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) OVERRIDE; @@ -603,6 +615,16 @@ class CodeGeneratorARMVIXL : public CodeGenerator { // is added only for AOT compilation if link-time generated thunks for fields are enabled. void MaybeAddBakerCcEntrypointTempForFields(LocationSummary* locations); + // Generate a GC root reference load: + // + // root <- *(obj + offset) + // + // while honoring read barriers based on read_barrier_option. + void GenerateGcRootFieldLoad(HInstruction* instruction, + Location root, + vixl::aarch32::Register obj, + uint32_t offset, + ReadBarrierOption read_barrier_option); // Fast path implementation of ReadBarrier::Barrier for a heap // reference field load when Baker's read barriers are used. void GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction, @@ -757,6 +779,83 @@ class CodeGeneratorARMVIXL : public CodeGenerator { vixl::aarch32::Register temp = vixl32::Register()); private: + // Encoding of thunk type and data for link-time generated thunks for Baker read barriers. + + enum class BakerReadBarrierKind : uint8_t { + kField, // Field get or array get with constant offset (i.e. constant index). + kArray, // Array get with index in register. + kGcRoot, // GC root load. + kLast = kGcRoot + }; + + enum class BakerReadBarrierWidth : uint8_t { + kWide, // 32-bit LDR (and 32-bit NEG if heap poisoning is enabled). + kNarrow, // 16-bit LDR (and 16-bit NEG if heap poisoning is enabled). + kLast = kNarrow + }; + + static constexpr uint32_t kBakerReadBarrierInvalidEncodedReg = /* pc is invalid */ 15u; + + static constexpr size_t kBitsForBakerReadBarrierKind = + MinimumBitsToStore(static_cast<size_t>(BakerReadBarrierKind::kLast)); + static constexpr size_t kBakerReadBarrierBitsForRegister = + MinimumBitsToStore(kBakerReadBarrierInvalidEncodedReg); + using BakerReadBarrierKindField = + BitField<BakerReadBarrierKind, 0, kBitsForBakerReadBarrierKind>; + using BakerReadBarrierFirstRegField = + BitField<uint32_t, kBitsForBakerReadBarrierKind, kBakerReadBarrierBitsForRegister>; + using BakerReadBarrierSecondRegField = + BitField<uint32_t, + kBitsForBakerReadBarrierKind + kBakerReadBarrierBitsForRegister, + kBakerReadBarrierBitsForRegister>; + static constexpr size_t kBitsForBakerReadBarrierWidth = + MinimumBitsToStore(static_cast<size_t>(BakerReadBarrierWidth::kLast)); + using BakerReadBarrierWidthField = + BitField<BakerReadBarrierWidth, + kBitsForBakerReadBarrierKind + 2 * kBakerReadBarrierBitsForRegister, + kBitsForBakerReadBarrierWidth>; + + static void CheckValidReg(uint32_t reg) { + DCHECK(reg < vixl::aarch32::ip.GetCode() && reg != kBakerCcEntrypointRegister.GetCode()) << reg; + } + + static uint32_t EncodeBakerReadBarrierFieldData(uint32_t base_reg, + uint32_t holder_reg, + bool narrow) { + CheckValidReg(base_reg); + CheckValidReg(holder_reg); + DCHECK(!narrow || base_reg < 8u) << base_reg; + BakerReadBarrierWidth width = + narrow ? BakerReadBarrierWidth::kNarrow : BakerReadBarrierWidth::kWide; + return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kField) | + BakerReadBarrierFirstRegField::Encode(base_reg) | + BakerReadBarrierSecondRegField::Encode(holder_reg) | + BakerReadBarrierWidthField::Encode(width); + } + + static uint32_t EncodeBakerReadBarrierArrayData(uint32_t base_reg) { + CheckValidReg(base_reg); + return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kArray) | + BakerReadBarrierFirstRegField::Encode(base_reg) | + BakerReadBarrierSecondRegField::Encode(kBakerReadBarrierInvalidEncodedReg) | + BakerReadBarrierWidthField::Encode(BakerReadBarrierWidth::kWide); + } + + static uint32_t EncodeBakerReadBarrierGcRootData(uint32_t root_reg, bool narrow) { + CheckValidReg(root_reg); + DCHECK(!narrow || root_reg < 8u) << root_reg; + BakerReadBarrierWidth width = + narrow ? BakerReadBarrierWidth::kNarrow : BakerReadBarrierWidth::kWide; + return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kGcRoot) | + BakerReadBarrierFirstRegField::Encode(root_reg) | + BakerReadBarrierSecondRegField::Encode(kBakerReadBarrierInvalidEncodedReg) | + BakerReadBarrierWidthField::Encode(width); + } + + void CompileBakerReadBarrierThunk(ArmVIXLAssembler& assembler, + uint32_t encoded_data, + /*out*/ std::string* debug_name); + vixl::aarch32::Register GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOrDirect* invoke, vixl::aarch32::Register temp); @@ -798,7 +897,8 @@ class CodeGeneratorARMVIXL : public CodeGenerator { // Deduplication map for 32-bit literals, used for non-patchable boot image addresses. Uint32ToLiteralMap uint32_literals_; - // PC-relative method patch info for kBootImageLinkTimePcRelative. + // PC-relative method patch info for kBootImageLinkTimePcRelative/kBootImageRelRo. + // Also used for type/string patches for kBootImageRelRo (same linker patch as for methods). ArenaDeque<PcRelativePatchInfo> boot_image_method_patches_; // PC-relative method patch info for kBssEntry. ArenaDeque<PcRelativePatchInfo> method_bss_entry_patches_; @@ -806,7 +906,7 @@ class CodeGeneratorARMVIXL : public CodeGenerator { ArenaDeque<PcRelativePatchInfo> boot_image_type_patches_; // PC-relative type patch info for kBssEntry. ArenaDeque<PcRelativePatchInfo> type_bss_entry_patches_; - // PC-relative String patch info; type depends on configuration (intern table or boot image PIC). + // PC-relative String patch info for kBootImageLinkTimePcRelative. ArenaDeque<PcRelativePatchInfo> boot_image_string_patches_; // PC-relative String patch info for kBssEntry. ArenaDeque<PcRelativePatchInfo> string_bss_entry_patches_; @@ -818,6 +918,7 @@ class CodeGeneratorARMVIXL : public CodeGenerator { // Patches for class literals in JIT compiled code. TypeToLiteralMap jit_class_patches_; + friend class linker::Thumb2RelativePatcherTest; DISALLOW_COPY_AND_ASSIGN(CodeGeneratorARMVIXL); }; diff --git a/compiler/optimizing/code_generator_mips.cc b/compiler/optimizing/code_generator_mips.cc index 87e6d6834b7..25e2eddbfab 100644 --- a/compiler/optimizing/code_generator_mips.cc +++ b/compiler/optimizing/code_generator_mips.cc @@ -1597,6 +1597,14 @@ inline void CodeGeneratorMIPS::EmitPcRelativeLinkerPatches( } } +linker::LinkerPatch DataBimgRelRoPatchAdapter(size_t literal_offset, + const DexFile* target_dex_file, + uint32_t pc_insn_offset, + uint32_t boot_image_offset) { + DCHECK(target_dex_file == nullptr); // Unused for DataBimgRelRoPatch(), should be null. + return linker::LinkerPatch::DataBimgRelRoPatch(literal_offset, pc_insn_offset, boot_image_offset); +} + void CodeGeneratorMIPS::EmitLinkerPatches(ArenaVector<linker::LinkerPatch>* linker_patches) { DCHECK(linker_patches->empty()); size_t size = @@ -1615,11 +1623,10 @@ void CodeGeneratorMIPS::EmitLinkerPatches(ArenaVector<linker::LinkerPatch>* link EmitPcRelativeLinkerPatches<linker::LinkerPatch::RelativeStringPatch>( boot_image_string_patches_, linker_patches); } else { - DCHECK(boot_image_method_patches_.empty()); - EmitPcRelativeLinkerPatches<linker::LinkerPatch::TypeClassTablePatch>( - boot_image_type_patches_, linker_patches); - EmitPcRelativeLinkerPatches<linker::LinkerPatch::StringInternTablePatch>( - boot_image_string_patches_, linker_patches); + EmitPcRelativeLinkerPatches<DataBimgRelRoPatchAdapter>( + boot_image_method_patches_, linker_patches); + DCHECK(boot_image_type_patches_.empty()); + DCHECK(boot_image_string_patches_.empty()); } EmitPcRelativeLinkerPatches<linker::LinkerPatch::MethodBssEntryPatch>( method_bss_entry_patches_, linker_patches); @@ -1630,6 +1637,13 @@ void CodeGeneratorMIPS::EmitLinkerPatches(ArenaVector<linker::LinkerPatch>* link DCHECK_EQ(size, linker_patches->size()); } +CodeGeneratorMIPS::PcRelativePatchInfo* CodeGeneratorMIPS::NewBootImageRelRoPatch( + uint32_t boot_image_offset, + const PcRelativePatchInfo* info_high) { + return NewPcRelativePatch( + /* dex_file */ nullptr, boot_image_offset, info_high, &boot_image_method_patches_); +} + CodeGeneratorMIPS::PcRelativePatchInfo* CodeGeneratorMIPS::NewBootImageMethodPatch( MethodReference target_method, const PcRelativePatchInfo* info_high) { @@ -1936,6 +1950,34 @@ void InstructionCodeGeneratorMIPS::GenerateClassInitializationCheck(SlowPathCode __ Bind(slow_path->GetExitLabel()); } +void InstructionCodeGeneratorMIPS::GenerateBitstringTypeCheckCompare(HTypeCheckInstruction* check, + Register temp) { + uint32_t path_to_root = check->GetBitstringPathToRoot(); + uint32_t mask = check->GetBitstringMask(); + DCHECK(IsPowerOfTwo(mask + 1)); + size_t mask_bits = WhichPowerOf2(mask + 1); + + if (mask_bits == 16u) { + // Load only the bitstring part of the status word. + __ LoadFromOffset( + kLoadUnsignedHalfword, temp, temp, mirror::Class::StatusOffset().Int32Value()); + // Compare the bitstring bits using XOR. + __ Xori(temp, temp, dchecked_integral_cast<uint16_t>(path_to_root)); + } else { + // /* uint32_t */ temp = temp->status_ + __ LoadFromOffset(kLoadWord, temp, temp, mirror::Class::StatusOffset().Int32Value()); + // Compare the bitstring bits using XOR. + if (IsUint<16>(path_to_root)) { + __ Xori(temp, temp, dchecked_integral_cast<uint16_t>(path_to_root)); + } else { + __ LoadConst32(TMP, path_to_root); + __ Xor(temp, temp, TMP); + } + // Shift out bits that do not contribute to the comparison. + __ Sll(temp, temp, 32 - mask_bits); + } +} + void InstructionCodeGeneratorMIPS::GenerateMemoryBarrier(MemBarrierKind kind ATTRIBUTE_UNUSED) { __ Sync(0); // Only stype 0 is supported. } @@ -3287,7 +3329,13 @@ void LocationsBuilderMIPS::VisitCheckCast(HCheckCast* instruction) { LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction, call_kind); locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); + if (type_check_kind == TypeCheckKind::kBitstringCheck) { + locations->SetInAt(1, Location::ConstantLocation(instruction->InputAt(1)->AsConstant())); + locations->SetInAt(2, Location::ConstantLocation(instruction->InputAt(2)->AsConstant())); + locations->SetInAt(3, Location::ConstantLocation(instruction->InputAt(3)->AsConstant())); + } else { + locations->SetInAt(1, Location::RequiresRegister()); + } locations->AddRegisterTemps(NumberOfCheckCastTemps(type_check_kind)); } @@ -3296,7 +3344,7 @@ void InstructionCodeGeneratorMIPS::VisitCheckCast(HCheckCast* instruction) { LocationSummary* locations = instruction->GetLocations(); Location obj_loc = locations->InAt(0); Register obj = obj_loc.AsRegister<Register>(); - Register cls = locations->InAt(1).AsRegister<Register>(); + Location cls = locations->InAt(1); Location temp_loc = locations->GetTemp(0); Register temp = temp_loc.AsRegister<Register>(); const size_t num_temps = NumberOfCheckCastTemps(type_check_kind); @@ -3335,7 +3383,7 @@ void InstructionCodeGeneratorMIPS::VisitCheckCast(HCheckCast* instruction) { kWithoutReadBarrier); // Jump to slow path for throwing the exception or doing a // more involved array check. - __ Bne(temp, cls, slow_path->GetEntryLabel()); + __ Bne(temp, cls.AsRegister<Register>(), slow_path->GetEntryLabel()); break; } @@ -3361,7 +3409,7 @@ void InstructionCodeGeneratorMIPS::VisitCheckCast(HCheckCast* instruction) { // exception. __ Beqz(temp, slow_path->GetEntryLabel()); // Otherwise, compare the classes. - __ Bne(temp, cls, &loop); + __ Bne(temp, cls.AsRegister<Register>(), &loop); break; } @@ -3376,7 +3424,7 @@ void InstructionCodeGeneratorMIPS::VisitCheckCast(HCheckCast* instruction) { // Walk over the class hierarchy to find a match. MipsLabel loop; __ Bind(&loop); - __ Beq(temp, cls, &done); + __ Beq(temp, cls.AsRegister<Register>(), &done); // /* HeapReference<Class> */ temp = temp->super_class_ GenerateReferenceLoadOneRegister(instruction, temp_loc, @@ -3399,7 +3447,7 @@ void InstructionCodeGeneratorMIPS::VisitCheckCast(HCheckCast* instruction) { maybe_temp2_loc, kWithoutReadBarrier); // Do an exact check. - __ Beq(temp, cls, &done); + __ Beq(temp, cls.AsRegister<Register>(), &done); // Otherwise, we need to check that the object's class is a non-primitive array. // /* HeapReference<Class> */ temp = temp->component_type_ GenerateReferenceLoadOneRegister(instruction, @@ -3458,7 +3506,21 @@ void InstructionCodeGeneratorMIPS::VisitCheckCast(HCheckCast* instruction) { // Go to next interface. __ Addiu(TMP, TMP, -2); // Compare the classes and continue the loop if they do not match. - __ Bne(AT, cls, &loop); + __ Bne(AT, cls.AsRegister<Register>(), &loop); + break; + } + + case TypeCheckKind::kBitstringCheck: { + // /* HeapReference<Class> */ temp = obj->klass_ + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + maybe_temp2_loc, + kWithoutReadBarrier); + + GenerateBitstringTypeCheckCompare(instruction, temp); + __ Bnez(temp, slow_path->GetEntryLabel()); break; } } @@ -7401,6 +7463,8 @@ void LocationsBuilderMIPS::VisitInstanceOf(HInstanceOf* instruction) { case TypeCheckKind::kInterfaceCheck: call_kind = LocationSummary::kCallOnSlowPath; break; + case TypeCheckKind::kBitstringCheck: + break; } LocationSummary* locations = @@ -7409,7 +7473,13 @@ void LocationsBuilderMIPS::VisitInstanceOf(HInstanceOf* instruction) { locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. } locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); + if (type_check_kind == TypeCheckKind::kBitstringCheck) { + locations->SetInAt(1, Location::ConstantLocation(instruction->InputAt(1)->AsConstant())); + locations->SetInAt(2, Location::ConstantLocation(instruction->InputAt(2)->AsConstant())); + locations->SetInAt(3, Location::ConstantLocation(instruction->InputAt(3)->AsConstant())); + } else { + locations->SetInAt(1, Location::RequiresRegister()); + } // The output does overlap inputs. // Note that TypeCheckSlowPathMIPS uses this register too. locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); @@ -7421,7 +7491,7 @@ void InstructionCodeGeneratorMIPS::VisitInstanceOf(HInstanceOf* instruction) { LocationSummary* locations = instruction->GetLocations(); Location obj_loc = locations->InAt(0); Register obj = obj_loc.AsRegister<Register>(); - Register cls = locations->InAt(1).AsRegister<Register>(); + Location cls = locations->InAt(1); Location out_loc = locations->Out(); Register out = out_loc.AsRegister<Register>(); const size_t num_temps = NumberOfInstanceOfTemps(type_check_kind); @@ -7453,7 +7523,7 @@ void InstructionCodeGeneratorMIPS::VisitInstanceOf(HInstanceOf* instruction) { maybe_temp_loc, read_barrier_option); // Classes must be equal for the instanceof to succeed. - __ Xor(out, out, cls); + __ Xor(out, out, cls.AsRegister<Register>()); __ Sltiu(out, out, 1); break; } @@ -7480,7 +7550,7 @@ void InstructionCodeGeneratorMIPS::VisitInstanceOf(HInstanceOf* instruction) { read_barrier_option); // If `out` is null, we use it for the result, and jump to `done`. __ Beqz(out, &done); - __ Bne(out, cls, &loop); + __ Bne(out, cls.AsRegister<Register>(), &loop); __ LoadConst32(out, 1); break; } @@ -7498,7 +7568,7 @@ void InstructionCodeGeneratorMIPS::VisitInstanceOf(HInstanceOf* instruction) { // Walk over the class hierarchy to find a match. MipsLabel loop, success; __ Bind(&loop); - __ Beq(out, cls, &success); + __ Beq(out, cls.AsRegister<Register>(), &success); // /* HeapReference<Class> */ out = out->super_class_ GenerateReferenceLoadOneRegister(instruction, out_loc, @@ -7525,7 +7595,7 @@ void InstructionCodeGeneratorMIPS::VisitInstanceOf(HInstanceOf* instruction) { read_barrier_option); // Do an exact check. MipsLabel success; - __ Beq(out, cls, &success); + __ Beq(out, cls.AsRegister<Register>(), &success); // Otherwise, we need to check that the object's class is a non-primitive array. // /* HeapReference<Class> */ out = out->component_type_ GenerateReferenceLoadOneRegister(instruction, @@ -7557,7 +7627,7 @@ void InstructionCodeGeneratorMIPS::VisitInstanceOf(HInstanceOf* instruction) { slow_path = new (codegen_->GetScopedAllocator()) TypeCheckSlowPathMIPS( instruction, /* is_fatal */ false); codegen_->AddSlowPath(slow_path); - __ Bne(out, cls, slow_path->GetEntryLabel()); + __ Bne(out, cls.AsRegister<Register>(), slow_path->GetEntryLabel()); __ LoadConst32(out, 1); break; } @@ -7589,6 +7659,20 @@ void InstructionCodeGeneratorMIPS::VisitInstanceOf(HInstanceOf* instruction) { __ B(slow_path->GetEntryLabel()); break; } + + case TypeCheckKind::kBitstringCheck: { + // /* HeapReference<Class> */ temp = obj->klass_ + GenerateReferenceLoadTwoRegisters(instruction, + out_loc, + obj_loc, + class_offset, + maybe_temp_loc, + kWithoutReadBarrier); + + GenerateBitstringTypeCheckCompare(instruction, out); + __ Sltiu(out, out, 1); + break; + } } __ Bind(&done); @@ -7725,7 +7809,7 @@ HLoadString::LoadKind CodeGeneratorMIPS::GetSupportedLoadStringKind( HLoadString::LoadKind desired_string_load_kind) { switch (desired_string_load_kind) { case HLoadString::LoadKind::kBootImageLinkTimePcRelative: - case HLoadString::LoadKind::kBootImageInternTable: + case HLoadString::LoadKind::kBootImageRelRo: case HLoadString::LoadKind::kBssEntry: DCHECK(!Runtime::Current()->UseJitCompilation()); break; @@ -7748,7 +7832,7 @@ HLoadClass::LoadKind CodeGeneratorMIPS::GetSupportedLoadClassKind( case HLoadClass::LoadKind::kReferrersClass: break; case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: - case HLoadClass::LoadKind::kBootImageClassTable: + case HLoadClass::LoadKind::kBootImageRelRo: case HLoadClass::LoadKind::kBssEntry: DCHECK(!Runtime::Current()->UseJitCompilation()); break; @@ -7835,6 +7919,15 @@ void CodeGeneratorMIPS::GenerateStaticOrDirectCall( case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress: __ LoadConst32(temp.AsRegister<Register>(), invoke->GetMethodAddress()); break; + case HInvokeStaticOrDirect::MethodLoadKind::kBootImageRelRo: { + uint32_t boot_image_offset = GetBootImageOffset(invoke); + PcRelativePatchInfo* info_high = NewBootImageRelRoPatch(boot_image_offset); + PcRelativePatchInfo* info_low = NewBootImageRelRoPatch(boot_image_offset, info_high); + Register temp_reg = temp.AsRegister<Register>(); + EmitPcRelativeAddressPlaceholderHigh(info_high, TMP, base_reg); + __ Lw(temp_reg, TMP, /* placeholder */ 0x5678, &info_low->label); + break; + } case HInvokeStaticOrDirect::MethodLoadKind::kBssEntry: { PcRelativePatchInfo* info_high = NewMethodBssEntryPatch( MethodReference(&GetGraph()->GetDexFile(), invoke->GetDexMethodIndex())); @@ -7956,7 +8049,7 @@ void LocationsBuilderMIPS::VisitLoadClass(HLoadClass* cls) { // We need an extra register for PC-relative literals on R2. case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: case HLoadClass::LoadKind::kBootImageAddress: - case HLoadClass::LoadKind::kBootImageClassTable: + case HLoadClass::LoadKind::kBootImageRelRo: case HLoadClass::LoadKind::kBssEntry: if (isR6) { break; @@ -8008,7 +8101,7 @@ void InstructionCodeGeneratorMIPS::VisitLoadClass(HLoadClass* cls) NO_THREAD_SAF // We need an extra register for PC-relative literals on R2. case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: case HLoadClass::LoadKind::kBootImageAddress: - case HLoadClass::LoadKind::kBootImageClassTable: + case HLoadClass::LoadKind::kBootImageRelRo: case HLoadClass::LoadKind::kBssEntry: base_or_current_method_reg = (isR6 || has_irreducible_loops) ? ZERO : locations->InAt(0).AsRegister<Register>(); @@ -8065,22 +8158,17 @@ void InstructionCodeGeneratorMIPS::VisitLoadClass(HLoadClass* cls) NO_THREAD_SAF } break; } - case HLoadClass::LoadKind::kBootImageClassTable: { + case HLoadClass::LoadKind::kBootImageRelRo: { DCHECK(!codegen_->GetCompilerOptions().IsBootImage()); + uint32_t boot_image_offset = codegen_->GetBootImageOffset(cls); CodeGeneratorMIPS::PcRelativePatchInfo* info_high = - codegen_->NewBootImageTypePatch(cls->GetDexFile(), cls->GetTypeIndex()); + codegen_->NewBootImageRelRoPatch(boot_image_offset); CodeGeneratorMIPS::PcRelativePatchInfo* info_low = - codegen_->NewBootImageTypePatch(cls->GetDexFile(), cls->GetTypeIndex(), info_high); + codegen_->NewBootImageRelRoPatch(boot_image_offset, info_high); codegen_->EmitPcRelativeAddressPlaceholderHigh(info_high, out, base_or_current_method_reg); __ Lw(out, out, /* placeholder */ 0x5678, &info_low->label); - // Extract the reference from the slot data, i.e. clear the hash bits. - int32_t masked_hash = ClassTable::TableSlot::MaskHash( - ComputeModifiedUtf8Hash(cls->GetDexFile().StringByTypeIdx(cls->GetTypeIndex()))); - if (masked_hash != 0) { - __ Addiu(out, out, -masked_hash); - } break; } case HLoadClass::LoadKind::kBssEntry: { @@ -8171,7 +8259,7 @@ void LocationsBuilderMIPS::VisitLoadString(HLoadString* load) { // We need an extra register for PC-relative literals on R2. case HLoadString::LoadKind::kBootImageAddress: case HLoadString::LoadKind::kBootImageLinkTimePcRelative: - case HLoadString::LoadKind::kBootImageInternTable: + case HLoadString::LoadKind::kBootImageRelRo: case HLoadString::LoadKind::kBssEntry: if (isR6) { break; @@ -8223,7 +8311,7 @@ void InstructionCodeGeneratorMIPS::VisitLoadString(HLoadString* load) NO_THREAD_ // We need an extra register for PC-relative literals on R2. case HLoadString::LoadKind::kBootImageAddress: case HLoadString::LoadKind::kBootImageLinkTimePcRelative: - case HLoadString::LoadKind::kBootImageInternTable: + case HLoadString::LoadKind::kBootImageRelRo: case HLoadString::LoadKind::kBssEntry: base_or_current_method_reg = (isR6 || has_irreducible_loops) ? ZERO : locations->InAt(0).AsRegister<Register>(); @@ -8259,12 +8347,13 @@ void InstructionCodeGeneratorMIPS::VisitLoadString(HLoadString* load) NO_THREAD_ } return; } - case HLoadString::LoadKind::kBootImageInternTable: { + case HLoadString::LoadKind::kBootImageRelRo: { DCHECK(!codegen_->GetCompilerOptions().IsBootImage()); + uint32_t boot_image_offset = codegen_->GetBootImageOffset(load); CodeGeneratorMIPS::PcRelativePatchInfo* info_high = - codegen_->NewBootImageStringPatch(load->GetDexFile(), load->GetStringIndex()); + codegen_->NewBootImageRelRoPatch(boot_image_offset); CodeGeneratorMIPS::PcRelativePatchInfo* info_low = - codegen_->NewBootImageStringPatch(load->GetDexFile(), load->GetStringIndex(), info_high); + codegen_->NewBootImageRelRoPatch(boot_image_offset, info_high); codegen_->EmitPcRelativeAddressPlaceholderHigh(info_high, out, base_or_current_method_reg); @@ -8779,6 +8868,501 @@ void InstructionCodeGeneratorMIPS::VisitRem(HRem* instruction) { } } +static void CreateMinMaxLocations(ArenaAllocator* allocator, HBinaryOperation* minmax) { + LocationSummary* locations = new (allocator) LocationSummary(minmax); + switch (minmax->GetResultType()) { + case DataType::Type::kInt32: + case DataType::Type::kInt64: + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); + break; + case DataType::Type::kFloat32: + case DataType::Type::kFloat64: + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetInAt(1, Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresFpuRegister(), Location::kOutputOverlap); + break; + default: + LOG(FATAL) << "Unexpected type for HMinMax " << minmax->GetResultType(); + } +} + +void InstructionCodeGeneratorMIPS::GenerateMinMaxInt(LocationSummary* locations, + bool is_min, + bool isR6, + DataType::Type type) { + if (isR6) { + // Some architectures, such as ARM and MIPS (prior to r6), have a + // conditional move instruction which only changes the target + // (output) register if the condition is true (MIPS prior to r6 had + // MOVF, MOVT, MOVN, and MOVZ). The SELEQZ and SELNEZ instructions + // always change the target (output) register. If the condition is + // true the output register gets the contents of the "rs" register; + // otherwise, the output register is set to zero. One consequence + // of this is that to implement something like "rd = c==0 ? rs : rt" + // MIPS64r6 needs to use a pair of SELEQZ/SELNEZ instructions. + // After executing this pair of instructions one of the output + // registers from the pair will necessarily contain zero. Then the + // code ORs the output registers from the SELEQZ/SELNEZ instructions + // to get the final result. + // + // The initial test to see if the output register is same as the + // first input register is needed to make sure that value in the + // first input register isn't clobbered before we've finished + // computing the output value. The logic in the corresponding else + // clause performs the same task but makes sure the second input + // register isn't clobbered in the event that it's the same register + // as the output register; the else clause also handles the case + // where the output register is distinct from both the first, and the + // second input registers. + if (type == DataType::Type::kInt64) { + Register a_lo = locations->InAt(0).AsRegisterPairLow<Register>(); + Register a_hi = locations->InAt(0).AsRegisterPairHigh<Register>(); + Register b_lo = locations->InAt(1).AsRegisterPairLow<Register>(); + Register b_hi = locations->InAt(1).AsRegisterPairHigh<Register>(); + Register out_lo = locations->Out().AsRegisterPairLow<Register>(); + Register out_hi = locations->Out().AsRegisterPairHigh<Register>(); + + MipsLabel compare_done; + + if (a_lo == b_lo) { + if (out_lo != a_lo) { + __ Move(out_lo, a_lo); + __ Move(out_hi, a_hi); + } + } else { + __ Slt(TMP, b_hi, a_hi); + __ Bne(b_hi, a_hi, &compare_done); + + __ Sltu(TMP, b_lo, a_lo); + + __ Bind(&compare_done); + + if (is_min) { + __ Seleqz(AT, a_lo, TMP); + __ Selnez(out_lo, b_lo, TMP); // Safe even if out_lo == a_lo/b_lo + // because at this point we're + // done using a_lo/b_lo. + } else { + __ Selnez(AT, a_lo, TMP); + __ Seleqz(out_lo, b_lo, TMP); // ditto + } + __ Or(out_lo, out_lo, AT); + if (is_min) { + __ Seleqz(AT, a_hi, TMP); + __ Selnez(out_hi, b_hi, TMP); // ditto but for out_hi & a_hi/b_hi + } else { + __ Selnez(AT, a_hi, TMP); + __ Seleqz(out_hi, b_hi, TMP); // ditto but for out_hi & a_hi/b_hi + } + __ Or(out_hi, out_hi, AT); + } + } else { + DCHECK_EQ(type, DataType::Type::kInt32); + Register a = locations->InAt(0).AsRegister<Register>(); + Register b = locations->InAt(1).AsRegister<Register>(); + Register out = locations->Out().AsRegister<Register>(); + + if (a == b) { + if (out != a) { + __ Move(out, a); + } + } else { + __ Slt(AT, b, a); + if (is_min) { + __ Seleqz(TMP, a, AT); + __ Selnez(AT, b, AT); + } else { + __ Selnez(TMP, a, AT); + __ Seleqz(AT, b, AT); + } + __ Or(out, TMP, AT); + } + } + } else { // !isR6 + if (type == DataType::Type::kInt64) { + Register a_lo = locations->InAt(0).AsRegisterPairLow<Register>(); + Register a_hi = locations->InAt(0).AsRegisterPairHigh<Register>(); + Register b_lo = locations->InAt(1).AsRegisterPairLow<Register>(); + Register b_hi = locations->InAt(1).AsRegisterPairHigh<Register>(); + Register out_lo = locations->Out().AsRegisterPairLow<Register>(); + Register out_hi = locations->Out().AsRegisterPairHigh<Register>(); + + MipsLabel compare_done; + + if (a_lo == b_lo) { + if (out_lo != a_lo) { + __ Move(out_lo, a_lo); + __ Move(out_hi, a_hi); + } + } else { + __ Slt(TMP, a_hi, b_hi); + __ Bne(a_hi, b_hi, &compare_done); + + __ Sltu(TMP, a_lo, b_lo); + + __ Bind(&compare_done); + + if (is_min) { + if (out_lo != a_lo) { + __ Movn(out_hi, a_hi, TMP); + __ Movn(out_lo, a_lo, TMP); + } + if (out_lo != b_lo) { + __ Movz(out_hi, b_hi, TMP); + __ Movz(out_lo, b_lo, TMP); + } + } else { + if (out_lo != a_lo) { + __ Movz(out_hi, a_hi, TMP); + __ Movz(out_lo, a_lo, TMP); + } + if (out_lo != b_lo) { + __ Movn(out_hi, b_hi, TMP); + __ Movn(out_lo, b_lo, TMP); + } + } + } + } else { + DCHECK_EQ(type, DataType::Type::kInt32); + Register a = locations->InAt(0).AsRegister<Register>(); + Register b = locations->InAt(1).AsRegister<Register>(); + Register out = locations->Out().AsRegister<Register>(); + + if (a == b) { + if (out != a) { + __ Move(out, a); + } + } else { + __ Slt(AT, a, b); + if (is_min) { + if (out != a) { + __ Movn(out, a, AT); + } + if (out != b) { + __ Movz(out, b, AT); + } + } else { + if (out != a) { + __ Movz(out, a, AT); + } + if (out != b) { + __ Movn(out, b, AT); + } + } + } + } + } +} + +void InstructionCodeGeneratorMIPS::GenerateMinMaxFP(LocationSummary* locations, + bool is_min, + bool isR6, + DataType::Type type) { + FRegister out = locations->Out().AsFpuRegister<FRegister>(); + FRegister a = locations->InAt(0).AsFpuRegister<FRegister>(); + FRegister b = locations->InAt(1).AsFpuRegister<FRegister>(); + + if (isR6) { + MipsLabel noNaNs; + MipsLabel done; + FRegister ftmp = ((out != a) && (out != b)) ? out : FTMP; + + // When Java computes min/max it prefers a NaN to a number; the + // behavior of MIPSR6 is to prefer numbers to NaNs, i.e., if one of + // the inputs is a NaN and the other is a valid number, the MIPS + // instruction will return the number; Java wants the NaN value + // returned. This is why there is extra logic preceding the use of + // the MIPS min.fmt/max.fmt instructions. If either a, or b holds a + // NaN, return the NaN, otherwise return the min/max. + if (type == DataType::Type::kFloat64) { + __ CmpUnD(FTMP, a, b); + __ Bc1eqz(FTMP, &noNaNs); + + // One of the inputs is a NaN + __ CmpEqD(ftmp, a, a); + // If a == a then b is the NaN, otherwise a is the NaN. + __ SelD(ftmp, a, b); + + if (ftmp != out) { + __ MovD(out, ftmp); + } + + __ B(&done); + + __ Bind(&noNaNs); + + if (is_min) { + __ MinD(out, a, b); + } else { + __ MaxD(out, a, b); + } + } else { + DCHECK_EQ(type, DataType::Type::kFloat32); + __ CmpUnS(FTMP, a, b); + __ Bc1eqz(FTMP, &noNaNs); + + // One of the inputs is a NaN + __ CmpEqS(ftmp, a, a); + // If a == a then b is the NaN, otherwise a is the NaN. + __ SelS(ftmp, a, b); + + if (ftmp != out) { + __ MovS(out, ftmp); + } + + __ B(&done); + + __ Bind(&noNaNs); + + if (is_min) { + __ MinS(out, a, b); + } else { + __ MaxS(out, a, b); + } + } + + __ Bind(&done); + + } else { // !isR6 + MipsLabel ordered; + MipsLabel compare; + MipsLabel select; + MipsLabel done; + + if (type == DataType::Type::kFloat64) { + __ CunD(a, b); + } else { + DCHECK_EQ(type, DataType::Type::kFloat32); + __ CunS(a, b); + } + __ Bc1f(&ordered); + + // a or b (or both) is a NaN. Return one, which is a NaN. + if (type == DataType::Type::kFloat64) { + __ CeqD(b, b); + } else { + __ CeqS(b, b); + } + __ B(&select); + + __ Bind(&ordered); + + // Neither is a NaN. + // a == b? (-0.0 compares equal with +0.0) + // If equal, handle zeroes, else compare further. + if (type == DataType::Type::kFloat64) { + __ CeqD(a, b); + } else { + __ CeqS(a, b); + } + __ Bc1f(&compare); + + // a == b either bit for bit or one is -0.0 and the other is +0.0. + if (type == DataType::Type::kFloat64) { + __ MoveFromFpuHigh(TMP, a); + __ MoveFromFpuHigh(AT, b); + } else { + __ Mfc1(TMP, a); + __ Mfc1(AT, b); + } + + if (is_min) { + // -0.0 prevails over +0.0. + __ Or(TMP, TMP, AT); + } else { + // +0.0 prevails over -0.0. + __ And(TMP, TMP, AT); + } + + if (type == DataType::Type::kFloat64) { + __ Mfc1(AT, a); + __ Mtc1(AT, out); + __ MoveToFpuHigh(TMP, out); + } else { + __ Mtc1(TMP, out); + } + __ B(&done); + + __ Bind(&compare); + + if (type == DataType::Type::kFloat64) { + if (is_min) { + // return (a <= b) ? a : b; + __ ColeD(a, b); + } else { + // return (a >= b) ? a : b; + __ ColeD(b, a); // b <= a + } + } else { + if (is_min) { + // return (a <= b) ? a : b; + __ ColeS(a, b); + } else { + // return (a >= b) ? a : b; + __ ColeS(b, a); // b <= a + } + } + + __ Bind(&select); + + if (type == DataType::Type::kFloat64) { + __ MovtD(out, a); + __ MovfD(out, b); + } else { + __ MovtS(out, a); + __ MovfS(out, b); + } + + __ Bind(&done); + } +} + +void InstructionCodeGeneratorMIPS::GenerateMinMax(HBinaryOperation* minmax, bool is_min) { + bool isR6 = codegen_->GetInstructionSetFeatures().IsR6(); + DataType::Type type = minmax->GetResultType(); + switch (type) { + case DataType::Type::kInt32: + case DataType::Type::kInt64: + GenerateMinMaxInt(minmax->GetLocations(), is_min, isR6, type); + break; + case DataType::Type::kFloat32: + case DataType::Type::kFloat64: + GenerateMinMaxFP(minmax->GetLocations(), is_min, isR6, type); + break; + default: + LOG(FATAL) << "Unexpected type for HMinMax " << type; + } +} + +void LocationsBuilderMIPS::VisitMin(HMin* min) { + CreateMinMaxLocations(GetGraph()->GetAllocator(), min); +} + +void InstructionCodeGeneratorMIPS::VisitMin(HMin* min) { + GenerateMinMax(min, /*is_min*/ true); +} + +void LocationsBuilderMIPS::VisitMax(HMax* max) { + CreateMinMaxLocations(GetGraph()->GetAllocator(), max); +} + +void InstructionCodeGeneratorMIPS::VisitMax(HMax* max) { + GenerateMinMax(max, /*is_min*/ false); +} + +void LocationsBuilderMIPS::VisitAbs(HAbs* abs) { + LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(abs); + switch (abs->GetResultType()) { + case DataType::Type::kInt32: + case DataType::Type::kInt64: + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); + break; + case DataType::Type::kFloat32: + case DataType::Type::kFloat64: + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); + break; + default: + LOG(FATAL) << "Unexpected abs type " << abs->GetResultType(); + } +} + +void InstructionCodeGeneratorMIPS::GenerateAbsFP(LocationSummary* locations, + DataType::Type type, + bool isR2OrNewer, + bool isR6) { + FRegister in = locations->InAt(0).AsFpuRegister<FRegister>(); + FRegister out = locations->Out().AsFpuRegister<FRegister>(); + + // Note, as a "quality of implementation", rather than pure "spec compliance", we require that + // Math.abs() clears the sign bit (but changes nothing else) for all numbers, including NaN + // (signaling NaN may become quiet though). + // + // The ABS.fmt instructions (abs.s and abs.d) do exactly that when NAN2008=1 (R6). For this case, + // both regular floating point numbers and NAN values are treated alike, only the sign bit is + // affected by this instruction. + // But when NAN2008=0 (R2 and before), the ABS.fmt instructions can't be used. For this case, any + // NaN operand signals invalid operation. This means that other bits (not just sign bit) might be + // changed when doing abs(NaN). Because of that, we clear sign bit in a different way. + if (isR6) { + if (type == DataType::Type::kFloat64) { + __ AbsD(out, in); + } else { + DCHECK_EQ(type, DataType::Type::kFloat32); + __ AbsS(out, in); + } + } else { + if (type == DataType::Type::kFloat64) { + if (in != out) { + __ MovD(out, in); + } + __ MoveFromFpuHigh(TMP, in); + // ins instruction is not available for R1. + if (isR2OrNewer) { + __ Ins(TMP, ZERO, 31, 1); + } else { + __ Sll(TMP, TMP, 1); + __ Srl(TMP, TMP, 1); + } + __ MoveToFpuHigh(TMP, out); + } else { + DCHECK_EQ(type, DataType::Type::kFloat32); + __ Mfc1(TMP, in); + // ins instruction is not available for R1. + if (isR2OrNewer) { + __ Ins(TMP, ZERO, 31, 1); + } else { + __ Sll(TMP, TMP, 1); + __ Srl(TMP, TMP, 1); + } + __ Mtc1(TMP, out); + } + } +} + +void InstructionCodeGeneratorMIPS::VisitAbs(HAbs* abs) { + LocationSummary* locations = abs->GetLocations(); + bool isR2OrNewer = codegen_->GetInstructionSetFeatures().IsMipsIsaRevGreaterThanEqual2(); + bool isR6 = codegen_->GetInstructionSetFeatures().IsR6(); + switch (abs->GetResultType()) { + case DataType::Type::kInt32: { + Register in = locations->InAt(0).AsRegister<Register>(); + Register out = locations->Out().AsRegister<Register>(); + __ Sra(AT, in, 31); + __ Xor(out, in, AT); + __ Subu(out, out, AT); + break; + } + case DataType::Type::kInt64: { + Register in_lo = locations->InAt(0).AsRegisterPairLow<Register>(); + Register in_hi = locations->InAt(0).AsRegisterPairHigh<Register>(); + Register out_lo = locations->Out().AsRegisterPairLow<Register>(); + Register out_hi = locations->Out().AsRegisterPairHigh<Register>(); + // The comments in this section show the analogous operations which would + // be performed if we had 64-bit registers "in", and "out". + // __ Dsra32(AT, in, 31); + __ Sra(AT, in_hi, 31); + // __ Xor(out, in, AT); + __ Xor(TMP, in_lo, AT); + __ Xor(out_hi, in_hi, AT); + // __ Dsubu(out, out, AT); + __ Subu(out_lo, TMP, AT); + __ Sltu(TMP, out_lo, TMP); + __ Addu(out_hi, out_hi, TMP); + break; + } + case DataType::Type::kFloat32: + case DataType::Type::kFloat64: + GenerateAbsFP(locations, abs->GetResultType(), isR2OrNewer, isR6); + break; + default: + LOG(FATAL) << "Unexpected abs type " << abs->GetResultType(); + } +} + void LocationsBuilderMIPS::VisitConstructorFence(HConstructorFence* constructor_fence) { constructor_fence->SetLocations(nullptr); } diff --git a/compiler/optimizing/code_generator_mips.h b/compiler/optimizing/code_generator_mips.h index c91cb62eda5..2e7c736dbd3 100644 --- a/compiler/optimizing/code_generator_mips.h +++ b/compiler/optimizing/code_generator_mips.h @@ -237,6 +237,7 @@ class InstructionCodeGeneratorMIPS : public InstructionCodeGenerator { private: void GenerateClassInitializationCheck(SlowPathCodeMIPS* slow_path, Register class_reg); void GenerateSuspendCheck(HSuspendCheck* check, HBasicBlock* successor); + void GenerateBitstringTypeCheckCompare(HTypeCheckInstruction* check, Register temp); void HandleBinaryOp(HBinaryOperation* operation); void HandleCondition(HCondition* instruction); void HandleShift(HBinaryOperation* operation); @@ -246,6 +247,11 @@ class InstructionCodeGeneratorMIPS : public InstructionCodeGenerator { bool value_can_be_null); void HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info, uint32_t dex_pc); + void GenerateMinMaxInt(LocationSummary* locations, bool is_min, bool isR6, DataType::Type type); + void GenerateMinMaxFP(LocationSummary* locations, bool is_min, bool isR6, DataType::Type type); + void GenerateMinMax(HBinaryOperation*, bool is_min); + void GenerateAbsFP(LocationSummary* locations, DataType::Type type, bool isR2OrNewer, bool isR6); + // Generate a heap reference load using one register `out`: // // out <- *(out + offset) @@ -615,6 +621,8 @@ class CodeGeneratorMIPS : public CodeGenerator { DISALLOW_COPY_AND_ASSIGN(PcRelativePatchInfo); }; + PcRelativePatchInfo* NewBootImageRelRoPatch(uint32_t boot_image_offset, + const PcRelativePatchInfo* info_high = nullptr); PcRelativePatchInfo* NewBootImageMethodPatch(MethodReference target_method, const PcRelativePatchInfo* info_high = nullptr); PcRelativePatchInfo* NewMethodBssEntryPatch(MethodReference target_method, @@ -689,7 +697,8 @@ class CodeGeneratorMIPS : public CodeGenerator { // Deduplication map for 32-bit literals, used for non-patchable boot image addresses. Uint32ToLiteralMap uint32_literals_; - // PC-relative method patch info for kBootImageLinkTimePcRelative. + // PC-relative method patch info for kBootImageLinkTimePcRelative/kBootImageRelRo. + // Also used for type/string patches for kBootImageRelRo (same linker patch as for methods). ArenaDeque<PcRelativePatchInfo> boot_image_method_patches_; // PC-relative method patch info for kBssEntry. ArenaDeque<PcRelativePatchInfo> method_bss_entry_patches_; @@ -697,7 +706,7 @@ class CodeGeneratorMIPS : public CodeGenerator { ArenaDeque<PcRelativePatchInfo> boot_image_type_patches_; // PC-relative type patch info for kBssEntry. ArenaDeque<PcRelativePatchInfo> type_bss_entry_patches_; - // PC-relative String patch info; type depends on configuration (intern table or boot image PIC). + // PC-relative String patch info for kBootImageLinkTimePcRelative. ArenaDeque<PcRelativePatchInfo> boot_image_string_patches_; // PC-relative String patch info for kBssEntry. ArenaDeque<PcRelativePatchInfo> string_bss_entry_patches_; diff --git a/compiler/optimizing/code_generator_mips64.cc b/compiler/optimizing/code_generator_mips64.cc index 985ac2ca554..5b07b55cbbb 100644 --- a/compiler/optimizing/code_generator_mips64.cc +++ b/compiler/optimizing/code_generator_mips64.cc @@ -1509,6 +1509,14 @@ inline void CodeGeneratorMIPS64::EmitPcRelativeLinkerPatches( } } +linker::LinkerPatch DataBimgRelRoPatchAdapter(size_t literal_offset, + const DexFile* target_dex_file, + uint32_t pc_insn_offset, + uint32_t boot_image_offset) { + DCHECK(target_dex_file == nullptr); // Unused for DataBimgRelRoPatch(), should be null. + return linker::LinkerPatch::DataBimgRelRoPatch(literal_offset, pc_insn_offset, boot_image_offset); +} + void CodeGeneratorMIPS64::EmitLinkerPatches(ArenaVector<linker::LinkerPatch>* linker_patches) { DCHECK(linker_patches->empty()); size_t size = @@ -1527,11 +1535,10 @@ void CodeGeneratorMIPS64::EmitLinkerPatches(ArenaVector<linker::LinkerPatch>* li EmitPcRelativeLinkerPatches<linker::LinkerPatch::RelativeStringPatch>( boot_image_string_patches_, linker_patches); } else { - DCHECK(boot_image_method_patches_.empty()); - EmitPcRelativeLinkerPatches<linker::LinkerPatch::TypeClassTablePatch>( - boot_image_type_patches_, linker_patches); - EmitPcRelativeLinkerPatches<linker::LinkerPatch::StringInternTablePatch>( - boot_image_string_patches_, linker_patches); + EmitPcRelativeLinkerPatches<DataBimgRelRoPatchAdapter>( + boot_image_method_patches_, linker_patches); + DCHECK(boot_image_type_patches_.empty()); + DCHECK(boot_image_string_patches_.empty()); } EmitPcRelativeLinkerPatches<linker::LinkerPatch::MethodBssEntryPatch>( method_bss_entry_patches_, linker_patches); @@ -1542,6 +1549,13 @@ void CodeGeneratorMIPS64::EmitLinkerPatches(ArenaVector<linker::LinkerPatch>* li DCHECK_EQ(size, linker_patches->size()); } +CodeGeneratorMIPS64::PcRelativePatchInfo* CodeGeneratorMIPS64::NewBootImageRelRoPatch( + uint32_t boot_image_offset, + const PcRelativePatchInfo* info_high) { + return NewPcRelativePatch( + /* dex_file */ nullptr, boot_image_offset, info_high, &boot_image_method_patches_); +} + CodeGeneratorMIPS64::PcRelativePatchInfo* CodeGeneratorMIPS64::NewBootImageMethodPatch( MethodReference target_method, const PcRelativePatchInfo* info_high) { @@ -1780,6 +1794,34 @@ void InstructionCodeGeneratorMIPS64::GenerateClassInitializationCheck(SlowPathCo __ Bind(slow_path->GetExitLabel()); } +void InstructionCodeGeneratorMIPS64::GenerateBitstringTypeCheckCompare(HTypeCheckInstruction* check, + GpuRegister temp) { + uint32_t path_to_root = check->GetBitstringPathToRoot(); + uint32_t mask = check->GetBitstringMask(); + DCHECK(IsPowerOfTwo(mask + 1)); + size_t mask_bits = WhichPowerOf2(mask + 1); + + if (mask_bits == 16u) { + // Load only the bitstring part of the status word. + __ LoadFromOffset( + kLoadUnsignedHalfword, temp, temp, mirror::Class::StatusOffset().Int32Value()); + // Compare the bitstring bits using XOR. + __ Xori(temp, temp, dchecked_integral_cast<uint16_t>(path_to_root)); + } else { + // /* uint32_t */ temp = temp->status_ + __ LoadFromOffset(kLoadWord, temp, temp, mirror::Class::StatusOffset().Int32Value()); + // Compare the bitstring bits using XOR. + if (IsUint<16>(path_to_root)) { + __ Xori(temp, temp, dchecked_integral_cast<uint16_t>(path_to_root)); + } else { + __ LoadConst32(TMP, path_to_root); + __ Xor(temp, temp, TMP); + } + // Shift out bits that do not contribute to the comparison. + __ Sll(temp, temp, 32 - mask_bits); + } +} + void InstructionCodeGeneratorMIPS64::GenerateMemoryBarrier(MemBarrierKind kind ATTRIBUTE_UNUSED) { __ Sync(0); // only stype 0 is supported } @@ -2840,7 +2882,13 @@ void LocationsBuilderMIPS64::VisitCheckCast(HCheckCast* instruction) { LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction, call_kind); locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); + if (type_check_kind == TypeCheckKind::kBitstringCheck) { + locations->SetInAt(1, Location::ConstantLocation(instruction->InputAt(1)->AsConstant())); + locations->SetInAt(2, Location::ConstantLocation(instruction->InputAt(2)->AsConstant())); + locations->SetInAt(3, Location::ConstantLocation(instruction->InputAt(3)->AsConstant())); + } else { + locations->SetInAt(1, Location::RequiresRegister()); + } locations->AddRegisterTemps(NumberOfCheckCastTemps(type_check_kind)); } @@ -2849,7 +2897,7 @@ void InstructionCodeGeneratorMIPS64::VisitCheckCast(HCheckCast* instruction) { LocationSummary* locations = instruction->GetLocations(); Location obj_loc = locations->InAt(0); GpuRegister obj = obj_loc.AsRegister<GpuRegister>(); - GpuRegister cls = locations->InAt(1).AsRegister<GpuRegister>(); + Location cls = locations->InAt(1); Location temp_loc = locations->GetTemp(0); GpuRegister temp = temp_loc.AsRegister<GpuRegister>(); const size_t num_temps = NumberOfCheckCastTemps(type_check_kind); @@ -2888,7 +2936,7 @@ void InstructionCodeGeneratorMIPS64::VisitCheckCast(HCheckCast* instruction) { kWithoutReadBarrier); // Jump to slow path for throwing the exception or doing a // more involved array check. - __ Bnec(temp, cls, slow_path->GetEntryLabel()); + __ Bnec(temp, cls.AsRegister<GpuRegister>(), slow_path->GetEntryLabel()); break; } @@ -2914,7 +2962,7 @@ void InstructionCodeGeneratorMIPS64::VisitCheckCast(HCheckCast* instruction) { // exception. __ Beqzc(temp, slow_path->GetEntryLabel()); // Otherwise, compare the classes. - __ Bnec(temp, cls, &loop); + __ Bnec(temp, cls.AsRegister<GpuRegister>(), &loop); break; } @@ -2929,7 +2977,7 @@ void InstructionCodeGeneratorMIPS64::VisitCheckCast(HCheckCast* instruction) { // Walk over the class hierarchy to find a match. Mips64Label loop; __ Bind(&loop); - __ Beqc(temp, cls, &done); + __ Beqc(temp, cls.AsRegister<GpuRegister>(), &done); // /* HeapReference<Class> */ temp = temp->super_class_ GenerateReferenceLoadOneRegister(instruction, temp_loc, @@ -2952,7 +3000,7 @@ void InstructionCodeGeneratorMIPS64::VisitCheckCast(HCheckCast* instruction) { maybe_temp2_loc, kWithoutReadBarrier); // Do an exact check. - __ Beqc(temp, cls, &done); + __ Beqc(temp, cls.AsRegister<GpuRegister>(), &done); // Otherwise, we need to check that the object's class is a non-primitive array. // /* HeapReference<Class> */ temp = temp->component_type_ GenerateReferenceLoadOneRegister(instruction, @@ -3011,7 +3059,21 @@ void InstructionCodeGeneratorMIPS64::VisitCheckCast(HCheckCast* instruction) { __ Daddiu(temp, temp, 2 * kHeapReferenceSize); __ Addiu(TMP, TMP, -2); // Compare the classes and continue the loop if they do not match. - __ Bnec(AT, cls, &loop); + __ Bnec(AT, cls.AsRegister<GpuRegister>(), &loop); + break; + } + + case TypeCheckKind::kBitstringCheck: { + // /* HeapReference<Class> */ temp = obj->klass_ + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + maybe_temp2_loc, + kWithoutReadBarrier); + + GenerateBitstringTypeCheckCompare(instruction, temp); + __ Bnezc(temp, slow_path->GetEntryLabel()); break; } } @@ -5515,6 +5577,8 @@ void LocationsBuilderMIPS64::VisitInstanceOf(HInstanceOf* instruction) { case TypeCheckKind::kInterfaceCheck: call_kind = LocationSummary::kCallOnSlowPath; break; + case TypeCheckKind::kBitstringCheck: + break; } LocationSummary* locations = @@ -5523,7 +5587,13 @@ void LocationsBuilderMIPS64::VisitInstanceOf(HInstanceOf* instruction) { locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. } locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); + if (type_check_kind == TypeCheckKind::kBitstringCheck) { + locations->SetInAt(1, Location::ConstantLocation(instruction->InputAt(1)->AsConstant())); + locations->SetInAt(2, Location::ConstantLocation(instruction->InputAt(2)->AsConstant())); + locations->SetInAt(3, Location::ConstantLocation(instruction->InputAt(3)->AsConstant())); + } else { + locations->SetInAt(1, Location::RequiresRegister()); + } // The output does overlap inputs. // Note that TypeCheckSlowPathMIPS64 uses this register too. locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); @@ -5535,7 +5605,7 @@ void InstructionCodeGeneratorMIPS64::VisitInstanceOf(HInstanceOf* instruction) { LocationSummary* locations = instruction->GetLocations(); Location obj_loc = locations->InAt(0); GpuRegister obj = obj_loc.AsRegister<GpuRegister>(); - GpuRegister cls = locations->InAt(1).AsRegister<GpuRegister>(); + Location cls = locations->InAt(1); Location out_loc = locations->Out(); GpuRegister out = out_loc.AsRegister<GpuRegister>(); const size_t num_temps = NumberOfInstanceOfTemps(type_check_kind); @@ -5567,7 +5637,7 @@ void InstructionCodeGeneratorMIPS64::VisitInstanceOf(HInstanceOf* instruction) { maybe_temp_loc, read_barrier_option); // Classes must be equal for the instanceof to succeed. - __ Xor(out, out, cls); + __ Xor(out, out, cls.AsRegister<GpuRegister>()); __ Sltiu(out, out, 1); break; } @@ -5594,7 +5664,7 @@ void InstructionCodeGeneratorMIPS64::VisitInstanceOf(HInstanceOf* instruction) { read_barrier_option); // If `out` is null, we use it for the result, and jump to `done`. __ Beqzc(out, &done); - __ Bnec(out, cls, &loop); + __ Bnec(out, cls.AsRegister<GpuRegister>(), &loop); __ LoadConst32(out, 1); break; } @@ -5612,7 +5682,7 @@ void InstructionCodeGeneratorMIPS64::VisitInstanceOf(HInstanceOf* instruction) { // Walk over the class hierarchy to find a match. Mips64Label loop, success; __ Bind(&loop); - __ Beqc(out, cls, &success); + __ Beqc(out, cls.AsRegister<GpuRegister>(), &success); // /* HeapReference<Class> */ out = out->super_class_ GenerateReferenceLoadOneRegister(instruction, out_loc, @@ -5639,7 +5709,7 @@ void InstructionCodeGeneratorMIPS64::VisitInstanceOf(HInstanceOf* instruction) { read_barrier_option); // Do an exact check. Mips64Label success; - __ Beqc(out, cls, &success); + __ Beqc(out, cls.AsRegister<GpuRegister>(), &success); // Otherwise, we need to check that the object's class is a non-primitive array. // /* HeapReference<Class> */ out = out->component_type_ GenerateReferenceLoadOneRegister(instruction, @@ -5671,7 +5741,7 @@ void InstructionCodeGeneratorMIPS64::VisitInstanceOf(HInstanceOf* instruction) { slow_path = new (codegen_->GetScopedAllocator()) TypeCheckSlowPathMIPS64( instruction, /* is_fatal */ false); codegen_->AddSlowPath(slow_path); - __ Bnec(out, cls, slow_path->GetEntryLabel()); + __ Bnec(out, cls.AsRegister<GpuRegister>(), slow_path->GetEntryLabel()); __ LoadConst32(out, 1); break; } @@ -5703,6 +5773,20 @@ void InstructionCodeGeneratorMIPS64::VisitInstanceOf(HInstanceOf* instruction) { __ Bc(slow_path->GetEntryLabel()); break; } + + case TypeCheckKind::kBitstringCheck: { + // /* HeapReference<Class> */ temp = obj->klass_ + GenerateReferenceLoadTwoRegisters(instruction, + out_loc, + obj_loc, + class_offset, + maybe_temp_loc, + kWithoutReadBarrier); + + GenerateBitstringTypeCheckCompare(instruction, out); + __ Sltiu(out, out, 1); + break; + } } __ Bind(&done); @@ -5839,7 +5923,7 @@ HLoadString::LoadKind CodeGeneratorMIPS64::GetSupportedLoadStringKind( bool fallback_load = false; switch (desired_string_load_kind) { case HLoadString::LoadKind::kBootImageLinkTimePcRelative: - case HLoadString::LoadKind::kBootImageInternTable: + case HLoadString::LoadKind::kBootImageRelRo: case HLoadString::LoadKind::kBssEntry: DCHECK(!Runtime::Current()->UseJitCompilation()); break; @@ -5866,7 +5950,7 @@ HLoadClass::LoadKind CodeGeneratorMIPS64::GetSupportedLoadClassKind( case HLoadClass::LoadKind::kReferrersClass: break; case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: - case HLoadClass::LoadKind::kBootImageClassTable: + case HLoadClass::LoadKind::kBootImageRelRo: case HLoadClass::LoadKind::kBssEntry: DCHECK(!Runtime::Current()->UseJitCompilation()); break; @@ -5926,6 +6010,15 @@ void CodeGeneratorMIPS64::GenerateStaticOrDirectCall( kLoadDoubleword, DeduplicateUint64Literal(invoke->GetMethodAddress())); break; + case HInvokeStaticOrDirect::MethodLoadKind::kBootImageRelRo: { + uint32_t boot_image_offset = GetBootImageOffset(invoke); + PcRelativePatchInfo* info_high = NewBootImageRelRoPatch(boot_image_offset); + PcRelativePatchInfo* info_low = NewBootImageRelRoPatch(boot_image_offset, info_high); + EmitPcRelativeAddressPlaceholderHigh(info_high, AT, info_low); + // Note: Boot image is in the low 4GiB and the entry is 32-bit, so emit a 32-bit load. + __ Lwu(temp.AsRegister<GpuRegister>(), AT, /* placeholder */ 0x5678); + break; + } case HInvokeStaticOrDirect::MethodLoadKind::kBssEntry: { PcRelativePatchInfo* info_high = NewMethodBssEntryPatch( MethodReference(&GetGraph()->GetDexFile(), invoke->GetDexMethodIndex())); @@ -6113,20 +6206,15 @@ void InstructionCodeGeneratorMIPS64::VisitLoadClass(HLoadClass* cls) NO_THREAD_S codegen_->DeduplicateBootImageAddressLiteral(address)); break; } - case HLoadClass::LoadKind::kBootImageClassTable: { + case HLoadClass::LoadKind::kBootImageRelRo: { DCHECK(!codegen_->GetCompilerOptions().IsBootImage()); + uint32_t boot_image_offset = codegen_->GetBootImageOffset(cls); CodeGeneratorMIPS64::PcRelativePatchInfo* info_high = - codegen_->NewBootImageTypePatch(cls->GetDexFile(), cls->GetTypeIndex()); + codegen_->NewBootImageRelRoPatch(boot_image_offset); CodeGeneratorMIPS64::PcRelativePatchInfo* info_low = - codegen_->NewBootImageTypePatch(cls->GetDexFile(), cls->GetTypeIndex(), info_high); + codegen_->NewBootImageRelRoPatch(boot_image_offset, info_high); codegen_->EmitPcRelativeAddressPlaceholderHigh(info_high, AT, info_low); __ Lwu(out, AT, /* placeholder */ 0x5678); - // Extract the reference from the slot data, i.e. clear the hash bits. - int32_t masked_hash = ClassTable::TableSlot::MaskHash( - ComputeModifiedUtf8Hash(cls->GetDexFile().StringByTypeIdx(cls->GetTypeIndex()))); - if (masked_hash != 0) { - __ Daddiu(out, out, -masked_hash); - } break; } case HLoadClass::LoadKind::kBssEntry: { @@ -6248,12 +6336,13 @@ void InstructionCodeGeneratorMIPS64::VisitLoadString(HLoadString* load) NO_THREA codegen_->DeduplicateBootImageAddressLiteral(address)); return; } - case HLoadString::LoadKind::kBootImageInternTable: { + case HLoadString::LoadKind::kBootImageRelRo: { DCHECK(!codegen_->GetCompilerOptions().IsBootImage()); + uint32_t boot_image_offset = codegen_->GetBootImageOffset(load); CodeGeneratorMIPS64::PcRelativePatchInfo* info_high = - codegen_->NewBootImageStringPatch(load->GetDexFile(), load->GetStringIndex()); + codegen_->NewBootImageRelRoPatch(boot_image_offset); CodeGeneratorMIPS64::PcRelativePatchInfo* info_low = - codegen_->NewBootImageStringPatch(load->GetDexFile(), load->GetStringIndex(), info_high); + codegen_->NewBootImageRelRoPatch(boot_image_offset, info_high); codegen_->EmitPcRelativeAddressPlaceholderHigh(info_high, AT, info_low); __ Lwu(out, AT, /* placeholder */ 0x5678); return; @@ -6665,6 +6754,236 @@ void InstructionCodeGeneratorMIPS64::VisitRem(HRem* instruction) { } } +static void CreateMinMaxLocations(ArenaAllocator* allocator, HBinaryOperation* minmax) { + LocationSummary* locations = new (allocator) LocationSummary(minmax); + switch (minmax->GetResultType()) { + case DataType::Type::kInt32: + case DataType::Type::kInt64: + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); + break; + case DataType::Type::kFloat32: + case DataType::Type::kFloat64: + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetInAt(1, Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); + break; + default: + LOG(FATAL) << "Unexpected type for HMinMax " << minmax->GetResultType(); + } +} + +void InstructionCodeGeneratorMIPS64::GenerateMinMaxInt(LocationSummary* locations, bool is_min) { + GpuRegister lhs = locations->InAt(0).AsRegister<GpuRegister>(); + GpuRegister rhs = locations->InAt(1).AsRegister<GpuRegister>(); + GpuRegister out = locations->Out().AsRegister<GpuRegister>(); + + if (lhs == rhs) { + if (out != lhs) { + __ Move(out, lhs); + } + } else { + // Some architectures, such as ARM and MIPS (prior to r6), have a + // conditional move instruction which only changes the target + // (output) register if the condition is true (MIPS prior to r6 had + // MOVF, MOVT, and MOVZ). The SELEQZ and SELNEZ instructions always + // change the target (output) register. If the condition is true the + // output register gets the contents of the "rs" register; otherwise, + // the output register is set to zero. One consequence of this is + // that to implement something like "rd = c==0 ? rs : rt" MIPS64r6 + // needs to use a pair of SELEQZ/SELNEZ instructions. After + // executing this pair of instructions one of the output registers + // from the pair will necessarily contain zero. Then the code ORs the + // output registers from the SELEQZ/SELNEZ instructions to get the + // final result. + // + // The initial test to see if the output register is same as the + // first input register is needed to make sure that value in the + // first input register isn't clobbered before we've finished + // computing the output value. The logic in the corresponding else + // clause performs the same task but makes sure the second input + // register isn't clobbered in the event that it's the same register + // as the output register; the else clause also handles the case + // where the output register is distinct from both the first, and the + // second input registers. + if (out == lhs) { + __ Slt(AT, rhs, lhs); + if (is_min) { + __ Seleqz(out, lhs, AT); + __ Selnez(AT, rhs, AT); + } else { + __ Selnez(out, lhs, AT); + __ Seleqz(AT, rhs, AT); + } + } else { + __ Slt(AT, lhs, rhs); + if (is_min) { + __ Seleqz(out, rhs, AT); + __ Selnez(AT, lhs, AT); + } else { + __ Selnez(out, rhs, AT); + __ Seleqz(AT, lhs, AT); + } + } + __ Or(out, out, AT); + } +} + +void InstructionCodeGeneratorMIPS64::GenerateMinMaxFP(LocationSummary* locations, + bool is_min, + DataType::Type type) { + FpuRegister a = locations->InAt(0).AsFpuRegister<FpuRegister>(); + FpuRegister b = locations->InAt(1).AsFpuRegister<FpuRegister>(); + FpuRegister out = locations->Out().AsFpuRegister<FpuRegister>(); + + Mips64Label noNaNs; + Mips64Label done; + FpuRegister ftmp = ((out != a) && (out != b)) ? out : FTMP; + + // When Java computes min/max it prefers a NaN to a number; the + // behavior of MIPSR6 is to prefer numbers to NaNs, i.e., if one of + // the inputs is a NaN and the other is a valid number, the MIPS + // instruction will return the number; Java wants the NaN value + // returned. This is why there is extra logic preceding the use of + // the MIPS min.fmt/max.fmt instructions. If either a, or b holds a + // NaN, return the NaN, otherwise return the min/max. + if (type == DataType::Type::kFloat64) { + __ CmpUnD(FTMP, a, b); + __ Bc1eqz(FTMP, &noNaNs); + + // One of the inputs is a NaN + __ CmpEqD(ftmp, a, a); + // If a == a then b is the NaN, otherwise a is the NaN. + __ SelD(ftmp, a, b); + + if (ftmp != out) { + __ MovD(out, ftmp); + } + + __ Bc(&done); + + __ Bind(&noNaNs); + + if (is_min) { + __ MinD(out, a, b); + } else { + __ MaxD(out, a, b); + } + } else { + DCHECK_EQ(type, DataType::Type::kFloat32); + __ CmpUnS(FTMP, a, b); + __ Bc1eqz(FTMP, &noNaNs); + + // One of the inputs is a NaN + __ CmpEqS(ftmp, a, a); + // If a == a then b is the NaN, otherwise a is the NaN. + __ SelS(ftmp, a, b); + + if (ftmp != out) { + __ MovS(out, ftmp); + } + + __ Bc(&done); + + __ Bind(&noNaNs); + + if (is_min) { + __ MinS(out, a, b); + } else { + __ MaxS(out, a, b); + } + } + + __ Bind(&done); +} + +void InstructionCodeGeneratorMIPS64::GenerateMinMax(HBinaryOperation* minmax, bool is_min) { + DataType::Type type = minmax->GetResultType(); + switch (type) { + case DataType::Type::kInt32: + case DataType::Type::kInt64: + GenerateMinMaxInt(minmax->GetLocations(), is_min); + break; + case DataType::Type::kFloat32: + case DataType::Type::kFloat64: + GenerateMinMaxFP(minmax->GetLocations(), is_min, type); + break; + default: + LOG(FATAL) << "Unexpected type for HMinMax " << type; + } +} + +void LocationsBuilderMIPS64::VisitMin(HMin* min) { + CreateMinMaxLocations(GetGraph()->GetAllocator(), min); +} + +void InstructionCodeGeneratorMIPS64::VisitMin(HMin* min) { + GenerateMinMax(min, /*is_min*/ true); +} + +void LocationsBuilderMIPS64::VisitMax(HMax* max) { + CreateMinMaxLocations(GetGraph()->GetAllocator(), max); +} + +void InstructionCodeGeneratorMIPS64::VisitMax(HMax* max) { + GenerateMinMax(max, /*is_min*/ false); +} + +void LocationsBuilderMIPS64::VisitAbs(HAbs* abs) { + LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(abs); + switch (abs->GetResultType()) { + case DataType::Type::kInt32: + case DataType::Type::kInt64: + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); + break; + case DataType::Type::kFloat32: + case DataType::Type::kFloat64: + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); + break; + default: + LOG(FATAL) << "Unexpected abs type " << abs->GetResultType(); + } +} + +void InstructionCodeGeneratorMIPS64::VisitAbs(HAbs* abs) { + LocationSummary* locations = abs->GetLocations(); + switch (abs->GetResultType()) { + case DataType::Type::kInt32: { + GpuRegister in = locations->InAt(0).AsRegister<GpuRegister>(); + GpuRegister out = locations->Out().AsRegister<GpuRegister>(); + __ Sra(AT, in, 31); + __ Xor(out, in, AT); + __ Subu(out, out, AT); + break; + } + case DataType::Type::kInt64: { + GpuRegister in = locations->InAt(0).AsRegister<GpuRegister>(); + GpuRegister out = locations->Out().AsRegister<GpuRegister>(); + __ Dsra32(AT, in, 31); + __ Xor(out, in, AT); + __ Dsubu(out, out, AT); + break; + } + case DataType::Type::kFloat32: { + FpuRegister in = locations->InAt(0).AsFpuRegister<FpuRegister>(); + FpuRegister out = locations->Out().AsFpuRegister<FpuRegister>(); + __ AbsS(out, in); + break; + } + case DataType::Type::kFloat64: { + FpuRegister in = locations->InAt(0).AsFpuRegister<FpuRegister>(); + FpuRegister out = locations->Out().AsFpuRegister<FpuRegister>(); + __ AbsD(out, in); + break; + } + default: + LOG(FATAL) << "Unexpected abs type " << abs->GetResultType(); + } +} + void LocationsBuilderMIPS64::VisitConstructorFence(HConstructorFence* constructor_fence) { constructor_fence->SetLocations(nullptr); } diff --git a/compiler/optimizing/code_generator_mips64.h b/compiler/optimizing/code_generator_mips64.h index e6b69c469fd..6e69e4611a7 100644 --- a/compiler/optimizing/code_generator_mips64.h +++ b/compiler/optimizing/code_generator_mips64.h @@ -233,6 +233,7 @@ class InstructionCodeGeneratorMIPS64 : public InstructionCodeGenerator { private: void GenerateClassInitializationCheck(SlowPathCodeMIPS64* slow_path, GpuRegister class_reg); + void GenerateBitstringTypeCheckCompare(HTypeCheckInstruction* check, GpuRegister temp); void GenerateSuspendCheck(HSuspendCheck* check, HBasicBlock* successor); void HandleBinaryOp(HBinaryOperation* operation); void HandleCondition(HCondition* instruction); @@ -242,6 +243,10 @@ class InstructionCodeGeneratorMIPS64 : public InstructionCodeGenerator { bool value_can_be_null); void HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info); + void GenerateMinMaxInt(LocationSummary* locations, bool is_min); + void GenerateMinMaxFP(LocationSummary* locations, bool is_min, DataType::Type type); + void GenerateMinMax(HBinaryOperation* minmax, bool is_min); + // Generate a heap reference load using one register `out`: // // out <- *(out + offset) @@ -586,6 +591,8 @@ class CodeGeneratorMIPS64 : public CodeGenerator { DISALLOW_COPY_AND_ASSIGN(PcRelativePatchInfo); }; + PcRelativePatchInfo* NewBootImageRelRoPatch(uint32_t boot_image_offset, + const PcRelativePatchInfo* info_high = nullptr); PcRelativePatchInfo* NewBootImageMethodPatch(MethodReference target_method, const PcRelativePatchInfo* info_high = nullptr); PcRelativePatchInfo* NewMethodBssEntryPatch(MethodReference target_method, @@ -655,7 +662,8 @@ class CodeGeneratorMIPS64 : public CodeGenerator { // Deduplication map for 64-bit literals, used for non-patchable method address or method code // address. Uint64ToLiteralMap uint64_literals_; - // PC-relative method patch info for kBootImageLinkTimePcRelative. + // PC-relative method patch info for kBootImageLinkTimePcRelative/kBootImageRelRo. + // Also used for type/string patches for kBootImageRelRo (same linker patch as for methods). ArenaDeque<PcRelativePatchInfo> boot_image_method_patches_; // PC-relative method patch info for kBssEntry. ArenaDeque<PcRelativePatchInfo> method_bss_entry_patches_; @@ -663,7 +671,7 @@ class CodeGeneratorMIPS64 : public CodeGenerator { ArenaDeque<PcRelativePatchInfo> boot_image_type_patches_; // PC-relative type patch info for kBssEntry. ArenaDeque<PcRelativePatchInfo> type_bss_entry_patches_; - // PC-relative String patch info; type depends on configuration (intern table or boot image PIC). + // PC-relative String patch info for kBootImageLinkTimePcRelative. ArenaDeque<PcRelativePatchInfo> boot_image_string_patches_; // PC-relative type patch info for kBssEntry. ArenaDeque<PcRelativePatchInfo> string_bss_entry_patches_; diff --git a/compiler/optimizing/code_generator_vector_arm64.cc b/compiler/optimizing/code_generator_vector_arm64.cc index 174efdf1155..6b0ec253e99 100644 --- a/compiler/optimizing/code_generator_vector_arm64.cc +++ b/compiler/optimizing/code_generator_vector_arm64.cc @@ -63,7 +63,7 @@ void LocationsBuilderARM64::VisitVecReplicateScalar(HVecReplicateScalar* instruc } break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -125,7 +125,7 @@ void InstructionCodeGeneratorARM64::VisitVecReplicateScalar(HVecReplicateScalar* } break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -149,7 +149,7 @@ void LocationsBuilderARM64::VisitVecExtractScalar(HVecExtractScalar* instruction locations->SetOut(Location::SameAsFirstInput()); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -173,7 +173,7 @@ void InstructionCodeGeneratorARM64::VisitVecExtractScalar(HVecExtractScalar* ins DCHECK(locations->InAt(0).Equals(locations->Out())); // no code required break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -200,7 +200,7 @@ static void CreateVecUnOpLocations(ArenaAllocator* allocator, HVecUnaryOperation locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -240,7 +240,7 @@ void InstructionCodeGeneratorARM64::VisitVecReduce(HVecReduce* instruction) { } break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -259,7 +259,7 @@ void InstructionCodeGeneratorARM64::VisitVecCnv(HVecCnv* instruction) { DCHECK_EQ(4u, instruction->GetVectorLength()); __ Scvtf(dst.V4S(), src.V4S()); } else { - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); } } @@ -299,7 +299,7 @@ void InstructionCodeGeneratorARM64::VisitVecNeg(HVecNeg* instruction) { __ Fneg(dst.V2D(), src.V2D()); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -338,7 +338,7 @@ void InstructionCodeGeneratorARM64::VisitVecAbs(HVecAbs* instruction) { __ Fabs(dst.V2D(), src.V2D()); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -366,7 +366,7 @@ void InstructionCodeGeneratorARM64::VisitVecNot(HVecNot* instruction) { __ Not(dst.V16B(), src.V16B()); // lanes do not matter break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -389,7 +389,7 @@ static void CreateVecBinOpLocations(ArenaAllocator* allocator, HVecBinaryOperati locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -431,7 +431,39 @@ void InstructionCodeGeneratorARM64::VisitVecAdd(HVecAdd* instruction) { __ Fadd(dst.V2D(), lhs.V2D(), rhs.V2D()); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); + UNREACHABLE(); + } +} + +void LocationsBuilderARM64::VisitVecSaturationAdd(HVecSaturationAdd* instruction) { + CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); +} + +void InstructionCodeGeneratorARM64::VisitVecSaturationAdd(HVecSaturationAdd* instruction) { + LocationSummary* locations = instruction->GetLocations(); + VRegister lhs = VRegisterFrom(locations->InAt(0)); + VRegister rhs = VRegisterFrom(locations->InAt(1)); + VRegister dst = VRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case DataType::Type::kUint8: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ Uqadd(dst.V16B(), lhs.V16B(), rhs.V16B()); + break; + case DataType::Type::kInt8: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ Sqadd(dst.V16B(), lhs.V16B(), rhs.V16B()); + break; + case DataType::Type::kUint16: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ Uqadd(dst.V8H(), lhs.V8H(), rhs.V8H()); + break; + case DataType::Type::kInt16: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ Sqadd(dst.V8H(), lhs.V8H(), rhs.V8H()); + break; + default: + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -471,7 +503,7 @@ void InstructionCodeGeneratorARM64::VisitVecHalvingAdd(HVecHalvingAdd* instructi : __ Shadd(dst.V8H(), lhs.V8H(), rhs.V8H()); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -513,7 +545,39 @@ void InstructionCodeGeneratorARM64::VisitVecSub(HVecSub* instruction) { __ Fsub(dst.V2D(), lhs.V2D(), rhs.V2D()); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); + UNREACHABLE(); + } +} + +void LocationsBuilderARM64::VisitVecSaturationSub(HVecSaturationSub* instruction) { + CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); +} + +void InstructionCodeGeneratorARM64::VisitVecSaturationSub(HVecSaturationSub* instruction) { + LocationSummary* locations = instruction->GetLocations(); + VRegister lhs = VRegisterFrom(locations->InAt(0)); + VRegister rhs = VRegisterFrom(locations->InAt(1)); + VRegister dst = VRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case DataType::Type::kUint8: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ Uqsub(dst.V16B(), lhs.V16B(), rhs.V16B()); + break; + case DataType::Type::kInt8: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ Sqsub(dst.V16B(), lhs.V16B(), rhs.V16B()); + break; + case DataType::Type::kUint16: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ Uqsub(dst.V8H(), lhs.V8H(), rhs.V8H()); + break; + case DataType::Type::kInt16: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ Sqsub(dst.V8H(), lhs.V8H(), rhs.V8H()); + break; + default: + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -551,7 +615,7 @@ void InstructionCodeGeneratorARM64::VisitVecMul(HVecMul* instruction) { __ Fmul(dst.V2D(), lhs.V2D(), rhs.V2D()); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -575,7 +639,7 @@ void InstructionCodeGeneratorARM64::VisitVecDiv(HVecDiv* instruction) { __ Fdiv(dst.V2D(), lhs.V2D(), rhs.V2D()); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -623,7 +687,7 @@ void InstructionCodeGeneratorARM64::VisitVecMin(HVecMin* instruction) { __ Fmin(dst.V2D(), lhs.V2D(), rhs.V2D()); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -671,7 +735,7 @@ void InstructionCodeGeneratorARM64::VisitVecMax(HVecMax* instruction) { __ Fmax(dst.V2D(), lhs.V2D(), rhs.V2D()); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -699,7 +763,7 @@ void InstructionCodeGeneratorARM64::VisitVecAnd(HVecAnd* instruction) { __ And(dst.V16B(), lhs.V16B(), rhs.V16B()); // lanes do not matter break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -735,7 +799,7 @@ void InstructionCodeGeneratorARM64::VisitVecOr(HVecOr* instruction) { __ Orr(dst.V16B(), lhs.V16B(), rhs.V16B()); // lanes do not matter break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -762,7 +826,7 @@ void InstructionCodeGeneratorARM64::VisitVecXor(HVecXor* instruction) { __ Eor(dst.V16B(), lhs.V16B(), rhs.V16B()); // lanes do not matter break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -782,7 +846,7 @@ static void CreateVecShiftLocations(ArenaAllocator* allocator, HVecBinaryOperati locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -816,7 +880,7 @@ void InstructionCodeGeneratorARM64::VisitVecShl(HVecShl* instruction) { __ Shl(dst.V2D(), lhs.V2D(), value); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -850,7 +914,7 @@ void InstructionCodeGeneratorARM64::VisitVecShr(HVecShr* instruction) { __ Sshr(dst.V2D(), lhs.V2D(), value); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -884,7 +948,7 @@ void InstructionCodeGeneratorARM64::VisitVecUShr(HVecUShr* instruction) { __ Ushr(dst.V2D(), lhs.V2D(), value); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -916,7 +980,7 @@ void LocationsBuilderARM64::VisitVecSetScalars(HVecSetScalars* instruction) { locations->SetOut(Location::RequiresFpuRegister()); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -957,7 +1021,7 @@ void InstructionCodeGeneratorARM64::VisitVecSetScalars(HVecSetScalars* instructi __ Mov(dst.V2D(), 0, InputRegisterAt(instruction, 0)); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -978,7 +1042,7 @@ static void CreateVecAccumLocations(ArenaAllocator* allocator, HVecOperation* in locations->SetOut(Location::SameAsFirstInput()); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -1026,7 +1090,7 @@ void InstructionCodeGeneratorARM64::VisitVecMultiplyAccumulate(HVecMultiplyAccum } break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -1139,7 +1203,7 @@ void InstructionCodeGeneratorARM64::VisitVecSADAccumulate(HVecSADAccumulate* ins break; } default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } break; @@ -1167,7 +1231,7 @@ void InstructionCodeGeneratorARM64::VisitVecSADAccumulate(HVecSADAccumulate* ins break; } default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } break; @@ -1188,7 +1252,7 @@ void InstructionCodeGeneratorARM64::VisitVecSADAccumulate(HVecSADAccumulate* ins __ Sabal2(acc.V2D(), left.V4S(), right.V4S()); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } break; @@ -1204,12 +1268,12 @@ void InstructionCodeGeneratorARM64::VisitVecSADAccumulate(HVecSADAccumulate* ins break; } default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); } } @@ -1237,7 +1301,7 @@ static void CreateVecMemLocations(ArenaAllocator* allocator, } break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -1331,7 +1395,7 @@ void InstructionCodeGeneratorARM64::VisitVecLoad(HVecLoad* instruction) { __ Ldr(reg, VecAddress(instruction, &temps, size, instruction->IsStringCharAt(), &scratch)); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -1362,7 +1426,7 @@ void InstructionCodeGeneratorARM64::VisitVecStore(HVecStore* instruction) { __ Str(reg, VecAddress(instruction, &temps, size, /*is_string_char_at*/ false, &scratch)); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } diff --git a/compiler/optimizing/code_generator_vector_arm_vixl.cc b/compiler/optimizing/code_generator_vector_arm_vixl.cc index 7c3155ab73b..7b66b179839 100644 --- a/compiler/optimizing/code_generator_vector_arm_vixl.cc +++ b/compiler/optimizing/code_generator_vector_arm_vixl.cc @@ -46,7 +46,7 @@ void LocationsBuilderARMVIXL::VisitVecReplicateScalar(HVecReplicateScalar* instr locations->SetOut(Location::RequiresFpuRegister()); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -71,7 +71,7 @@ void InstructionCodeGeneratorARMVIXL::VisitVecReplicateScalar(HVecReplicateScala __ Vdup(Untyped32, dst, InputRegisterAt(instruction, 0)); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -84,7 +84,7 @@ void LocationsBuilderARMVIXL::VisitVecExtractScalar(HVecExtractScalar* instructi locations->SetOut(Location::RequiresRegister()); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -98,7 +98,7 @@ void InstructionCodeGeneratorARMVIXL::VisitVecExtractScalar(HVecExtractScalar* i __ Vmov(OutputRegister(instruction), DRegisterLane(src, 0)); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -122,7 +122,7 @@ static void CreateVecUnOpLocations(ArenaAllocator* allocator, HVecUnaryOperation locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -151,7 +151,7 @@ void InstructionCodeGeneratorARMVIXL::VisitVecReduce(HVecReduce* instruction) { } break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -188,7 +188,7 @@ void InstructionCodeGeneratorARMVIXL::VisitVecNeg(HVecNeg* instruction) { __ Vneg(DataTypeValue::S32, dst, src); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -215,7 +215,7 @@ void InstructionCodeGeneratorARMVIXL::VisitVecAbs(HVecAbs* instruction) { __ Vabs(DataTypeValue::S32, dst, src); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -242,7 +242,7 @@ void InstructionCodeGeneratorARMVIXL::VisitVecNot(HVecNot* instruction) { __ Vmvn(I8, dst, src); // lanes do not matter break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -262,7 +262,7 @@ static void CreateVecBinOpLocations(ArenaAllocator* allocator, HVecBinaryOperati locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -292,7 +292,39 @@ void InstructionCodeGeneratorARMVIXL::VisitVecAdd(HVecAdd* instruction) { __ Vadd(I32, dst, lhs, rhs); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); + UNREACHABLE(); + } +} + +void LocationsBuilderARMVIXL::VisitVecSaturationAdd(HVecSaturationAdd* instruction) { + CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); +} + +void InstructionCodeGeneratorARMVIXL::VisitVecSaturationAdd(HVecSaturationAdd* instruction) { + LocationSummary* locations = instruction->GetLocations(); + vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0)); + vixl32::DRegister rhs = DRegisterFrom(locations->InAt(1)); + vixl32::DRegister dst = DRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case DataType::Type::kUint8: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ Vqadd(DataTypeValue::U8, dst, lhs, rhs); + break; + case DataType::Type::kInt8: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ Vqadd(DataTypeValue::S8, dst, lhs, rhs); + break; + case DataType::Type::kUint16: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ Vqadd(DataTypeValue::U16, dst, lhs, rhs); + break; + case DataType::Type::kInt16: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ Vqadd(DataTypeValue::S16, dst, lhs, rhs); + break; + default: + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -332,7 +364,7 @@ void InstructionCodeGeneratorARMVIXL::VisitVecHalvingAdd(HVecHalvingAdd* instruc : __ Vhadd(DataTypeValue::S16, dst, lhs, rhs); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -362,7 +394,39 @@ void InstructionCodeGeneratorARMVIXL::VisitVecSub(HVecSub* instruction) { __ Vsub(I32, dst, lhs, rhs); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); + UNREACHABLE(); + } +} + +void LocationsBuilderARMVIXL::VisitVecSaturationSub(HVecSaturationSub* instruction) { + CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); +} + +void InstructionCodeGeneratorARMVIXL::VisitVecSaturationSub(HVecSaturationSub* instruction) { + LocationSummary* locations = instruction->GetLocations(); + vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0)); + vixl32::DRegister rhs = DRegisterFrom(locations->InAt(1)); + vixl32::DRegister dst = DRegisterFrom(locations->Out()); + switch (instruction->GetPackedType()) { + case DataType::Type::kUint8: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ Vqsub(DataTypeValue::U8, dst, lhs, rhs); + break; + case DataType::Type::kInt8: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ Vqsub(DataTypeValue::S8, dst, lhs, rhs); + break; + case DataType::Type::kUint16: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ Vqsub(DataTypeValue::U16, dst, lhs, rhs); + break; + case DataType::Type::kInt16: + DCHECK_EQ(4u, instruction->GetVectorLength()); + __ Vqsub(DataTypeValue::S16, dst, lhs, rhs); + break; + default: + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -392,7 +456,7 @@ void InstructionCodeGeneratorARMVIXL::VisitVecMul(HVecMul* instruction) { __ Vmul(I32, dst, lhs, rhs); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -440,7 +504,7 @@ void InstructionCodeGeneratorARMVIXL::VisitVecMin(HVecMin* instruction) { __ Vmin(DataTypeValue::S32, dst, lhs, rhs); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -480,7 +544,7 @@ void InstructionCodeGeneratorARMVIXL::VisitVecMax(HVecMax* instruction) { __ Vmax(DataTypeValue::S32, dst, lhs, rhs); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -505,7 +569,7 @@ void InstructionCodeGeneratorARMVIXL::VisitVecAnd(HVecAnd* instruction) { __ Vand(I8, dst, lhs, rhs); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -537,7 +601,7 @@ void InstructionCodeGeneratorARMVIXL::VisitVecOr(HVecOr* instruction) { __ Vorr(I8, dst, lhs, rhs); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -561,7 +625,7 @@ void InstructionCodeGeneratorARMVIXL::VisitVecXor(HVecXor* instruction) { __ Veor(I8, dst, lhs, rhs); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -580,7 +644,7 @@ static void CreateVecShiftLocations(ArenaAllocator* allocator, HVecBinaryOperati locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -610,7 +674,7 @@ void InstructionCodeGeneratorARMVIXL::VisitVecShl(HVecShl* instruction) { __ Vshl(I32, dst, lhs, value); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -640,7 +704,7 @@ void InstructionCodeGeneratorARMVIXL::VisitVecShr(HVecShr* instruction) { __ Vshr(DataTypeValue::S32, dst, lhs, value); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -670,7 +734,7 @@ void InstructionCodeGeneratorARMVIXL::VisitVecUShr(HVecUShr* instruction) { __ Vshr(DataTypeValue::U32, dst, lhs, value); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -690,7 +754,7 @@ void LocationsBuilderARMVIXL::VisitVecSetScalars(HVecSetScalars* instruction) { locations->SetOut(Location::RequiresFpuRegister()); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -716,7 +780,7 @@ void InstructionCodeGeneratorARMVIXL::VisitVecSetScalars(HVecSetScalars* instruc __ Vmov(Untyped32, DRegisterLane(dst, 0), InputRegisterAt(instruction, 0)); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -737,7 +801,7 @@ static void CreateVecAccumLocations(ArenaAllocator* allocator, HVecOperation* in locations->SetOut(Location::SameAsFirstInput()); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -780,12 +844,12 @@ void InstructionCodeGeneratorARMVIXL::VisitVecSADAccumulate(HVecSADAccumulate* i break; } default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -817,7 +881,7 @@ static void CreateVecMemLocations(ArenaAllocator* allocator, } break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -923,7 +987,7 @@ void InstructionCodeGeneratorARMVIXL::VisitVecLoad(HVecLoad* instruction) { } break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -971,7 +1035,7 @@ void InstructionCodeGeneratorARMVIXL::VisitVecStore(HVecStore* instruction) { } break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } diff --git a/compiler/optimizing/code_generator_vector_mips.cc b/compiler/optimizing/code_generator_vector_mips.cc index ed9de964965..df0e1485d69 100644 --- a/compiler/optimizing/code_generator_vector_mips.cc +++ b/compiler/optimizing/code_generator_vector_mips.cc @@ -42,7 +42,7 @@ void LocationsBuilderMIPS::VisitVecReplicateScalar(HVecReplicateScalar* instruct locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -89,7 +89,7 @@ void InstructionCodeGeneratorMIPS::VisitVecReplicateScalar(HVecReplicateScalar* /* is_double */ true); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -113,7 +113,7 @@ void LocationsBuilderMIPS::VisitVecExtractScalar(HVecExtractScalar* instruction) locations->SetOut(Location::SameAsFirstInput()); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -138,7 +138,7 @@ void InstructionCodeGeneratorMIPS::VisitVecExtractScalar(HVecExtractScalar* inst DCHECK(locations->InAt(0).Equals(locations->Out())); // no code required break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -170,7 +170,7 @@ static void CreateVecUnOpLocations(ArenaAllocator* allocator, HVecUnaryOperation : Location::kNoOutputOverlap); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -225,7 +225,7 @@ void InstructionCodeGeneratorMIPS::VisitVecReduce(HVecReduce* instruction) { } break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -244,7 +244,7 @@ void InstructionCodeGeneratorMIPS::VisitVecCnv(HVecCnv* instruction) { DCHECK_EQ(4u, instruction->GetVectorLength()); __ Ffint_sW(dst, src); } else { - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); } } @@ -290,7 +290,7 @@ void InstructionCodeGeneratorMIPS::VisitVecNeg(HVecNeg* instruction) { __ FsubD(dst, dst, src); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -337,7 +337,7 @@ void InstructionCodeGeneratorMIPS::VisitVecAbs(HVecAbs* instruction) { __ AndV(dst, dst, src); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -369,7 +369,7 @@ void InstructionCodeGeneratorMIPS::VisitVecNot(HVecNot* instruction) { __ NorV(dst, src, src); // lanes do not matter break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -392,7 +392,7 @@ static void CreateVecBinOpLocations(ArenaAllocator* allocator, HVecBinaryOperati locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -434,11 +434,19 @@ void InstructionCodeGeneratorMIPS::VisitVecAdd(HVecAdd* instruction) { __ FaddD(dst, lhs, rhs); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } +void LocationsBuilderMIPS::VisitVecSaturationAdd(HVecSaturationAdd* instruction) { + CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); +} + +void InstructionCodeGeneratorMIPS::VisitVecSaturationAdd(HVecSaturationAdd* instruction) { + LOG(FATAL) << "Unsupported SIMD " << instruction->GetId(); +} + void LocationsBuilderMIPS::VisitVecHalvingAdd(HVecHalvingAdd* instruction) { CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); } @@ -474,7 +482,7 @@ void InstructionCodeGeneratorMIPS::VisitVecHalvingAdd(HVecHalvingAdd* instructio : __ Ave_sH(dst, lhs, rhs); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -516,11 +524,19 @@ void InstructionCodeGeneratorMIPS::VisitVecSub(HVecSub* instruction) { __ FsubD(dst, lhs, rhs); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } +void LocationsBuilderMIPS::VisitVecSaturationSub(HVecSaturationSub* instruction) { + CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); +} + +void InstructionCodeGeneratorMIPS::VisitVecSaturationSub(HVecSaturationSub* instruction) { + LOG(FATAL) << "Unsupported SIMD " << instruction->GetId(); +} + void LocationsBuilderMIPS::VisitVecMul(HVecMul* instruction) { CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); } @@ -558,7 +574,7 @@ void InstructionCodeGeneratorMIPS::VisitVecMul(HVecMul* instruction) { __ FmulD(dst, lhs, rhs); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -582,7 +598,7 @@ void InstructionCodeGeneratorMIPS::VisitVecDiv(HVecDiv* instruction) { __ FdivD(dst, lhs, rhs); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -640,7 +656,7 @@ void InstructionCodeGeneratorMIPS::VisitVecMin(HVecMin* instruction) { __ FminD(dst, lhs, rhs); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -698,7 +714,7 @@ void InstructionCodeGeneratorMIPS::VisitVecMax(HVecMax* instruction) { __ FmaxD(dst, lhs, rhs); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -727,7 +743,7 @@ void InstructionCodeGeneratorMIPS::VisitVecAnd(HVecAnd* instruction) { __ AndV(dst, lhs, rhs); // lanes do not matter break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -764,7 +780,7 @@ void InstructionCodeGeneratorMIPS::VisitVecOr(HVecOr* instruction) { __ OrV(dst, lhs, rhs); // lanes do not matter break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -793,7 +809,7 @@ void InstructionCodeGeneratorMIPS::VisitVecXor(HVecXor* instruction) { __ XorV(dst, lhs, rhs); // lanes do not matter break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -813,7 +829,7 @@ static void CreateVecShiftLocations(ArenaAllocator* allocator, HVecBinaryOperati locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -847,7 +863,7 @@ void InstructionCodeGeneratorMIPS::VisitVecShl(HVecShl* instruction) { __ SlliD(dst, lhs, value); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -881,7 +897,7 @@ void InstructionCodeGeneratorMIPS::VisitVecShr(HVecShr* instruction) { __ SraiD(dst, lhs, value); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -915,7 +931,7 @@ void InstructionCodeGeneratorMIPS::VisitVecUShr(HVecUShr* instruction) { __ SrliD(dst, lhs, value); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -947,7 +963,7 @@ void LocationsBuilderMIPS::VisitVecSetScalars(HVecSetScalars* instruction) { locations->SetOut(Location::RequiresFpuRegister()); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -989,7 +1005,7 @@ void InstructionCodeGeneratorMIPS::VisitVecSetScalars(HVecSetScalars* instructio __ InsertW(dst, locations->InAt(0).AsRegisterPairHigh<Register>(), 1); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -1010,7 +1026,7 @@ static void CreateVecAccumLocations(ArenaAllocator* allocator, HVecOperation* in locations->SetOut(Location::SameAsFirstInput()); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -1060,7 +1076,7 @@ void InstructionCodeGeneratorMIPS::VisitVecMultiplyAccumulate(HVecMultiplyAccumu } break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -1162,7 +1178,7 @@ void InstructionCodeGeneratorMIPS::VisitVecSADAccumulate(HVecSADAccumulate* inst break; } default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } break; @@ -1201,7 +1217,7 @@ void InstructionCodeGeneratorMIPS::VisitVecSADAccumulate(HVecSADAccumulate* inst break; } default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } break; @@ -1231,7 +1247,7 @@ void InstructionCodeGeneratorMIPS::VisitVecSADAccumulate(HVecSADAccumulate* inst break; } default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } break; @@ -1247,13 +1263,13 @@ void InstructionCodeGeneratorMIPS::VisitVecSADAccumulate(HVecSADAccumulate* inst break; } default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } break; } default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -1282,7 +1298,7 @@ static void CreateVecMemLocations(ArenaAllocator* allocator, } break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -1357,7 +1373,7 @@ void InstructionCodeGeneratorMIPS::VisitVecLoad(HVecLoad* instruction) { __ LdD(reg, base, offset); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -1395,7 +1411,7 @@ void InstructionCodeGeneratorMIPS::VisitVecStore(HVecStore* instruction) { __ StD(reg, base, offset); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } diff --git a/compiler/optimizing/code_generator_vector_mips64.cc b/compiler/optimizing/code_generator_vector_mips64.cc index 9ea55ec8d79..de354b63a15 100644 --- a/compiler/optimizing/code_generator_vector_mips64.cc +++ b/compiler/optimizing/code_generator_vector_mips64.cc @@ -47,7 +47,7 @@ void LocationsBuilderMIPS64::VisitVecReplicateScalar(HVecReplicateScalar* instru locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -88,7 +88,7 @@ void InstructionCodeGeneratorMIPS64::VisitVecReplicateScalar(HVecReplicateScalar /* is_double */ true); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -112,7 +112,7 @@ void LocationsBuilderMIPS64::VisitVecExtractScalar(HVecExtractScalar* instructio locations->SetOut(Location::SameAsFirstInput()); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -136,7 +136,7 @@ void InstructionCodeGeneratorMIPS64::VisitVecExtractScalar(HVecExtractScalar* in DCHECK(locations->InAt(0).Equals(locations->Out())); // no code required break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -168,7 +168,7 @@ static void CreateVecUnOpLocations(ArenaAllocator* allocator, HVecUnaryOperation : Location::kNoOutputOverlap); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -223,7 +223,7 @@ void InstructionCodeGeneratorMIPS64::VisitVecReduce(HVecReduce* instruction) { } break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -242,7 +242,7 @@ void InstructionCodeGeneratorMIPS64::VisitVecCnv(HVecCnv* instruction) { DCHECK_EQ(4u, instruction->GetVectorLength()); __ Ffint_sW(dst, src); } else { - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -289,7 +289,7 @@ void InstructionCodeGeneratorMIPS64::VisitVecNeg(HVecNeg* instruction) { __ FsubD(dst, dst, src); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -336,7 +336,7 @@ void InstructionCodeGeneratorMIPS64::VisitVecAbs(HVecAbs* instruction) { __ AndV(dst, dst, src); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -368,7 +368,7 @@ void InstructionCodeGeneratorMIPS64::VisitVecNot(HVecNot* instruction) { __ NorV(dst, src, src); // lanes do not matter break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -391,7 +391,7 @@ static void CreateVecBinOpLocations(ArenaAllocator* allocator, HVecBinaryOperati locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -433,11 +433,19 @@ void InstructionCodeGeneratorMIPS64::VisitVecAdd(HVecAdd* instruction) { __ FaddD(dst, lhs, rhs); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } +void LocationsBuilderMIPS64::VisitVecSaturationAdd(HVecSaturationAdd* instruction) { + CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); +} + +void InstructionCodeGeneratorMIPS64::VisitVecSaturationAdd(HVecSaturationAdd* instruction) { + LOG(FATAL) << "Unsupported SIMD " << instruction->GetId(); +} + void LocationsBuilderMIPS64::VisitVecHalvingAdd(HVecHalvingAdd* instruction) { CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); } @@ -473,7 +481,7 @@ void InstructionCodeGeneratorMIPS64::VisitVecHalvingAdd(HVecHalvingAdd* instruct : __ Ave_sH(dst, lhs, rhs); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -515,11 +523,19 @@ void InstructionCodeGeneratorMIPS64::VisitVecSub(HVecSub* instruction) { __ FsubD(dst, lhs, rhs); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } +void LocationsBuilderMIPS64::VisitVecSaturationSub(HVecSaturationSub* instruction) { + CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); +} + +void InstructionCodeGeneratorMIPS64::VisitVecSaturationSub(HVecSaturationSub* instruction) { + LOG(FATAL) << "Unsupported SIMD " << instruction->GetId(); +} + void LocationsBuilderMIPS64::VisitVecMul(HVecMul* instruction) { CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); } @@ -557,7 +573,7 @@ void InstructionCodeGeneratorMIPS64::VisitVecMul(HVecMul* instruction) { __ FmulD(dst, lhs, rhs); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -581,7 +597,7 @@ void InstructionCodeGeneratorMIPS64::VisitVecDiv(HVecDiv* instruction) { __ FdivD(dst, lhs, rhs); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -639,7 +655,7 @@ void InstructionCodeGeneratorMIPS64::VisitVecMin(HVecMin* instruction) { __ FminD(dst, lhs, rhs); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -697,7 +713,7 @@ void InstructionCodeGeneratorMIPS64::VisitVecMax(HVecMax* instruction) { __ FmaxD(dst, lhs, rhs); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -726,7 +742,7 @@ void InstructionCodeGeneratorMIPS64::VisitVecAnd(HVecAnd* instruction) { __ AndV(dst, lhs, rhs); // lanes do not matter break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -763,7 +779,7 @@ void InstructionCodeGeneratorMIPS64::VisitVecOr(HVecOr* instruction) { __ OrV(dst, lhs, rhs); // lanes do not matter break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -792,7 +808,7 @@ void InstructionCodeGeneratorMIPS64::VisitVecXor(HVecXor* instruction) { __ XorV(dst, lhs, rhs); // lanes do not matter break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -812,7 +828,7 @@ static void CreateVecShiftLocations(ArenaAllocator* allocator, HVecBinaryOperati locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -846,7 +862,7 @@ void InstructionCodeGeneratorMIPS64::VisitVecShl(HVecShl* instruction) { __ SlliD(dst, lhs, value); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -880,7 +896,7 @@ void InstructionCodeGeneratorMIPS64::VisitVecShr(HVecShr* instruction) { __ SraiD(dst, lhs, value); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -914,7 +930,7 @@ void InstructionCodeGeneratorMIPS64::VisitVecUShr(HVecUShr* instruction) { __ SrliD(dst, lhs, value); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -946,7 +962,7 @@ void LocationsBuilderMIPS64::VisitVecSetScalars(HVecSetScalars* instruction) { locations->SetOut(Location::RequiresFpuRegister()); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -987,7 +1003,7 @@ void InstructionCodeGeneratorMIPS64::VisitVecSetScalars(HVecSetScalars* instruct __ InsertD(dst, locations->InAt(0).AsRegister<GpuRegister>(), 0); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -1008,7 +1024,7 @@ static void CreateVecAccumLocations(ArenaAllocator* allocator, HVecOperation* in locations->SetOut(Location::SameAsFirstInput()); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -1058,7 +1074,7 @@ void InstructionCodeGeneratorMIPS64::VisitVecMultiplyAccumulate(HVecMultiplyAccu } break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -1160,7 +1176,7 @@ void InstructionCodeGeneratorMIPS64::VisitVecSADAccumulate(HVecSADAccumulate* in break; } default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } break; @@ -1199,7 +1215,7 @@ void InstructionCodeGeneratorMIPS64::VisitVecSADAccumulate(HVecSADAccumulate* in break; } default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } break; @@ -1229,7 +1245,7 @@ void InstructionCodeGeneratorMIPS64::VisitVecSADAccumulate(HVecSADAccumulate* in break; } default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } break; @@ -1245,13 +1261,13 @@ void InstructionCodeGeneratorMIPS64::VisitVecSADAccumulate(HVecSADAccumulate* in break; } default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } break; } default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -1280,7 +1296,7 @@ static void CreateVecMemLocations(ArenaAllocator* allocator, } break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -1355,7 +1371,7 @@ void InstructionCodeGeneratorMIPS64::VisitVecLoad(HVecLoad* instruction) { __ LdD(reg, base, offset); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -1393,7 +1409,7 @@ void InstructionCodeGeneratorMIPS64::VisitVecStore(HVecStore* instruction) { __ StD(reg, base, offset); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } diff --git a/compiler/optimizing/code_generator_vector_x86.cc b/compiler/optimizing/code_generator_vector_x86.cc index f2ffccc8879..086ae07a064 100644 --- a/compiler/optimizing/code_generator_vector_x86.cc +++ b/compiler/optimizing/code_generator_vector_x86.cc @@ -54,7 +54,7 @@ void LocationsBuilderX86::VisitVecReplicateScalar(HVecReplicateScalar* instructi : Location::SameAsFirstInput()); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -111,7 +111,7 @@ void InstructionCodeGeneratorX86::VisitVecReplicateScalar(HVecReplicateScalar* i __ shufpd(dst, dst, Immediate(0)); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -138,7 +138,7 @@ void LocationsBuilderX86::VisitVecExtractScalar(HVecExtractScalar* instruction) locations->SetOut(Location::SameAsFirstInput()); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -152,7 +152,7 @@ void InstructionCodeGeneratorX86::VisitVecExtractScalar(HVecExtractScalar* instr case DataType::Type::kInt8: case DataType::Type::kUint16: case DataType::Type::kInt16: // TODO: up to here, and? - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); case DataType::Type::kInt32: DCHECK_LE(4u, instruction->GetVectorLength()); @@ -174,7 +174,7 @@ void InstructionCodeGeneratorX86::VisitVecExtractScalar(HVecExtractScalar* instr DCHECK(locations->InAt(0).Equals(locations->Out())); // no code required break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -196,7 +196,7 @@ static void CreateVecUnOpLocations(ArenaAllocator* allocator, HVecUnaryOperation locations->SetOut(Location::RequiresFpuRegister()); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -258,12 +258,12 @@ void InstructionCodeGeneratorX86::VisitVecReduce(HVecReduce* instruction) { break; case HVecReduce::kMin: case HVecReduce::kMax: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); } break; } default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -282,7 +282,7 @@ void InstructionCodeGeneratorX86::VisitVecCnv(HVecCnv* instruction) { DCHECK_EQ(4u, instruction->GetVectorLength()); __ cvtdq2ps(dst, src); } else { - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); } } @@ -328,7 +328,7 @@ void InstructionCodeGeneratorX86::VisitVecNeg(HVecNeg* instruction) { __ subpd(dst, src); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -369,7 +369,7 @@ void InstructionCodeGeneratorX86::VisitVecAbs(HVecAbs* instruction) { __ andpd(dst, src); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -418,7 +418,7 @@ void InstructionCodeGeneratorX86::VisitVecNot(HVecNot* instruction) { __ xorpd(dst, src); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -441,7 +441,7 @@ static void CreateVecBinOpLocations(ArenaAllocator* allocator, HVecBinaryOperati locations->SetOut(Location::SameAsFirstInput()); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -483,7 +483,39 @@ void InstructionCodeGeneratorX86::VisitVecAdd(HVecAdd* instruction) { __ addpd(dst, src); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); + UNREACHABLE(); + } +} + +void LocationsBuilderX86::VisitVecSaturationAdd(HVecSaturationAdd* instruction) { + CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); +} + +void InstructionCodeGeneratorX86::VisitVecSaturationAdd(HVecSaturationAdd* instruction) { + LocationSummary* locations = instruction->GetLocations(); + DCHECK(locations->InAt(0).Equals(locations->Out())); + XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); + XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); + switch (instruction->GetPackedType()) { + case DataType::Type::kUint8: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ paddusb(dst, src); + break; + case DataType::Type::kInt8: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ paddsb(dst, src); + break; + case DataType::Type::kUint16: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ paddusw(dst, src); + break; + case DataType::Type::kInt16: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ paddsw(dst, src); + break; + default: + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -503,14 +535,14 @@ void InstructionCodeGeneratorX86::VisitVecHalvingAdd(HVecHalvingAdd* instruction switch (instruction->GetPackedType()) { case DataType::Type::kUint8: DCHECK_EQ(16u, instruction->GetVectorLength()); - __ pavgb(dst, src); - return; + __ pavgb(dst, src); + break; case DataType::Type::kUint16: DCHECK_EQ(8u, instruction->GetVectorLength()); __ pavgw(dst, src); - return; + break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -552,7 +584,39 @@ void InstructionCodeGeneratorX86::VisitVecSub(HVecSub* instruction) { __ subpd(dst, src); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); + UNREACHABLE(); + } +} + +void LocationsBuilderX86::VisitVecSaturationSub(HVecSaturationSub* instruction) { + CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); +} + +void InstructionCodeGeneratorX86::VisitVecSaturationSub(HVecSaturationSub* instruction) { + LocationSummary* locations = instruction->GetLocations(); + DCHECK(locations->InAt(0).Equals(locations->Out())); + XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); + XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); + switch (instruction->GetPackedType()) { + case DataType::Type::kUint8: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ psubusb(dst, src); + break; + case DataType::Type::kInt8: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ psubsb(dst, src); + break; + case DataType::Type::kUint16: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ psubusw(dst, src); + break; + case DataType::Type::kInt16: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ psubsw(dst, src); + break; + default: + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -585,7 +649,7 @@ void InstructionCodeGeneratorX86::VisitVecMul(HVecMul* instruction) { __ mulpd(dst, src); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -609,7 +673,7 @@ void InstructionCodeGeneratorX86::VisitVecDiv(HVecDiv* instruction) { __ divpd(dst, src); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -658,7 +722,7 @@ void InstructionCodeGeneratorX86::VisitVecMin(HVecMin* instruction) { __ minpd(dst, src); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -707,7 +771,7 @@ void InstructionCodeGeneratorX86::VisitVecMax(HVecMax* instruction) { __ maxpd(dst, src); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -742,7 +806,7 @@ void InstructionCodeGeneratorX86::VisitVecAnd(HVecAnd* instruction) { __ andpd(dst, src); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -777,7 +841,7 @@ void InstructionCodeGeneratorX86::VisitVecAndNot(HVecAndNot* instruction) { __ andnpd(dst, src); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -812,7 +876,7 @@ void InstructionCodeGeneratorX86::VisitVecOr(HVecOr* instruction) { __ orpd(dst, src); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -847,7 +911,7 @@ void InstructionCodeGeneratorX86::VisitVecXor(HVecXor* instruction) { __ xorpd(dst, src); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -865,7 +929,7 @@ static void CreateVecShiftLocations(ArenaAllocator* allocator, HVecBinaryOperati locations->SetOut(Location::SameAsFirstInput()); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -894,7 +958,7 @@ void InstructionCodeGeneratorX86::VisitVecShl(HVecShl* instruction) { __ psllq(dst, Immediate(static_cast<uint8_t>(value))); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -919,7 +983,7 @@ void InstructionCodeGeneratorX86::VisitVecShr(HVecShr* instruction) { __ psrad(dst, Immediate(static_cast<uint8_t>(value))); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -948,7 +1012,7 @@ void InstructionCodeGeneratorX86::VisitVecUShr(HVecUShr* instruction) { __ psrlq(dst, Immediate(static_cast<uint8_t>(value))); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -985,7 +1049,7 @@ void LocationsBuilderX86::VisitVecSetScalars(HVecSetScalars* instruction) { locations->SetOut(Location::RequiresFpuRegister()); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -1011,7 +1075,7 @@ void InstructionCodeGeneratorX86::VisitVecSetScalars(HVecSetScalars* instruction case DataType::Type::kInt8: case DataType::Type::kUint16: case DataType::Type::kInt16: // TODO: up to here, and? - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); case DataType::Type::kInt32: DCHECK_EQ(4u, instruction->GetVectorLength()); @@ -1035,7 +1099,7 @@ void InstructionCodeGeneratorX86::VisitVecSetScalars(HVecSetScalars* instruction __ movsd(dst, locations->InAt(1).AsFpuRegister<XmmRegister>()); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -1056,7 +1120,7 @@ static void CreateVecAccumLocations(ArenaAllocator* allocator, HVecOperation* in locations->SetOut(Location::SameAsFirstInput()); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -1103,7 +1167,7 @@ static void CreateVecMemLocations(ArenaAllocator* allocator, } break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -1184,7 +1248,7 @@ void InstructionCodeGeneratorX86::VisitVecLoad(HVecLoad* instruction) { is_aligned16 ? __ movapd(reg, address) : __ movupd(reg, address); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -1220,7 +1284,7 @@ void InstructionCodeGeneratorX86::VisitVecStore(HVecStore* instruction) { is_aligned16 ? __ movapd(address, reg) : __ movupd(address, reg); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } diff --git a/compiler/optimizing/code_generator_vector_x86_64.cc b/compiler/optimizing/code_generator_vector_x86_64.cc index e2b0485f890..4d31ab68d11 100644 --- a/compiler/optimizing/code_generator_vector_x86_64.cc +++ b/compiler/optimizing/code_generator_vector_x86_64.cc @@ -49,7 +49,7 @@ void LocationsBuilderX86_64::VisitVecReplicateScalar(HVecReplicateScalar* instru : Location::SameAsFirstInput()); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -102,7 +102,7 @@ void InstructionCodeGeneratorX86_64::VisitVecReplicateScalar(HVecReplicateScalar __ shufpd(dst, dst, Immediate(0)); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -126,7 +126,7 @@ void LocationsBuilderX86_64::VisitVecExtractScalar(HVecExtractScalar* instructio locations->SetOut(Location::SameAsFirstInput()); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -140,7 +140,7 @@ void InstructionCodeGeneratorX86_64::VisitVecExtractScalar(HVecExtractScalar* in case DataType::Type::kInt8: case DataType::Type::kUint16: case DataType::Type::kInt16: // TODO: up to here, and? - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); case DataType::Type::kInt32: DCHECK_EQ(4u, instruction->GetVectorLength()); @@ -157,7 +157,7 @@ void InstructionCodeGeneratorX86_64::VisitVecExtractScalar(HVecExtractScalar* in DCHECK(locations->InAt(0).Equals(locations->Out())); // no code required break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -179,7 +179,7 @@ static void CreateVecUnOpLocations(ArenaAllocator* allocator, HVecUnaryOperation locations->SetOut(Location::RequiresFpuRegister()); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -241,12 +241,12 @@ void InstructionCodeGeneratorX86_64::VisitVecReduce(HVecReduce* instruction) { break; case HVecReduce::kMin: case HVecReduce::kMax: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); } break; } default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -265,7 +265,7 @@ void InstructionCodeGeneratorX86_64::VisitVecCnv(HVecCnv* instruction) { DCHECK_EQ(4u, instruction->GetVectorLength()); __ cvtdq2ps(dst, src); } else { - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); } } @@ -311,7 +311,7 @@ void InstructionCodeGeneratorX86_64::VisitVecNeg(HVecNeg* instruction) { __ subpd(dst, src); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -352,7 +352,7 @@ void InstructionCodeGeneratorX86_64::VisitVecAbs(HVecAbs* instruction) { __ andpd(dst, src); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -401,7 +401,7 @@ void InstructionCodeGeneratorX86_64::VisitVecNot(HVecNot* instruction) { __ xorpd(dst, src); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -424,7 +424,7 @@ static void CreateVecBinOpLocations(ArenaAllocator* allocator, HVecBinaryOperati locations->SetOut(Location::SameAsFirstInput()); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -466,7 +466,39 @@ void InstructionCodeGeneratorX86_64::VisitVecAdd(HVecAdd* instruction) { __ addpd(dst, src); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); + UNREACHABLE(); + } +} + +void LocationsBuilderX86_64::VisitVecSaturationAdd(HVecSaturationAdd* instruction) { + CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); +} + +void InstructionCodeGeneratorX86_64::VisitVecSaturationAdd(HVecSaturationAdd* instruction) { + LocationSummary* locations = instruction->GetLocations(); + DCHECK(locations->InAt(0).Equals(locations->Out())); + XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); + XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); + switch (instruction->GetPackedType()) { + case DataType::Type::kUint8: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ paddusb(dst, src); + break; + case DataType::Type::kInt8: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ paddsb(dst, src); + break; + case DataType::Type::kUint16: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ paddusw(dst, src); + break; + case DataType::Type::kInt16: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ paddsw(dst, src); + break; + default: + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -486,14 +518,14 @@ void InstructionCodeGeneratorX86_64::VisitVecHalvingAdd(HVecHalvingAdd* instruct switch (instruction->GetPackedType()) { case DataType::Type::kUint8: DCHECK_EQ(16u, instruction->GetVectorLength()); - __ pavgb(dst, src); - return; + __ pavgb(dst, src); + break; case DataType::Type::kUint16: DCHECK_EQ(8u, instruction->GetVectorLength()); __ pavgw(dst, src); - return; + break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -535,7 +567,39 @@ void InstructionCodeGeneratorX86_64::VisitVecSub(HVecSub* instruction) { __ subpd(dst, src); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); + UNREACHABLE(); + } +} + +void LocationsBuilderX86_64::VisitVecSaturationSub(HVecSaturationSub* instruction) { + CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction); +} + +void InstructionCodeGeneratorX86_64::VisitVecSaturationSub(HVecSaturationSub* instruction) { + LocationSummary* locations = instruction->GetLocations(); + DCHECK(locations->InAt(0).Equals(locations->Out())); + XmmRegister src = locations->InAt(1).AsFpuRegister<XmmRegister>(); + XmmRegister dst = locations->Out().AsFpuRegister<XmmRegister>(); + switch (instruction->GetPackedType()) { + case DataType::Type::kUint8: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ psubusb(dst, src); + break; + case DataType::Type::kInt8: + DCHECK_EQ(16u, instruction->GetVectorLength()); + __ psubsb(dst, src); + break; + case DataType::Type::kUint16: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ psubusw(dst, src); + break; + case DataType::Type::kInt16: + DCHECK_EQ(8u, instruction->GetVectorLength()); + __ psubsw(dst, src); + break; + default: + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -568,7 +632,7 @@ void InstructionCodeGeneratorX86_64::VisitVecMul(HVecMul* instruction) { __ mulpd(dst, src); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -592,7 +656,7 @@ void InstructionCodeGeneratorX86_64::VisitVecDiv(HVecDiv* instruction) { __ divpd(dst, src); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -641,7 +705,7 @@ void InstructionCodeGeneratorX86_64::VisitVecMin(HVecMin* instruction) { __ minpd(dst, src); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -690,7 +754,7 @@ void InstructionCodeGeneratorX86_64::VisitVecMax(HVecMax* instruction) { __ maxpd(dst, src); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -725,7 +789,7 @@ void InstructionCodeGeneratorX86_64::VisitVecAnd(HVecAnd* instruction) { __ andpd(dst, src); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -760,7 +824,7 @@ void InstructionCodeGeneratorX86_64::VisitVecAndNot(HVecAndNot* instruction) { __ andnpd(dst, src); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -795,7 +859,7 @@ void InstructionCodeGeneratorX86_64::VisitVecOr(HVecOr* instruction) { __ orpd(dst, src); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -830,7 +894,7 @@ void InstructionCodeGeneratorX86_64::VisitVecXor(HVecXor* instruction) { __ xorpd(dst, src); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -848,7 +912,7 @@ static void CreateVecShiftLocations(ArenaAllocator* allocator, HVecBinaryOperati locations->SetOut(Location::SameAsFirstInput()); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -877,7 +941,7 @@ void InstructionCodeGeneratorX86_64::VisitVecShl(HVecShl* instruction) { __ psllq(dst, Immediate(static_cast<int8_t>(value))); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -902,7 +966,7 @@ void InstructionCodeGeneratorX86_64::VisitVecShr(HVecShr* instruction) { __ psrad(dst, Immediate(static_cast<int8_t>(value))); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -931,7 +995,7 @@ void InstructionCodeGeneratorX86_64::VisitVecUShr(HVecUShr* instruction) { __ psrlq(dst, Immediate(static_cast<int8_t>(value))); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -963,7 +1027,7 @@ void LocationsBuilderX86_64::VisitVecSetScalars(HVecSetScalars* instruction) { locations->SetOut(Location::RequiresFpuRegister()); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -989,7 +1053,7 @@ void InstructionCodeGeneratorX86_64::VisitVecSetScalars(HVecSetScalars* instruct case DataType::Type::kInt8: case DataType::Type::kUint16: case DataType::Type::kInt16: // TODO: up to here, and? - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); case DataType::Type::kInt32: DCHECK_EQ(4u, instruction->GetVectorLength()); @@ -1008,7 +1072,7 @@ void InstructionCodeGeneratorX86_64::VisitVecSetScalars(HVecSetScalars* instruct __ movsd(dst, locations->InAt(0).AsFpuRegister<XmmRegister>()); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -1029,7 +1093,7 @@ static void CreateVecAccumLocations(ArenaAllocator* allocator, HVecOperation* in locations->SetOut(Location::SameAsFirstInput()); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -1076,7 +1140,7 @@ static void CreateVecMemLocations(ArenaAllocator* allocator, } break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -1157,7 +1221,7 @@ void InstructionCodeGeneratorX86_64::VisitVecLoad(HVecLoad* instruction) { is_aligned16 ? __ movapd(reg, address) : __ movupd(reg, address); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } @@ -1193,7 +1257,7 @@ void InstructionCodeGeneratorX86_64::VisitVecStore(HVecStore* instruction) { is_aligned16 ? __ movapd(address, reg) : __ movupd(address, reg); break; default: - LOG(FATAL) << "Unsupported SIMD type"; + LOG(FATAL) << "Unsupported SIMD type: " << instruction->GetPackedType(); UNREACHABLE(); } } diff --git a/compiler/optimizing/code_generator_x86.cc b/compiler/optimizing/code_generator_x86.cc index 6bf045885d6..82d1fda8789 100644 --- a/compiler/optimizing/code_generator_x86.cc +++ b/compiler/optimizing/code_generator_x86.cc @@ -51,6 +51,9 @@ static constexpr int kC2ConditionMask = 0x400; static constexpr int kFakeReturnRegister = Register(8); +static constexpr int64_t kDoubleNaN = INT64_C(0x7FF8000000000000); +static constexpr int32_t kFloatNaN = INT32_C(0x7FC00000); + // NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. #define __ down_cast<X86Assembler*>(codegen->GetAssembler())-> // NOLINT #define QUICK_ENTRY_POINT(x) QUICK_ENTRYPOINT_OFFSET(kX86PointerSize, x).Int32Value() @@ -3802,6 +3805,301 @@ void InstructionCodeGeneratorX86::VisitRem(HRem* rem) { } } +static void CreateMinMaxLocations(ArenaAllocator* allocator, HBinaryOperation* minmax) { + LocationSummary* locations = new (allocator) LocationSummary(minmax); + switch (minmax->GetResultType()) { + case DataType::Type::kInt32: + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RequiresRegister()); + locations->SetOut(Location::SameAsFirstInput()); + break; + case DataType::Type::kInt64: + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RequiresRegister()); + locations->SetOut(Location::SameAsFirstInput()); + // Register to use to perform a long subtract to set cc. + locations->AddTemp(Location::RequiresRegister()); + break; + case DataType::Type::kFloat32: + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetInAt(1, Location::RequiresFpuRegister()); + locations->SetOut(Location::SameAsFirstInput()); + locations->AddTemp(Location::RequiresRegister()); + break; + case DataType::Type::kFloat64: + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetInAt(1, Location::RequiresFpuRegister()); + locations->SetOut(Location::SameAsFirstInput()); + break; + default: + LOG(FATAL) << "Unexpected type for HMinMax " << minmax->GetResultType(); + } +} + +void InstructionCodeGeneratorX86::GenerateMinMaxInt(LocationSummary* locations, + bool is_min, + DataType::Type type) { + Location op1_loc = locations->InAt(0); + Location op2_loc = locations->InAt(1); + + // Shortcut for same input locations. + if (op1_loc.Equals(op2_loc)) { + // Can return immediately, as op1_loc == out_loc. + // Note: if we ever support separate registers, e.g., output into memory, we need to check for + // a copy here. + DCHECK(locations->Out().Equals(op1_loc)); + return; + } + + if (type == DataType::Type::kInt64) { + // Need to perform a subtract to get the sign right. + // op1 is already in the same location as the output. + Location output = locations->Out(); + Register output_lo = output.AsRegisterPairLow<Register>(); + Register output_hi = output.AsRegisterPairHigh<Register>(); + + Register op2_lo = op2_loc.AsRegisterPairLow<Register>(); + Register op2_hi = op2_loc.AsRegisterPairHigh<Register>(); + + // The comparison is performed by subtracting the second operand from + // the first operand and then setting the status flags in the same + // manner as the SUB instruction." + __ cmpl(output_lo, op2_lo); + + // Now use a temp and the borrow to finish the subtraction of op2_hi. + Register temp = locations->GetTemp(0).AsRegister<Register>(); + __ movl(temp, output_hi); + __ sbbl(temp, op2_hi); + + // Now the condition code is correct. + Condition cond = is_min ? Condition::kGreaterEqual : Condition::kLess; + __ cmovl(cond, output_lo, op2_lo); + __ cmovl(cond, output_hi, op2_hi); + } else { + DCHECK_EQ(type, DataType::Type::kInt32); + Register out = locations->Out().AsRegister<Register>(); + Register op2 = op2_loc.AsRegister<Register>(); + + // (out := op1) + // out <=? op2 + // if out is min jmp done + // out := op2 + // done: + + __ cmpl(out, op2); + Condition cond = is_min ? Condition::kGreater : Condition::kLess; + __ cmovl(cond, out, op2); + } +} + +void InstructionCodeGeneratorX86::GenerateMinMaxFP(LocationSummary* locations, + bool is_min, + DataType::Type type) { + Location op1_loc = locations->InAt(0); + Location op2_loc = locations->InAt(1); + Location out_loc = locations->Out(); + XmmRegister out = out_loc.AsFpuRegister<XmmRegister>(); + + // Shortcut for same input locations. + if (op1_loc.Equals(op2_loc)) { + DCHECK(out_loc.Equals(op1_loc)); + return; + } + + // (out := op1) + // out <=? op2 + // if Nan jmp Nan_label + // if out is min jmp done + // if op2 is min jmp op2_label + // handle -0/+0 + // jmp done + // Nan_label: + // out := NaN + // op2_label: + // out := op2 + // done: + // + // This removes one jmp, but needs to copy one input (op1) to out. + // + // TODO: This is straight from Quick (except literal pool). Make NaN an out-of-line slowpath? + + XmmRegister op2 = op2_loc.AsFpuRegister<XmmRegister>(); + + NearLabel nan, done, op2_label; + if (type == DataType::Type::kFloat64) { + __ ucomisd(out, op2); + } else { + DCHECK_EQ(type, DataType::Type::kFloat32); + __ ucomiss(out, op2); + } + + __ j(Condition::kParityEven, &nan); + + __ j(is_min ? Condition::kAbove : Condition::kBelow, &op2_label); + __ j(is_min ? Condition::kBelow : Condition::kAbove, &done); + + // Handle 0.0/-0.0. + if (is_min) { + if (type == DataType::Type::kFloat64) { + __ orpd(out, op2); + } else { + __ orps(out, op2); + } + } else { + if (type == DataType::Type::kFloat64) { + __ andpd(out, op2); + } else { + __ andps(out, op2); + } + } + __ jmp(&done); + + // NaN handling. + __ Bind(&nan); + if (type == DataType::Type::kFloat64) { + // TODO: Use a constant from the constant table (requires extra input). + __ LoadLongConstant(out, kDoubleNaN); + } else { + Register constant = locations->GetTemp(0).AsRegister<Register>(); + __ movl(constant, Immediate(kFloatNaN)); + __ movd(out, constant); + } + __ jmp(&done); + + // out := op2; + __ Bind(&op2_label); + if (type == DataType::Type::kFloat64) { + __ movsd(out, op2); + } else { + __ movss(out, op2); + } + + // Done. + __ Bind(&done); +} + +void InstructionCodeGeneratorX86::GenerateMinMax(HBinaryOperation* minmax, bool is_min) { + DataType::Type type = minmax->GetResultType(); + switch (type) { + case DataType::Type::kInt32: + case DataType::Type::kInt64: + GenerateMinMaxInt(minmax->GetLocations(), is_min, type); + break; + case DataType::Type::kFloat32: + case DataType::Type::kFloat64: + GenerateMinMaxFP(minmax->GetLocations(), is_min, type); + break; + default: + LOG(FATAL) << "Unexpected type for HMinMax " << type; + } +} + +void LocationsBuilderX86::VisitMin(HMin* min) { + CreateMinMaxLocations(GetGraph()->GetAllocator(), min); +} + +void InstructionCodeGeneratorX86::VisitMin(HMin* min) { + GenerateMinMax(min, /*is_min*/ true); +} + +void LocationsBuilderX86::VisitMax(HMax* max) { + CreateMinMaxLocations(GetGraph()->GetAllocator(), max); +} + +void InstructionCodeGeneratorX86::VisitMax(HMax* max) { + GenerateMinMax(max, /*is_min*/ false); +} + +void LocationsBuilderX86::VisitAbs(HAbs* abs) { + LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(abs); + switch (abs->GetResultType()) { + case DataType::Type::kInt32: + locations->SetInAt(0, Location::RegisterLocation(EAX)); + locations->SetOut(Location::SameAsFirstInput()); + locations->AddTemp(Location::RegisterLocation(EDX)); + break; + case DataType::Type::kInt64: + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); + locations->AddTemp(Location::RequiresRegister()); + break; + case DataType::Type::kFloat32: + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetOut(Location::SameAsFirstInput()); + locations->AddTemp(Location::RequiresFpuRegister()); + locations->AddTemp(Location::RequiresRegister()); + break; + case DataType::Type::kFloat64: + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetOut(Location::SameAsFirstInput()); + locations->AddTemp(Location::RequiresFpuRegister()); + break; + default: + LOG(FATAL) << "Unexpected type for HAbs " << abs->GetResultType(); + } +} + +void InstructionCodeGeneratorX86::VisitAbs(HAbs* abs) { + LocationSummary* locations = abs->GetLocations(); + switch (abs->GetResultType()) { + case DataType::Type::kInt32: { + Register out = locations->Out().AsRegister<Register>(); + DCHECK_EQ(out, EAX); + Register temp = locations->GetTemp(0).AsRegister<Register>(); + DCHECK_EQ(temp, EDX); + // Sign extend EAX into EDX. + __ cdq(); + // XOR EAX with sign. + __ xorl(EAX, EDX); + // Subtract out sign to correct. + __ subl(EAX, EDX); + // The result is in EAX. + break; + } + case DataType::Type::kInt64: { + Location input = locations->InAt(0); + Register input_lo = input.AsRegisterPairLow<Register>(); + Register input_hi = input.AsRegisterPairHigh<Register>(); + Location output = locations->Out(); + Register output_lo = output.AsRegisterPairLow<Register>(); + Register output_hi = output.AsRegisterPairHigh<Register>(); + Register temp = locations->GetTemp(0).AsRegister<Register>(); + // Compute the sign into the temporary. + __ movl(temp, input_hi); + __ sarl(temp, Immediate(31)); + // Store the sign into the output. + __ movl(output_lo, temp); + __ movl(output_hi, temp); + // XOR the input to the output. + __ xorl(output_lo, input_lo); + __ xorl(output_hi, input_hi); + // Subtract the sign. + __ subl(output_lo, temp); + __ sbbl(output_hi, temp); + break; + } + case DataType::Type::kFloat32: { + XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>(); + XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>(); + Register constant = locations->GetTemp(1).AsRegister<Register>(); + __ movl(constant, Immediate(INT32_C(0x7FFFFFFF))); + __ movd(temp, constant); + __ andps(out, temp); + break; + } + case DataType::Type::kFloat64: { + XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>(); + XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>(); + // TODO: Use a constant from the constant table (requires extra input). + __ LoadLongConstant(temp, INT64_C(0x7FFFFFFFFFFFFFFF)); + __ andpd(out, temp); + break; + } + default: + LOG(FATAL) << "Unexpected type for HAbs " << abs->GetResultType(); + } +} + void LocationsBuilderX86::VisitDivZeroCheck(HDivZeroCheck* instruction) { LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction); switch (instruction->GetType()) { @@ -4534,6 +4832,15 @@ void CodeGeneratorX86::GenerateStaticOrDirectCall( case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress: __ movl(temp.AsRegister<Register>(), Immediate(invoke->GetMethodAddress())); break; + case HInvokeStaticOrDirect::MethodLoadKind::kBootImageRelRo: { + Register base_reg = GetInvokeStaticOrDirectExtraParameter(invoke, + temp.AsRegister<Register>()); + __ movl(temp.AsRegister<Register>(), Address(base_reg, kDummy32BitOffset)); + RecordBootImageRelRoPatch( + invoke->InputAt(invoke->GetSpecialInputIndex())->AsX86ComputeBaseMethodAddress(), + GetBootImageOffset(invoke)); + break; + } case HInvokeStaticOrDirect::MethodLoadKind::kBssEntry: { Register base_reg = GetInvokeStaticOrDirectExtraParameter(invoke, temp.AsRegister<Register>()); @@ -4595,6 +4902,13 @@ void CodeGeneratorX86::GenerateVirtualCall( RecordPcInfo(invoke, invoke->GetDexPc(), slow_path); } +void CodeGeneratorX86::RecordBootImageRelRoPatch(HX86ComputeBaseMethodAddress* method_address, + uint32_t boot_image_offset) { + boot_image_method_patches_.emplace_back( + method_address, /* target_dex_file */ nullptr, boot_image_offset); + __ Bind(&boot_image_method_patches_.back().label); +} + void CodeGeneratorX86::RecordBootImageMethodPatch(HInvokeStaticOrDirect* invoke) { DCHECK_EQ(invoke->InputCount(), invoke->GetNumberOfArguments() + 1u); HX86ComputeBaseMethodAddress* method_address = @@ -4664,6 +4978,14 @@ inline void CodeGeneratorX86::EmitPcRelativeLinkerPatches( } } +linker::LinkerPatch DataBimgRelRoPatchAdapter(size_t literal_offset, + const DexFile* target_dex_file, + uint32_t pc_insn_offset, + uint32_t boot_image_offset) { + DCHECK(target_dex_file == nullptr); // Unused for DataBimgRelRoPatch(), should be null. + return linker::LinkerPatch::DataBimgRelRoPatch(literal_offset, pc_insn_offset, boot_image_offset); +} + void CodeGeneratorX86::EmitLinkerPatches(ArenaVector<linker::LinkerPatch>* linker_patches) { DCHECK(linker_patches->empty()); size_t size = @@ -4682,11 +5004,10 @@ void CodeGeneratorX86::EmitLinkerPatches(ArenaVector<linker::LinkerPatch>* linke EmitPcRelativeLinkerPatches<linker::LinkerPatch::RelativeStringPatch>( boot_image_string_patches_, linker_patches); } else { - DCHECK(boot_image_method_patches_.empty()); - EmitPcRelativeLinkerPatches<linker::LinkerPatch::TypeClassTablePatch>( - boot_image_type_patches_, linker_patches); - EmitPcRelativeLinkerPatches<linker::LinkerPatch::StringInternTablePatch>( - boot_image_string_patches_, linker_patches); + EmitPcRelativeLinkerPatches<DataBimgRelRoPatchAdapter>( + boot_image_method_patches_, linker_patches); + DCHECK(boot_image_type_patches_.empty()); + DCHECK(boot_image_string_patches_.empty()); } EmitPcRelativeLinkerPatches<linker::LinkerPatch::MethodBssEntryPatch>( method_bss_entry_patches_, linker_patches); @@ -6055,7 +6376,7 @@ HLoadClass::LoadKind CodeGeneratorX86::GetSupportedLoadClassKind( case HLoadClass::LoadKind::kReferrersClass: break; case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: - case HLoadClass::LoadKind::kBootImageClassTable: + case HLoadClass::LoadKind::kBootImageRelRo: case HLoadClass::LoadKind::kBssEntry: DCHECK(!Runtime::Current()->UseJitCompilation()); break; @@ -6093,7 +6414,7 @@ void LocationsBuilderX86::VisitLoadClass(HLoadClass* cls) { if (load_kind == HLoadClass::LoadKind::kReferrersClass || load_kind == HLoadClass::LoadKind::kBootImageLinkTimePcRelative || - load_kind == HLoadClass::LoadKind::kBootImageClassTable || + load_kind == HLoadClass::LoadKind::kBootImageRelRo || load_kind == HLoadClass::LoadKind::kBssEntry) { locations->SetInAt(0, Location::RequiresRegister()); } @@ -6169,17 +6490,12 @@ void InstructionCodeGeneratorX86::VisitLoadClass(HLoadClass* cls) NO_THREAD_SAFE __ movl(out, Immediate(address)); break; } - case HLoadClass::LoadKind::kBootImageClassTable: { + case HLoadClass::LoadKind::kBootImageRelRo: { DCHECK(!codegen_->GetCompilerOptions().IsBootImage()); Register method_address = locations->InAt(0).AsRegister<Register>(); __ movl(out, Address(method_address, CodeGeneratorX86::kDummy32BitOffset)); - codegen_->RecordBootImageTypePatch(cls); - // Extract the reference from the slot data, i.e. clear the hash bits. - int32_t masked_hash = ClassTable::TableSlot::MaskHash( - ComputeModifiedUtf8Hash(cls->GetDexFile().StringByTypeIdx(cls->GetTypeIndex()))); - if (masked_hash != 0) { - __ subl(out, Immediate(masked_hash)); - } + codegen_->RecordBootImageRelRoPatch(cls->InputAt(0)->AsX86ComputeBaseMethodAddress(), + codegen_->GetBootImageOffset(cls)); break; } case HLoadClass::LoadKind::kBssEntry: { @@ -6255,11 +6571,31 @@ void InstructionCodeGeneratorX86::GenerateClassInitializationCheck( // No need for memory fence, thanks to the X86 memory model. } +void InstructionCodeGeneratorX86::GenerateBitstringTypeCheckCompare(HTypeCheckInstruction* check, + Register temp) { + uint32_t path_to_root = check->GetBitstringPathToRoot(); + uint32_t mask = check->GetBitstringMask(); + DCHECK(IsPowerOfTwo(mask + 1)); + size_t mask_bits = WhichPowerOf2(mask + 1); + + if (mask_bits == 16u) { + // Compare the bitstring in memory. + __ cmpw(Address(temp, mirror::Class::StatusOffset()), Immediate(path_to_root)); + } else { + // /* uint32_t */ temp = temp->status_ + __ movl(temp, Address(temp, mirror::Class::StatusOffset())); + // Compare the bitstring bits using SUB. + __ subl(temp, Immediate(path_to_root)); + // Shift out bits that do not contribute to the comparison. + __ shll(temp, Immediate(32u - mask_bits)); + } +} + HLoadString::LoadKind CodeGeneratorX86::GetSupportedLoadStringKind( HLoadString::LoadKind desired_string_load_kind) { switch (desired_string_load_kind) { case HLoadString::LoadKind::kBootImageLinkTimePcRelative: - case HLoadString::LoadKind::kBootImageInternTable: + case HLoadString::LoadKind::kBootImageRelRo: case HLoadString::LoadKind::kBssEntry: DCHECK(!Runtime::Current()->UseJitCompilation()); break; @@ -6278,7 +6614,7 @@ void LocationsBuilderX86::VisitLoadString(HLoadString* load) { LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(load, call_kind); HLoadString::LoadKind load_kind = load->GetLoadKind(); if (load_kind == HLoadString::LoadKind::kBootImageLinkTimePcRelative || - load_kind == HLoadString::LoadKind::kBootImageInternTable || + load_kind == HLoadString::LoadKind::kBootImageRelRo || load_kind == HLoadString::LoadKind::kBssEntry) { locations->SetInAt(0, Location::RequiresRegister()); } @@ -6332,11 +6668,12 @@ void InstructionCodeGeneratorX86::VisitLoadString(HLoadString* load) NO_THREAD_S __ movl(out, Immediate(address)); return; } - case HLoadString::LoadKind::kBootImageInternTable: { + case HLoadString::LoadKind::kBootImageRelRo: { DCHECK(!codegen_->GetCompilerOptions().IsBootImage()); Register method_address = locations->InAt(0).AsRegister<Register>(); __ movl(out, Address(method_address, CodeGeneratorX86::kDummy32BitOffset)); - codegen_->RecordBootImageStringPatch(load); + codegen_->RecordBootImageRelRoPatch(load->InputAt(0)->AsX86ComputeBaseMethodAddress(), + codegen_->GetBootImageOffset(load)); return; } case HLoadString::LoadKind::kBssEntry: { @@ -6418,8 +6755,8 @@ static size_t NumberOfInstanceOfTemps(TypeCheckKind type_check_kind) { return 0; } -// Interface case has 3 temps, one for holding the number of interfaces, one for the current -// interface pointer, one for loading the current interface. +// Interface case has 2 temps, one for holding the number of interfaces, one for the current +// interface pointer, the current interface is compared in memory. // The other checks have one temp for loading the object's class. static size_t NumberOfCheckCastTemps(TypeCheckKind type_check_kind) { if (type_check_kind == TypeCheckKind::kInterfaceCheck) { @@ -6447,6 +6784,8 @@ void LocationsBuilderX86::VisitInstanceOf(HInstanceOf* instruction) { case TypeCheckKind::kInterfaceCheck: call_kind = LocationSummary::kCallOnSlowPath; break; + case TypeCheckKind::kBitstringCheck: + break; } LocationSummary* locations = @@ -6455,7 +6794,13 @@ void LocationsBuilderX86::VisitInstanceOf(HInstanceOf* instruction) { locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. } locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::Any()); + if (type_check_kind == TypeCheckKind::kBitstringCheck) { + locations->SetInAt(1, Location::ConstantLocation(instruction->InputAt(1)->AsConstant())); + locations->SetInAt(2, Location::ConstantLocation(instruction->InputAt(2)->AsConstant())); + locations->SetInAt(3, Location::ConstantLocation(instruction->InputAt(3)->AsConstant())); + } else { + locations->SetInAt(1, Location::Any()); + } // Note that TypeCheckSlowPathX86 uses this "out" register too. locations->SetOut(Location::RequiresRegister()); // When read barriers are enabled, we need a temporary register for some cases. @@ -6676,6 +7021,21 @@ void InstructionCodeGeneratorX86::VisitInstanceOf(HInstanceOf* instruction) { } break; } + + case TypeCheckKind::kBitstringCheck: { + // /* HeapReference<Class> */ temp = obj->klass_ + GenerateReferenceLoadTwoRegisters(instruction, + out_loc, + obj_loc, + class_offset, + kWithoutReadBarrier); + + GenerateBitstringTypeCheckCompare(instruction, out); + __ j(kNotEqual, &zero); + __ movl(out, Immediate(1)); + __ jmp(&done); + break; + } } if (zero.IsLinked()) { @@ -6702,12 +7062,14 @@ void LocationsBuilderX86::VisitCheckCast(HCheckCast* instruction) { // Require a register for the interface check since there is a loop that compares the class to // a memory address. locations->SetInAt(1, Location::RequiresRegister()); + } else if (type_check_kind == TypeCheckKind::kBitstringCheck) { + locations->SetInAt(1, Location::ConstantLocation(instruction->InputAt(1)->AsConstant())); + locations->SetInAt(2, Location::ConstantLocation(instruction->InputAt(2)->AsConstant())); + locations->SetInAt(3, Location::ConstantLocation(instruction->InputAt(3)->AsConstant())); } else { locations->SetInAt(1, Location::Any()); } - // Note that TypeCheckSlowPathX86 uses this "temp" register too. - locations->AddTemp(Location::RequiresRegister()); - // When read barriers are enabled, we need an additional temporary register for some cases. + // Add temps for read barriers and other uses. One is used by TypeCheckSlowPathX86. locations->AddRegisterTemps(NumberOfCheckCastTemps(type_check_kind)); } @@ -6921,6 +7283,19 @@ void InstructionCodeGeneratorX86::VisitCheckCast(HCheckCast* instruction) { __ MaybeUnpoisonHeapReference(cls.AsRegister<Register>()); break; } + + case TypeCheckKind::kBitstringCheck: { + // /* HeapReference<Class> */ temp = obj->klass_ + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + kWithoutReadBarrier); + + GenerateBitstringTypeCheckCompare(instruction, temp); + __ j(kNotEqual, type_check_slow_path->GetEntryLabel()); + break; + } } __ Bind(&done); diff --git a/compiler/optimizing/code_generator_x86.h b/compiler/optimizing/code_generator_x86.h index 51e5bca00b6..6c76e27d35f 100644 --- a/compiler/optimizing/code_generator_x86.h +++ b/compiler/optimizing/code_generator_x86.h @@ -211,6 +211,7 @@ class InstructionCodeGeneratorX86 : public InstructionCodeGenerator { // the suspend call. void GenerateSuspendCheck(HSuspendCheck* check, HBasicBlock* successor); void GenerateClassInitializationCheck(SlowPathCode* slow_path, Register class_reg); + void GenerateBitstringTypeCheckCompare(HTypeCheckInstruction* check, Register temp); void HandleBitwiseOperation(HBinaryOperation* instruction); void GenerateDivRemIntegral(HBinaryOperation* instruction); void DivRemOneOrMinusOne(HBinaryOperation* instruction); @@ -225,6 +226,9 @@ class InstructionCodeGeneratorX86 : public InstructionCodeGenerator { void GenerateShlLong(const Location& loc, int shift); void GenerateShrLong(const Location& loc, int shift); void GenerateUShrLong(const Location& loc, int shift); + void GenerateMinMaxInt(LocationSummary* locations, bool is_min, DataType::Type type); + void GenerateMinMaxFP(LocationSummary* locations, bool is_min, DataType::Type type); + void GenerateMinMax(HBinaryOperation* minmax, bool is_min); void HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info, @@ -414,6 +418,8 @@ class CodeGeneratorX86 : public CodeGenerator { void GenerateVirtualCall( HInvokeVirtual* invoke, Location temp, SlowPathCode* slow_path = nullptr) OVERRIDE; + void RecordBootImageRelRoPatch(HX86ComputeBaseMethodAddress* method_address, + uint32_t boot_image_offset); void RecordBootImageMethodPatch(HInvokeStaticOrDirect* invoke); void RecordMethodBssEntryPatch(HInvokeStaticOrDirect* invoke); void RecordBootImageTypePatch(HLoadClass* load_class); @@ -631,17 +637,18 @@ class CodeGeneratorX86 : public CodeGenerator { X86Assembler assembler_; const X86InstructionSetFeatures& isa_features_; - // PC-relative method patch info for kBootImageLinkTimePcRelative. + // PC-relative method patch info for kBootImageLinkTimePcRelative/kBootImageRelRo. + // Also used for type/string patches for kBootImageRelRo (same linker patch as for methods). ArenaDeque<X86PcRelativePatchInfo> boot_image_method_patches_; // PC-relative method patch info for kBssEntry. ArenaDeque<X86PcRelativePatchInfo> method_bss_entry_patches_; // PC-relative type patch info for kBootImageLinkTimePcRelative. ArenaDeque<X86PcRelativePatchInfo> boot_image_type_patches_; - // Type patch locations for kBssEntry. + // PC-relative type patch info for kBssEntry. ArenaDeque<X86PcRelativePatchInfo> type_bss_entry_patches_; - // String patch locations; type depends on configuration (intern table or boot image PIC). + // PC-relative String patch info for kBootImageLinkTimePcRelative. ArenaDeque<X86PcRelativePatchInfo> boot_image_string_patches_; - // String patch locations for kBssEntry. + // PC-relative String patch info for kBssEntry. ArenaDeque<X86PcRelativePatchInfo> string_bss_entry_patches_; // Patches for string root accesses in JIT compiled code. diff --git a/compiler/optimizing/code_generator_x86_64.cc b/compiler/optimizing/code_generator_x86_64.cc index 7be360536b2..322b0cfc4c1 100644 --- a/compiler/optimizing/code_generator_x86_64.cc +++ b/compiler/optimizing/code_generator_x86_64.cc @@ -998,6 +998,13 @@ void CodeGeneratorX86_64::GenerateStaticOrDirectCall( case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress: Load64BitValue(temp.AsRegister<CpuRegister>(), invoke->GetMethodAddress()); break; + case HInvokeStaticOrDirect::MethodLoadKind::kBootImageRelRo: { + // Note: Boot image is in the low 4GiB and the entry is 32-bit, so emit a 32-bit load. + __ movl(temp.AsRegister<CpuRegister>(), + Address::Absolute(kDummy32BitOffset, /* no_rip */ false)); + RecordBootImageRelRoPatch(GetBootImageOffset(invoke)); + break; + } case HInvokeStaticOrDirect::MethodLoadKind::kBssEntry: { __ movq(temp.AsRegister<CpuRegister>(), Address::Absolute(kDummy32BitOffset, /* no_rip */ false)); @@ -1059,6 +1066,11 @@ void CodeGeneratorX86_64::GenerateVirtualCall( RecordPcInfo(invoke, invoke->GetDexPc(), slow_path); } +void CodeGeneratorX86_64::RecordBootImageRelRoPatch(uint32_t boot_image_offset) { + boot_image_method_patches_.emplace_back(/* target_dex_file */ nullptr, boot_image_offset); + __ Bind(&boot_image_method_patches_.back().label); +} + void CodeGeneratorX86_64::RecordBootImageMethodPatch(HInvokeStaticOrDirect* invoke) { boot_image_method_patches_.emplace_back( invoke->GetTargetMethod().dex_file, invoke->GetTargetMethod().index); @@ -1110,6 +1122,14 @@ inline void CodeGeneratorX86_64::EmitPcRelativeLinkerPatches( } } +linker::LinkerPatch DataBimgRelRoPatchAdapter(size_t literal_offset, + const DexFile* target_dex_file, + uint32_t pc_insn_offset, + uint32_t boot_image_offset) { + DCHECK(target_dex_file == nullptr); // Unused for DataBimgRelRoPatch(), should be null. + return linker::LinkerPatch::DataBimgRelRoPatch(literal_offset, pc_insn_offset, boot_image_offset); +} + void CodeGeneratorX86_64::EmitLinkerPatches(ArenaVector<linker::LinkerPatch>* linker_patches) { DCHECK(linker_patches->empty()); size_t size = @@ -1128,11 +1148,10 @@ void CodeGeneratorX86_64::EmitLinkerPatches(ArenaVector<linker::LinkerPatch>* li EmitPcRelativeLinkerPatches<linker::LinkerPatch::RelativeStringPatch>( boot_image_string_patches_, linker_patches); } else { - DCHECK(boot_image_method_patches_.empty()); - EmitPcRelativeLinkerPatches<linker::LinkerPatch::TypeClassTablePatch>( - boot_image_type_patches_, linker_patches); - EmitPcRelativeLinkerPatches<linker::LinkerPatch::StringInternTablePatch>( - boot_image_string_patches_, linker_patches); + EmitPcRelativeLinkerPatches<DataBimgRelRoPatchAdapter>( + boot_image_method_patches_, linker_patches); + DCHECK(boot_image_type_patches_.empty()); + DCHECK(boot_image_string_patches_.empty()); } EmitPcRelativeLinkerPatches<linker::LinkerPatch::MethodBssEntryPatch>( method_bss_entry_patches_, linker_patches); @@ -3821,6 +3840,241 @@ void InstructionCodeGeneratorX86_64::VisitRem(HRem* rem) { } } +static void CreateMinMaxLocations(ArenaAllocator* allocator, HBinaryOperation* minmax) { + LocationSummary* locations = new (allocator) LocationSummary(minmax); + switch (minmax->GetResultType()) { + case DataType::Type::kInt32: + case DataType::Type::kInt64: + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetInAt(1, Location::RequiresRegister()); + locations->SetOut(Location::SameAsFirstInput()); + break; + case DataType::Type::kFloat32: + case DataType::Type::kFloat64: + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetInAt(1, Location::RequiresFpuRegister()); + // The following is sub-optimal, but all we can do for now. It would be fine to also accept + // the second input to be the output (we can simply swap inputs). + locations->SetOut(Location::SameAsFirstInput()); + break; + default: + LOG(FATAL) << "Unexpected type for HMinMax " << minmax->GetResultType(); + } +} + +void InstructionCodeGeneratorX86_64::GenerateMinMaxInt(LocationSummary* locations, + bool is_min, + DataType::Type type) { + Location op1_loc = locations->InAt(0); + Location op2_loc = locations->InAt(1); + + // Shortcut for same input locations. + if (op1_loc.Equals(op2_loc)) { + // Can return immediately, as op1_loc == out_loc. + // Note: if we ever support separate registers, e.g., output into memory, we need to check for + // a copy here. + DCHECK(locations->Out().Equals(op1_loc)); + return; + } + + CpuRegister out = locations->Out().AsRegister<CpuRegister>(); + CpuRegister op2 = op2_loc.AsRegister<CpuRegister>(); + + // (out := op1) + // out <=? op2 + // if out is min jmp done + // out := op2 + // done: + + if (type == DataType::Type::kInt64) { + __ cmpq(out, op2); + __ cmov(is_min ? Condition::kGreater : Condition::kLess, out, op2, /*is64bit*/ true); + } else { + DCHECK_EQ(type, DataType::Type::kInt32); + __ cmpl(out, op2); + __ cmov(is_min ? Condition::kGreater : Condition::kLess, out, op2, /*is64bit*/ false); + } +} + +void InstructionCodeGeneratorX86_64::GenerateMinMaxFP(LocationSummary* locations, + bool is_min, + DataType::Type type) { + Location op1_loc = locations->InAt(0); + Location op2_loc = locations->InAt(1); + Location out_loc = locations->Out(); + XmmRegister out = out_loc.AsFpuRegister<XmmRegister>(); + + // Shortcut for same input locations. + if (op1_loc.Equals(op2_loc)) { + DCHECK(out_loc.Equals(op1_loc)); + return; + } + + // (out := op1) + // out <=? op2 + // if Nan jmp Nan_label + // if out is min jmp done + // if op2 is min jmp op2_label + // handle -0/+0 + // jmp done + // Nan_label: + // out := NaN + // op2_label: + // out := op2 + // done: + // + // This removes one jmp, but needs to copy one input (op1) to out. + // + // TODO: This is straight from Quick. Make NaN an out-of-line slowpath? + + XmmRegister op2 = op2_loc.AsFpuRegister<XmmRegister>(); + + NearLabel nan, done, op2_label; + if (type == DataType::Type::kFloat64) { + __ ucomisd(out, op2); + } else { + DCHECK_EQ(type, DataType::Type::kFloat32); + __ ucomiss(out, op2); + } + + __ j(Condition::kParityEven, &nan); + + __ j(is_min ? Condition::kAbove : Condition::kBelow, &op2_label); + __ j(is_min ? Condition::kBelow : Condition::kAbove, &done); + + // Handle 0.0/-0.0. + if (is_min) { + if (type == DataType::Type::kFloat64) { + __ orpd(out, op2); + } else { + __ orps(out, op2); + } + } else { + if (type == DataType::Type::kFloat64) { + __ andpd(out, op2); + } else { + __ andps(out, op2); + } + } + __ jmp(&done); + + // NaN handling. + __ Bind(&nan); + if (type == DataType::Type::kFloat64) { + __ movsd(out, codegen_->LiteralInt64Address(INT64_C(0x7FF8000000000000))); + } else { + __ movss(out, codegen_->LiteralInt32Address(INT32_C(0x7FC00000))); + } + __ jmp(&done); + + // out := op2; + __ Bind(&op2_label); + if (type == DataType::Type::kFloat64) { + __ movsd(out, op2); + } else { + __ movss(out, op2); + } + + // Done. + __ Bind(&done); +} + +void InstructionCodeGeneratorX86_64::GenerateMinMax(HBinaryOperation* minmax, bool is_min) { + DataType::Type type = minmax->GetResultType(); + switch (type) { + case DataType::Type::kInt32: + case DataType::Type::kInt64: + GenerateMinMaxInt(minmax->GetLocations(), is_min, type); + break; + case DataType::Type::kFloat32: + case DataType::Type::kFloat64: + GenerateMinMaxFP(minmax->GetLocations(), is_min, type); + break; + default: + LOG(FATAL) << "Unexpected type for HMinMax " << type; + } +} + +void LocationsBuilderX86_64::VisitMin(HMin* min) { + CreateMinMaxLocations(GetGraph()->GetAllocator(), min); +} + +void InstructionCodeGeneratorX86_64::VisitMin(HMin* min) { + GenerateMinMax(min, /*is_min*/ true); +} + +void LocationsBuilderX86_64::VisitMax(HMax* max) { + CreateMinMaxLocations(GetGraph()->GetAllocator(), max); +} + +void InstructionCodeGeneratorX86_64::VisitMax(HMax* max) { + GenerateMinMax(max, /*is_min*/ false); +} + +void LocationsBuilderX86_64::VisitAbs(HAbs* abs) { + LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(abs); + switch (abs->GetResultType()) { + case DataType::Type::kInt32: + case DataType::Type::kInt64: + locations->SetInAt(0, Location::RequiresRegister()); + locations->SetOut(Location::SameAsFirstInput()); + locations->AddTemp(Location::RequiresRegister()); + break; + case DataType::Type::kFloat32: + case DataType::Type::kFloat64: + locations->SetInAt(0, Location::RequiresFpuRegister()); + locations->SetOut(Location::SameAsFirstInput()); + locations->AddTemp(Location::RequiresFpuRegister()); + break; + default: + LOG(FATAL) << "Unexpected type for HAbs " << abs->GetResultType(); + } +} + +void InstructionCodeGeneratorX86_64::VisitAbs(HAbs* abs) { + LocationSummary* locations = abs->GetLocations(); + switch (abs->GetResultType()) { + case DataType::Type::kInt32: { + CpuRegister out = locations->Out().AsRegister<CpuRegister>(); + CpuRegister mask = locations->GetTemp(0).AsRegister<CpuRegister>(); + // Create mask. + __ movl(mask, out); + __ sarl(mask, Immediate(31)); + // Add mask. + __ addl(out, mask); + __ xorl(out, mask); + break; + } + case DataType::Type::kInt64: { + CpuRegister out = locations->Out().AsRegister<CpuRegister>(); + CpuRegister mask = locations->GetTemp(0).AsRegister<CpuRegister>(); + // Create mask. + __ movq(mask, out); + __ sarq(mask, Immediate(63)); + // Add mask. + __ addq(out, mask); + __ xorq(out, mask); + break; + } + case DataType::Type::kFloat32: { + XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>(); + XmmRegister mask = locations->GetTemp(0).AsFpuRegister<XmmRegister>(); + __ movss(mask, codegen_->LiteralInt32Address(INT32_C(0x7FFFFFFF))); + __ andps(out, mask); + break; + } + case DataType::Type::kFloat64: { + XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>(); + XmmRegister mask = locations->GetTemp(0).AsFpuRegister<XmmRegister>(); + __ movsd(mask, codegen_->LiteralInt64Address(INT64_C(0x7FFFFFFFFFFFFFFF))); + __ andpd(out, mask); + break; + } + default: + LOG(FATAL) << "Unexpected type for HAbs " << abs->GetResultType(); + } +} + void LocationsBuilderX86_64::VisitDivZeroCheck(HDivZeroCheck* instruction) { LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction); locations->SetInAt(0, Location::Any()); @@ -5462,6 +5716,26 @@ void InstructionCodeGeneratorX86_64::GenerateClassInitializationCheck( // No need for memory fence, thanks to the x86-64 memory model. } +void InstructionCodeGeneratorX86_64::GenerateBitstringTypeCheckCompare(HTypeCheckInstruction* check, + CpuRegister temp) { + uint32_t path_to_root = check->GetBitstringPathToRoot(); + uint32_t mask = check->GetBitstringMask(); + DCHECK(IsPowerOfTwo(mask + 1)); + size_t mask_bits = WhichPowerOf2(mask + 1); + + if (mask_bits == 16u) { + // Compare the bitstring in memory. + __ cmpw(Address(temp, mirror::Class::StatusOffset()), Immediate(path_to_root)); + } else { + // /* uint32_t */ temp = temp->status_ + __ movl(temp, Address(temp, mirror::Class::StatusOffset())); + // Compare the bitstring bits using SUB. + __ subl(temp, Immediate(path_to_root)); + // Shift out bits that do not contribute to the comparison. + __ shll(temp, Immediate(32u - mask_bits)); + } +} + HLoadClass::LoadKind CodeGeneratorX86_64::GetSupportedLoadClassKind( HLoadClass::LoadKind desired_class_load_kind) { switch (desired_class_load_kind) { @@ -5471,7 +5745,7 @@ HLoadClass::LoadKind CodeGeneratorX86_64::GetSupportedLoadClassKind( case HLoadClass::LoadKind::kReferrersClass: break; case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: - case HLoadClass::LoadKind::kBootImageClassTable: + case HLoadClass::LoadKind::kBootImageRelRo: case HLoadClass::LoadKind::kBssEntry: DCHECK(!Runtime::Current()->UseJitCompilation()); break; @@ -5579,16 +5853,10 @@ void InstructionCodeGeneratorX86_64::VisitLoadClass(HLoadClass* cls) NO_THREAD_S __ movl(out, Immediate(static_cast<int32_t>(address))); // Zero-extended. break; } - case HLoadClass::LoadKind::kBootImageClassTable: { + case HLoadClass::LoadKind::kBootImageRelRo: { DCHECK(!codegen_->GetCompilerOptions().IsBootImage()); __ movl(out, Address::Absolute(CodeGeneratorX86_64::kDummy32BitOffset, /* no_rip */ false)); - codegen_->RecordBootImageTypePatch(cls); - // Extract the reference from the slot data, i.e. clear the hash bits. - int32_t masked_hash = ClassTable::TableSlot::MaskHash( - ComputeModifiedUtf8Hash(cls->GetDexFile().StringByTypeIdx(cls->GetTypeIndex()))); - if (masked_hash != 0) { - __ subl(out, Immediate(masked_hash)); - } + codegen_->RecordBootImageRelRoPatch(codegen_->GetBootImageOffset(cls)); break; } case HLoadClass::LoadKind::kBssEntry: { @@ -5653,7 +5921,7 @@ HLoadString::LoadKind CodeGeneratorX86_64::GetSupportedLoadStringKind( HLoadString::LoadKind desired_string_load_kind) { switch (desired_string_load_kind) { case HLoadString::LoadKind::kBootImageLinkTimePcRelative: - case HLoadString::LoadKind::kBootImageInternTable: + case HLoadString::LoadKind::kBootImageRelRo: case HLoadString::LoadKind::kBssEntry: DCHECK(!Runtime::Current()->UseJitCompilation()); break; @@ -5719,10 +5987,10 @@ void InstructionCodeGeneratorX86_64::VisitLoadString(HLoadString* load) NO_THREA __ movl(out, Immediate(static_cast<int32_t>(address))); // Zero-extended. return; } - case HLoadString::LoadKind::kBootImageInternTable: { + case HLoadString::LoadKind::kBootImageRelRo: { DCHECK(!codegen_->GetCompilerOptions().IsBootImage()); __ movl(out, Address::Absolute(CodeGeneratorX86_64::kDummy32BitOffset, /* no_rip */ false)); - codegen_->RecordBootImageStringPatch(load); + codegen_->RecordBootImageRelRoPatch(codegen_->GetBootImageOffset(load)); return; } case HLoadString::LoadKind::kBssEntry: { @@ -5795,24 +6063,26 @@ void InstructionCodeGeneratorX86_64::VisitThrow(HThrow* instruction) { CheckEntrypointTypes<kQuickDeliverException, void, mirror::Object*>(); } -static bool CheckCastTypeCheckNeedsATemporary(TypeCheckKind type_check_kind) { - if (type_check_kind == TypeCheckKind::kInterfaceCheck) { - // We need a temporary for holding the iftable length. - return true; - } - return kEmitCompilerReadBarrier && +// Temp is used for read barrier. +static size_t NumberOfInstanceOfTemps(TypeCheckKind type_check_kind) { + if (kEmitCompilerReadBarrier && !kUseBakerReadBarrier && (type_check_kind == TypeCheckKind::kAbstractClassCheck || type_check_kind == TypeCheckKind::kClassHierarchyCheck || - type_check_kind == TypeCheckKind::kArrayObjectCheck); + type_check_kind == TypeCheckKind::kArrayObjectCheck)) { + return 1; + } + return 0; } -static bool InstanceOfTypeCheckNeedsATemporary(TypeCheckKind type_check_kind) { - return kEmitCompilerReadBarrier && - !kUseBakerReadBarrier && - (type_check_kind == TypeCheckKind::kAbstractClassCheck || - type_check_kind == TypeCheckKind::kClassHierarchyCheck || - type_check_kind == TypeCheckKind::kArrayObjectCheck); +// Interface case has 2 temps, one for holding the number of interfaces, one for the current +// interface pointer, the current interface is compared in memory. +// The other checks have one temp for loading the object's class. +static size_t NumberOfCheckCastTemps(TypeCheckKind type_check_kind) { + if (type_check_kind == TypeCheckKind::kInterfaceCheck) { + return 2; + } + return 1 + NumberOfInstanceOfTemps(type_check_kind); } void LocationsBuilderX86_64::VisitInstanceOf(HInstanceOf* instruction) { @@ -5834,6 +6104,8 @@ void LocationsBuilderX86_64::VisitInstanceOf(HInstanceOf* instruction) { case TypeCheckKind::kInterfaceCheck: call_kind = LocationSummary::kCallOnSlowPath; break; + case TypeCheckKind::kBitstringCheck: + break; } LocationSummary* locations = @@ -5842,14 +6114,16 @@ void LocationsBuilderX86_64::VisitInstanceOf(HInstanceOf* instruction) { locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. } locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::Any()); + if (type_check_kind == TypeCheckKind::kBitstringCheck) { + locations->SetInAt(1, Location::ConstantLocation(instruction->InputAt(1)->AsConstant())); + locations->SetInAt(2, Location::ConstantLocation(instruction->InputAt(2)->AsConstant())); + locations->SetInAt(3, Location::ConstantLocation(instruction->InputAt(3)->AsConstant())); + } else { + locations->SetInAt(1, Location::Any()); + } // Note that TypeCheckSlowPathX86_64 uses this "out" register too. locations->SetOut(Location::RequiresRegister()); - // When read barriers are enabled, we need a temporary register for - // some cases. - if (InstanceOfTypeCheckNeedsATemporary(type_check_kind)) { - locations->AddTemp(Location::RequiresRegister()); - } + locations->AddRegisterTemps(NumberOfInstanceOfTemps(type_check_kind)); } void InstructionCodeGeneratorX86_64::VisitInstanceOf(HInstanceOf* instruction) { @@ -5860,9 +6134,9 @@ void InstructionCodeGeneratorX86_64::VisitInstanceOf(HInstanceOf* instruction) { Location cls = locations->InAt(1); Location out_loc = locations->Out(); CpuRegister out = out_loc.AsRegister<CpuRegister>(); - Location maybe_temp_loc = InstanceOfTypeCheckNeedsATemporary(type_check_kind) ? - locations->GetTemp(0) : - Location::NoLocation(); + const size_t num_temps = NumberOfInstanceOfTemps(type_check_kind); + DCHECK_LE(num_temps, 1u); + Location maybe_temp_loc = (num_temps >= 1u) ? locations->GetTemp(0) : Location::NoLocation(); uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); @@ -6071,6 +6345,27 @@ void InstructionCodeGeneratorX86_64::VisitInstanceOf(HInstanceOf* instruction) { } break; } + + case TypeCheckKind::kBitstringCheck: { + // /* HeapReference<Class> */ temp = obj->klass_ + GenerateReferenceLoadTwoRegisters(instruction, + out_loc, + obj_loc, + class_offset, + kWithoutReadBarrier); + + GenerateBitstringTypeCheckCompare(instruction, out); + if (zero.IsLinked()) { + __ j(kNotEqual, &zero); + __ movl(out, Immediate(1)); + __ jmp(&done); + } else { + __ setcc(kEqual, out); + // setcc only sets the low byte. + __ andl(out, Immediate(1)); + } + break; + } } if (zero.IsLinked()) { @@ -6097,17 +6392,15 @@ void LocationsBuilderX86_64::VisitCheckCast(HCheckCast* instruction) { // Require a register for the interface check since there is a loop that compares the class to // a memory address. locations->SetInAt(1, Location::RequiresRegister()); + } else if (type_check_kind == TypeCheckKind::kBitstringCheck) { + locations->SetInAt(1, Location::ConstantLocation(instruction->InputAt(1)->AsConstant())); + locations->SetInAt(2, Location::ConstantLocation(instruction->InputAt(2)->AsConstant())); + locations->SetInAt(3, Location::ConstantLocation(instruction->InputAt(3)->AsConstant())); } else { locations->SetInAt(1, Location::Any()); } - - // Note that TypeCheckSlowPathX86_64 uses this "temp" register too. - locations->AddTemp(Location::RequiresRegister()); - // When read barriers are enabled, we need an additional temporary - // register for some cases. - if (CheckCastTypeCheckNeedsATemporary(type_check_kind)) { - locations->AddTemp(Location::RequiresRegister()); - } + // Add temps for read barriers and other uses. One is used by TypeCheckSlowPathX86. + locations->AddRegisterTemps(NumberOfCheckCastTemps(type_check_kind)); } void InstructionCodeGeneratorX86_64::VisitCheckCast(HCheckCast* instruction) { @@ -6118,9 +6411,10 @@ void InstructionCodeGeneratorX86_64::VisitCheckCast(HCheckCast* instruction) { Location cls = locations->InAt(1); Location temp_loc = locations->GetTemp(0); CpuRegister temp = temp_loc.AsRegister<CpuRegister>(); - Location maybe_temp2_loc = CheckCastTypeCheckNeedsATemporary(type_check_kind) ? - locations->GetTemp(1) : - Location::NoLocation(); + const size_t num_temps = NumberOfCheckCastTemps(type_check_kind); + DCHECK_GE(num_temps, 1u); + DCHECK_LE(num_temps, 2u); + Location maybe_temp2_loc = (num_temps >= 2u) ? locations->GetTemp(1) : Location::NoLocation(); const uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); const uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); const uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); @@ -6283,7 +6577,7 @@ void InstructionCodeGeneratorX86_64::VisitCheckCast(HCheckCast* instruction) { break; } - case TypeCheckKind::kInterfaceCheck: + case TypeCheckKind::kInterfaceCheck: { // Fast path for the interface check. Try to avoid read barriers to improve the fast path. // We can not get false positives by doing this. // /* HeapReference<Class> */ temp = obj->klass_ @@ -6319,6 +6613,20 @@ void InstructionCodeGeneratorX86_64::VisitCheckCast(HCheckCast* instruction) { // If `cls` was poisoned above, unpoison it. __ MaybeUnpoisonHeapReference(cls.AsRegister<CpuRegister>()); break; + } + + case TypeCheckKind::kBitstringCheck: { + // /* HeapReference<Class> */ temp = obj->klass_ + GenerateReferenceLoadTwoRegisters(instruction, + temp_loc, + obj_loc, + class_offset, + kWithoutReadBarrier); + + GenerateBitstringTypeCheckCompare(instruction, temp); + __ j(kNotEqual, type_check_slow_path->GetEntryLabel()); + break; + } } if (done.IsLinked()) { diff --git a/compiler/optimizing/code_generator_x86_64.h b/compiler/optimizing/code_generator_x86_64.h index 1079e94dfc2..9a4c53b5240 100644 --- a/compiler/optimizing/code_generator_x86_64.h +++ b/compiler/optimizing/code_generator_x86_64.h @@ -208,6 +208,7 @@ class InstructionCodeGeneratorX86_64 : public InstructionCodeGenerator { // the suspend call. void GenerateSuspendCheck(HSuspendCheck* instruction, HBasicBlock* successor); void GenerateClassInitializationCheck(SlowPathCode* slow_path, CpuRegister class_reg); + void GenerateBitstringTypeCheckCompare(HTypeCheckInstruction* check, CpuRegister temp); void HandleBitwiseOperation(HBinaryOperation* operation); void GenerateRemFP(HRem* rem); void DivRemOneOrMinusOne(HBinaryOperation* instruction); @@ -222,6 +223,10 @@ class InstructionCodeGeneratorX86_64 : public InstructionCodeGenerator { bool value_can_be_null); void HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info); + void GenerateMinMaxInt(LocationSummary* locations, bool is_min, DataType::Type type); + void GenerateMinMaxFP(LocationSummary* locations, bool is_min, DataType::Type type); + void GenerateMinMax(HBinaryOperation* minmax, bool is_min); + // Generate a heap reference load using one register `out`: // // out <- *(out + offset) @@ -410,6 +415,7 @@ class CodeGeneratorX86_64 : public CodeGenerator { void GenerateVirtualCall( HInvokeVirtual* invoke, Location temp, SlowPathCode* slow_path = nullptr) OVERRIDE; + void RecordBootImageRelRoPatch(uint32_t boot_image_offset); void RecordBootImageMethodPatch(HInvokeStaticOrDirect* invoke); void RecordMethodBssEntryPatch(HInvokeStaticOrDirect* invoke); void RecordBootImageTypePatch(HLoadClass* load_class); @@ -604,17 +610,18 @@ class CodeGeneratorX86_64 : public CodeGenerator { // Used for fixups to the constant area. int constant_area_start_; - // PC-relative method patch info for kBootImageLinkTimePcRelative. + // PC-relative method patch info for kBootImageLinkTimePcRelative/kBootImageRelRo. + // Also used for type/string patches for kBootImageRelRo (same linker patch as for methods). ArenaDeque<PatchInfo<Label>> boot_image_method_patches_; // PC-relative method patch info for kBssEntry. ArenaDeque<PatchInfo<Label>> method_bss_entry_patches_; // PC-relative type patch info for kBootImageLinkTimePcRelative. ArenaDeque<PatchInfo<Label>> boot_image_type_patches_; - // Type patch locations for kBssEntry. + // PC-relative type patch info for kBssEntry. ArenaDeque<PatchInfo<Label>> type_bss_entry_patches_; - // String patch locations; type depends on configuration (intern table or boot image PIC). + // PC-relative String patch info for kBootImageLinkTimePcRelative. ArenaDeque<PatchInfo<Label>> boot_image_string_patches_; - // String patch locations for kBssEntry. + // PC-relative String patch info for kBssEntry. ArenaDeque<PatchInfo<Label>> string_bss_entry_patches_; // Patches for string literals in JIT compiled code. diff --git a/compiler/optimizing/codegen_test_utils.h b/compiler/optimizing/codegen_test_utils.h index c41c290c8b4..792cfb539a6 100644 --- a/compiler/optimizing/codegen_test_utils.h +++ b/compiler/optimizing/codegen_test_utils.h @@ -195,7 +195,9 @@ class InternalCodeAllocator : public CodeAllocator { } size_t GetSize() const { return size_; } - uint8_t* GetMemory() const { return memory_.get(); } + ArrayRef<const uint8_t> GetMemory() const OVERRIDE { + return ArrayRef<const uint8_t>(memory_.get(), size_); + } private: size_t size_; @@ -269,8 +271,8 @@ static void Run(const InternalCodeAllocator& allocator, InstructionSet target_isa = codegen.GetInstructionSet(); typedef Expected (*fptr)(); - CommonCompilerTest::MakeExecutable(allocator.GetMemory(), allocator.GetSize()); - fptr f = reinterpret_cast<fptr>(allocator.GetMemory()); + CommonCompilerTest::MakeExecutable(allocator.GetMemory().data(), allocator.GetMemory().size()); + fptr f = reinterpret_cast<fptr>(reinterpret_cast<uintptr_t>(allocator.GetMemory().data())); if (target_isa == InstructionSet::kThumb2) { // For thumb we need the bottom bit set. f = reinterpret_cast<fptr>(reinterpret_cast<uintptr_t>(f) + 1); diff --git a/compiler/optimizing/data_type.h b/compiler/optimizing/data_type.h index 4a6c91459fc..be26e67af37 100644 --- a/compiler/optimizing/data_type.h +++ b/compiler/optimizing/data_type.h @@ -210,6 +210,12 @@ class DataType { static bool IsTypeConversionImplicit(Type input_type, Type result_type); static bool IsTypeConversionImplicit(int64_t value, Type result_type); + static bool IsZeroExtension(Type input_type, Type result_type) { + return IsIntOrLongType(result_type) && + IsUnsignedType(input_type) && + Size(result_type) > Size(input_type); + } + static const char* PrettyDescriptor(Type type); private: diff --git a/compiler/optimizing/graph_checker.cc b/compiler/optimizing/graph_checker.cc index c88baa8610f..fbcbe3608e6 100644 --- a/compiler/optimizing/graph_checker.cc +++ b/compiler/optimizing/graph_checker.cc @@ -25,6 +25,11 @@ #include "base/bit_vector-inl.h" #include "base/scoped_arena_allocator.h" #include "base/scoped_arena_containers.h" +#include "handle.h" +#include "mirror/class.h" +#include "obj_ptr-inl.h" +#include "scoped_thread_state_change-inl.h" +#include "subtype_check.h" namespace art { @@ -548,30 +553,85 @@ void GraphChecker::VisitReturnVoid(HReturnVoid* ret) { } } -void GraphChecker::VisitCheckCast(HCheckCast* check) { - VisitInstruction(check); - HInstruction* input = check->InputAt(1); - if (!input->IsLoadClass()) { - AddError(StringPrintf("%s:%d expects a HLoadClass as second input, not %s:%d.", +void GraphChecker::CheckTypeCheckBitstringInput(HTypeCheckInstruction* check, + size_t input_pos, + bool check_value, + uint32_t expected_value, + const char* name) { + if (!check->InputAt(input_pos)->IsIntConstant()) { + AddError(StringPrintf("%s:%d (bitstring) expects a HIntConstant input %zu (%s), not %s:%d.", check->DebugName(), check->GetId(), - input->DebugName(), - input->GetId())); + input_pos, + name, + check->InputAt(2)->DebugName(), + check->InputAt(2)->GetId())); + } else if (check_value) { + uint32_t actual_value = + static_cast<uint32_t>(check->InputAt(input_pos)->AsIntConstant()->GetValue()); + if (actual_value != expected_value) { + AddError(StringPrintf("%s:%d (bitstring) has %s 0x%x, not 0x%x as expected.", + check->DebugName(), + check->GetId(), + name, + actual_value, + expected_value)); + } } } -void GraphChecker::VisitInstanceOf(HInstanceOf* instruction) { - VisitInstruction(instruction); - HInstruction* input = instruction->InputAt(1); - if (!input->IsLoadClass()) { - AddError(StringPrintf("%s:%d expects a HLoadClass as second input, not %s:%d.", - instruction->DebugName(), - instruction->GetId(), - input->DebugName(), - input->GetId())); +void GraphChecker::HandleTypeCheckInstruction(HTypeCheckInstruction* check) { + VisitInstruction(check); + HInstruction* input = check->InputAt(1); + if (check->GetTypeCheckKind() == TypeCheckKind::kBitstringCheck) { + if (!input->IsNullConstant()) { + AddError(StringPrintf("%s:%d (bitstring) expects a HNullConstant as second input, not %s:%d.", + check->DebugName(), + check->GetId(), + input->DebugName(), + input->GetId())); + } + bool check_values = false; + BitString::StorageType expected_path_to_root = 0u; + BitString::StorageType expected_mask = 0u; + { + ScopedObjectAccess soa(Thread::Current()); + ObjPtr<mirror::Class> klass = check->GetClass().Get(); + MutexLock subtype_check_lock(Thread::Current(), *Locks::subtype_check_lock_); + SubtypeCheckInfo::State state = SubtypeCheck<ObjPtr<mirror::Class>>::GetState(klass); + if (state == SubtypeCheckInfo::kAssigned) { + expected_path_to_root = + SubtypeCheck<ObjPtr<mirror::Class>>::GetEncodedPathToRootForTarget(klass); + expected_mask = SubtypeCheck<ObjPtr<mirror::Class>>::GetEncodedPathToRootMask(klass); + check_values = true; + } else { + AddError(StringPrintf("%s:%d (bitstring) references a class with unassigned bitstring.", + check->DebugName(), + check->GetId())); + } + } + CheckTypeCheckBitstringInput( + check, /* input_pos */ 2, check_values, expected_path_to_root, "path_to_root"); + CheckTypeCheckBitstringInput(check, /* input_pos */ 3, check_values, expected_mask, "mask"); + } else { + if (!input->IsLoadClass()) { + AddError(StringPrintf("%s:%d (classic) expects a HLoadClass as second input, not %s:%d.", + check->DebugName(), + check->GetId(), + input->DebugName(), + input->GetId())); + } } } +void GraphChecker::VisitCheckCast(HCheckCast* check) { + HandleTypeCheckInstruction(check); +} + +void GraphChecker::VisitInstanceOf(HInstanceOf* instruction) { + HandleTypeCheckInstruction(instruction); +} + void GraphChecker::HandleLoop(HBasicBlock* loop_header) { int id = loop_header->GetBlockId(); HLoopInformation* loop_information = loop_header->GetLoopInformation(); diff --git a/compiler/optimizing/graph_checker.h b/compiler/optimizing/graph_checker.h index 0f0b49d240a..dbedc405185 100644 --- a/compiler/optimizing/graph_checker.h +++ b/compiler/optimizing/graph_checker.h @@ -71,6 +71,12 @@ class GraphChecker : public HGraphDelegateVisitor { void VisitTryBoundary(HTryBoundary* try_boundary) OVERRIDE; void VisitTypeConversion(HTypeConversion* instruction) OVERRIDE; + void CheckTypeCheckBitstringInput(HTypeCheckInstruction* check, + size_t input_pos, + bool check_value, + uint32_t expected_value, + const char* name); + void HandleTypeCheckInstruction(HTypeCheckInstruction* instruction); void HandleLoop(HBasicBlock* loop_header); void HandleBooleanInput(HInstruction* instruction, size_t input_index); diff --git a/compiler/optimizing/graph_visualizer.cc b/compiler/optimizing/graph_visualizer.cc index 5ff31cead58..54d46445804 100644 --- a/compiler/optimizing/graph_visualizer.cc +++ b/compiler/optimizing/graph_visualizer.cc @@ -390,16 +390,23 @@ class HGraphVisualizerPrinter : public HGraphDelegateVisitor { StartAttributeStream("load_kind") << load_string->GetLoadKind(); } - void VisitCheckCast(HCheckCast* check_cast) OVERRIDE { - StartAttributeStream("check_kind") << check_cast->GetTypeCheckKind(); + void HandleTypeCheckInstruction(HTypeCheckInstruction* check) { + StartAttributeStream("check_kind") << check->GetTypeCheckKind(); StartAttributeStream("must_do_null_check") << std::boolalpha - << check_cast->MustDoNullCheck() << std::noboolalpha; + << check->MustDoNullCheck() << std::noboolalpha; + if (check->GetTypeCheckKind() == TypeCheckKind::kBitstringCheck) { + StartAttributeStream("path_to_root") << std::hex + << "0x" << check->GetBitstringPathToRoot() << std::dec; + StartAttributeStream("mask") << std::hex << "0x" << check->GetBitstringMask() << std::dec; + } + } + + void VisitCheckCast(HCheckCast* check_cast) OVERRIDE { + HandleTypeCheckInstruction(check_cast); } void VisitInstanceOf(HInstanceOf* instance_of) OVERRIDE { - StartAttributeStream("check_kind") << instance_of->GetTypeCheckKind(); - StartAttributeStream("must_do_null_check") << std::boolalpha - << instance_of->MustDoNullCheck() << std::noboolalpha; + HandleTypeCheckInstruction(instance_of); } void VisitArrayLength(HArrayLength* array_length) OVERRIDE { @@ -576,6 +583,11 @@ class HGraphVisualizerPrinter : public HGraphDelegateVisitor { } StartAttributeStream() << input_list; } + if (instruction->GetDexPc() != kNoDexPc) { + StartAttributeStream("dex_pc") << instruction->GetDexPc(); + } else { + StartAttributeStream("dex_pc") << "n/a"; + } instruction->Accept(this); if (instruction->HasEnvironment()) { StringList envs; @@ -641,20 +653,32 @@ class HGraphVisualizerPrinter : public HGraphDelegateVisitor { << std::boolalpha << loop_info->IsIrreducible() << std::noboolalpha; } + // For the builder and the inliner, we want to add extra information on HInstructions + // that have reference types, and also HInstanceOf/HCheckcast. if ((IsPass(HGraphBuilder::kBuilderPassName) || IsPass(HInliner::kInlinerPassName)) - && (instruction->GetType() == DataType::Type::kReference)) { - ReferenceTypeInfo info = instruction->IsLoadClass() - ? instruction->AsLoadClass()->GetLoadedClassRTI() - : instruction->GetReferenceTypeInfo(); + && (instruction->GetType() == DataType::Type::kReference || + instruction->IsInstanceOf() || + instruction->IsCheckCast())) { + ReferenceTypeInfo info = (instruction->GetType() == DataType::Type::kReference) + ? instruction->IsLoadClass() + ? instruction->AsLoadClass()->GetLoadedClassRTI() + : instruction->GetReferenceTypeInfo() + : instruction->IsInstanceOf() + ? instruction->AsInstanceOf()->GetTargetClassRTI() + : instruction->AsCheckCast()->GetTargetClassRTI(); ScopedObjectAccess soa(Thread::Current()); if (info.IsValid()) { StartAttributeStream("klass") << mirror::Class::PrettyDescriptor(info.GetTypeHandle().Get()); - StartAttributeStream("can_be_null") - << std::boolalpha << instruction->CanBeNull() << std::noboolalpha; + if (instruction->GetType() == DataType::Type::kReference) { + StartAttributeStream("can_be_null") + << std::boolalpha << instruction->CanBeNull() << std::noboolalpha; + } StartAttributeStream("exact") << std::boolalpha << info.IsExact() << std::noboolalpha; - } else if (instruction->IsLoadClass()) { + } else if (instruction->IsLoadClass() || + instruction->IsInstanceOf() || + instruction->IsCheckCast()) { StartAttributeStream("klass") << "unresolved"; } else { // The NullConstant may be added to the graph during other passes that happen between diff --git a/compiler/optimizing/induction_var_range.cc b/compiler/optimizing/induction_var_range.cc index 99dec112400..55eca2316a1 100644 --- a/compiler/optimizing/induction_var_range.cc +++ b/compiler/optimizing/induction_var_range.cc @@ -78,22 +78,15 @@ static bool IsGEZero(HInstruction* instruction) { DCHECK(instruction != nullptr); if (instruction->IsArrayLength()) { return true; - } else if (instruction->IsInvokeStaticOrDirect()) { - switch (instruction->AsInvoke()->GetIntrinsic()) { - case Intrinsics::kMathMinIntInt: - case Intrinsics::kMathMinLongLong: - // Instruction MIN(>=0, >=0) is >= 0. - return IsGEZero(instruction->InputAt(0)) && - IsGEZero(instruction->InputAt(1)); - case Intrinsics::kMathAbsInt: - case Intrinsics::kMathAbsLong: - // Instruction ABS(>=0) is >= 0. - // NOTE: ABS(minint) = minint prevents assuming - // >= 0 without looking at the argument. - return IsGEZero(instruction->InputAt(0)); - default: - break; - } + } else if (instruction->IsMin()) { + // Instruction MIN(>=0, >=0) is >= 0. + return IsGEZero(instruction->InputAt(0)) && + IsGEZero(instruction->InputAt(1)); + } else if (instruction->IsAbs()) { + // Instruction ABS(>=0) is >= 0. + // NOTE: ABS(minint) = minint prevents assuming + // >= 0 without looking at the argument. + return IsGEZero(instruction->InputAt(0)); } int64_t value = -1; return IsInt64AndGet(instruction, &value) && value >= 0; @@ -102,21 +95,14 @@ static bool IsGEZero(HInstruction* instruction) { /** Hunts "under the hood" for a suitable instruction at the hint. */ static bool IsMaxAtHint( HInstruction* instruction, HInstruction* hint, /*out*/HInstruction** suitable) { - if (instruction->IsInvokeStaticOrDirect()) { - switch (instruction->AsInvoke()->GetIntrinsic()) { - case Intrinsics::kMathMinIntInt: - case Intrinsics::kMathMinLongLong: - // For MIN(x, y), return most suitable x or y as maximum. - return IsMaxAtHint(instruction->InputAt(0), hint, suitable) || - IsMaxAtHint(instruction->InputAt(1), hint, suitable); - default: - break; - } + if (instruction->IsMin()) { + // For MIN(x, y), return most suitable x or y as maximum. + return IsMaxAtHint(instruction->InputAt(0), hint, suitable) || + IsMaxAtHint(instruction->InputAt(1), hint, suitable); } else { *suitable = instruction; return HuntForDeclaration(instruction) == hint; } - return false; } /** Post-analysis simplification of a minimum value that makes the bound more useful to clients. */ @@ -365,14 +351,16 @@ void InductionVarRange::Replace(HInstruction* instruction, } } -bool InductionVarRange::IsFinite(HLoopInformation* loop, /*out*/ int64_t* tc) const { - HInductionVarAnalysis::InductionInfo *trip = - induction_analysis_->LookupInfo(loop, GetLoopControl(loop)); - if (trip != nullptr && !IsUnsafeTripCount(trip)) { - IsConstant(trip->op_a, kExact, tc); - return true; - } - return false; +bool InductionVarRange::IsFinite(HLoopInformation* loop, /*out*/ int64_t* trip_count) const { + bool is_constant_unused = false; + return CheckForFiniteAndConstantProps(loop, &is_constant_unused, trip_count); +} + +bool InductionVarRange::HasKnownTripCount(HLoopInformation* loop, + /*out*/ int64_t* trip_count) const { + bool is_constant = false; + CheckForFiniteAndConstantProps(loop, &is_constant, trip_count); + return is_constant; } bool InductionVarRange::IsUnitStride(HInstruction* context, @@ -431,6 +419,18 @@ HInstruction* InductionVarRange::GenerateTripCount(HLoopInformation* loop, // Private class methods. // +bool InductionVarRange::CheckForFiniteAndConstantProps(HLoopInformation* loop, + /*out*/ bool* is_constant, + /*out*/ int64_t* trip_count) const { + HInductionVarAnalysis::InductionInfo *trip = + induction_analysis_->LookupInfo(loop, GetLoopControl(loop)); + if (trip != nullptr && !IsUnsafeTripCount(trip)) { + *is_constant = IsConstant(trip->op_a, kExact, trip_count); + return true; + } + return false; +} + bool InductionVarRange::IsConstant(HInductionVarAnalysis::InductionInfo* info, ConstantRequest request, /*out*/ int64_t* value) const { diff --git a/compiler/optimizing/induction_var_range.h b/compiler/optimizing/induction_var_range.h index 0b980f596a3..906dc6bb7b9 100644 --- a/compiler/optimizing/induction_var_range.h +++ b/compiler/optimizing/induction_var_range.h @@ -161,9 +161,15 @@ class InductionVarRange { } /** - * Checks if header logic of a loop terminates. Sets trip-count tc if known. + * Checks if header logic of a loop terminates. If trip count is known sets 'trip_count' to its + * value. */ - bool IsFinite(HLoopInformation* loop, /*out*/ int64_t* tc) const; + bool IsFinite(HLoopInformation* loop, /*out*/ int64_t* trip_count) const; + + /** + * Checks if a trip count is known for the loop and sets 'trip_count' to its value in this case. + */ + bool HasKnownTripCount(HLoopInformation* loop, /*out*/ int64_t* trip_count) const; /** * Checks if the given instruction is a unit stride induction inside the closest enveloping @@ -194,6 +200,14 @@ class InductionVarRange { }; /** + * Checks if header logic of a loop terminates. If trip count is known (constant) sets + * 'is_constant' to true and 'trip_count' to the trip count value. + */ + bool CheckForFiniteAndConstantProps(HLoopInformation* loop, + /*out*/ bool* is_constant, + /*out*/ int64_t* trip_count) const; + + /** * Returns true if exact or upper/lower bound on the given induction * information is known as a 64-bit constant, which is returned in value. */ diff --git a/compiler/optimizing/inliner.cc b/compiler/optimizing/inliner.cc index 4fc7262265e..8b10a78212e 100644 --- a/compiler/optimizing/inliner.cc +++ b/compiler/optimizing/inliner.cc @@ -147,10 +147,11 @@ void HInliner::Run() { // that this method is actually inlined; // - if a method's name contains the substring "$noinline$", do not // inline that method. - // We limit this to AOT compilation, as the JIT may or may not inline + // We limit the latter to AOT compilation, as the JIT may or may not inline // depending on the state of classes at runtime. - const bool honor_inlining_directives = - IsCompilingWithCoreImage() && Runtime::Current()->IsAotCompiler(); + const bool honor_noinline_directives = IsCompilingWithCoreImage(); + const bool honor_inline_directives = + honor_noinline_directives && Runtime::Current()->IsAotCompiler(); // Keep a copy of all blocks when starting the visit. ArenaVector<HBasicBlock*> blocks = graph_->GetReversePostOrder(); @@ -164,18 +165,19 @@ void HInliner::Run() { HInvoke* call = instruction->AsInvoke(); // As long as the call is not intrinsified, it is worth trying to inline. if (call != nullptr && call->GetIntrinsic() == Intrinsics::kNone) { - if (honor_inlining_directives) { + if (honor_noinline_directives) { // Debugging case: directives in method names control or assert on inlining. std::string callee_name = outer_compilation_unit_.GetDexFile()->PrettyMethod( call->GetDexMethodIndex(), /* with_signature */ false); // Tests prevent inlining by having $noinline$ in their method names. if (callee_name.find("$noinline$") == std::string::npos) { - if (!TryInline(call)) { + if (!TryInline(call) && honor_inline_directives) { bool should_have_inlined = (callee_name.find("$inline$") != std::string::npos); CHECK(!should_have_inlined) << "Could not inline " << callee_name; } } } else { + DCHECK(!honor_inline_directives); // Normal case: try to inline. TryInline(call); } diff --git a/compiler/optimizing/instruction_builder.cc b/compiler/optimizing/instruction_builder.cc index c7aef3779d1..9647dd5d41c 100644 --- a/compiler/optimizing/instruction_builder.cc +++ b/compiler/optimizing/instruction_builder.cc @@ -1815,29 +1815,6 @@ void HInstructionBuilder::BuildFillWideArrayData(HInstruction* object, } } -static TypeCheckKind ComputeTypeCheckKind(Handle<mirror::Class> cls) - REQUIRES_SHARED(Locks::mutator_lock_) { - if (cls == nullptr) { - return TypeCheckKind::kUnresolvedCheck; - } else if (cls->IsInterface()) { - return TypeCheckKind::kInterfaceCheck; - } else if (cls->IsArrayClass()) { - if (cls->GetComponentType()->IsObjectClass()) { - return TypeCheckKind::kArrayObjectCheck; - } else if (cls->CannotBeAssignedFromOtherTypes()) { - return TypeCheckKind::kExactCheck; - } else { - return TypeCheckKind::kArrayCheck; - } - } else if (cls->IsFinal()) { - return TypeCheckKind::kExactCheck; - } else if (cls->IsAbstract()) { - return TypeCheckKind::kAbstractClassCheck; - } else { - return TypeCheckKind::kClassHierarchyCheck; - } -} - void HInstructionBuilder::BuildLoadString(dex::StringIndex string_index, uint32_t dex_pc) { HLoadString* load_string = new (allocator_) HLoadString(graph_->GetCurrentMethod(), string_index, *dex_file_, dex_pc); @@ -1852,22 +1829,8 @@ void HInstructionBuilder::BuildLoadString(dex::StringIndex string_index, uint32_ HLoadClass* HInstructionBuilder::BuildLoadClass(dex::TypeIndex type_index, uint32_t dex_pc) { ScopedObjectAccess soa(Thread::Current()); const DexFile& dex_file = *dex_compilation_unit_->GetDexFile(); - Handle<mirror::ClassLoader> class_loader = dex_compilation_unit_->GetClassLoader(); - Handle<mirror::Class> klass = handles_->NewHandle(compiler_driver_->ResolveClass( - soa, dex_compilation_unit_->GetDexCache(), class_loader, type_index, dex_compilation_unit_)); - - bool needs_access_check = true; - if (klass != nullptr) { - if (klass->IsPublic()) { - needs_access_check = false; - } else { - ObjPtr<mirror::Class> compiling_class = GetCompilingClass(); - if (compiling_class != nullptr && compiling_class->CanAccess(klass.Get())) { - needs_access_check = false; - } - } - } - + Handle<mirror::Class> klass = ResolveClass(soa, type_index); + bool needs_access_check = LoadClassNeedsAccessCheck(klass); return BuildLoadClass(type_index, dex_file, klass, dex_pc, needs_access_check); } @@ -1912,25 +1875,83 @@ HLoadClass* HInstructionBuilder::BuildLoadClass(dex::TypeIndex type_index, return load_class; } +Handle<mirror::Class> HInstructionBuilder::ResolveClass(ScopedObjectAccess& soa, + dex::TypeIndex type_index) { + Handle<mirror::ClassLoader> class_loader = dex_compilation_unit_->GetClassLoader(); + ObjPtr<mirror::Class> klass = compiler_driver_->ResolveClass( + soa, dex_compilation_unit_->GetDexCache(), class_loader, type_index, dex_compilation_unit_); + // TODO: Avoid creating excessive handles if the method references the same class repeatedly. + // (Use a map on the local_allocator_.) + return handles_->NewHandle(klass); +} + +bool HInstructionBuilder::LoadClassNeedsAccessCheck(Handle<mirror::Class> klass) { + if (klass == nullptr) { + return true; + } else if (klass->IsPublic()) { + return false; + } else { + ObjPtr<mirror::Class> compiling_class = GetCompilingClass(); + return compiling_class == nullptr || !compiling_class->CanAccess(klass.Get()); + } +} + void HInstructionBuilder::BuildTypeCheck(const Instruction& instruction, uint8_t destination, uint8_t reference, dex::TypeIndex type_index, uint32_t dex_pc) { HInstruction* object = LoadLocal(reference, DataType::Type::kReference); - HLoadClass* cls = BuildLoadClass(type_index, dex_pc); ScopedObjectAccess soa(Thread::Current()); - TypeCheckKind check_kind = ComputeTypeCheckKind(cls->GetClass()); + const DexFile& dex_file = *dex_compilation_unit_->GetDexFile(); + Handle<mirror::Class> klass = ResolveClass(soa, type_index); + bool needs_access_check = LoadClassNeedsAccessCheck(klass); + TypeCheckKind check_kind = HSharpening::ComputeTypeCheckKind( + klass.Get(), code_generator_, compiler_driver_, needs_access_check); + + HInstruction* class_or_null = nullptr; + HIntConstant* bitstring_path_to_root = nullptr; + HIntConstant* bitstring_mask = nullptr; + if (check_kind == TypeCheckKind::kBitstringCheck) { + // TODO: Allow using the bitstring check also if we need an access check. + DCHECK(!needs_access_check); + class_or_null = graph_->GetNullConstant(dex_pc); + MutexLock subtype_check_lock(Thread::Current(), *Locks::subtype_check_lock_); + uint32_t path_to_root = + SubtypeCheck<ObjPtr<mirror::Class>>::GetEncodedPathToRootForTarget(klass.Get()); + uint32_t mask = SubtypeCheck<ObjPtr<mirror::Class>>::GetEncodedPathToRootMask(klass.Get()); + bitstring_path_to_root = graph_->GetIntConstant(static_cast<int32_t>(path_to_root), dex_pc); + bitstring_mask = graph_->GetIntConstant(static_cast<int32_t>(mask), dex_pc); + } else { + class_or_null = BuildLoadClass(type_index, dex_file, klass, dex_pc, needs_access_check); + } + DCHECK(class_or_null != nullptr); + if (instruction.Opcode() == Instruction::INSTANCE_OF) { - AppendInstruction(new (allocator_) HInstanceOf(object, cls, check_kind, dex_pc)); + AppendInstruction(new (allocator_) HInstanceOf(object, + class_or_null, + check_kind, + klass, + dex_pc, + allocator_, + bitstring_path_to_root, + bitstring_mask)); UpdateLocal(destination, current_block_->GetLastInstruction()); } else { DCHECK_EQ(instruction.Opcode(), Instruction::CHECK_CAST); // We emit a CheckCast followed by a BoundType. CheckCast is a statement // which may throw. If it succeeds BoundType sets the new type of `object` // for all subsequent uses. - AppendInstruction(new (allocator_) HCheckCast(object, cls, check_kind, dex_pc)); + AppendInstruction( + new (allocator_) HCheckCast(object, + class_or_null, + check_kind, + klass, + dex_pc, + allocator_, + bitstring_path_to_root, + bitstring_mask)); AppendInstruction(new (allocator_) HBoundType(object, dex_pc)); UpdateLocal(reference, current_block_->GetLastInstruction()); } diff --git a/compiler/optimizing/instruction_builder.h b/compiler/optimizing/instruction_builder.h index 4428c532779..f78829232d4 100644 --- a/compiler/optimizing/instruction_builder.h +++ b/compiler/optimizing/instruction_builder.h @@ -39,6 +39,7 @@ class DexCompilationUnit; class HBasicBlockBuilder; class Instruction; class OptimizingCompilerStats; +class ScopedObjectAccess; class SsaBuilder; class VariableSizedHandleScope; @@ -232,6 +233,12 @@ class HInstructionBuilder : public ValueObject { bool needs_access_check) REQUIRES_SHARED(Locks::mutator_lock_); + Handle<mirror::Class> ResolveClass(ScopedObjectAccess& soa, dex::TypeIndex type_index) + REQUIRES_SHARED(Locks::mutator_lock_); + + bool LoadClassNeedsAccessCheck(Handle<mirror::Class> klass) + REQUIRES_SHARED(Locks::mutator_lock_); + // Returns the outer-most compiling method's class. ObjPtr<mirror::Class> GetOutermostCompilingClass() const; diff --git a/compiler/optimizing/instruction_simplifier.cc b/compiler/optimizing/instruction_simplifier.cc index a42a85dc1d2..d3cf9568c2d 100644 --- a/compiler/optimizing/instruction_simplifier.cc +++ b/compiler/optimizing/instruction_simplifier.cc @@ -67,7 +67,6 @@ class InstructionSimplifierVisitor : public HGraphDelegateVisitor { bool TryCombineVecMultiplyAccumulate(HVecMul* mul); void VisitShift(HBinaryOperation* shift); - void VisitEqual(HEqual* equal) OVERRIDE; void VisitNotEqual(HNotEqual* equal) OVERRIDE; void VisitBooleanNot(HBooleanNot* bool_not) OVERRIDE; @@ -78,6 +77,7 @@ class InstructionSimplifierVisitor : public HGraphDelegateVisitor { void VisitNullCheck(HNullCheck* instruction) OVERRIDE; void VisitArrayLength(HArrayLength* instruction) OVERRIDE; void VisitCheckCast(HCheckCast* instruction) OVERRIDE; + void VisitAbs(HAbs* instruction) OVERRIDE; void VisitAdd(HAdd* instruction) OVERRIDE; void VisitAnd(HAnd* instruction) OVERRIDE; void VisitCondition(HCondition* instruction) OVERRIDE; @@ -120,6 +120,9 @@ class InstructionSimplifierVisitor : public HGraphDelegateVisitor { void SimplifyReturnThis(HInvoke* invoke); void SimplifyAllocationIntrinsic(HInvoke* invoke); void SimplifyMemBarrier(HInvoke* invoke, MemBarrierKind barrier_kind); + void SimplifyMin(HInvoke* invoke, DataType::Type type); + void SimplifyMax(HInvoke* invoke, DataType::Type type); + void SimplifyAbs(HInvoke* invoke, DataType::Type type); CodeGenerator* codegen_; CompilerDriver* compiler_driver_; @@ -576,7 +579,9 @@ bool InstructionSimplifierVisitor::CanEnsureNotNullAt(HInstruction* input, HInst // Returns whether doing a type test between the class of `object` against `klass` has // a statically known outcome. The result of the test is stored in `outcome`. -static bool TypeCheckHasKnownOutcome(HLoadClass* klass, HInstruction* object, bool* outcome) { +static bool TypeCheckHasKnownOutcome(ReferenceTypeInfo class_rti, + HInstruction* object, + /*out*/bool* outcome) { DCHECK(!object->IsNullConstant()) << "Null constants should be special cased"; ReferenceTypeInfo obj_rti = object->GetReferenceTypeInfo(); ScopedObjectAccess soa(Thread::Current()); @@ -586,7 +591,6 @@ static bool TypeCheckHasKnownOutcome(HLoadClass* klass, HInstruction* object, bo return false; } - ReferenceTypeInfo class_rti = klass->GetLoadedClassRTI(); if (!class_rti.IsValid()) { // Happens when the loaded class is unresolved. return false; @@ -611,8 +615,8 @@ static bool TypeCheckHasKnownOutcome(HLoadClass* klass, HInstruction* object, bo void InstructionSimplifierVisitor::VisitCheckCast(HCheckCast* check_cast) { HInstruction* object = check_cast->InputAt(0); - HLoadClass* load_class = check_cast->InputAt(1)->AsLoadClass(); - if (load_class->NeedsAccessCheck()) { + if (check_cast->GetTypeCheckKind() != TypeCheckKind::kBitstringCheck && + check_cast->GetTargetClass()->NeedsAccessCheck()) { // If we need to perform an access check we cannot remove the instruction. return; } @@ -630,15 +634,18 @@ void InstructionSimplifierVisitor::VisitCheckCast(HCheckCast* check_cast) { // Note: The `outcome` is initialized to please valgrind - the compiler can reorder // the return value check with the `outcome` check, b/27651442 . bool outcome = false; - if (TypeCheckHasKnownOutcome(load_class, object, &outcome)) { + if (TypeCheckHasKnownOutcome(check_cast->GetTargetClassRTI(), object, &outcome)) { if (outcome) { check_cast->GetBlock()->RemoveInstruction(check_cast); MaybeRecordStat(stats_, MethodCompilationStat::kRemovedCheckedCast); - if (!load_class->HasUses()) { - // We cannot rely on DCE to remove the class because the `HLoadClass` thinks it can throw. - // However, here we know that it cannot because the checkcast was successfull, hence - // the class was already loaded. - load_class->GetBlock()->RemoveInstruction(load_class); + if (check_cast->GetTypeCheckKind() != TypeCheckKind::kBitstringCheck) { + HLoadClass* load_class = check_cast->GetTargetClass(); + if (!load_class->HasUses()) { + // We cannot rely on DCE to remove the class because the `HLoadClass` thinks it can throw. + // However, here we know that it cannot because the checkcast was successfull, hence + // the class was already loaded. + load_class->GetBlock()->RemoveInstruction(load_class); + } } } else { // Don't do anything for exceptional cases for now. Ideally we should remove @@ -649,8 +656,8 @@ void InstructionSimplifierVisitor::VisitCheckCast(HCheckCast* check_cast) { void InstructionSimplifierVisitor::VisitInstanceOf(HInstanceOf* instruction) { HInstruction* object = instruction->InputAt(0); - HLoadClass* load_class = instruction->InputAt(1)->AsLoadClass(); - if (load_class->NeedsAccessCheck()) { + if (instruction->GetTypeCheckKind() != TypeCheckKind::kBitstringCheck && + instruction->GetTargetClass()->NeedsAccessCheck()) { // If we need to perform an access check we cannot remove the instruction. return; } @@ -673,7 +680,7 @@ void InstructionSimplifierVisitor::VisitInstanceOf(HInstanceOf* instruction) { // Note: The `outcome` is initialized to please valgrind - the compiler can reorder // the return value check with the `outcome` check, b/27651442 . bool outcome = false; - if (TypeCheckHasKnownOutcome(load_class, object, &outcome)) { + if (TypeCheckHasKnownOutcome(instruction->GetTargetClassRTI(), object, &outcome)) { MaybeRecordStat(stats_, MethodCompilationStat::kRemovedInstanceOf); if (outcome && can_be_null) { // Type test will succeed, we just need a null test. @@ -686,11 +693,14 @@ void InstructionSimplifierVisitor::VisitInstanceOf(HInstanceOf* instruction) { } RecordSimplification(); instruction->GetBlock()->RemoveInstruction(instruction); - if (outcome && !load_class->HasUses()) { - // We cannot rely on DCE to remove the class because the `HLoadClass` thinks it can throw. - // However, here we know that it cannot because the instanceof check was successfull, hence - // the class was already loaded. - load_class->GetBlock()->RemoveInstruction(load_class); + if (outcome && instruction->GetTypeCheckKind() != TypeCheckKind::kBitstringCheck) { + HLoadClass* load_class = instruction->GetTargetClass(); + if (!load_class->HasUses()) { + // We cannot rely on DCE to remove the class because the `HLoadClass` thinks it can throw. + // However, here we know that it cannot because the instanceof check was successfull, hence + // the class was already loaded. + load_class->GetBlock()->RemoveInstruction(load_class); + } } } } @@ -849,35 +859,29 @@ void InstructionSimplifierVisitor::VisitBooleanNot(HBooleanNot* bool_not) { static HInstruction* NewIntegralAbs(ArenaAllocator* allocator, HInstruction* x, HInstruction* cursor) { - DataType::Type type = x->GetType(); - DCHECK(type == DataType::Type::kInt32 || type == DataType::Type::kInt64); - // Construct a fake intrinsic with as much context as is needed to allocate one. - // The intrinsic will always be lowered into code later anyway. - // TODO: b/65164101 : moving towards a real HAbs node makes more sense. - HInvokeStaticOrDirect::DispatchInfo dispatch_info = { - HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress, - HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod, - 0u - }; - HInvokeStaticOrDirect* invoke = new (allocator) HInvokeStaticOrDirect( - allocator, - 1, - type, - x->GetDexPc(), - /*method_idx*/ -1, - /*resolved_method*/ nullptr, - dispatch_info, - kStatic, - MethodReference(nullptr, dex::kDexNoIndex), - HInvokeStaticOrDirect::ClinitCheckRequirement::kNone); - invoke->SetArgumentAt(0, x); - invoke->SetIntrinsic(type == DataType::Type::kInt32 ? Intrinsics::kMathAbsInt - : Intrinsics::kMathAbsLong, - kNoEnvironmentOrCache, - kNoSideEffects, - kNoThrow); - cursor->GetBlock()->InsertInstructionBefore(invoke, cursor); - return invoke; + DataType::Type type = DataType::Kind(x->GetType()); + DCHECK(type == DataType::Type::kInt32 || type == DataType::Type::kInt64); + HAbs* abs = new (allocator) HAbs(type, x, cursor->GetDexPc()); + cursor->GetBlock()->InsertInstructionBefore(abs, cursor); + return abs; +} + +// Constructs a new MIN/MAX(x, y) node in the HIR. +static HInstruction* NewIntegralMinMax(ArenaAllocator* allocator, + HInstruction* x, + HInstruction* y, + HInstruction* cursor, + bool is_min) { + DataType::Type type = DataType::Kind(x->GetType()); + DCHECK(type == DataType::Type::kInt32 || type == DataType::Type::kInt64); + HBinaryOperation* minmax = nullptr; + if (is_min) { + minmax = new (allocator) HMin(type, x, y, cursor->GetDexPc()); + } else { + minmax = new (allocator) HMax(type, x, y, cursor->GetDexPc()); + } + cursor->GetBlock()->InsertInstructionBefore(minmax, cursor); + return minmax; } // Returns true if operands a and b consists of widening type conversions @@ -899,6 +903,30 @@ static bool AreLowerPrecisionArgs(DataType::Type to_type, HInstruction* a, HInst to_type == DataType::Type::kInt64); } +// Returns an acceptable substitution for "a" on the select +// construct "a <cmp> b ? c : .." during MIN/MAX recognition. +static HInstruction* AllowInMinMax(IfCondition cmp, + HInstruction* a, + HInstruction* b, + HInstruction* c) { + int64_t value = 0; + if (IsInt64AndGet(b, /*out*/ &value) && + (((cmp == kCondLT || cmp == kCondLE) && c->IsMax()) || + ((cmp == kCondGT || cmp == kCondGE) && c->IsMin()))) { + HConstant* other = c->AsBinaryOperation()->GetConstantRight(); + if (other != nullptr && a == c->AsBinaryOperation()->GetLeastConstantLeft()) { + int64_t other_value = Int64FromConstant(other); + bool is_max = (cmp == kCondLT || cmp == kCondLE); + // Allow the max for a < 100 ? max(a, -100) : .. + // or the min for a > -100 ? min(a, 100) : .. + if (is_max ? (value >= other_value) : (value <= other_value)) { + return c; + } + } + } + return nullptr; +} + void InstructionSimplifierVisitor::VisitSelect(HSelect* select) { HInstruction* replace_with = nullptr; HInstruction* condition = select->GetCondition(); @@ -942,23 +970,35 @@ void InstructionSimplifierVisitor::VisitSelect(HSelect* select) { DataType::Type t_type = true_value->GetType(); DataType::Type f_type = false_value->GetType(); // Here we have a <cmp> b ? true_value : false_value. - // Test if both values are same-typed int or long. - if (t_type == f_type && - (t_type == DataType::Type::kInt32 || t_type == DataType::Type::kInt64)) { - // Try to replace typical integral ABS constructs. - if (true_value->IsNeg()) { - HInstruction* negated = true_value->InputAt(0); - if ((cmp == kCondLT || cmp == kCondLE) && - (a == negated && a == false_value && IsInt64Value(b, 0))) { - // Found a < 0 ? -a : a which can be replaced by ABS(a). - replace_with = NewIntegralAbs(GetGraph()->GetAllocator(), false_value, select); - } - } else if (false_value->IsNeg()) { - HInstruction* negated = false_value->InputAt(0); - if ((cmp == kCondGT || cmp == kCondGE) && - (a == true_value && a == negated && IsInt64Value(b, 0))) { - // Found a > 0 ? a : -a which can be replaced by ABS(a). - replace_with = NewIntegralAbs(GetGraph()->GetAllocator(), true_value, select); + // Test if both values are compatible integral types (resulting MIN/MAX/ABS + // type will be int or long, like the condition). Replacements are general, + // but assume conditions prefer constants on the right. + if (DataType::IsIntegralType(t_type) && DataType::Kind(t_type) == DataType::Kind(f_type)) { + // Allow a < 100 ? max(a, -100) : .. + // or a > -100 ? min(a, 100) : .. + // to use min/max instead of a to detect nested min/max expressions. + HInstruction* new_a = AllowInMinMax(cmp, a, b, true_value); + if (new_a != nullptr) { + a = new_a; + } + // Try to replace typical integral MIN/MAX/ABS constructs. + if ((cmp == kCondLT || cmp == kCondLE || cmp == kCondGT || cmp == kCondGE) && + ((a == true_value && b == false_value) || + (b == true_value && a == false_value))) { + // Found a < b ? a : b (MIN) or a < b ? b : a (MAX) + // or a > b ? a : b (MAX) or a > b ? b : a (MIN). + bool is_min = (cmp == kCondLT || cmp == kCondLE) == (a == true_value); + replace_with = NewIntegralMinMax(GetGraph()->GetAllocator(), a, b, select, is_min); + } else if (((cmp == kCondLT || cmp == kCondLE) && true_value->IsNeg()) || + ((cmp == kCondGT || cmp == kCondGE) && false_value->IsNeg())) { + bool negLeft = (cmp == kCondLT || cmp == kCondLE); + HInstruction* the_negated = negLeft ? true_value->InputAt(0) : false_value->InputAt(0); + HInstruction* not_negated = negLeft ? false_value : true_value; + if (a == the_negated && a == not_negated && IsInt64Value(b, 0)) { + // Found a < 0 ? -a : a + // or a > 0 ? a : -a + // which can be replaced by ABS(a). + replace_with = NewIntegralAbs(GetGraph()->GetAllocator(), a, select); } } else if (true_value->IsSub() && false_value->IsSub()) { HInstruction* true_sub1 = true_value->InputAt(0); @@ -970,8 +1010,8 @@ void InstructionSimplifierVisitor::VisitSelect(HSelect* select) { ((cmp == kCondLT || cmp == kCondLE) && (a == true_sub2 && b == true_sub1 && a == false_sub1 && b == false_sub2))) && AreLowerPrecisionArgs(t_type, a, b)) { - // Found a > b ? a - b : b - a or - // a < b ? b - a : a - b + // Found a > b ? a - b : b - a + // or a < b ? b - a : a - b // which can be replaced by ABS(a - b) for lower precision operands a, b. replace_with = NewIntegralAbs(GetGraph()->GetAllocator(), true_value, select); } @@ -1230,6 +1270,17 @@ void InstructionSimplifierVisitor::VisitTypeConversion(HTypeConversion* instruct } } +void InstructionSimplifierVisitor::VisitAbs(HAbs* instruction) { + HInstruction* input = instruction->GetInput(); + if (DataType::IsZeroExtension(input->GetType(), instruction->GetResultType())) { + // Zero extension from narrow to wide can never set sign bit in the wider + // operand, making the subsequent Abs redundant (e.g., abs(b & 0xff) for byte b). + instruction->ReplaceWith(input); + instruction->GetBlock()->RemoveInstruction(instruction); + RecordSimplification(); + } +} + void InstructionSimplifierVisitor::VisitAdd(HAdd* instruction) { HConstant* input_cst = instruction->GetConstantRight(); HInstruction* input_other = instruction->GetLeastConstantLeft(); @@ -2430,6 +2481,27 @@ void InstructionSimplifierVisitor::SimplifyMemBarrier(HInvoke* invoke, invoke->GetBlock()->ReplaceAndRemoveInstructionWith(invoke, mem_barrier); } +void InstructionSimplifierVisitor::SimplifyMin(HInvoke* invoke, DataType::Type type) { + DCHECK(invoke->IsInvokeStaticOrDirect()); + HMin* min = new (GetGraph()->GetAllocator()) + HMin(type, invoke->InputAt(0), invoke->InputAt(1), invoke->GetDexPc()); + invoke->GetBlock()->ReplaceAndRemoveInstructionWith(invoke, min); +} + +void InstructionSimplifierVisitor::SimplifyMax(HInvoke* invoke, DataType::Type type) { + DCHECK(invoke->IsInvokeStaticOrDirect()); + HMax* max = new (GetGraph()->GetAllocator()) + HMax(type, invoke->InputAt(0), invoke->InputAt(1), invoke->GetDexPc()); + invoke->GetBlock()->ReplaceAndRemoveInstructionWith(invoke, max); +} + +void InstructionSimplifierVisitor::SimplifyAbs(HInvoke* invoke, DataType::Type type) { + DCHECK(invoke->IsInvokeStaticOrDirect()); + HAbs* abs = new (GetGraph()->GetAllocator()) + HAbs(type, invoke->InputAt(0), invoke->GetDexPc()); + invoke->GetBlock()->ReplaceAndRemoveInstructionWith(invoke, abs); +} + void InstructionSimplifierVisitor::VisitInvoke(HInvoke* instruction) { switch (instruction->GetIntrinsic()) { case Intrinsics::kStringEquals: @@ -2513,6 +2585,42 @@ void InstructionSimplifierVisitor::VisitInvoke(HInvoke* instruction) { case Intrinsics::kVarHandleStoreStoreFence: SimplifyMemBarrier(instruction, MemBarrierKind::kStoreStore); break; + case Intrinsics::kMathMinIntInt: + SimplifyMin(instruction, DataType::Type::kInt32); + break; + case Intrinsics::kMathMinLongLong: + SimplifyMin(instruction, DataType::Type::kInt64); + break; + case Intrinsics::kMathMinFloatFloat: + SimplifyMin(instruction, DataType::Type::kFloat32); + break; + case Intrinsics::kMathMinDoubleDouble: + SimplifyMin(instruction, DataType::Type::kFloat64); + break; + case Intrinsics::kMathMaxIntInt: + SimplifyMax(instruction, DataType::Type::kInt32); + break; + case Intrinsics::kMathMaxLongLong: + SimplifyMax(instruction, DataType::Type::kInt64); + break; + case Intrinsics::kMathMaxFloatFloat: + SimplifyMax(instruction, DataType::Type::kFloat32); + break; + case Intrinsics::kMathMaxDoubleDouble: + SimplifyMax(instruction, DataType::Type::kFloat64); + break; + case Intrinsics::kMathAbsInt: + SimplifyAbs(instruction, DataType::Type::kInt32); + break; + case Intrinsics::kMathAbsLong: + SimplifyAbs(instruction, DataType::Type::kInt64); + break; + case Intrinsics::kMathAbsFloat: + SimplifyAbs(instruction, DataType::Type::kFloat32); + break; + case Intrinsics::kMathAbsDouble: + SimplifyAbs(instruction, DataType::Type::kFloat64); + break; default: break; } diff --git a/compiler/optimizing/intrinsics.h b/compiler/optimizing/intrinsics.h index 62991435c7a..1035cbc2c46 100644 --- a/compiler/optimizing/intrinsics.h +++ b/compiler/optimizing/intrinsics.h @@ -266,6 +266,18 @@ void IntrinsicCodeGenerator ## Arch::Visit ## Name(HInvoke* invoke) { \ << " should have been converted to HIR"; \ } #define UNREACHABLE_INTRINSICS(Arch) \ +UNREACHABLE_INTRINSIC(Arch, MathMinIntInt) \ +UNREACHABLE_INTRINSIC(Arch, MathMinLongLong) \ +UNREACHABLE_INTRINSIC(Arch, MathMinFloatFloat) \ +UNREACHABLE_INTRINSIC(Arch, MathMinDoubleDouble) \ +UNREACHABLE_INTRINSIC(Arch, MathMaxIntInt) \ +UNREACHABLE_INTRINSIC(Arch, MathMaxLongLong) \ +UNREACHABLE_INTRINSIC(Arch, MathMaxFloatFloat) \ +UNREACHABLE_INTRINSIC(Arch, MathMaxDoubleDouble) \ +UNREACHABLE_INTRINSIC(Arch, MathAbsInt) \ +UNREACHABLE_INTRINSIC(Arch, MathAbsLong) \ +UNREACHABLE_INTRINSIC(Arch, MathAbsFloat) \ +UNREACHABLE_INTRINSIC(Arch, MathAbsDouble) \ UNREACHABLE_INTRINSIC(Arch, FloatFloatToIntBits) \ UNREACHABLE_INTRINSIC(Arch, DoubleDoubleToLongBits) \ UNREACHABLE_INTRINSIC(Arch, FloatIsNaN) \ diff --git a/compiler/optimizing/intrinsics_arm64.cc b/compiler/optimizing/intrinsics_arm64.cc index 0e6485be9f7..c3d643a7d18 100644 --- a/compiler/optimizing/intrinsics_arm64.cc +++ b/compiler/optimizing/intrinsics_arm64.cc @@ -344,14 +344,6 @@ void IntrinsicCodeGeneratorARM64::VisitShortReverseBytes(HInvoke* invoke) { GenReverseBytes(invoke->GetLocations(), DataType::Type::kInt16, GetVIXLAssembler()); } -static void CreateIntIntToIntLocations(ArenaAllocator* allocator, HInvoke* invoke) { - LocationSummary* locations = - new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); -} - static void GenNumberOfLeadingZeros(LocationSummary* locations, DataType::Type type, MacroAssembler* masm) { @@ -536,168 +528,6 @@ static void CreateFPToFPLocations(ArenaAllocator* allocator, HInvoke* invoke) { locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); } -static void MathAbsFP(LocationSummary* locations, bool is64bit, MacroAssembler* masm) { - Location in = locations->InAt(0); - Location out = locations->Out(); - - FPRegister in_reg = is64bit ? DRegisterFrom(in) : SRegisterFrom(in); - FPRegister out_reg = is64bit ? DRegisterFrom(out) : SRegisterFrom(out); - - __ Fabs(out_reg, in_reg); -} - -void IntrinsicLocationsBuilderARM64::VisitMathAbsDouble(HInvoke* invoke) { - CreateFPToFPLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorARM64::VisitMathAbsDouble(HInvoke* invoke) { - MathAbsFP(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler()); -} - -void IntrinsicLocationsBuilderARM64::VisitMathAbsFloat(HInvoke* invoke) { - CreateFPToFPLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorARM64::VisitMathAbsFloat(HInvoke* invoke) { - MathAbsFP(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler()); -} - -static void GenAbsInteger(LocationSummary* locations, - bool is64bit, - MacroAssembler* masm) { - Location in = locations->InAt(0); - Location output = locations->Out(); - - Register in_reg = is64bit ? XRegisterFrom(in) : WRegisterFrom(in); - Register out_reg = is64bit ? XRegisterFrom(output) : WRegisterFrom(output); - - __ Cmp(in_reg, Operand(0)); - __ Cneg(out_reg, in_reg, lt); -} - -void IntrinsicLocationsBuilderARM64::VisitMathAbsInt(HInvoke* invoke) { - CreateIntToIntLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorARM64::VisitMathAbsInt(HInvoke* invoke) { - GenAbsInteger(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler()); -} - -void IntrinsicLocationsBuilderARM64::VisitMathAbsLong(HInvoke* invoke) { - CreateIntToIntLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorARM64::VisitMathAbsLong(HInvoke* invoke) { - GenAbsInteger(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler()); -} - -static void GenMinMaxFP(LocationSummary* locations, - bool is_min, - bool is_double, - MacroAssembler* masm) { - Location op1 = locations->InAt(0); - Location op2 = locations->InAt(1); - Location out = locations->Out(); - - FPRegister op1_reg = is_double ? DRegisterFrom(op1) : SRegisterFrom(op1); - FPRegister op2_reg = is_double ? DRegisterFrom(op2) : SRegisterFrom(op2); - FPRegister out_reg = is_double ? DRegisterFrom(out) : SRegisterFrom(out); - if (is_min) { - __ Fmin(out_reg, op1_reg, op2_reg); - } else { - __ Fmax(out_reg, op1_reg, op2_reg); - } -} - -static void CreateFPFPToFPLocations(ArenaAllocator* allocator, HInvoke* invoke) { - LocationSummary* locations = - new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetInAt(1, Location::RequiresFpuRegister()); - locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); -} - -void IntrinsicLocationsBuilderARM64::VisitMathMinDoubleDouble(HInvoke* invoke) { - CreateFPFPToFPLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorARM64::VisitMathMinDoubleDouble(HInvoke* invoke) { - GenMinMaxFP(invoke->GetLocations(), /* is_min */ true, /* is_double */ true, GetVIXLAssembler()); -} - -void IntrinsicLocationsBuilderARM64::VisitMathMinFloatFloat(HInvoke* invoke) { - CreateFPFPToFPLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorARM64::VisitMathMinFloatFloat(HInvoke* invoke) { - GenMinMaxFP(invoke->GetLocations(), /* is_min */ true, /* is_double */ false, GetVIXLAssembler()); -} - -void IntrinsicLocationsBuilderARM64::VisitMathMaxDoubleDouble(HInvoke* invoke) { - CreateFPFPToFPLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorARM64::VisitMathMaxDoubleDouble(HInvoke* invoke) { - GenMinMaxFP(invoke->GetLocations(), /* is_min */ false, /* is_double */ true, GetVIXLAssembler()); -} - -void IntrinsicLocationsBuilderARM64::VisitMathMaxFloatFloat(HInvoke* invoke) { - CreateFPFPToFPLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorARM64::VisitMathMaxFloatFloat(HInvoke* invoke) { - GenMinMaxFP( - invoke->GetLocations(), /* is_min */ false, /* is_double */ false, GetVIXLAssembler()); -} - -static void GenMinMax(LocationSummary* locations, - bool is_min, - bool is_long, - MacroAssembler* masm) { - Location op1 = locations->InAt(0); - Location op2 = locations->InAt(1); - Location out = locations->Out(); - - Register op1_reg = is_long ? XRegisterFrom(op1) : WRegisterFrom(op1); - Register op2_reg = is_long ? XRegisterFrom(op2) : WRegisterFrom(op2); - Register out_reg = is_long ? XRegisterFrom(out) : WRegisterFrom(out); - - __ Cmp(op1_reg, op2_reg); - __ Csel(out_reg, op1_reg, op2_reg, is_min ? lt : gt); -} - -void IntrinsicLocationsBuilderARM64::VisitMathMinIntInt(HInvoke* invoke) { - CreateIntIntToIntLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorARM64::VisitMathMinIntInt(HInvoke* invoke) { - GenMinMax(invoke->GetLocations(), /* is_min */ true, /* is_long */ false, GetVIXLAssembler()); -} - -void IntrinsicLocationsBuilderARM64::VisitMathMinLongLong(HInvoke* invoke) { - CreateIntIntToIntLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorARM64::VisitMathMinLongLong(HInvoke* invoke) { - GenMinMax(invoke->GetLocations(), /* is_min */ true, /* is_long */ true, GetVIXLAssembler()); -} - -void IntrinsicLocationsBuilderARM64::VisitMathMaxIntInt(HInvoke* invoke) { - CreateIntIntToIntLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorARM64::VisitMathMaxIntInt(HInvoke* invoke) { - GenMinMax(invoke->GetLocations(), /* is_min */ false, /* is_long */ false, GetVIXLAssembler()); -} - -void IntrinsicLocationsBuilderARM64::VisitMathMaxLongLong(HInvoke* invoke) { - CreateIntIntToIntLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorARM64::VisitMathMaxLongLong(HInvoke* invoke) { - GenMinMax(invoke->GetLocations(), /* is_min */ false, /* is_long */ true, GetVIXLAssembler()); -} - void IntrinsicLocationsBuilderARM64::VisitMathSqrt(HInvoke* invoke) { CreateFPToFPLocations(allocator_, invoke); } diff --git a/compiler/optimizing/intrinsics_arm_vixl.cc b/compiler/optimizing/intrinsics_arm_vixl.cc index 97a145664c3..29aecbc0975 100644 --- a/compiler/optimizing/intrinsics_arm_vixl.cc +++ b/compiler/optimizing/intrinsics_arm_vixl.cc @@ -432,341 +432,6 @@ void IntrinsicCodeGeneratorARMVIXL::VisitLongNumberOfTrailingZeros(HInvoke* invo GenNumberOfTrailingZeros(invoke, DataType::Type::kInt64, codegen_); } -static void MathAbsFP(HInvoke* invoke, ArmVIXLAssembler* assembler) { - __ Vabs(OutputVRegister(invoke), InputVRegisterAt(invoke, 0)); -} - -void IntrinsicLocationsBuilderARMVIXL::VisitMathAbsDouble(HInvoke* invoke) { - CreateFPToFPLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorARMVIXL::VisitMathAbsDouble(HInvoke* invoke) { - MathAbsFP(invoke, GetAssembler()); -} - -void IntrinsicLocationsBuilderARMVIXL::VisitMathAbsFloat(HInvoke* invoke) { - CreateFPToFPLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorARMVIXL::VisitMathAbsFloat(HInvoke* invoke) { - MathAbsFP(invoke, GetAssembler()); -} - -static void CreateIntToIntPlusTemp(ArenaAllocator* allocator, HInvoke* invoke) { - LocationSummary* locations = - new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); - - locations->AddTemp(Location::RequiresRegister()); -} - -static void GenAbsInteger(LocationSummary* locations, - bool is64bit, - ArmVIXLAssembler* assembler) { - Location in = locations->InAt(0); - Location output = locations->Out(); - - vixl32::Register mask = RegisterFrom(locations->GetTemp(0)); - - if (is64bit) { - vixl32::Register in_reg_lo = LowRegisterFrom(in); - vixl32::Register in_reg_hi = HighRegisterFrom(in); - vixl32::Register out_reg_lo = LowRegisterFrom(output); - vixl32::Register out_reg_hi = HighRegisterFrom(output); - - DCHECK(!out_reg_lo.Is(in_reg_hi)) << "Diagonal overlap unexpected."; - - __ Asr(mask, in_reg_hi, 31); - __ Adds(out_reg_lo, in_reg_lo, mask); - __ Adc(out_reg_hi, in_reg_hi, mask); - __ Eor(out_reg_lo, mask, out_reg_lo); - __ Eor(out_reg_hi, mask, out_reg_hi); - } else { - vixl32::Register in_reg = RegisterFrom(in); - vixl32::Register out_reg = RegisterFrom(output); - - __ Asr(mask, in_reg, 31); - __ Add(out_reg, in_reg, mask); - __ Eor(out_reg, mask, out_reg); - } -} - -void IntrinsicLocationsBuilderARMVIXL::VisitMathAbsInt(HInvoke* invoke) { - CreateIntToIntPlusTemp(allocator_, invoke); -} - -void IntrinsicCodeGeneratorARMVIXL::VisitMathAbsInt(HInvoke* invoke) { - GenAbsInteger(invoke->GetLocations(), /* is64bit */ false, GetAssembler()); -} - - -void IntrinsicLocationsBuilderARMVIXL::VisitMathAbsLong(HInvoke* invoke) { - CreateIntToIntPlusTemp(allocator_, invoke); -} - -void IntrinsicCodeGeneratorARMVIXL::VisitMathAbsLong(HInvoke* invoke) { - GenAbsInteger(invoke->GetLocations(), /* is64bit */ true, GetAssembler()); -} - -static void GenMinMaxFloat(HInvoke* invoke, bool is_min, CodeGeneratorARMVIXL* codegen) { - ArmVIXLAssembler* assembler = codegen->GetAssembler(); - Location op1_loc = invoke->GetLocations()->InAt(0); - Location op2_loc = invoke->GetLocations()->InAt(1); - Location out_loc = invoke->GetLocations()->Out(); - - // Optimization: don't generate any code if inputs are the same. - if (op1_loc.Equals(op2_loc)) { - DCHECK(out_loc.Equals(op1_loc)); // out_loc is set as SameAsFirstInput() in location builder. - return; - } - - vixl32::SRegister op1 = SRegisterFrom(op1_loc); - vixl32::SRegister op2 = SRegisterFrom(op2_loc); - vixl32::SRegister out = OutputSRegister(invoke); - UseScratchRegisterScope temps(assembler->GetVIXLAssembler()); - const vixl32::Register temp1 = temps.Acquire(); - vixl32::Register temp2 = RegisterFrom(invoke->GetLocations()->GetTemp(0)); - vixl32::Label nan, done; - vixl32::Label* final_label = codegen->GetFinalLabel(invoke, &done); - - DCHECK(op1.Is(out)); - - __ Vcmp(op1, op2); - __ Vmrs(RegisterOrAPSR_nzcv(kPcCode), FPSCR); - __ B(vs, &nan, /* far_target */ false); // if un-ordered, go to NaN handling. - - // op1 <> op2 - vixl32::ConditionType cond = is_min ? gt : lt; - { - ExactAssemblyScope it_scope(assembler->GetVIXLAssembler(), - 2 * kMaxInstructionSizeInBytes, - CodeBufferCheckScope::kMaximumSize); - __ it(cond); - __ vmov(cond, F32, out, op2); - } - // for <>(not equal), we've done min/max calculation. - __ B(ne, final_label, /* far_target */ false); - - // handle op1 == op2, max(+0.0,-0.0), min(+0.0,-0.0). - __ Vmov(temp1, op1); - __ Vmov(temp2, op2); - if (is_min) { - __ Orr(temp1, temp1, temp2); - } else { - __ And(temp1, temp1, temp2); - } - __ Vmov(out, temp1); - __ B(final_label); - - // handle NaN input. - __ Bind(&nan); - __ Movt(temp1, High16Bits(kNanFloat)); // 0x7FC0xxxx is a NaN. - __ Vmov(out, temp1); - - if (done.IsReferenced()) { - __ Bind(&done); - } -} - -static void CreateFPFPToFPLocations(ArenaAllocator* allocator, HInvoke* invoke) { - LocationSummary* locations = - new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetInAt(1, Location::RequiresFpuRegister()); - locations->SetOut(Location::SameAsFirstInput()); -} - -void IntrinsicLocationsBuilderARMVIXL::VisitMathMinFloatFloat(HInvoke* invoke) { - CreateFPFPToFPLocations(allocator_, invoke); - invoke->GetLocations()->AddTemp(Location::RequiresRegister()); -} - -void IntrinsicCodeGeneratorARMVIXL::VisitMathMinFloatFloat(HInvoke* invoke) { - GenMinMaxFloat(invoke, /* is_min */ true, codegen_); -} - -void IntrinsicLocationsBuilderARMVIXL::VisitMathMaxFloatFloat(HInvoke* invoke) { - CreateFPFPToFPLocations(allocator_, invoke); - invoke->GetLocations()->AddTemp(Location::RequiresRegister()); -} - -void IntrinsicCodeGeneratorARMVIXL::VisitMathMaxFloatFloat(HInvoke* invoke) { - GenMinMaxFloat(invoke, /* is_min */ false, codegen_); -} - -static void GenMinMaxDouble(HInvoke* invoke, bool is_min, CodeGeneratorARMVIXL* codegen) { - ArmVIXLAssembler* assembler = codegen->GetAssembler(); - Location op1_loc = invoke->GetLocations()->InAt(0); - Location op2_loc = invoke->GetLocations()->InAt(1); - Location out_loc = invoke->GetLocations()->Out(); - - // Optimization: don't generate any code if inputs are the same. - if (op1_loc.Equals(op2_loc)) { - DCHECK(out_loc.Equals(op1_loc)); // out_loc is set as SameAsFirstInput() in. - return; - } - - vixl32::DRegister op1 = DRegisterFrom(op1_loc); - vixl32::DRegister op2 = DRegisterFrom(op2_loc); - vixl32::DRegister out = OutputDRegister(invoke); - vixl32::Label handle_nan_eq, done; - vixl32::Label* final_label = codegen->GetFinalLabel(invoke, &done); - - DCHECK(op1.Is(out)); - - __ Vcmp(op1, op2); - __ Vmrs(RegisterOrAPSR_nzcv(kPcCode), FPSCR); - __ B(vs, &handle_nan_eq, /* far_target */ false); // if un-ordered, go to NaN handling. - - // op1 <> op2 - vixl32::ConditionType cond = is_min ? gt : lt; - { - ExactAssemblyScope it_scope(assembler->GetVIXLAssembler(), - 2 * kMaxInstructionSizeInBytes, - CodeBufferCheckScope::kMaximumSize); - __ it(cond); - __ vmov(cond, F64, out, op2); - } - // for <>(not equal), we've done min/max calculation. - __ B(ne, final_label, /* far_target */ false); - - // handle op1 == op2, max(+0.0,-0.0). - if (!is_min) { - __ Vand(F64, out, op1, op2); - __ B(final_label); - } - - // handle op1 == op2, min(+0.0,-0.0), NaN input. - __ Bind(&handle_nan_eq); - __ Vorr(F64, out, op1, op2); // assemble op1/-0.0/NaN. - - if (done.IsReferenced()) { - __ Bind(&done); - } -} - -void IntrinsicLocationsBuilderARMVIXL::VisitMathMinDoubleDouble(HInvoke* invoke) { - CreateFPFPToFPLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorARMVIXL::VisitMathMinDoubleDouble(HInvoke* invoke) { - GenMinMaxDouble(invoke, /* is_min */ true , codegen_); -} - -void IntrinsicLocationsBuilderARMVIXL::VisitMathMaxDoubleDouble(HInvoke* invoke) { - CreateFPFPToFPLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorARMVIXL::VisitMathMaxDoubleDouble(HInvoke* invoke) { - GenMinMaxDouble(invoke, /* is_min */ false, codegen_); -} - -static void GenMinMaxLong(HInvoke* invoke, bool is_min, ArmVIXLAssembler* assembler) { - Location op1_loc = invoke->GetLocations()->InAt(0); - Location op2_loc = invoke->GetLocations()->InAt(1); - Location out_loc = invoke->GetLocations()->Out(); - - // Optimization: don't generate any code if inputs are the same. - if (op1_loc.Equals(op2_loc)) { - DCHECK(out_loc.Equals(op1_loc)); // out_loc is set as SameAsFirstInput() in location builder. - return; - } - - vixl32::Register op1_lo = LowRegisterFrom(op1_loc); - vixl32::Register op1_hi = HighRegisterFrom(op1_loc); - vixl32::Register op2_lo = LowRegisterFrom(op2_loc); - vixl32::Register op2_hi = HighRegisterFrom(op2_loc); - vixl32::Register out_lo = LowRegisterFrom(out_loc); - vixl32::Register out_hi = HighRegisterFrom(out_loc); - UseScratchRegisterScope temps(assembler->GetVIXLAssembler()); - const vixl32::Register temp = temps.Acquire(); - - DCHECK(op1_lo.Is(out_lo)); - DCHECK(op1_hi.Is(out_hi)); - - // Compare op1 >= op2, or op1 < op2. - __ Cmp(out_lo, op2_lo); - __ Sbcs(temp, out_hi, op2_hi); - - // Now GE/LT condition code is correct for the long comparison. - { - vixl32::ConditionType cond = is_min ? ge : lt; - ExactAssemblyScope it_scope(assembler->GetVIXLAssembler(), - 3 * kMaxInstructionSizeInBytes, - CodeBufferCheckScope::kMaximumSize); - __ itt(cond); - __ mov(cond, out_lo, op2_lo); - __ mov(cond, out_hi, op2_hi); - } -} - -static void CreateLongLongToLongLocations(ArenaAllocator* allocator, HInvoke* invoke) { - LocationSummary* locations = - new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); - locations->SetOut(Location::SameAsFirstInput()); -} - -void IntrinsicLocationsBuilderARMVIXL::VisitMathMinLongLong(HInvoke* invoke) { - CreateLongLongToLongLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorARMVIXL::VisitMathMinLongLong(HInvoke* invoke) { - GenMinMaxLong(invoke, /* is_min */ true, GetAssembler()); -} - -void IntrinsicLocationsBuilderARMVIXL::VisitMathMaxLongLong(HInvoke* invoke) { - CreateLongLongToLongLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorARMVIXL::VisitMathMaxLongLong(HInvoke* invoke) { - GenMinMaxLong(invoke, /* is_min */ false, GetAssembler()); -} - -static void GenMinMax(HInvoke* invoke, bool is_min, ArmVIXLAssembler* assembler) { - vixl32::Register op1 = InputRegisterAt(invoke, 0); - vixl32::Register op2 = InputRegisterAt(invoke, 1); - vixl32::Register out = OutputRegister(invoke); - - __ Cmp(op1, op2); - - { - ExactAssemblyScope aas(assembler->GetVIXLAssembler(), - 3 * kMaxInstructionSizeInBytes, - CodeBufferCheckScope::kMaximumSize); - - __ ite(is_min ? lt : gt); - __ mov(is_min ? lt : gt, out, op1); - __ mov(is_min ? ge : le, out, op2); - } -} - -static void CreateIntIntToIntLocations(ArenaAllocator* allocator, HInvoke* invoke) { - LocationSummary* locations = - new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); -} - -void IntrinsicLocationsBuilderARMVIXL::VisitMathMinIntInt(HInvoke* invoke) { - CreateIntIntToIntLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorARMVIXL::VisitMathMinIntInt(HInvoke* invoke) { - GenMinMax(invoke, /* is_min */ true, GetAssembler()); -} - -void IntrinsicLocationsBuilderARMVIXL::VisitMathMaxIntInt(HInvoke* invoke) { - CreateIntIntToIntLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorARMVIXL::VisitMathMaxIntInt(HInvoke* invoke) { - GenMinMax(invoke, /* is_min */ false, GetAssembler()); -} - void IntrinsicLocationsBuilderARMVIXL::VisitMathSqrt(HInvoke* invoke) { CreateFPToFPLocations(allocator_, invoke); } diff --git a/compiler/optimizing/intrinsics_mips.cc b/compiler/optimizing/intrinsics_mips.cc index b7936b9c8ec..ae248a3e5c7 100644 --- a/compiler/optimizing/intrinsics_mips.cc +++ b/compiler/optimizing/intrinsics_mips.cc @@ -58,6 +58,10 @@ inline bool IntrinsicCodeGeneratorMIPS::Is32BitFPU() const { return codegen_->GetInstructionSetFeatures().Is32BitFloatingPoint(); } +inline bool IntrinsicCodeGeneratorMIPS::HasMsa() const { + return codegen_->GetInstructionSetFeatures().HasMsa(); +} + #define __ codegen->GetAssembler()-> static void MoveFromReturnRegister(Location trg, @@ -612,6 +616,7 @@ static void CreateFPToFPLocations(ArenaAllocator* allocator, HInvoke* invoke) { static void GenBitCount(LocationSummary* locations, DataType::Type type, bool isR6, + bool hasMsa, MipsAssembler* assembler) { Register out = locations->Out().AsRegister<Register>(); @@ -637,85 +642,102 @@ static void GenBitCount(LocationSummary* locations, // instructions compared to a loop-based algorithm which required 47 // instructions. - if (type == DataType::Type::kInt32) { - Register in = locations->InAt(0).AsRegister<Register>(); - - __ Srl(TMP, in, 1); - __ LoadConst32(AT, 0x55555555); - __ And(TMP, TMP, AT); - __ Subu(TMP, in, TMP); - __ LoadConst32(AT, 0x33333333); - __ And(out, TMP, AT); - __ Srl(TMP, TMP, 2); - __ And(TMP, TMP, AT); - __ Addu(TMP, out, TMP); - __ Srl(out, TMP, 4); - __ Addu(out, out, TMP); - __ LoadConst32(AT, 0x0F0F0F0F); - __ And(out, out, AT); - __ LoadConst32(TMP, 0x01010101); - if (isR6) { - __ MulR6(out, out, TMP); + if (hasMsa) { + if (type == DataType::Type::kInt32) { + Register in = locations->InAt(0).AsRegister<Register>(); + __ Mtc1(in, FTMP); + __ PcntW(static_cast<VectorRegister>(FTMP), static_cast<VectorRegister>(FTMP)); + __ Mfc1(out, FTMP); } else { - __ MulR2(out, out, TMP); + DCHECK_EQ(type, DataType::Type::kInt64); + Register in_lo = locations->InAt(0).AsRegisterPairLow<Register>(); + Register in_hi = locations->InAt(0).AsRegisterPairHigh<Register>(); + __ Mtc1(in_lo, FTMP); + __ Mthc1(in_hi, FTMP); + __ PcntD(static_cast<VectorRegister>(FTMP), static_cast<VectorRegister>(FTMP)); + __ Mfc1(out, FTMP); } - __ Srl(out, out, 24); } else { - DCHECK_EQ(type, DataType::Type::kInt64); - Register in_lo = locations->InAt(0).AsRegisterPairLow<Register>(); - Register in_hi = locations->InAt(0).AsRegisterPairHigh<Register>(); - Register tmp_hi = locations->GetTemp(0).AsRegister<Register>(); - Register out_hi = locations->GetTemp(1).AsRegister<Register>(); - Register tmp_lo = TMP; - Register out_lo = out; + if (type == DataType::Type::kInt32) { + Register in = locations->InAt(0).AsRegister<Register>(); + + __ Srl(TMP, in, 1); + __ LoadConst32(AT, 0x55555555); + __ And(TMP, TMP, AT); + __ Subu(TMP, in, TMP); + __ LoadConst32(AT, 0x33333333); + __ And(out, TMP, AT); + __ Srl(TMP, TMP, 2); + __ And(TMP, TMP, AT); + __ Addu(TMP, out, TMP); + __ Srl(out, TMP, 4); + __ Addu(out, out, TMP); + __ LoadConst32(AT, 0x0F0F0F0F); + __ And(out, out, AT); + __ LoadConst32(TMP, 0x01010101); + if (isR6) { + __ MulR6(out, out, TMP); + } else { + __ MulR2(out, out, TMP); + } + __ Srl(out, out, 24); + } else { + DCHECK_EQ(type, DataType::Type::kInt64); + Register in_lo = locations->InAt(0).AsRegisterPairLow<Register>(); + Register in_hi = locations->InAt(0).AsRegisterPairHigh<Register>(); + Register tmp_hi = locations->GetTemp(0).AsRegister<Register>(); + Register out_hi = locations->GetTemp(1).AsRegister<Register>(); + Register tmp_lo = TMP; + Register out_lo = out; - __ Srl(tmp_lo, in_lo, 1); - __ Srl(tmp_hi, in_hi, 1); + __ Srl(tmp_lo, in_lo, 1); + __ Srl(tmp_hi, in_hi, 1); - __ LoadConst32(AT, 0x55555555); + __ LoadConst32(AT, 0x55555555); - __ And(tmp_lo, tmp_lo, AT); - __ Subu(tmp_lo, in_lo, tmp_lo); + __ And(tmp_lo, tmp_lo, AT); + __ Subu(tmp_lo, in_lo, tmp_lo); - __ And(tmp_hi, tmp_hi, AT); - __ Subu(tmp_hi, in_hi, tmp_hi); + __ And(tmp_hi, tmp_hi, AT); + __ Subu(tmp_hi, in_hi, tmp_hi); - __ LoadConst32(AT, 0x33333333); + __ LoadConst32(AT, 0x33333333); - __ And(out_lo, tmp_lo, AT); - __ Srl(tmp_lo, tmp_lo, 2); - __ And(tmp_lo, tmp_lo, AT); - __ Addu(tmp_lo, out_lo, tmp_lo); + __ And(out_lo, tmp_lo, AT); + __ Srl(tmp_lo, tmp_lo, 2); + __ And(tmp_lo, tmp_lo, AT); + __ Addu(tmp_lo, out_lo, tmp_lo); - __ And(out_hi, tmp_hi, AT); - __ Srl(tmp_hi, tmp_hi, 2); - __ And(tmp_hi, tmp_hi, AT); - __ Addu(tmp_hi, out_hi, tmp_hi); + __ And(out_hi, tmp_hi, AT); + __ Srl(tmp_hi, tmp_hi, 2); + __ And(tmp_hi, tmp_hi, AT); + __ Addu(tmp_hi, out_hi, tmp_hi); - // Here we deviate from the original algorithm a bit. We've reached - // the stage where the bitfields holding the subtotals are large - // enough to hold the combined subtotals for both the low word, and - // the high word. This means that we can add the subtotals for the - // the high, and low words into a single word, and compute the final - // result for both the high, and low words using fewer instructions. - __ LoadConst32(AT, 0x0F0F0F0F); + // Here we deviate from the original algorithm a bit. We've reached + // the stage where the bitfields holding the subtotals are large + // enough to hold the combined subtotals for both the low word, and + // the high word. This means that we can add the subtotals for the + // the high, and low words into a single word, and compute the final + // result for both the high, and low words using fewer instructions. + __ LoadConst32(AT, 0x0F0F0F0F); - __ Addu(TMP, tmp_hi, tmp_lo); + __ Addu(TMP, tmp_hi, tmp_lo); - __ Srl(out, TMP, 4); - __ And(out, out, AT); - __ And(TMP, TMP, AT); - __ Addu(out, out, TMP); + __ Srl(out, TMP, 4); + __ And(out, out, AT); + __ And(TMP, TMP, AT); + __ Addu(out, out, TMP); - __ LoadConst32(AT, 0x01010101); + __ LoadConst32(AT, 0x01010101); - if (isR6) { - __ MulR6(out, out, AT); - } else { - __ MulR2(out, out, AT); - } + if (isR6) { + __ MulR6(out, out, AT); + } else { + __ MulR2(out, out, AT); + } - __ Srl(out, out, 24); + __ Srl(out, out, 24); + } } } @@ -725,7 +747,7 @@ void IntrinsicLocationsBuilderMIPS::VisitIntegerBitCount(HInvoke* invoke) { } void IntrinsicCodeGeneratorMIPS::VisitIntegerBitCount(HInvoke* invoke) { - GenBitCount(invoke->GetLocations(), DataType::Type::kInt32, IsR6(), GetAssembler()); + GenBitCount(invoke->GetLocations(), DataType::Type::kInt32, IsR6(), HasMsa(), GetAssembler()); } // int java.lang.Long.bitCount(int) @@ -739,575 +761,7 @@ void IntrinsicLocationsBuilderMIPS::VisitLongBitCount(HInvoke* invoke) { } void IntrinsicCodeGeneratorMIPS::VisitLongBitCount(HInvoke* invoke) { - GenBitCount(invoke->GetLocations(), DataType::Type::kInt64, IsR6(), GetAssembler()); -} - -static void MathAbsFP(LocationSummary* locations, - bool is64bit, - bool isR2OrNewer, - bool isR6, - MipsAssembler* assembler) { - FRegister in = locations->InAt(0).AsFpuRegister<FRegister>(); - FRegister out = locations->Out().AsFpuRegister<FRegister>(); - - // Note, as a "quality of implementation", rather than pure "spec compliance", we require that - // Math.abs() clears the sign bit (but changes nothing else) for all numbers, including NaN - // (signaling NaN may become quiet though). - // - // The ABS.fmt instructions (abs.s and abs.d) do exactly that when NAN2008=1 (R6). For this case, - // both regular floating point numbers and NAN values are treated alike, only the sign bit is - // affected by this instruction. - // But when NAN2008=0 (R2 and before), the ABS.fmt instructions can't be used. For this case, any - // NaN operand signals invalid operation. This means that other bits (not just sign bit) might be - // changed when doing abs(NaN). Because of that, we clear sign bit in a different way. - if (isR6) { - if (is64bit) { - __ AbsD(out, in); - } else { - __ AbsS(out, in); - } - } else { - if (is64bit) { - if (in != out) { - __ MovD(out, in); - } - __ MoveFromFpuHigh(TMP, in); - // ins instruction is not available for R1. - if (isR2OrNewer) { - __ Ins(TMP, ZERO, 31, 1); - } else { - __ Sll(TMP, TMP, 1); - __ Srl(TMP, TMP, 1); - } - __ MoveToFpuHigh(TMP, out); - } else { - __ Mfc1(TMP, in); - // ins instruction is not available for R1. - if (isR2OrNewer) { - __ Ins(TMP, ZERO, 31, 1); - } else { - __ Sll(TMP, TMP, 1); - __ Srl(TMP, TMP, 1); - } - __ Mtc1(TMP, out); - } - } -} - -// double java.lang.Math.abs(double) -void IntrinsicLocationsBuilderMIPS::VisitMathAbsDouble(HInvoke* invoke) { - CreateFPToFPLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorMIPS::VisitMathAbsDouble(HInvoke* invoke) { - MathAbsFP(invoke->GetLocations(), /* is64bit */ true, IsR2OrNewer(), IsR6(), GetAssembler()); -} - -// float java.lang.Math.abs(float) -void IntrinsicLocationsBuilderMIPS::VisitMathAbsFloat(HInvoke* invoke) { - CreateFPToFPLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorMIPS::VisitMathAbsFloat(HInvoke* invoke) { - MathAbsFP(invoke->GetLocations(), /* is64bit */ false, IsR2OrNewer(), IsR6(), GetAssembler()); -} - -static void GenAbsInteger(LocationSummary* locations, bool is64bit, MipsAssembler* assembler) { - if (is64bit) { - Register in_lo = locations->InAt(0).AsRegisterPairLow<Register>(); - Register in_hi = locations->InAt(0).AsRegisterPairHigh<Register>(); - Register out_lo = locations->Out().AsRegisterPairLow<Register>(); - Register out_hi = locations->Out().AsRegisterPairHigh<Register>(); - - // The comments in this section show the analogous operations which would - // be performed if we had 64-bit registers "in", and "out". - // __ Dsra32(AT, in, 31); - __ Sra(AT, in_hi, 31); - // __ Xor(out, in, AT); - __ Xor(TMP, in_lo, AT); - __ Xor(out_hi, in_hi, AT); - // __ Dsubu(out, out, AT); - __ Subu(out_lo, TMP, AT); - __ Sltu(TMP, out_lo, TMP); - __ Addu(out_hi, out_hi, TMP); - } else { - Register in = locations->InAt(0).AsRegister<Register>(); - Register out = locations->Out().AsRegister<Register>(); - - __ Sra(AT, in, 31); - __ Xor(out, in, AT); - __ Subu(out, out, AT); - } -} - -// int java.lang.Math.abs(int) -void IntrinsicLocationsBuilderMIPS::VisitMathAbsInt(HInvoke* invoke) { - CreateIntToIntLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorMIPS::VisitMathAbsInt(HInvoke* invoke) { - GenAbsInteger(invoke->GetLocations(), /* is64bit */ false, GetAssembler()); -} - -// long java.lang.Math.abs(long) -void IntrinsicLocationsBuilderMIPS::VisitMathAbsLong(HInvoke* invoke) { - CreateIntToIntLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorMIPS::VisitMathAbsLong(HInvoke* invoke) { - GenAbsInteger(invoke->GetLocations(), /* is64bit */ true, GetAssembler()); -} - -static void GenMinMaxFP(LocationSummary* locations, - bool is_min, - DataType::Type type, - bool is_R6, - MipsAssembler* assembler) { - FRegister out = locations->Out().AsFpuRegister<FRegister>(); - FRegister a = locations->InAt(0).AsFpuRegister<FRegister>(); - FRegister b = locations->InAt(1).AsFpuRegister<FRegister>(); - - if (is_R6) { - MipsLabel noNaNs; - MipsLabel done; - FRegister ftmp = ((out != a) && (out != b)) ? out : FTMP; - - // When Java computes min/max it prefers a NaN to a number; the - // behavior of MIPSR6 is to prefer numbers to NaNs, i.e., if one of - // the inputs is a NaN and the other is a valid number, the MIPS - // instruction will return the number; Java wants the NaN value - // returned. This is why there is extra logic preceding the use of - // the MIPS min.fmt/max.fmt instructions. If either a, or b holds a - // NaN, return the NaN, otherwise return the min/max. - if (type == DataType::Type::kFloat64) { - __ CmpUnD(FTMP, a, b); - __ Bc1eqz(FTMP, &noNaNs); - - // One of the inputs is a NaN - __ CmpEqD(ftmp, a, a); - // If a == a then b is the NaN, otherwise a is the NaN. - __ SelD(ftmp, a, b); - - if (ftmp != out) { - __ MovD(out, ftmp); - } - - __ B(&done); - - __ Bind(&noNaNs); - - if (is_min) { - __ MinD(out, a, b); - } else { - __ MaxD(out, a, b); - } - } else { - DCHECK_EQ(type, DataType::Type::kFloat32); - __ CmpUnS(FTMP, a, b); - __ Bc1eqz(FTMP, &noNaNs); - - // One of the inputs is a NaN - __ CmpEqS(ftmp, a, a); - // If a == a then b is the NaN, otherwise a is the NaN. - __ SelS(ftmp, a, b); - - if (ftmp != out) { - __ MovS(out, ftmp); - } - - __ B(&done); - - __ Bind(&noNaNs); - - if (is_min) { - __ MinS(out, a, b); - } else { - __ MaxS(out, a, b); - } - } - - __ Bind(&done); - } else { - MipsLabel ordered; - MipsLabel compare; - MipsLabel select; - MipsLabel done; - - if (type == DataType::Type::kFloat64) { - __ CunD(a, b); - } else { - DCHECK_EQ(type, DataType::Type::kFloat32); - __ CunS(a, b); - } - __ Bc1f(&ordered); - - // a or b (or both) is a NaN. Return one, which is a NaN. - if (type == DataType::Type::kFloat64) { - __ CeqD(b, b); - } else { - __ CeqS(b, b); - } - __ B(&select); - - __ Bind(&ordered); - - // Neither is a NaN. - // a == b? (-0.0 compares equal with +0.0) - // If equal, handle zeroes, else compare further. - if (type == DataType::Type::kFloat64) { - __ CeqD(a, b); - } else { - __ CeqS(a, b); - } - __ Bc1f(&compare); - - // a == b either bit for bit or one is -0.0 and the other is +0.0. - if (type == DataType::Type::kFloat64) { - __ MoveFromFpuHigh(TMP, a); - __ MoveFromFpuHigh(AT, b); - } else { - __ Mfc1(TMP, a); - __ Mfc1(AT, b); - } - - if (is_min) { - // -0.0 prevails over +0.0. - __ Or(TMP, TMP, AT); - } else { - // +0.0 prevails over -0.0. - __ And(TMP, TMP, AT); - } - - if (type == DataType::Type::kFloat64) { - __ Mfc1(AT, a); - __ Mtc1(AT, out); - __ MoveToFpuHigh(TMP, out); - } else { - __ Mtc1(TMP, out); - } - __ B(&done); - - __ Bind(&compare); - - if (type == DataType::Type::kFloat64) { - if (is_min) { - // return (a <= b) ? a : b; - __ ColeD(a, b); - } else { - // return (a >= b) ? a : b; - __ ColeD(b, a); // b <= a - } - } else { - if (is_min) { - // return (a <= b) ? a : b; - __ ColeS(a, b); - } else { - // return (a >= b) ? a : b; - __ ColeS(b, a); // b <= a - } - } - - __ Bind(&select); - - if (type == DataType::Type::kFloat64) { - __ MovtD(out, a); - __ MovfD(out, b); - } else { - __ MovtS(out, a); - __ MovfS(out, b); - } - - __ Bind(&done); - } -} - -static void CreateFPFPToFPLocations(ArenaAllocator* allocator, HInvoke* invoke) { - LocationSummary* locations = - new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetInAt(1, Location::RequiresFpuRegister()); - locations->SetOut(Location::RequiresFpuRegister(), Location::kOutputOverlap); -} - -// double java.lang.Math.min(double, double) -void IntrinsicLocationsBuilderMIPS::VisitMathMinDoubleDouble(HInvoke* invoke) { - CreateFPFPToFPLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorMIPS::VisitMathMinDoubleDouble(HInvoke* invoke) { - GenMinMaxFP(invoke->GetLocations(), - /* is_min */ true, - DataType::Type::kFloat64, - IsR6(), - GetAssembler()); -} - -// float java.lang.Math.min(float, float) -void IntrinsicLocationsBuilderMIPS::VisitMathMinFloatFloat(HInvoke* invoke) { - CreateFPFPToFPLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorMIPS::VisitMathMinFloatFloat(HInvoke* invoke) { - GenMinMaxFP(invoke->GetLocations(), - /* is_min */ true, - DataType::Type::kFloat32, - IsR6(), - GetAssembler()); -} - -// double java.lang.Math.max(double, double) -void IntrinsicLocationsBuilderMIPS::VisitMathMaxDoubleDouble(HInvoke* invoke) { - CreateFPFPToFPLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorMIPS::VisitMathMaxDoubleDouble(HInvoke* invoke) { - GenMinMaxFP(invoke->GetLocations(), - /* is_min */ false, - DataType::Type::kFloat64, - IsR6(), - GetAssembler()); -} - -// float java.lang.Math.max(float, float) -void IntrinsicLocationsBuilderMIPS::VisitMathMaxFloatFloat(HInvoke* invoke) { - CreateFPFPToFPLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorMIPS::VisitMathMaxFloatFloat(HInvoke* invoke) { - GenMinMaxFP(invoke->GetLocations(), - /* is_min */ false, - DataType::Type::kFloat32, - IsR6(), - GetAssembler()); -} - -static void CreateIntIntToIntLocations(ArenaAllocator* allocator, HInvoke* invoke) { - LocationSummary* locations = - new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); -} - -static void GenMinMax(LocationSummary* locations, - bool is_min, - DataType::Type type, - bool is_R6, - MipsAssembler* assembler) { - if (is_R6) { - // Some architectures, such as ARM and MIPS (prior to r6), have a - // conditional move instruction which only changes the target - // (output) register if the condition is true (MIPS prior to r6 had - // MOVF, MOVT, MOVN, and MOVZ). The SELEQZ and SELNEZ instructions - // always change the target (output) register. If the condition is - // true the output register gets the contents of the "rs" register; - // otherwise, the output register is set to zero. One consequence - // of this is that to implement something like "rd = c==0 ? rs : rt" - // MIPS64r6 needs to use a pair of SELEQZ/SELNEZ instructions. - // After executing this pair of instructions one of the output - // registers from the pair will necessarily contain zero. Then the - // code ORs the output registers from the SELEQZ/SELNEZ instructions - // to get the final result. - // - // The initial test to see if the output register is same as the - // first input register is needed to make sure that value in the - // first input register isn't clobbered before we've finished - // computing the output value. The logic in the corresponding else - // clause performs the same task but makes sure the second input - // register isn't clobbered in the event that it's the same register - // as the output register; the else clause also handles the case - // where the output register is distinct from both the first, and the - // second input registers. - if (type == DataType::Type::kInt64) { - Register a_lo = locations->InAt(0).AsRegisterPairLow<Register>(); - Register a_hi = locations->InAt(0).AsRegisterPairHigh<Register>(); - Register b_lo = locations->InAt(1).AsRegisterPairLow<Register>(); - Register b_hi = locations->InAt(1).AsRegisterPairHigh<Register>(); - Register out_lo = locations->Out().AsRegisterPairLow<Register>(); - Register out_hi = locations->Out().AsRegisterPairHigh<Register>(); - - MipsLabel compare_done; - - if (a_lo == b_lo) { - if (out_lo != a_lo) { - __ Move(out_lo, a_lo); - __ Move(out_hi, a_hi); - } - } else { - __ Slt(TMP, b_hi, a_hi); - __ Bne(b_hi, a_hi, &compare_done); - - __ Sltu(TMP, b_lo, a_lo); - - __ Bind(&compare_done); - - if (is_min) { - __ Seleqz(AT, a_lo, TMP); - __ Selnez(out_lo, b_lo, TMP); // Safe even if out_lo == a_lo/b_lo - // because at this point we're - // done using a_lo/b_lo. - } else { - __ Selnez(AT, a_lo, TMP); - __ Seleqz(out_lo, b_lo, TMP); // ditto - } - __ Or(out_lo, out_lo, AT); - if (is_min) { - __ Seleqz(AT, a_hi, TMP); - __ Selnez(out_hi, b_hi, TMP); // ditto but for out_hi & a_hi/b_hi - } else { - __ Selnez(AT, a_hi, TMP); - __ Seleqz(out_hi, b_hi, TMP); // ditto but for out_hi & a_hi/b_hi - } - __ Or(out_hi, out_hi, AT); - } - } else { - DCHECK_EQ(type, DataType::Type::kInt32); - Register a = locations->InAt(0).AsRegister<Register>(); - Register b = locations->InAt(1).AsRegister<Register>(); - Register out = locations->Out().AsRegister<Register>(); - - if (a == b) { - if (out != a) { - __ Move(out, a); - } - } else { - __ Slt(AT, b, a); - if (is_min) { - __ Seleqz(TMP, a, AT); - __ Selnez(AT, b, AT); - } else { - __ Selnez(TMP, a, AT); - __ Seleqz(AT, b, AT); - } - __ Or(out, TMP, AT); - } - } - } else { - if (type == DataType::Type::kInt64) { - Register a_lo = locations->InAt(0).AsRegisterPairLow<Register>(); - Register a_hi = locations->InAt(0).AsRegisterPairHigh<Register>(); - Register b_lo = locations->InAt(1).AsRegisterPairLow<Register>(); - Register b_hi = locations->InAt(1).AsRegisterPairHigh<Register>(); - Register out_lo = locations->Out().AsRegisterPairLow<Register>(); - Register out_hi = locations->Out().AsRegisterPairHigh<Register>(); - - MipsLabel compare_done; - - if (a_lo == b_lo) { - if (out_lo != a_lo) { - __ Move(out_lo, a_lo); - __ Move(out_hi, a_hi); - } - } else { - __ Slt(TMP, a_hi, b_hi); - __ Bne(a_hi, b_hi, &compare_done); - - __ Sltu(TMP, a_lo, b_lo); - - __ Bind(&compare_done); - - if (is_min) { - if (out_lo != a_lo) { - __ Movn(out_hi, a_hi, TMP); - __ Movn(out_lo, a_lo, TMP); - } - if (out_lo != b_lo) { - __ Movz(out_hi, b_hi, TMP); - __ Movz(out_lo, b_lo, TMP); - } - } else { - if (out_lo != a_lo) { - __ Movz(out_hi, a_hi, TMP); - __ Movz(out_lo, a_lo, TMP); - } - if (out_lo != b_lo) { - __ Movn(out_hi, b_hi, TMP); - __ Movn(out_lo, b_lo, TMP); - } - } - } - } else { - DCHECK_EQ(type, DataType::Type::kInt32); - Register a = locations->InAt(0).AsRegister<Register>(); - Register b = locations->InAt(1).AsRegister<Register>(); - Register out = locations->Out().AsRegister<Register>(); - - if (a == b) { - if (out != a) { - __ Move(out, a); - } - } else { - __ Slt(AT, a, b); - if (is_min) { - if (out != a) { - __ Movn(out, a, AT); - } - if (out != b) { - __ Movz(out, b, AT); - } - } else { - if (out != a) { - __ Movz(out, a, AT); - } - if (out != b) { - __ Movn(out, b, AT); - } - } - } - } - } -} - -// int java.lang.Math.min(int, int) -void IntrinsicLocationsBuilderMIPS::VisitMathMinIntInt(HInvoke* invoke) { - CreateIntIntToIntLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorMIPS::VisitMathMinIntInt(HInvoke* invoke) { - GenMinMax(invoke->GetLocations(), - /* is_min */ true, - DataType::Type::kInt32, - IsR6(), - GetAssembler()); -} - -// long java.lang.Math.min(long, long) -void IntrinsicLocationsBuilderMIPS::VisitMathMinLongLong(HInvoke* invoke) { - CreateIntIntToIntLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorMIPS::VisitMathMinLongLong(HInvoke* invoke) { - GenMinMax(invoke->GetLocations(), - /* is_min */ true, - DataType::Type::kInt64, - IsR6(), - GetAssembler()); -} - -// int java.lang.Math.max(int, int) -void IntrinsicLocationsBuilderMIPS::VisitMathMaxIntInt(HInvoke* invoke) { - CreateIntIntToIntLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorMIPS::VisitMathMaxIntInt(HInvoke* invoke) { - GenMinMax(invoke->GetLocations(), - /* is_min */ false, - DataType::Type::kInt32, - IsR6(), - GetAssembler()); -} - -// long java.lang.Math.max(long, long) -void IntrinsicLocationsBuilderMIPS::VisitMathMaxLongLong(HInvoke* invoke) { - CreateIntIntToIntLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorMIPS::VisitMathMaxLongLong(HInvoke* invoke) { - GenMinMax(invoke->GetLocations(), - /* is_min */ false, - DataType::Type::kInt64, - IsR6(), - GetAssembler()); + GenBitCount(invoke->GetLocations(), DataType::Type::kInt64, IsR6(), HasMsa(), GetAssembler()); } // double java.lang.Math.sqrt(double) diff --git a/compiler/optimizing/intrinsics_mips.h b/compiler/optimizing/intrinsics_mips.h index 13397f11d4c..1c1ba401325 100644 --- a/compiler/optimizing/intrinsics_mips.h +++ b/compiler/optimizing/intrinsics_mips.h @@ -71,6 +71,7 @@ class IntrinsicCodeGeneratorMIPS FINAL : public IntrinsicVisitor { bool IsR2OrNewer() const; bool IsR6() const; bool Is32BitFPU() const; + bool HasMsa() const; private: MipsAssembler* GetAssembler(); diff --git a/compiler/optimizing/intrinsics_mips64.cc b/compiler/optimizing/intrinsics_mips64.cc index 4668c561ed3..9a9ae714bc6 100644 --- a/compiler/optimizing/intrinsics_mips64.cc +++ b/compiler/optimizing/intrinsics_mips64.cc @@ -46,6 +46,10 @@ ArenaAllocator* IntrinsicCodeGeneratorMIPS64::GetAllocator() { return codegen_->GetGraph()->GetAllocator(); } +inline bool IntrinsicCodeGeneratorMIPS64::HasMsa() const { + return codegen_->GetInstructionSetFeatures().HasMsa(); +} + #define __ codegen->GetAssembler()-> static void MoveFromReturnRegister(Location trg, @@ -386,6 +390,7 @@ static void CreateFPToFPLocations(ArenaAllocator* allocator, HInvoke* invoke) { static void GenBitCount(LocationSummary* locations, const DataType::Type type, + const bool hasMsa, Mips64Assembler* assembler) { GpuRegister out = locations->Out().AsRegister<GpuRegister>(); GpuRegister in = locations->InAt(0).AsRegister<GpuRegister>(); @@ -414,41 +419,52 @@ static void GenBitCount(LocationSummary* locations, // bits are set but the algorithm here attempts to minimize the total // number of instructions executed even when a large number of bits // are set. - - if (type == DataType::Type::kInt32) { - __ Srl(TMP, in, 1); - __ LoadConst32(AT, 0x55555555); - __ And(TMP, TMP, AT); - __ Subu(TMP, in, TMP); - __ LoadConst32(AT, 0x33333333); - __ And(out, TMP, AT); - __ Srl(TMP, TMP, 2); - __ And(TMP, TMP, AT); - __ Addu(TMP, out, TMP); - __ Srl(out, TMP, 4); - __ Addu(out, out, TMP); - __ LoadConst32(AT, 0x0F0F0F0F); - __ And(out, out, AT); - __ LoadConst32(TMP, 0x01010101); - __ MulR6(out, out, TMP); - __ Srl(out, out, 24); - } else if (type == DataType::Type::kInt64) { - __ Dsrl(TMP, in, 1); - __ LoadConst64(AT, 0x5555555555555555L); - __ And(TMP, TMP, AT); - __ Dsubu(TMP, in, TMP); - __ LoadConst64(AT, 0x3333333333333333L); - __ And(out, TMP, AT); - __ Dsrl(TMP, TMP, 2); - __ And(TMP, TMP, AT); - __ Daddu(TMP, out, TMP); - __ Dsrl(out, TMP, 4); - __ Daddu(out, out, TMP); - __ LoadConst64(AT, 0x0F0F0F0F0F0F0F0FL); - __ And(out, out, AT); - __ LoadConst64(TMP, 0x0101010101010101L); - __ Dmul(out, out, TMP); - __ Dsrl32(out, out, 24); + if (hasMsa) { + if (type == DataType::Type::kInt32) { + __ Mtc1(in, FTMP); + __ PcntW(static_cast<VectorRegister>(FTMP), static_cast<VectorRegister>(FTMP)); + __ Mfc1(out, FTMP); + } else { + __ Dmtc1(in, FTMP); + __ PcntD(static_cast<VectorRegister>(FTMP), static_cast<VectorRegister>(FTMP)); + __ Dmfc1(out, FTMP); + } + } else { + if (type == DataType::Type::kInt32) { + __ Srl(TMP, in, 1); + __ LoadConst32(AT, 0x55555555); + __ And(TMP, TMP, AT); + __ Subu(TMP, in, TMP); + __ LoadConst32(AT, 0x33333333); + __ And(out, TMP, AT); + __ Srl(TMP, TMP, 2); + __ And(TMP, TMP, AT); + __ Addu(TMP, out, TMP); + __ Srl(out, TMP, 4); + __ Addu(out, out, TMP); + __ LoadConst32(AT, 0x0F0F0F0F); + __ And(out, out, AT); + __ LoadConst32(TMP, 0x01010101); + __ MulR6(out, out, TMP); + __ Srl(out, out, 24); + } else { + __ Dsrl(TMP, in, 1); + __ LoadConst64(AT, 0x5555555555555555L); + __ And(TMP, TMP, AT); + __ Dsubu(TMP, in, TMP); + __ LoadConst64(AT, 0x3333333333333333L); + __ And(out, TMP, AT); + __ Dsrl(TMP, TMP, 2); + __ And(TMP, TMP, AT); + __ Daddu(TMP, out, TMP); + __ Dsrl(out, TMP, 4); + __ Daddu(out, out, TMP); + __ LoadConst64(AT, 0x0F0F0F0F0F0F0F0FL); + __ And(out, out, AT); + __ LoadConst64(TMP, 0x0101010101010101L); + __ Dmul(out, out, TMP); + __ Dsrl32(out, out, 24); + } } } @@ -458,7 +474,7 @@ void IntrinsicLocationsBuilderMIPS64::VisitIntegerBitCount(HInvoke* invoke) { } void IntrinsicCodeGeneratorMIPS64::VisitIntegerBitCount(HInvoke* invoke) { - GenBitCount(invoke->GetLocations(), DataType::Type::kInt32, GetAssembler()); + GenBitCount(invoke->GetLocations(), DataType::Type::kInt32, HasMsa(), GetAssembler()); } // int java.lang.Long.bitCount(long) @@ -467,291 +483,7 @@ void IntrinsicLocationsBuilderMIPS64::VisitLongBitCount(HInvoke* invoke) { } void IntrinsicCodeGeneratorMIPS64::VisitLongBitCount(HInvoke* invoke) { - GenBitCount(invoke->GetLocations(), DataType::Type::kInt64, GetAssembler()); -} - -static void MathAbsFP(LocationSummary* locations, bool is64bit, Mips64Assembler* assembler) { - FpuRegister in = locations->InAt(0).AsFpuRegister<FpuRegister>(); - FpuRegister out = locations->Out().AsFpuRegister<FpuRegister>(); - - if (is64bit) { - __ AbsD(out, in); - } else { - __ AbsS(out, in); - } -} - -// double java.lang.Math.abs(double) -void IntrinsicLocationsBuilderMIPS64::VisitMathAbsDouble(HInvoke* invoke) { - CreateFPToFPLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorMIPS64::VisitMathAbsDouble(HInvoke* invoke) { - MathAbsFP(invoke->GetLocations(), /* is64bit */ true, GetAssembler()); -} - -// float java.lang.Math.abs(float) -void IntrinsicLocationsBuilderMIPS64::VisitMathAbsFloat(HInvoke* invoke) { - CreateFPToFPLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorMIPS64::VisitMathAbsFloat(HInvoke* invoke) { - MathAbsFP(invoke->GetLocations(), /* is64bit */ false, GetAssembler()); -} - -static void CreateIntToInt(ArenaAllocator* allocator, HInvoke* invoke) { - LocationSummary* locations = - new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); -} - -static void GenAbsInteger(LocationSummary* locations, bool is64bit, Mips64Assembler* assembler) { - GpuRegister in = locations->InAt(0).AsRegister<GpuRegister>(); - GpuRegister out = locations->Out().AsRegister<GpuRegister>(); - - if (is64bit) { - __ Dsra32(AT, in, 31); - __ Xor(out, in, AT); - __ Dsubu(out, out, AT); - } else { - __ Sra(AT, in, 31); - __ Xor(out, in, AT); - __ Subu(out, out, AT); - } -} - -// int java.lang.Math.abs(int) -void IntrinsicLocationsBuilderMIPS64::VisitMathAbsInt(HInvoke* invoke) { - CreateIntToInt(allocator_, invoke); -} - -void IntrinsicCodeGeneratorMIPS64::VisitMathAbsInt(HInvoke* invoke) { - GenAbsInteger(invoke->GetLocations(), /* is64bit */ false, GetAssembler()); -} - -// long java.lang.Math.abs(long) -void IntrinsicLocationsBuilderMIPS64::VisitMathAbsLong(HInvoke* invoke) { - CreateIntToInt(allocator_, invoke); -} - -void IntrinsicCodeGeneratorMIPS64::VisitMathAbsLong(HInvoke* invoke) { - GenAbsInteger(invoke->GetLocations(), /* is64bit */ true, GetAssembler()); -} - -static void GenMinMaxFP(LocationSummary* locations, - bool is_min, - DataType::Type type, - Mips64Assembler* assembler) { - FpuRegister a = locations->InAt(0).AsFpuRegister<FpuRegister>(); - FpuRegister b = locations->InAt(1).AsFpuRegister<FpuRegister>(); - FpuRegister out = locations->Out().AsFpuRegister<FpuRegister>(); - - Mips64Label noNaNs; - Mips64Label done; - FpuRegister ftmp = ((out != a) && (out != b)) ? out : FTMP; - - // When Java computes min/max it prefers a NaN to a number; the - // behavior of MIPSR6 is to prefer numbers to NaNs, i.e., if one of - // the inputs is a NaN and the other is a valid number, the MIPS - // instruction will return the number; Java wants the NaN value - // returned. This is why there is extra logic preceding the use of - // the MIPS min.fmt/max.fmt instructions. If either a, or b holds a - // NaN, return the NaN, otherwise return the min/max. - if (type == DataType::Type::kFloat64) { - __ CmpUnD(FTMP, a, b); - __ Bc1eqz(FTMP, &noNaNs); - - // One of the inputs is a NaN - __ CmpEqD(ftmp, a, a); - // If a == a then b is the NaN, otherwise a is the NaN. - __ SelD(ftmp, a, b); - - if (ftmp != out) { - __ MovD(out, ftmp); - } - - __ Bc(&done); - - __ Bind(&noNaNs); - - if (is_min) { - __ MinD(out, a, b); - } else { - __ MaxD(out, a, b); - } - } else { - DCHECK_EQ(type, DataType::Type::kFloat32); - __ CmpUnS(FTMP, a, b); - __ Bc1eqz(FTMP, &noNaNs); - - // One of the inputs is a NaN - __ CmpEqS(ftmp, a, a); - // If a == a then b is the NaN, otherwise a is the NaN. - __ SelS(ftmp, a, b); - - if (ftmp != out) { - __ MovS(out, ftmp); - } - - __ Bc(&done); - - __ Bind(&noNaNs); - - if (is_min) { - __ MinS(out, a, b); - } else { - __ MaxS(out, a, b); - } - } - - __ Bind(&done); -} - -static void CreateFPFPToFPLocations(ArenaAllocator* allocator, HInvoke* invoke) { - LocationSummary* locations = - new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetInAt(1, Location::RequiresFpuRegister()); - locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); -} - -// double java.lang.Math.min(double, double) -void IntrinsicLocationsBuilderMIPS64::VisitMathMinDoubleDouble(HInvoke* invoke) { - CreateFPFPToFPLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorMIPS64::VisitMathMinDoubleDouble(HInvoke* invoke) { - GenMinMaxFP(invoke->GetLocations(), /* is_min */ true, DataType::Type::kFloat64, GetAssembler()); -} - -// float java.lang.Math.min(float, float) -void IntrinsicLocationsBuilderMIPS64::VisitMathMinFloatFloat(HInvoke* invoke) { - CreateFPFPToFPLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorMIPS64::VisitMathMinFloatFloat(HInvoke* invoke) { - GenMinMaxFP(invoke->GetLocations(), /* is_min */ true, DataType::Type::kFloat32, GetAssembler()); -} - -// double java.lang.Math.max(double, double) -void IntrinsicLocationsBuilderMIPS64::VisitMathMaxDoubleDouble(HInvoke* invoke) { - CreateFPFPToFPLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorMIPS64::VisitMathMaxDoubleDouble(HInvoke* invoke) { - GenMinMaxFP(invoke->GetLocations(), /* is_min */ false, DataType::Type::kFloat64, GetAssembler()); -} - -// float java.lang.Math.max(float, float) -void IntrinsicLocationsBuilderMIPS64::VisitMathMaxFloatFloat(HInvoke* invoke) { - CreateFPFPToFPLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorMIPS64::VisitMathMaxFloatFloat(HInvoke* invoke) { - GenMinMaxFP(invoke->GetLocations(), /* is_min */ false, DataType::Type::kFloat32, GetAssembler()); -} - -static void GenMinMax(LocationSummary* locations, - bool is_min, - Mips64Assembler* assembler) { - GpuRegister lhs = locations->InAt(0).AsRegister<GpuRegister>(); - GpuRegister rhs = locations->InAt(1).AsRegister<GpuRegister>(); - GpuRegister out = locations->Out().AsRegister<GpuRegister>(); - - if (lhs == rhs) { - if (out != lhs) { - __ Move(out, lhs); - } - } else { - // Some architectures, such as ARM and MIPS (prior to r6), have a - // conditional move instruction which only changes the target - // (output) register if the condition is true (MIPS prior to r6 had - // MOVF, MOVT, and MOVZ). The SELEQZ and SELNEZ instructions always - // change the target (output) register. If the condition is true the - // output register gets the contents of the "rs" register; otherwise, - // the output register is set to zero. One consequence of this is - // that to implement something like "rd = c==0 ? rs : rt" MIPS64r6 - // needs to use a pair of SELEQZ/SELNEZ instructions. After - // executing this pair of instructions one of the output registers - // from the pair will necessarily contain zero. Then the code ORs the - // output registers from the SELEQZ/SELNEZ instructions to get the - // final result. - // - // The initial test to see if the output register is same as the - // first input register is needed to make sure that value in the - // first input register isn't clobbered before we've finished - // computing the output value. The logic in the corresponding else - // clause performs the same task but makes sure the second input - // register isn't clobbered in the event that it's the same register - // as the output register; the else clause also handles the case - // where the output register is distinct from both the first, and the - // second input registers. - if (out == lhs) { - __ Slt(AT, rhs, lhs); - if (is_min) { - __ Seleqz(out, lhs, AT); - __ Selnez(AT, rhs, AT); - } else { - __ Selnez(out, lhs, AT); - __ Seleqz(AT, rhs, AT); - } - } else { - __ Slt(AT, lhs, rhs); - if (is_min) { - __ Seleqz(out, rhs, AT); - __ Selnez(AT, lhs, AT); - } else { - __ Selnez(out, rhs, AT); - __ Seleqz(AT, lhs, AT); - } - } - __ Or(out, out, AT); - } -} - -static void CreateIntIntToIntLocations(ArenaAllocator* allocator, HInvoke* invoke) { - LocationSummary* locations = - new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); -} - -// int java.lang.Math.min(int, int) -void IntrinsicLocationsBuilderMIPS64::VisitMathMinIntInt(HInvoke* invoke) { - CreateIntIntToIntLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorMIPS64::VisitMathMinIntInt(HInvoke* invoke) { - GenMinMax(invoke->GetLocations(), /* is_min */ true, GetAssembler()); -} - -// long java.lang.Math.min(long, long) -void IntrinsicLocationsBuilderMIPS64::VisitMathMinLongLong(HInvoke* invoke) { - CreateIntIntToIntLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorMIPS64::VisitMathMinLongLong(HInvoke* invoke) { - GenMinMax(invoke->GetLocations(), /* is_min */ true, GetAssembler()); -} - -// int java.lang.Math.max(int, int) -void IntrinsicLocationsBuilderMIPS64::VisitMathMaxIntInt(HInvoke* invoke) { - CreateIntIntToIntLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorMIPS64::VisitMathMaxIntInt(HInvoke* invoke) { - GenMinMax(invoke->GetLocations(), /* is_min */ false, GetAssembler()); -} - -// long java.lang.Math.max(long, long) -void IntrinsicLocationsBuilderMIPS64::VisitMathMaxLongLong(HInvoke* invoke) { - CreateIntIntToIntLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorMIPS64::VisitMathMaxLongLong(HInvoke* invoke) { - GenMinMax(invoke->GetLocations(), /* is_min */ false, GetAssembler()); + GenBitCount(invoke->GetLocations(), DataType::Type::kInt64, HasMsa(), GetAssembler()); } // double java.lang.Math.sqrt(double) diff --git a/compiler/optimizing/intrinsics_mips64.h b/compiler/optimizing/intrinsics_mips64.h index 6f40d90ddbf..748b0b02b2e 100644 --- a/compiler/optimizing/intrinsics_mips64.h +++ b/compiler/optimizing/intrinsics_mips64.h @@ -68,6 +68,8 @@ class IntrinsicCodeGeneratorMIPS64 FINAL : public IntrinsicVisitor { #undef INTRINSICS_LIST #undef OPTIMIZING_INTRINSICS + bool HasMsa() const; + private: Mips64Assembler* GetAssembler(); diff --git a/compiler/optimizing/intrinsics_x86.cc b/compiler/optimizing/intrinsics_x86.cc index 0763ef23529..f84a33bb8e3 100644 --- a/compiler/optimizing/intrinsics_x86.cc +++ b/compiler/optimizing/intrinsics_x86.cc @@ -40,11 +40,6 @@ namespace art { namespace x86 { -static constexpr int kDoubleNaNHigh = 0x7FF80000; -static constexpr int kDoubleNaNLow = 0x00000000; -static constexpr int64_t kDoubleNaN = INT64_C(0x7FF8000000000000); -static constexpr int32_t kFloatNaN = INT32_C(0x7FC00000); - IntrinsicLocationsBuilderX86::IntrinsicLocationsBuilderX86(CodeGeneratorX86* codegen) : allocator_(codegen->GetGraph()->GetAllocator()), codegen_(codegen) { @@ -333,432 +328,6 @@ void IntrinsicCodeGeneratorX86::VisitShortReverseBytes(HInvoke* invoke) { GenReverseBytes(invoke->GetLocations(), DataType::Type::kInt16, GetAssembler()); } - -// TODO: Consider Quick's way of doing Double abs through integer operations, as the immediate we -// need is 64b. - -static void CreateFloatToFloat(ArenaAllocator* allocator, HInvoke* invoke) { - // TODO: Enable memory operations when the assembler supports them. - LocationSummary* locations = - new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetOut(Location::SameAsFirstInput()); - HInvokeStaticOrDirect* static_or_direct = invoke->AsInvokeStaticOrDirect(); - DCHECK(static_or_direct != nullptr); - if (static_or_direct->HasSpecialInput() && - invoke->InputAt(static_or_direct->GetSpecialInputIndex())->IsX86ComputeBaseMethodAddress()) { - // We need addressibility for the constant area. - locations->SetInAt(1, Location::RequiresRegister()); - // We need a temporary to hold the constant. - locations->AddTemp(Location::RequiresFpuRegister()); - } -} - -static void MathAbsFP(HInvoke* invoke, - bool is64bit, - X86Assembler* assembler, - CodeGeneratorX86* codegen) { - LocationSummary* locations = invoke->GetLocations(); - Location output = locations->Out(); - - DCHECK(output.IsFpuRegister()); - if (locations->GetInputCount() == 2 && locations->InAt(1).IsValid()) { - HX86ComputeBaseMethodAddress* method_address = - invoke->InputAt(1)->AsX86ComputeBaseMethodAddress(); - DCHECK(locations->InAt(1).IsRegister()); - // We also have a constant area pointer. - Register constant_area = locations->InAt(1).AsRegister<Register>(); - XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>(); - if (is64bit) { - __ movsd(temp, codegen->LiteralInt64Address( - INT64_C(0x7FFFFFFFFFFFFFFF), method_address, constant_area)); - __ andpd(output.AsFpuRegister<XmmRegister>(), temp); - } else { - __ movss(temp, codegen->LiteralInt32Address( - INT32_C(0x7FFFFFFF), method_address, constant_area)); - __ andps(output.AsFpuRegister<XmmRegister>(), temp); - } - } else { - // Create the right constant on an aligned stack. - if (is64bit) { - __ subl(ESP, Immediate(8)); - __ pushl(Immediate(0x7FFFFFFF)); - __ pushl(Immediate(0xFFFFFFFF)); - __ andpd(output.AsFpuRegister<XmmRegister>(), Address(ESP, 0)); - } else { - __ subl(ESP, Immediate(12)); - __ pushl(Immediate(0x7FFFFFFF)); - __ andps(output.AsFpuRegister<XmmRegister>(), Address(ESP, 0)); - } - __ addl(ESP, Immediate(16)); - } -} - -void IntrinsicLocationsBuilderX86::VisitMathAbsDouble(HInvoke* invoke) { - CreateFloatToFloat(allocator_, invoke); -} - -void IntrinsicCodeGeneratorX86::VisitMathAbsDouble(HInvoke* invoke) { - MathAbsFP(invoke, /* is64bit */ true, GetAssembler(), codegen_); -} - -void IntrinsicLocationsBuilderX86::VisitMathAbsFloat(HInvoke* invoke) { - CreateFloatToFloat(allocator_, invoke); -} - -void IntrinsicCodeGeneratorX86::VisitMathAbsFloat(HInvoke* invoke) { - MathAbsFP(invoke, /* is64bit */ false, GetAssembler(), codegen_); -} - -static void CreateAbsIntLocation(ArenaAllocator* allocator, HInvoke* invoke) { - LocationSummary* locations = - new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); - locations->SetInAt(0, Location::RegisterLocation(EAX)); - locations->SetOut(Location::SameAsFirstInput()); - locations->AddTemp(Location::RegisterLocation(EDX)); -} - -static void GenAbsInteger(LocationSummary* locations, X86Assembler* assembler) { - Location output = locations->Out(); - Register out = output.AsRegister<Register>(); - DCHECK_EQ(out, EAX); - Register temp = locations->GetTemp(0).AsRegister<Register>(); - DCHECK_EQ(temp, EDX); - - // Sign extend EAX into EDX. - __ cdq(); - - // XOR EAX with sign. - __ xorl(EAX, EDX); - - // Subtract out sign to correct. - __ subl(EAX, EDX); - - // The result is in EAX. -} - -static void CreateAbsLongLocation(ArenaAllocator* allocator, HInvoke* invoke) { - LocationSummary* locations = - new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); - locations->AddTemp(Location::RequiresRegister()); -} - -static void GenAbsLong(LocationSummary* locations, X86Assembler* assembler) { - Location input = locations->InAt(0); - Register input_lo = input.AsRegisterPairLow<Register>(); - Register input_hi = input.AsRegisterPairHigh<Register>(); - Location output = locations->Out(); - Register output_lo = output.AsRegisterPairLow<Register>(); - Register output_hi = output.AsRegisterPairHigh<Register>(); - Register temp = locations->GetTemp(0).AsRegister<Register>(); - - // Compute the sign into the temporary. - __ movl(temp, input_hi); - __ sarl(temp, Immediate(31)); - - // Store the sign into the output. - __ movl(output_lo, temp); - __ movl(output_hi, temp); - - // XOR the input to the output. - __ xorl(output_lo, input_lo); - __ xorl(output_hi, input_hi); - - // Subtract the sign. - __ subl(output_lo, temp); - __ sbbl(output_hi, temp); -} - -void IntrinsicLocationsBuilderX86::VisitMathAbsInt(HInvoke* invoke) { - CreateAbsIntLocation(allocator_, invoke); -} - -void IntrinsicCodeGeneratorX86::VisitMathAbsInt(HInvoke* invoke) { - GenAbsInteger(invoke->GetLocations(), GetAssembler()); -} - -void IntrinsicLocationsBuilderX86::VisitMathAbsLong(HInvoke* invoke) { - CreateAbsLongLocation(allocator_, invoke); -} - -void IntrinsicCodeGeneratorX86::VisitMathAbsLong(HInvoke* invoke) { - GenAbsLong(invoke->GetLocations(), GetAssembler()); -} - -static void GenMinMaxFP(HInvoke* invoke, - bool is_min, - bool is_double, - X86Assembler* assembler, - CodeGeneratorX86* codegen) { - LocationSummary* locations = invoke->GetLocations(); - Location op1_loc = locations->InAt(0); - Location op2_loc = locations->InAt(1); - Location out_loc = locations->Out(); - XmmRegister out = out_loc.AsFpuRegister<XmmRegister>(); - - // Shortcut for same input locations. - if (op1_loc.Equals(op2_loc)) { - DCHECK(out_loc.Equals(op1_loc)); - return; - } - - // (out := op1) - // out <=? op2 - // if Nan jmp Nan_label - // if out is min jmp done - // if op2 is min jmp op2_label - // handle -0/+0 - // jmp done - // Nan_label: - // out := NaN - // op2_label: - // out := op2 - // done: - // - // This removes one jmp, but needs to copy one input (op1) to out. - // - // TODO: This is straight from Quick (except literal pool). Make NaN an out-of-line slowpath? - - XmmRegister op2 = op2_loc.AsFpuRegister<XmmRegister>(); - - NearLabel nan, done, op2_label; - if (is_double) { - __ ucomisd(out, op2); - } else { - __ ucomiss(out, op2); - } - - __ j(Condition::kParityEven, &nan); - - __ j(is_min ? Condition::kAbove : Condition::kBelow, &op2_label); - __ j(is_min ? Condition::kBelow : Condition::kAbove, &done); - - // Handle 0.0/-0.0. - if (is_min) { - if (is_double) { - __ orpd(out, op2); - } else { - __ orps(out, op2); - } - } else { - if (is_double) { - __ andpd(out, op2); - } else { - __ andps(out, op2); - } - } - __ jmp(&done); - - // NaN handling. - __ Bind(&nan); - // Do we have a constant area pointer? - if (locations->GetInputCount() == 3 && locations->InAt(2).IsValid()) { - HX86ComputeBaseMethodAddress* method_address = - invoke->InputAt(2)->AsX86ComputeBaseMethodAddress(); - DCHECK(locations->InAt(2).IsRegister()); - Register constant_area = locations->InAt(2).AsRegister<Register>(); - if (is_double) { - __ movsd(out, codegen->LiteralInt64Address(kDoubleNaN, method_address, constant_area)); - } else { - __ movss(out, codegen->LiteralInt32Address(kFloatNaN, method_address, constant_area)); - } - } else { - if (is_double) { - __ pushl(Immediate(kDoubleNaNHigh)); - __ pushl(Immediate(kDoubleNaNLow)); - __ movsd(out, Address(ESP, 0)); - __ addl(ESP, Immediate(8)); - } else { - __ pushl(Immediate(kFloatNaN)); - __ movss(out, Address(ESP, 0)); - __ addl(ESP, Immediate(4)); - } - } - __ jmp(&done); - - // out := op2; - __ Bind(&op2_label); - if (is_double) { - __ movsd(out, op2); - } else { - __ movss(out, op2); - } - - // Done. - __ Bind(&done); -} - -static void CreateFPFPToFPLocations(ArenaAllocator* allocator, HInvoke* invoke) { - LocationSummary* locations = - new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetInAt(1, Location::RequiresFpuRegister()); - // The following is sub-optimal, but all we can do for now. It would be fine to also accept - // the second input to be the output (we can simply swap inputs). - locations->SetOut(Location::SameAsFirstInput()); - HInvokeStaticOrDirect* static_or_direct = invoke->AsInvokeStaticOrDirect(); - DCHECK(static_or_direct != nullptr); - if (static_or_direct->HasSpecialInput() && - invoke->InputAt(static_or_direct->GetSpecialInputIndex())->IsX86ComputeBaseMethodAddress()) { - locations->SetInAt(2, Location::RequiresRegister()); - } -} - -void IntrinsicLocationsBuilderX86::VisitMathMinDoubleDouble(HInvoke* invoke) { - CreateFPFPToFPLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorX86::VisitMathMinDoubleDouble(HInvoke* invoke) { - GenMinMaxFP(invoke, - /* is_min */ true, - /* is_double */ true, - GetAssembler(), - codegen_); -} - -void IntrinsicLocationsBuilderX86::VisitMathMinFloatFloat(HInvoke* invoke) { - CreateFPFPToFPLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorX86::VisitMathMinFloatFloat(HInvoke* invoke) { - GenMinMaxFP(invoke, - /* is_min */ true, - /* is_double */ false, - GetAssembler(), - codegen_); -} - -void IntrinsicLocationsBuilderX86::VisitMathMaxDoubleDouble(HInvoke* invoke) { - CreateFPFPToFPLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorX86::VisitMathMaxDoubleDouble(HInvoke* invoke) { - GenMinMaxFP(invoke, - /* is_min */ false, - /* is_double */ true, - GetAssembler(), - codegen_); -} - -void IntrinsicLocationsBuilderX86::VisitMathMaxFloatFloat(HInvoke* invoke) { - CreateFPFPToFPLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorX86::VisitMathMaxFloatFloat(HInvoke* invoke) { - GenMinMaxFP(invoke, - /* is_min */ false, - /* is_double */ false, - GetAssembler(), - codegen_); -} - -static void GenMinMax(LocationSummary* locations, bool is_min, bool is_long, - X86Assembler* assembler) { - Location op1_loc = locations->InAt(0); - Location op2_loc = locations->InAt(1); - - // Shortcut for same input locations. - if (op1_loc.Equals(op2_loc)) { - // Can return immediately, as op1_loc == out_loc. - // Note: if we ever support separate registers, e.g., output into memory, we need to check for - // a copy here. - DCHECK(locations->Out().Equals(op1_loc)); - return; - } - - if (is_long) { - // Need to perform a subtract to get the sign right. - // op1 is already in the same location as the output. - Location output = locations->Out(); - Register output_lo = output.AsRegisterPairLow<Register>(); - Register output_hi = output.AsRegisterPairHigh<Register>(); - - Register op2_lo = op2_loc.AsRegisterPairLow<Register>(); - Register op2_hi = op2_loc.AsRegisterPairHigh<Register>(); - - // Spare register to compute the subtraction to set condition code. - Register temp = locations->GetTemp(0).AsRegister<Register>(); - - // Subtract off op2_low. - __ movl(temp, output_lo); - __ subl(temp, op2_lo); - - // Now use the same tempo and the borrow to finish the subtraction of op2_hi. - __ movl(temp, output_hi); - __ sbbl(temp, op2_hi); - - // Now the condition code is correct. - Condition cond = is_min ? Condition::kGreaterEqual : Condition::kLess; - __ cmovl(cond, output_lo, op2_lo); - __ cmovl(cond, output_hi, op2_hi); - } else { - Register out = locations->Out().AsRegister<Register>(); - Register op2 = op2_loc.AsRegister<Register>(); - - // (out := op1) - // out <=? op2 - // if out is min jmp done - // out := op2 - // done: - - __ cmpl(out, op2); - Condition cond = is_min ? Condition::kGreater : Condition::kLess; - __ cmovl(cond, out, op2); - } -} - -static void CreateIntIntToIntLocations(ArenaAllocator* allocator, HInvoke* invoke) { - LocationSummary* locations = - new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); - locations->SetOut(Location::SameAsFirstInput()); -} - -static void CreateLongLongToLongLocations(ArenaAllocator* allocator, HInvoke* invoke) { - LocationSummary* locations = - new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); - locations->SetOut(Location::SameAsFirstInput()); - // Register to use to perform a long subtract to set cc. - locations->AddTemp(Location::RequiresRegister()); -} - -void IntrinsicLocationsBuilderX86::VisitMathMinIntInt(HInvoke* invoke) { - CreateIntIntToIntLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorX86::VisitMathMinIntInt(HInvoke* invoke) { - GenMinMax(invoke->GetLocations(), /* is_min */ true, /* is_long */ false, GetAssembler()); -} - -void IntrinsicLocationsBuilderX86::VisitMathMinLongLong(HInvoke* invoke) { - CreateLongLongToLongLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorX86::VisitMathMinLongLong(HInvoke* invoke) { - GenMinMax(invoke->GetLocations(), /* is_min */ true, /* is_long */ true, GetAssembler()); -} - -void IntrinsicLocationsBuilderX86::VisitMathMaxIntInt(HInvoke* invoke) { - CreateIntIntToIntLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorX86::VisitMathMaxIntInt(HInvoke* invoke) { - GenMinMax(invoke->GetLocations(), /* is_min */ false, /* is_long */ false, GetAssembler()); -} - -void IntrinsicLocationsBuilderX86::VisitMathMaxLongLong(HInvoke* invoke) { - CreateLongLongToLongLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorX86::VisitMathMaxLongLong(HInvoke* invoke) { - GenMinMax(invoke->GetLocations(), /* is_min */ false, /* is_long */ true, GetAssembler()); -} - static void CreateFPToFPLocations(ArenaAllocator* allocator, HInvoke* invoke) { LocationSummary* locations = new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); diff --git a/compiler/optimizing/intrinsics_x86_64.cc b/compiler/optimizing/intrinsics_x86_64.cc index 91a505ede1a..7627dc9490a 100644 --- a/compiler/optimizing/intrinsics_x86_64.cc +++ b/compiler/optimizing/intrinsics_x86_64.cc @@ -236,304 +236,6 @@ void IntrinsicCodeGeneratorX86_64::VisitShortReverseBytes(HInvoke* invoke) { GenReverseBytes(invoke->GetLocations(), DataType::Type::kInt16, GetAssembler()); } - -// TODO: Consider Quick's way of doing Double abs through integer operations, as the immediate we -// need is 64b. - -static void CreateFloatToFloatPlusTemps(ArenaAllocator* allocator, HInvoke* invoke) { - // TODO: Enable memory operations when the assembler supports them. - LocationSummary* locations = - new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetOut(Location::SameAsFirstInput()); - locations->AddTemp(Location::RequiresFpuRegister()); // FP reg to hold mask. -} - -static void MathAbsFP(LocationSummary* locations, - bool is64bit, - X86_64Assembler* assembler, - CodeGeneratorX86_64* codegen) { - Location output = locations->Out(); - - DCHECK(output.IsFpuRegister()); - XmmRegister xmm_temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>(); - - // TODO: Can mask directly with constant area using pand if we can guarantee - // that the literal is aligned on a 16 byte boundary. This will avoid a - // temporary. - if (is64bit) { - __ movsd(xmm_temp, codegen->LiteralInt64Address(INT64_C(0x7FFFFFFFFFFFFFFF))); - __ andpd(output.AsFpuRegister<XmmRegister>(), xmm_temp); - } else { - __ movss(xmm_temp, codegen->LiteralInt32Address(INT32_C(0x7FFFFFFF))); - __ andps(output.AsFpuRegister<XmmRegister>(), xmm_temp); - } -} - -void IntrinsicLocationsBuilderX86_64::VisitMathAbsDouble(HInvoke* invoke) { - CreateFloatToFloatPlusTemps(allocator_, invoke); -} - -void IntrinsicCodeGeneratorX86_64::VisitMathAbsDouble(HInvoke* invoke) { - MathAbsFP(invoke->GetLocations(), /* is64bit */ true, GetAssembler(), codegen_); -} - -void IntrinsicLocationsBuilderX86_64::VisitMathAbsFloat(HInvoke* invoke) { - CreateFloatToFloatPlusTemps(allocator_, invoke); -} - -void IntrinsicCodeGeneratorX86_64::VisitMathAbsFloat(HInvoke* invoke) { - MathAbsFP(invoke->GetLocations(), /* is64bit */ false, GetAssembler(), codegen_); -} - -static void CreateIntToIntPlusTemp(ArenaAllocator* allocator, HInvoke* invoke) { - LocationSummary* locations = - new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetOut(Location::SameAsFirstInput()); - locations->AddTemp(Location::RequiresRegister()); -} - -static void GenAbsInteger(LocationSummary* locations, bool is64bit, X86_64Assembler* assembler) { - Location output = locations->Out(); - CpuRegister out = output.AsRegister<CpuRegister>(); - CpuRegister mask = locations->GetTemp(0).AsRegister<CpuRegister>(); - - if (is64bit) { - // Create mask. - __ movq(mask, out); - __ sarq(mask, Immediate(63)); - // Add mask. - __ addq(out, mask); - __ xorq(out, mask); - } else { - // Create mask. - __ movl(mask, out); - __ sarl(mask, Immediate(31)); - // Add mask. - __ addl(out, mask); - __ xorl(out, mask); - } -} - -void IntrinsicLocationsBuilderX86_64::VisitMathAbsInt(HInvoke* invoke) { - CreateIntToIntPlusTemp(allocator_, invoke); -} - -void IntrinsicCodeGeneratorX86_64::VisitMathAbsInt(HInvoke* invoke) { - GenAbsInteger(invoke->GetLocations(), /* is64bit */ false, GetAssembler()); -} - -void IntrinsicLocationsBuilderX86_64::VisitMathAbsLong(HInvoke* invoke) { - CreateIntToIntPlusTemp(allocator_, invoke); -} - -void IntrinsicCodeGeneratorX86_64::VisitMathAbsLong(HInvoke* invoke) { - GenAbsInteger(invoke->GetLocations(), /* is64bit */ true, GetAssembler()); -} - -static void GenMinMaxFP(LocationSummary* locations, - bool is_min, - bool is_double, - X86_64Assembler* assembler, - CodeGeneratorX86_64* codegen) { - Location op1_loc = locations->InAt(0); - Location op2_loc = locations->InAt(1); - Location out_loc = locations->Out(); - XmmRegister out = out_loc.AsFpuRegister<XmmRegister>(); - - // Shortcut for same input locations. - if (op1_loc.Equals(op2_loc)) { - DCHECK(out_loc.Equals(op1_loc)); - return; - } - - // (out := op1) - // out <=? op2 - // if Nan jmp Nan_label - // if out is min jmp done - // if op2 is min jmp op2_label - // handle -0/+0 - // jmp done - // Nan_label: - // out := NaN - // op2_label: - // out := op2 - // done: - // - // This removes one jmp, but needs to copy one input (op1) to out. - // - // TODO: This is straight from Quick. Make NaN an out-of-line slowpath? - - XmmRegister op2 = op2_loc.AsFpuRegister<XmmRegister>(); - - NearLabel nan, done, op2_label; - if (is_double) { - __ ucomisd(out, op2); - } else { - __ ucomiss(out, op2); - } - - __ j(Condition::kParityEven, &nan); - - __ j(is_min ? Condition::kAbove : Condition::kBelow, &op2_label); - __ j(is_min ? Condition::kBelow : Condition::kAbove, &done); - - // Handle 0.0/-0.0. - if (is_min) { - if (is_double) { - __ orpd(out, op2); - } else { - __ orps(out, op2); - } - } else { - if (is_double) { - __ andpd(out, op2); - } else { - __ andps(out, op2); - } - } - __ jmp(&done); - - // NaN handling. - __ Bind(&nan); - if (is_double) { - __ movsd(out, codegen->LiteralInt64Address(INT64_C(0x7FF8000000000000))); - } else { - __ movss(out, codegen->LiteralInt32Address(INT32_C(0x7FC00000))); - } - __ jmp(&done); - - // out := op2; - __ Bind(&op2_label); - if (is_double) { - __ movsd(out, op2); - } else { - __ movss(out, op2); - } - - // Done. - __ Bind(&done); -} - -static void CreateFPFPToFP(ArenaAllocator* allocator, HInvoke* invoke) { - LocationSummary* locations = - new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); - locations->SetInAt(0, Location::RequiresFpuRegister()); - locations->SetInAt(1, Location::RequiresFpuRegister()); - // The following is sub-optimal, but all we can do for now. It would be fine to also accept - // the second input to be the output (we can simply swap inputs). - locations->SetOut(Location::SameAsFirstInput()); -} - -void IntrinsicLocationsBuilderX86_64::VisitMathMinDoubleDouble(HInvoke* invoke) { - CreateFPFPToFP(allocator_, invoke); -} - -void IntrinsicCodeGeneratorX86_64::VisitMathMinDoubleDouble(HInvoke* invoke) { - GenMinMaxFP( - invoke->GetLocations(), /* is_min */ true, /* is_double */ true, GetAssembler(), codegen_); -} - -void IntrinsicLocationsBuilderX86_64::VisitMathMinFloatFloat(HInvoke* invoke) { - CreateFPFPToFP(allocator_, invoke); -} - -void IntrinsicCodeGeneratorX86_64::VisitMathMinFloatFloat(HInvoke* invoke) { - GenMinMaxFP( - invoke->GetLocations(), /* is_min */ true, /* is_double */ false, GetAssembler(), codegen_); -} - -void IntrinsicLocationsBuilderX86_64::VisitMathMaxDoubleDouble(HInvoke* invoke) { - CreateFPFPToFP(allocator_, invoke); -} - -void IntrinsicCodeGeneratorX86_64::VisitMathMaxDoubleDouble(HInvoke* invoke) { - GenMinMaxFP( - invoke->GetLocations(), /* is_min */ false, /* is_double */ true, GetAssembler(), codegen_); -} - -void IntrinsicLocationsBuilderX86_64::VisitMathMaxFloatFloat(HInvoke* invoke) { - CreateFPFPToFP(allocator_, invoke); -} - -void IntrinsicCodeGeneratorX86_64::VisitMathMaxFloatFloat(HInvoke* invoke) { - GenMinMaxFP( - invoke->GetLocations(), /* is_min */ false, /* is_double */ false, GetAssembler(), codegen_); -} - -static void GenMinMax(LocationSummary* locations, bool is_min, bool is_long, - X86_64Assembler* assembler) { - Location op1_loc = locations->InAt(0); - Location op2_loc = locations->InAt(1); - - // Shortcut for same input locations. - if (op1_loc.Equals(op2_loc)) { - // Can return immediately, as op1_loc == out_loc. - // Note: if we ever support separate registers, e.g., output into memory, we need to check for - // a copy here. - DCHECK(locations->Out().Equals(op1_loc)); - return; - } - - CpuRegister out = locations->Out().AsRegister<CpuRegister>(); - CpuRegister op2 = op2_loc.AsRegister<CpuRegister>(); - - // (out := op1) - // out <=? op2 - // if out is min jmp done - // out := op2 - // done: - - if (is_long) { - __ cmpq(out, op2); - } else { - __ cmpl(out, op2); - } - - __ cmov(is_min ? Condition::kGreater : Condition::kLess, out, op2, is_long); -} - -static void CreateIntIntToIntLocations(ArenaAllocator* allocator, HInvoke* invoke) { - LocationSummary* locations = - new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); - locations->SetInAt(0, Location::RequiresRegister()); - locations->SetInAt(1, Location::RequiresRegister()); - locations->SetOut(Location::SameAsFirstInput()); -} - -void IntrinsicLocationsBuilderX86_64::VisitMathMinIntInt(HInvoke* invoke) { - CreateIntIntToIntLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorX86_64::VisitMathMinIntInt(HInvoke* invoke) { - GenMinMax(invoke->GetLocations(), /* is_min */ true, /* is_long */ false, GetAssembler()); -} - -void IntrinsicLocationsBuilderX86_64::VisitMathMinLongLong(HInvoke* invoke) { - CreateIntIntToIntLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorX86_64::VisitMathMinLongLong(HInvoke* invoke) { - GenMinMax(invoke->GetLocations(), /* is_min */ true, /* is_long */ true, GetAssembler()); -} - -void IntrinsicLocationsBuilderX86_64::VisitMathMaxIntInt(HInvoke* invoke) { - CreateIntIntToIntLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorX86_64::VisitMathMaxIntInt(HInvoke* invoke) { - GenMinMax(invoke->GetLocations(), /* is_min */ false, /* is_long */ false, GetAssembler()); -} - -void IntrinsicLocationsBuilderX86_64::VisitMathMaxLongLong(HInvoke* invoke) { - CreateIntIntToIntLocations(allocator_, invoke); -} - -void IntrinsicCodeGeneratorX86_64::VisitMathMaxLongLong(HInvoke* invoke) { - GenMinMax(invoke->GetLocations(), /* is_min */ false, /* is_long */ true, GetAssembler()); -} - static void CreateFPToFPLocations(ArenaAllocator* allocator, HInvoke* invoke) { LocationSummary* locations = new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); diff --git a/compiler/optimizing/loop_analysis.cc b/compiler/optimizing/loop_analysis.cc new file mode 100644 index 00000000000..a0760eff691 --- /dev/null +++ b/compiler/optimizing/loop_analysis.cc @@ -0,0 +1,141 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "loop_analysis.h" + +#include "base/bit_vector-inl.h" + +namespace art { + +void LoopAnalysis::CalculateLoopBasicProperties(HLoopInformation* loop_info, + LoopAnalysisInfo* analysis_results) { + for (HBlocksInLoopIterator block_it(*loop_info); + !block_it.Done(); + block_it.Advance()) { + HBasicBlock* block = block_it.Current(); + + for (HBasicBlock* successor : block->GetSuccessors()) { + if (!loop_info->Contains(*successor)) { + analysis_results->exits_num_++; + } + } + + for (HInstructionIterator it(block->GetInstructions()); !it.Done(); it.Advance()) { + HInstruction* instruction = it.Current(); + if (MakesScalarPeelingUnrollingNonBeneficial(instruction)) { + analysis_results->has_instructions_preventing_scalar_peeling_ = true; + analysis_results->has_instructions_preventing_scalar_unrolling_ = true; + } + analysis_results->instr_num_++; + } + analysis_results->bb_num_++; + } +} + +bool LoopAnalysis::HasLoopAtLeastOneInvariantExit(HLoopInformation* loop_info) { + HGraph* graph = loop_info->GetHeader()->GetGraph(); + for (uint32_t block_id : loop_info->GetBlocks().Indexes()) { + HBasicBlock* block = graph->GetBlocks()[block_id]; + DCHECK(block != nullptr); + if (block->EndsWithIf()) { + HIf* hif = block->GetLastInstruction()->AsIf(); + HInstruction* input = hif->InputAt(0); + if (IsLoopExit(loop_info, hif) && !loop_info->Contains(*input->GetBlock())) { + return true; + } + } + } + return false; +} + +class Arm64LoopHelper : public ArchDefaultLoopHelper { + public: + // Scalar loop unrolling parameters and heuristics. + // + // Maximum possible unrolling factor. + static constexpr uint32_t kArm64ScalarMaxUnrollFactor = 2; + // Loop's maximum instruction count. Loops with higher count will not be peeled/unrolled. + static constexpr uint32_t kArm64ScalarHeuristicMaxBodySizeInstr = 40; + // Loop's maximum basic block count. Loops with higher count will not be peeled/unrolled. + static constexpr uint32_t kArm64ScalarHeuristicMaxBodySizeBlocks = 8; + + // SIMD loop unrolling parameters and heuristics. + // + // Maximum possible unrolling factor. + static constexpr uint32_t kArm64SimdMaxUnrollFactor = 8; + // Loop's maximum instruction count. Loops with higher count will not be unrolled. + static constexpr uint32_t kArm64SimdHeuristicMaxBodySizeInstr = 50; + + bool IsLoopTooBigForScalarPeelingUnrolling(LoopAnalysisInfo* loop_analysis_info) const OVERRIDE { + size_t instr_num = loop_analysis_info->GetNumberOfInstructions(); + size_t bb_num = loop_analysis_info->GetNumberOfBasicBlocks(); + return (instr_num >= kArm64ScalarHeuristicMaxBodySizeInstr || + bb_num >= kArm64ScalarHeuristicMaxBodySizeBlocks); + } + + uint32_t GetScalarUnrollingFactor(HLoopInformation* loop_info ATTRIBUTE_UNUSED, + uint64_t trip_count) const OVERRIDE { + uint32_t desired_unrolling_factor = kArm64ScalarMaxUnrollFactor; + if (trip_count < desired_unrolling_factor || trip_count % desired_unrolling_factor != 0) { + return kNoUnrollingFactor; + } + + return desired_unrolling_factor; + } + + bool IsLoopPeelingEnabled() const OVERRIDE { return true; } + + uint32_t GetSIMDUnrollingFactor(HBasicBlock* block, + int64_t trip_count, + uint32_t max_peel, + uint32_t vector_length) const OVERRIDE { + // Don't unroll with insufficient iterations. + // TODO: Unroll loops with unknown trip count. + DCHECK_NE(vector_length, 0u); + if (trip_count < (2 * vector_length + max_peel)) { + return kNoUnrollingFactor; + } + // Don't unroll for large loop body size. + uint32_t instruction_count = block->GetInstructions().CountSize(); + if (instruction_count >= kArm64SimdHeuristicMaxBodySizeInstr) { + return kNoUnrollingFactor; + } + // Find a beneficial unroll factor with the following restrictions: + // - At least one iteration of the transformed loop should be executed. + // - The loop body shouldn't be "too big" (heuristic). + + uint32_t uf1 = kArm64SimdHeuristicMaxBodySizeInstr / instruction_count; + uint32_t uf2 = (trip_count - max_peel) / vector_length; + uint32_t unroll_factor = + TruncToPowerOfTwo(std::min({uf1, uf2, kArm64SimdMaxUnrollFactor})); + DCHECK_GE(unroll_factor, 1u); + return unroll_factor; + } +}; + +ArchDefaultLoopHelper* ArchDefaultLoopHelper::Create(InstructionSet isa, + ArenaAllocator* allocator) { + switch (isa) { + case InstructionSet::kArm64: { + return new (allocator) Arm64LoopHelper; + } + default: { + return new (allocator) ArchDefaultLoopHelper; + } + } +} + +} // namespace art diff --git a/compiler/optimizing/loop_analysis.h b/compiler/optimizing/loop_analysis.h new file mode 100644 index 00000000000..ece98581367 --- /dev/null +++ b/compiler/optimizing/loop_analysis.h @@ -0,0 +1,164 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_COMPILER_OPTIMIZING_LOOP_ANALYSIS_H_ +#define ART_COMPILER_OPTIMIZING_LOOP_ANALYSIS_H_ + +#include "nodes.h" + +namespace art { + +class LoopAnalysis; + +// No loop unrolling factor (just one copy of the loop-body). +static constexpr uint32_t kNoUnrollingFactor = 1; + +// Class to hold cached information on properties of the loop. +class LoopAnalysisInfo : public ValueObject { + public: + explicit LoopAnalysisInfo(HLoopInformation* loop_info) + : bb_num_(0), + instr_num_(0), + exits_num_(0), + has_instructions_preventing_scalar_peeling_(false), + has_instructions_preventing_scalar_unrolling_(false), + loop_info_(loop_info) {} + + size_t GetNumberOfBasicBlocks() const { return bb_num_; } + size_t GetNumberOfInstructions() const { return instr_num_; } + size_t GetNumberOfExits() const { return exits_num_; } + + bool HasInstructionsPreventingScalarPeeling() const { + return has_instructions_preventing_scalar_peeling_; + } + + bool HasInstructionsPreventingScalarUnrolling() const { + return has_instructions_preventing_scalar_unrolling_; + } + + const HLoopInformation* GetLoopInfo() const { return loop_info_; } + + private: + // Number of basic blocks in the loop body. + size_t bb_num_; + // Number of instructions in the loop body. + size_t instr_num_; + // Number of loop's exits. + size_t exits_num_; + // Whether the loop has instructions which make scalar loop peeling non-beneficial. + bool has_instructions_preventing_scalar_peeling_; + // Whether the loop has instructions which make scalar loop unrolling non-beneficial. + bool has_instructions_preventing_scalar_unrolling_; + + // Corresponding HLoopInformation. + const HLoopInformation* loop_info_; + + friend class LoopAnalysis; +}; + +// Placeholder class for methods and routines used to analyse loops, calculate loop properties +// and characteristics. +class LoopAnalysis : public ValueObject { + public: + // Calculates loops basic properties like body size, exits number, etc. and fills + // 'analysis_results' with this information. + static void CalculateLoopBasicProperties(HLoopInformation* loop_info, + LoopAnalysisInfo* analysis_results); + + // Returns whether the loop has at least one loop invariant exit. + static bool HasLoopAtLeastOneInvariantExit(HLoopInformation* loop_info); + + // Returns whether HIf's true or false successor is outside the specified loop. + // + // Prerequisite: HIf must be in the specified loop. + static bool IsLoopExit(HLoopInformation* loop_info, const HIf* hif) { + DCHECK(loop_info->Contains(*hif->GetBlock())); + HBasicBlock* true_succ = hif->IfTrueSuccessor(); + HBasicBlock* false_succ = hif->IfFalseSuccessor(); + return (!loop_info->Contains(*true_succ) || !loop_info->Contains(*false_succ)); + } + + private: + // Returns whether an instruction makes scalar loop peeling/unrolling non-beneficial. + // + // If in the loop body we have a dex/runtime call then its contribution to the whole + // loop performance will probably prevail. So peeling/unrolling optimization will not bring + // any noticeable performance improvement. It will increase the code size. + static bool MakesScalarPeelingUnrollingNonBeneficial(HInstruction* instruction) { + return (instruction->IsNewArray() || + instruction->IsNewInstance() || + instruction->IsUnresolvedInstanceFieldGet() || + instruction->IsUnresolvedInstanceFieldSet() || + instruction->IsUnresolvedStaticFieldGet() || + instruction->IsUnresolvedStaticFieldSet() || + // TODO: Support loops with intrinsified invokes. + instruction->IsInvoke() || + // TODO: Support loops with ClinitChecks. + instruction->IsClinitCheck()); + } +}; + +// +// Helper class which holds target-dependent methods and constants needed for loop optimizations. +// +// To support peeling/unrolling for a new architecture one needs to create new helper class, +// inherit it from this and add implementation for the following methods. +// +class ArchDefaultLoopHelper : public ArenaObject<kArenaAllocOptimization> { + public: + virtual ~ArchDefaultLoopHelper() {} + + // Creates an instance of specialised helper for the target or default helper if the target + // doesn't support loop peeling and unrolling. + static ArchDefaultLoopHelper* Create(InstructionSet isa, ArenaAllocator* allocator); + + // Returns whether the loop is too big for loop peeling/unrolling by checking its total number of + // basic blocks and instructions. + // + // If the loop body has too many instructions then peeling/unrolling optimization will not bring + // any noticeable performance improvement however will increase the code size. + // + // Returns 'true' by default, should be overridden by particular target loop helper. + virtual bool IsLoopTooBigForScalarPeelingUnrolling( + LoopAnalysisInfo* loop_analysis_info ATTRIBUTE_UNUSED) const { return true; } + + // Returns optimal scalar unrolling factor for the loop. + // + // Returns kNoUnrollingFactor by default, should be overridden by particular target loop helper. + virtual uint32_t GetScalarUnrollingFactor(HLoopInformation* loop_info ATTRIBUTE_UNUSED, + uint64_t trip_count ATTRIBUTE_UNUSED) const { + return kNoUnrollingFactor; + } + + // Returns whether scalar loop peeling is enabled, + // + // Returns 'false' by default, should be overridden by particular target loop helper. + virtual bool IsLoopPeelingEnabled() const { return false; } + + // Returns optimal SIMD unrolling factor for the loop. + // + // Returns kNoUnrollingFactor by default, should be overridden by particular target loop helper. + virtual uint32_t GetSIMDUnrollingFactor(HBasicBlock* block ATTRIBUTE_UNUSED, + int64_t trip_count ATTRIBUTE_UNUSED, + uint32_t max_peel ATTRIBUTE_UNUSED, + uint32_t vector_length ATTRIBUTE_UNUSED) const { + return kNoUnrollingFactor; + } +}; + +} // namespace art + +#endif // ART_COMPILER_OPTIMIZING_LOOP_ANALYSIS_H_ diff --git a/compiler/optimizing/loop_optimization.cc b/compiler/optimizing/loop_optimization.cc index 899496328eb..1462404932e 100644 --- a/compiler/optimizing/loop_optimization.cc +++ b/compiler/optimizing/loop_optimization.cc @@ -33,8 +33,8 @@ namespace art { // Enables vectorization (SIMDization) in the loop optimizer. static constexpr bool kEnableVectorization = true; -// No loop unrolling factor (just one copy of the loop-body). -static constexpr uint32_t kNoUnrollingFactor = 1; +// Enables scalar loop unrolling in the loop optimizer. +static constexpr bool kEnableScalarPeelingUnrolling = false; // // Static helpers. @@ -153,6 +153,18 @@ static bool IsSignExtensionAndGet(HInstruction* instruction, return false; } } + // A MIN-MAX on narrower operands qualifies as well + // (returning the operator itself). + if (instruction->IsMin() || instruction->IsMax()) { + HBinaryOperation* min_max = instruction->AsBinaryOperation(); + DCHECK(min_max->GetType() == DataType::Type::kInt32 || + min_max->GetType() == DataType::Type::kInt64); + if (IsSignExtensionAndGet(min_max->InputAt(0), type, operand) && + IsSignExtensionAndGet(min_max->InputAt(1), type, operand)) { + *operand = min_max; + return true; + } + } return false; } @@ -216,6 +228,18 @@ static bool IsZeroExtensionAndGet(HInstruction* instruction, return false; } } + // A MIN-MAX on narrower operands qualifies as well + // (returning the operator itself). + if (instruction->IsMin() || instruction->IsMax()) { + HBinaryOperation* min_max = instruction->AsBinaryOperation(); + DCHECK(min_max->GetType() == DataType::Type::kInt32 || + min_max->GetType() == DataType::Type::kInt64); + if (IsZeroExtensionAndGet(min_max->InputAt(0), type, operand) && + IsZeroExtensionAndGet(min_max->InputAt(1), type, operand)) { + *operand = min_max; + return true; + } + } return false; } @@ -227,6 +251,7 @@ static bool IsNarrowerOperands(HInstruction* a, /*out*/ HInstruction** r, /*out*/ HInstruction** s, /*out*/ bool* is_unsigned) { + DCHECK(a != nullptr && b != nullptr); // Look for a matching sign extension. DataType::Type stype = HVecOperation::ToSignedType(type); if (IsSignExtensionAndGet(a, stype, r) && IsSignExtensionAndGet(b, stype, s)) { @@ -247,6 +272,7 @@ static bool IsNarrowerOperand(HInstruction* a, DataType::Type type, /*out*/ HInstruction** r, /*out*/ bool* is_unsigned) { + DCHECK(a != nullptr); // Look for a matching sign extension. DataType::Type stype = HVecOperation::ToSignedType(type); if (IsSignExtensionAndGet(a, stype, r)) { @@ -270,20 +296,28 @@ static uint32_t GetOtherVL(DataType::Type other_type, DataType::Type vector_type return vl >> (DataType::SizeShift(other_type) - DataType::SizeShift(vector_type)); } -// Detect up to two instructions a and b, and an acccumulated constant c. -static bool IsAddConstHelper(HInstruction* instruction, - /*out*/ HInstruction** a, - /*out*/ HInstruction** b, - /*out*/ int64_t* c, - int32_t depth) { - static constexpr int32_t kMaxDepth = 8; // don't search too deep +// Detect up to two added operands a and b and an acccumulated constant c. +static bool IsAddConst(HInstruction* instruction, + /*out*/ HInstruction** a, + /*out*/ HInstruction** b, + /*out*/ int64_t* c, + int32_t depth = 8) { // don't search too deep int64_t value = 0; + // Enter add/sub while still within reasonable depth. + if (depth > 0) { + if (instruction->IsAdd()) { + return IsAddConst(instruction->InputAt(0), a, b, c, depth - 1) && + IsAddConst(instruction->InputAt(1), a, b, c, depth - 1); + } else if (instruction->IsSub() && + IsInt64AndGet(instruction->InputAt(1), &value)) { + *c -= value; + return IsAddConst(instruction->InputAt(0), a, b, c, depth - 1); + } + } + // Otherwise, deal with leaf nodes. if (IsInt64AndGet(instruction, &value)) { *c += value; return true; - } else if (instruction->IsAdd() && depth <= kMaxDepth) { - return IsAddConstHelper(instruction->InputAt(0), a, b, c, depth + 1) && - IsAddConstHelper(instruction->InputAt(1), a, b, c, depth + 1); } else if (*a == nullptr) { *a = instruction; return true; @@ -291,72 +325,170 @@ static bool IsAddConstHelper(HInstruction* instruction, *b = instruction; return true; } - return false; // too many non-const operands + return false; // too many operands } -// Detect a + b + c for an optional constant c. -static bool IsAddConst(HInstruction* instruction, - /*out*/ HInstruction** a, - /*out*/ HInstruction** b, - /*out*/ int64_t* c) { - if (instruction->IsAdd()) { - // Try to find a + b and accumulated c. - if (IsAddConstHelper(instruction->InputAt(0), a, b, c, /*depth*/ 0) && - IsAddConstHelper(instruction->InputAt(1), a, b, c, /*depth*/ 0) && - *b != nullptr) { - return true; +// Detect a + b + c with optional constant c. +static bool IsAddConst2(HGraph* graph, + HInstruction* instruction, + /*out*/ HInstruction** a, + /*out*/ HInstruction** b, + /*out*/ int64_t* c) { + if (IsAddConst(instruction, a, b, c) && *a != nullptr) { + if (*b == nullptr) { + // Constant is usually already present, unless accumulated. + *b = graph->GetConstant(instruction->GetType(), (*c)); + *c = 0; } - // Found a + b. + return true; + } + return false; +} + +// Detect a direct a - b or a hidden a - (-c). +static bool IsSubConst2(HGraph* graph, + HInstruction* instruction, + /*out*/ HInstruction** a, + /*out*/ HInstruction** b) { + int64_t c = 0; + if (instruction->IsSub()) { *a = instruction->InputAt(0); *b = instruction->InputAt(1); - *c = 0; + return true; + } else if (IsAddConst(instruction, a, b, &c) && *a != nullptr && *b == nullptr) { + // Constant for the hidden subtraction. + *b = graph->GetConstant(instruction->GetType(), -c); return true; } return false; } -// Detect a + c for constant c. -static bool IsAddConst(HInstruction* instruction, - /*out*/ HInstruction** a, - /*out*/ int64_t* c) { - if (instruction->IsAdd()) { - if (IsInt64AndGet(instruction->InputAt(0), c)) { - *a = instruction->InputAt(1); - return true; - } else if (IsInt64AndGet(instruction->InputAt(1), c)) { - *a = instruction->InputAt(0); - return true; +// Detect clipped [lo, hi] range for nested MIN-MAX operations on a clippee, +// such as MIN(hi, MAX(lo, clippee)) for an arbitrary clippee expression. +// Example: MIN(10, MIN(20, MAX(0, x))) yields [0, 10] with clippee x. +static HInstruction* FindClippee(HInstruction* instruction, + /*out*/ int64_t* lo, + /*out*/ int64_t* hi) { + // Iterate into MIN(.., c)-MAX(.., c) expressions and 'tighten' the range [lo, hi]. + while (instruction->IsMin() || instruction->IsMax()) { + HBinaryOperation* min_max = instruction->AsBinaryOperation(); + DCHECK(min_max->GetType() == DataType::Type::kInt32 || + min_max->GetType() == DataType::Type::kInt64); + // Process the constant. + HConstant* right = min_max->GetConstantRight(); + if (right == nullptr) { + break; + } else if (instruction->IsMin()) { + *hi = std::min(*hi, Int64FromConstant(right)); + } else { + *lo = std::max(*lo, Int64FromConstant(right)); } + instruction = min_max->GetLeastConstantLeft(); + } + // Iteration ends in any other expression (possibly MIN/MAX without constant). + // This leaf expression is the clippee with range [lo, hi]. + return instruction; +} + +// Set value range for type (or fail). +static bool CanSetRange(DataType::Type type, + /*out*/ int64_t* uhi, + /*out*/ int64_t* slo, + /*out*/ int64_t* shi) { + if (DataType::Size(type) == 1) { + *uhi = std::numeric_limits<uint8_t>::max(); + *slo = std::numeric_limits<int8_t>::min(); + *shi = std::numeric_limits<int8_t>::max(); + return true; + } else if (DataType::Size(type) == 2) { + *uhi = std::numeric_limits<uint16_t>::max(); + *slo = std::numeric_limits<int16_t>::min(); + *shi = std::numeric_limits<int16_t>::max(); + return true; } return false; } +// Accept various saturated addition forms. +static bool IsSaturatedAdd(HInstruction* a, + HInstruction* b, + DataType::Type type, + int64_t lo, + int64_t hi, + bool is_unsigned) { + int64_t ulo = 0, uhi = 0, slo = 0, shi = 0; + if (!CanSetRange(type, &uhi, &slo, &shi)) { + return false; + } + // Tighten the range for signed single clipping on constant. + if (!is_unsigned) { + int64_t c = 0; + if (IsInt64AndGet(a, &c) || IsInt64AndGet(b, &c)) { + // For c in proper range and narrower operand r: + // MIN(r + c, 127) c > 0 + // or MAX(r + c, -128) c < 0 (and possibly redundant bound). + if (0 < c && c <= shi && hi == shi) { + if (lo <= (slo + c)) { + return true; + } + } else if (slo <= c && c < 0 && lo == slo) { + if (hi >= (shi + c)) { + return true; + } + } + } + } + // Detect for narrower operands r and s: + // MIN(r + s, 255) => SAT_ADD_unsigned + // MAX(MIN(r + s, 127), -128) => SAT_ADD_signed. + return is_unsigned ? (lo <= ulo && hi == uhi) : (lo == slo && hi == shi); +} + +// Accept various saturated subtraction forms. +static bool IsSaturatedSub(HInstruction* a, + DataType::Type type, + int64_t lo, + int64_t hi, + bool is_unsigned) { + int64_t ulo = 0, uhi = 0, slo = 0, shi = 0; + if (!CanSetRange(type, &uhi, &slo, &shi)) { + return false; + } + // Tighten the range for signed single clipping on constant. + if (!is_unsigned) { + int64_t c = 0; + if (IsInt64AndGet(a, /*out*/ &c)) { + // For c in proper range and narrower operand r: + // MIN(c - r, 127) c > 0 + // or MAX(c - r, -128) c < 0 (and possibly redundant bound). + if (0 < c && c <= shi && hi == shi) { + if (lo <= (c - shi)) { + return true; + } + } else if (slo <= c && c < 0 && lo == slo) { + if (hi >= (c - slo)) { + return true; + } + } + } + } + // Detect for narrower operands r and s: + // MAX(r - s, 0) => SAT_SUB_unsigned + // MIN(MAX(r - s, -128), 127) => SAT_ADD_signed. + return is_unsigned ? (lo == ulo && hi >= uhi) : (lo == slo && hi == shi); +} + // Detect reductions of the following forms, // x = x_phi + .. // x = x_phi - .. -// x = max(x_phi, ..) // x = min(x_phi, ..) +// x = max(x_phi, ..) static bool HasReductionFormat(HInstruction* reduction, HInstruction* phi) { - if (reduction->IsAdd()) { + if (reduction->IsAdd() || reduction->IsMin() || reduction->IsMax()) { return (reduction->InputAt(0) == phi && reduction->InputAt(1) != phi) || (reduction->InputAt(0) != phi && reduction->InputAt(1) == phi); } else if (reduction->IsSub()) { return (reduction->InputAt(0) == phi && reduction->InputAt(1) != phi); - } else if (reduction->IsInvokeStaticOrDirect()) { - switch (reduction->AsInvokeStaticOrDirect()->GetIntrinsic()) { - case Intrinsics::kMathMinIntInt: - case Intrinsics::kMathMinLongLong: - case Intrinsics::kMathMinFloatFloat: - case Intrinsics::kMathMinDoubleDouble: - case Intrinsics::kMathMaxIntInt: - case Intrinsics::kMathMaxLongLong: - case Intrinsics::kMathMaxFloatFloat: - case Intrinsics::kMathMaxDoubleDouble: - return (reduction->InputAt(0) == phi && reduction->InputAt(1) != phi) || - (reduction->InputAt(0) != phi && reduction->InputAt(1) == phi); - default: - return false; - } } return false; } @@ -401,6 +533,43 @@ static bool CheckInductionSetFullyRemoved(ScopedArenaSet<HInstruction*>* iset) { return true; } +// Tries to statically evaluate condition of the specified "HIf" for other condition checks. +static void TryToEvaluateIfCondition(HIf* instruction, HGraph* graph) { + HInstruction* cond = instruction->InputAt(0); + + // If a condition 'cond' is evaluated in an HIf instruction then in the successors of the + // IF_BLOCK we statically know the value of the condition 'cond' (TRUE in TRUE_SUCC, FALSE in + // FALSE_SUCC). Using that we can replace another evaluation (use) EVAL of the same 'cond' + // with TRUE value (FALSE value) if every path from the ENTRY_BLOCK to EVAL_BLOCK contains the + // edge HIF_BLOCK->TRUE_SUCC (HIF_BLOCK->FALSE_SUCC). + // if (cond) { if(cond) { + // if (cond) {} if (1) {} + // } else { =======> } else { + // if (cond) {} if (0) {} + // } } + if (!cond->IsConstant()) { + HBasicBlock* true_succ = instruction->IfTrueSuccessor(); + HBasicBlock* false_succ = instruction->IfFalseSuccessor(); + + DCHECK_EQ(true_succ->GetPredecessors().size(), 1u); + DCHECK_EQ(false_succ->GetPredecessors().size(), 1u); + + const HUseList<HInstruction*>& uses = cond->GetUses(); + for (auto it = uses.begin(), end = uses.end(); it != end; /* ++it below */) { + HInstruction* user = it->GetUser(); + size_t index = it->GetIndex(); + HBasicBlock* user_block = user->GetBlock(); + // Increment `it` now because `*it` may disappear thanks to user->ReplaceInput(). + ++it; + if (true_succ->Dominates(user_block)) { + user->ReplaceInput(graph->GetIntConstant(1), index); + } else if (false_succ->Dominates(user_block)) { + user->ReplaceInput(graph->GetIntConstant(0), index); + } + } + } +} + // // Public methods. // @@ -432,7 +601,11 @@ HLoopOptimization::HLoopOptimization(HGraph* graph, vector_preheader_(nullptr), vector_header_(nullptr), vector_body_(nullptr), - vector_index_(nullptr) { + vector_index_(nullptr), + arch_loop_helper_(ArchDefaultLoopHelper::Create(compiler_driver_ != nullptr + ? compiler_driver_->GetInstructionSet() + : InstructionSet::kNone, + global_allocator_)) { } void HLoopOptimization::Run() { @@ -643,7 +816,7 @@ void HLoopOptimization::SimplifyBlocks(LoopNode* node) { } } -bool HLoopOptimization::OptimizeInnerLoop(LoopNode* node) { +bool HLoopOptimization::TryOptimizeInnerLoopFinite(LoopNode* node) { HBasicBlock* header = node->loop_info->GetHeader(); HBasicBlock* preheader = node->loop_info->GetPreHeader(); // Ensure loop header logic is finite. @@ -713,6 +886,103 @@ bool HLoopOptimization::OptimizeInnerLoop(LoopNode* node) { return false; } +bool HLoopOptimization::OptimizeInnerLoop(LoopNode* node) { + return TryOptimizeInnerLoopFinite(node) || + TryPeelingForLoopInvariantExitsElimination(node) || + TryUnrollingForBranchPenaltyReduction(node); +} + + + +// +// Loop unrolling: generic part methods. +// + +bool HLoopOptimization::TryUnrollingForBranchPenaltyReduction(LoopNode* node) { + // Don't run peeling/unrolling if compiler_driver_ is nullptr (i.e., running under tests) + // as InstructionSet is needed. + if (!kEnableScalarPeelingUnrolling || compiler_driver_ == nullptr) { + return false; + } + + HLoopInformation* loop_info = node->loop_info; + int64_t trip_count = 0; + // Only unroll loops with a known tripcount. + if (!induction_range_.HasKnownTripCount(loop_info, &trip_count)) { + return false; + } + + uint32_t unrolling_factor = arch_loop_helper_->GetScalarUnrollingFactor(loop_info, trip_count); + if (unrolling_factor == kNoUnrollingFactor) { + return false; + } + + LoopAnalysisInfo loop_analysis_info(loop_info); + LoopAnalysis::CalculateLoopBasicProperties(loop_info, &loop_analysis_info); + + // Check "IsLoopClonable" last as it can be time-consuming. + if (arch_loop_helper_->IsLoopTooBigForScalarPeelingUnrolling(&loop_analysis_info) || + (loop_analysis_info.GetNumberOfExits() > 1) || + loop_analysis_info.HasInstructionsPreventingScalarUnrolling() || + !PeelUnrollHelper::IsLoopClonable(loop_info)) { + return false; + } + + // TODO: support other unrolling factors. + DCHECK_EQ(unrolling_factor, 2u); + + // Perform unrolling. + PeelUnrollSimpleHelper helper(loop_info); + helper.DoUnrolling(); + + // Remove the redundant loop check after unrolling. + HIf* copy_hif = + helper.GetBasicBlockMap()->Get(loop_info->GetHeader())->GetLastInstruction()->AsIf(); + int32_t constant = loop_info->Contains(*copy_hif->IfTrueSuccessor()) ? 1 : 0; + copy_hif->ReplaceInput(graph_->GetIntConstant(constant), 0u); + + return true; +} + +bool HLoopOptimization::TryPeelingForLoopInvariantExitsElimination(LoopNode* node) { + // Don't run peeling/unrolling if compiler_driver_ is nullptr (i.e., running under tests) + // as InstructionSet is needed. + if (!kEnableScalarPeelingUnrolling || compiler_driver_ == nullptr) { + return false; + } + + HLoopInformation* loop_info = node->loop_info; + // Check 'IsLoopClonable' the last as it might be time-consuming. + if (!arch_loop_helper_->IsLoopPeelingEnabled()) { + return false; + } + + LoopAnalysisInfo loop_analysis_info(loop_info); + LoopAnalysis::CalculateLoopBasicProperties(loop_info, &loop_analysis_info); + + // Check "IsLoopClonable" last as it can be time-consuming. + if (arch_loop_helper_->IsLoopTooBigForScalarPeelingUnrolling(&loop_analysis_info) || + loop_analysis_info.HasInstructionsPreventingScalarPeeling() || + !LoopAnalysis::HasLoopAtLeastOneInvariantExit(loop_info) || + !PeelUnrollHelper::IsLoopClonable(loop_info)) { + return false; + } + + // Perform peeling. + PeelUnrollSimpleHelper helper(loop_info); + helper.DoPeeling(); + + const SuperblockCloner::HInstructionMap* hir_map = helper.GetInstructionMap(); + for (auto entry : *hir_map) { + HInstruction* copy = entry.second; + if (copy->IsIf()) { + TryToEvaluateIfCondition(copy->AsIf(), graph_); + } + } + + return true; +} + // // Loop vectorization. The implementation is based on the book by Aart J.C. Bik: // "The Software Vectorization Handbook. Applying Multimedia Extensions for Maximum Performance." @@ -843,7 +1113,8 @@ void HLoopOptimization::Vectorize(LoopNode* node, HBasicBlock* preheader = node->loop_info->GetPreHeader(); // Pick a loop unrolling factor for the vector loop. - uint32_t unroll = GetUnrollingFactor(block, trip_count); + uint32_t unroll = arch_loop_helper_->GetSIMDUnrollingFactor( + block, trip_count, MaxNumberPeeled(), vector_length_); uint32_t chunk = vector_length_ * unroll; DCHECK(trip_count == 0 || (trip_count >= MaxNumberPeeled() + chunk)); @@ -1082,6 +1353,11 @@ bool HLoopOptimization::VectorizeDef(LoopNode* node, HInstruction* index = instruction->InputAt(1); HInstruction* value = instruction->InputAt(2); HInstruction* offset = nullptr; + // For narrow types, explicit type conversion may have been + // optimized way, so set the no hi bits restriction here. + if (DataType::Size(type) <= 2) { + restrictions |= kNoHiBits; + } if (TrySetVectorType(type, &restrictions) && node->loop_info->IsDefinedOutOfTheLoop(base) && induction_range_.IsUnitStride(instruction, index, graph_, &offset) && @@ -1124,7 +1400,6 @@ bool HLoopOptimization::VectorizeDef(LoopNode* node, return !IsUsedOutsideLoop(node->loop_info, instruction) && !instruction->DoesAnyWrite(); } -// TODO: saturation arithmetic. bool HLoopOptimization::VectorizeUse(LoopNode* node, HInstruction* instruction, bool generate_code, @@ -1297,80 +1572,62 @@ bool HLoopOptimization::VectorizeUse(LoopNode* node, return true; } } - } else if (instruction->IsInvokeStaticOrDirect()) { - // Accept particular intrinsics. - HInvokeStaticOrDirect* invoke = instruction->AsInvokeStaticOrDirect(); - switch (invoke->GetIntrinsic()) { - case Intrinsics::kMathAbsInt: - case Intrinsics::kMathAbsLong: - case Intrinsics::kMathAbsFloat: - case Intrinsics::kMathAbsDouble: { - // Deal with vector restrictions. - HInstruction* opa = instruction->InputAt(0); - HInstruction* r = opa; - bool is_unsigned = false; - if (HasVectorRestrictions(restrictions, kNoAbs)) { - return false; - } else if (HasVectorRestrictions(restrictions, kNoHiBits) && - (!IsNarrowerOperand(opa, type, &r, &is_unsigned) || is_unsigned)) { - return false; // reject, unless operand is sign-extension narrower - } - // Accept ABS(x) for vectorizable operand. - DCHECK(r != nullptr); - if (generate_code && vector_mode_ != kVector) { // de-idiom - r = opa; - } - if (VectorizeUse(node, r, generate_code, type, restrictions)) { - if (generate_code) { - GenerateVecOp(instruction, - vector_map_->Get(r), - nullptr, - HVecOperation::ToProperType(type, is_unsigned)); - } - return true; - } - return false; + } else if (instruction->IsAbs()) { + // Deal with vector restrictions. + HInstruction* opa = instruction->InputAt(0); + HInstruction* r = opa; + bool is_unsigned = false; + if (HasVectorRestrictions(restrictions, kNoAbs)) { + return false; + } else if (HasVectorRestrictions(restrictions, kNoHiBits) && + (!IsNarrowerOperand(opa, type, &r, &is_unsigned) || is_unsigned)) { + return false; // reject, unless operand is sign-extension narrower + } + // Accept ABS(x) for vectorizable operand. + DCHECK(r != nullptr); + if (generate_code && vector_mode_ != kVector) { // de-idiom + r = opa; + } + if (VectorizeUse(node, r, generate_code, type, restrictions)) { + if (generate_code) { + GenerateVecOp(instruction, + vector_map_->Get(r), + nullptr, + HVecOperation::ToProperType(type, is_unsigned)); } - case Intrinsics::kMathMinIntInt: - case Intrinsics::kMathMinLongLong: - case Intrinsics::kMathMinFloatFloat: - case Intrinsics::kMathMinDoubleDouble: - case Intrinsics::kMathMaxIntInt: - case Intrinsics::kMathMaxLongLong: - case Intrinsics::kMathMaxFloatFloat: - case Intrinsics::kMathMaxDoubleDouble: { - // Deal with vector restrictions. - HInstruction* opa = instruction->InputAt(0); - HInstruction* opb = instruction->InputAt(1); - HInstruction* r = opa; - HInstruction* s = opb; - bool is_unsigned = false; - if (HasVectorRestrictions(restrictions, kNoMinMax)) { - return false; - } else if (HasVectorRestrictions(restrictions, kNoHiBits) && - !IsNarrowerOperands(opa, opb, type, &r, &s, &is_unsigned)) { - return false; // reject, unless all operands are same-extension narrower - } - // Accept MIN/MAX(x, y) for vectorizable operands. - DCHECK(r != nullptr); - DCHECK(s != nullptr); - if (generate_code && vector_mode_ != kVector) { // de-idiom - r = opa; - s = opb; - } - if (VectorizeUse(node, r, generate_code, type, restrictions) && - VectorizeUse(node, s, generate_code, type, restrictions)) { - if (generate_code) { - GenerateVecOp( - instruction, vector_map_->Get(r), vector_map_->Get(s), type, is_unsigned); - } - return true; - } - return false; + return true; + } + } else if (instruction->IsMin() || instruction->IsMax()) { + // Recognize saturation arithmetic. + if (VectorizeSaturationIdiom(node, instruction, generate_code, type, restrictions)) { + return true; + } + // Deal with vector restrictions. + HInstruction* opa = instruction->InputAt(0); + HInstruction* opb = instruction->InputAt(1); + HInstruction* r = opa; + HInstruction* s = opb; + bool is_unsigned = false; + if (HasVectorRestrictions(restrictions, kNoMinMax)) { + return false; + } else if (HasVectorRestrictions(restrictions, kNoHiBits) && + !IsNarrowerOperands(opa, opb, type, &r, &s, &is_unsigned)) { + return false; // reject, unless all operands are same-extension narrower + } + // Accept MIN/MAX(x, y) for vectorizable operands. + DCHECK(r != nullptr && s != nullptr); + if (generate_code && vector_mode_ != kVector) { // de-idiom + r = opa; + s = opb; + } + if (VectorizeUse(node, r, generate_code, type, restrictions) && + VectorizeUse(node, s, generate_code, type, restrictions)) { + if (generate_code) { + GenerateVecOp( + instruction, vector_map_->Get(r), vector_map_->Get(s), type, is_unsigned); } - default: - return false; - } // switch + return true; + } } return false; } @@ -1475,11 +1732,11 @@ bool HLoopOptimization::TrySetVectorType(DataType::Type type, uint64_t* restrict case DataType::Type::kBool: case DataType::Type::kUint8: case DataType::Type::kInt8: - *restrictions |= kNoDiv; + *restrictions |= kNoDiv | kNoSaturation; return TrySetVectorLength(16); case DataType::Type::kUint16: case DataType::Type::kInt16: - *restrictions |= kNoDiv | kNoStringCharAt; + *restrictions |= kNoDiv | kNoSaturation | kNoStringCharAt; return TrySetVectorLength(8); case DataType::Type::kInt32: *restrictions |= kNoDiv; @@ -1504,11 +1761,11 @@ bool HLoopOptimization::TrySetVectorType(DataType::Type type, uint64_t* restrict case DataType::Type::kBool: case DataType::Type::kUint8: case DataType::Type::kInt8: - *restrictions |= kNoDiv; + *restrictions |= kNoDiv | kNoSaturation; return TrySetVectorLength(16); case DataType::Type::kUint16: case DataType::Type::kInt16: - *restrictions |= kNoDiv | kNoStringCharAt; + *restrictions |= kNoDiv | kNoSaturation | kNoStringCharAt; return TrySetVectorLength(8); case DataType::Type::kInt32: *restrictions |= kNoDiv; @@ -1811,83 +2068,29 @@ void HLoopOptimization::GenerateVecOp(HInstruction* org, GENERATE_VEC( new (global_allocator_) HVecUShr(global_allocator_, opa, opb, type, vector_length_, dex_pc), new (global_allocator_) HUShr(org_type, opa, opb, dex_pc)); - case HInstruction::kInvokeStaticOrDirect: { - HInvokeStaticOrDirect* invoke = org->AsInvokeStaticOrDirect(); - if (vector_mode_ == kVector) { - switch (invoke->GetIntrinsic()) { - case Intrinsics::kMathAbsInt: - case Intrinsics::kMathAbsLong: - case Intrinsics::kMathAbsFloat: - case Intrinsics::kMathAbsDouble: - DCHECK(opb == nullptr); - vector = new (global_allocator_) - HVecAbs(global_allocator_, opa, type, vector_length_, dex_pc); - break; - case Intrinsics::kMathMinIntInt: - case Intrinsics::kMathMinLongLong: - case Intrinsics::kMathMinFloatFloat: - case Intrinsics::kMathMinDoubleDouble: { - vector = new (global_allocator_) - HVecMin(global_allocator_, - opa, - opb, - HVecOperation::ToProperType(type, is_unsigned), - vector_length_, - dex_pc); - break; - } - case Intrinsics::kMathMaxIntInt: - case Intrinsics::kMathMaxLongLong: - case Intrinsics::kMathMaxFloatFloat: - case Intrinsics::kMathMaxDoubleDouble: { - vector = new (global_allocator_) - HVecMax(global_allocator_, - opa, - opb, - HVecOperation::ToProperType(type, is_unsigned), - vector_length_, - dex_pc); - break; - } - default: - LOG(FATAL) << "Unsupported SIMD intrinsic " << org->GetId(); - UNREACHABLE(); - } // switch invoke - } else { - // In scalar code, simply clone the method invoke, and replace its operands with the - // corresponding new scalar instructions in the loop. The instruction will get an - // environment while being inserted from the instruction map in original program order. - DCHECK(vector_mode_ == kSequential); - size_t num_args = invoke->GetNumberOfArguments(); - HInvokeStaticOrDirect* new_invoke = new (global_allocator_) HInvokeStaticOrDirect( - global_allocator_, - num_args, - invoke->GetType(), - invoke->GetDexPc(), - invoke->GetDexMethodIndex(), - invoke->GetResolvedMethod(), - invoke->GetDispatchInfo(), - invoke->GetInvokeType(), - invoke->GetTargetMethod(), - invoke->GetClinitCheckRequirement()); - HInputsRef inputs = invoke->GetInputs(); - size_t num_inputs = inputs.size(); - DCHECK_LE(num_args, num_inputs); - DCHECK_EQ(num_inputs, new_invoke->GetInputs().size()); // both invokes agree - for (size_t index = 0; index < num_inputs; ++index) { - HInstruction* new_input = index < num_args - ? vector_map_->Get(inputs[index]) - : inputs[index]; // beyond arguments: just pass through - new_invoke->SetArgumentAt(index, new_input); - } - new_invoke->SetIntrinsic(invoke->GetIntrinsic(), - kNeedsEnvironmentOrCache, - kNoSideEffects, - kNoThrow); - vector = new_invoke; - } - break; - } + case HInstruction::kMin: + GENERATE_VEC( + new (global_allocator_) HVecMin(global_allocator_, + opa, + opb, + HVecOperation::ToProperType(type, is_unsigned), + vector_length_, + dex_pc), + new (global_allocator_) HMin(org_type, opa, opb, dex_pc)); + case HInstruction::kMax: + GENERATE_VEC( + new (global_allocator_) HVecMax(global_allocator_, + opa, + opb, + HVecOperation::ToProperType(type, is_unsigned), + vector_length_, + dex_pc), + new (global_allocator_) HMax(org_type, opa, opb, dex_pc)); + case HInstruction::kAbs: + DCHECK(opb == nullptr); + GENERATE_VEC( + new (global_allocator_) HVecAbs(global_allocator_, opa, type, vector_length_, dex_pc), + new (global_allocator_) HAbs(org_type, opa, dex_pc)); default: break; } // switch @@ -1901,6 +2104,79 @@ void HLoopOptimization::GenerateVecOp(HInstruction* org, // Vectorization idioms. // +// Method recognizes single and double clipping saturation arithmetic. +bool HLoopOptimization::VectorizeSaturationIdiom(LoopNode* node, + HInstruction* instruction, + bool generate_code, + DataType::Type type, + uint64_t restrictions) { + // Deal with vector restrictions. + if (HasVectorRestrictions(restrictions, kNoSaturation)) { + return false; + } + // Restrict type (generalize if one day we generalize allowed MIN/MAX integral types). + if (instruction->GetType() != DataType::Type::kInt32 && + instruction->GetType() != DataType::Type::kInt64) { + return false; + } + // Clipped addition or subtraction on narrower operands? We will try both + // formats since, e.g., x+c can be interpreted as x+c and x-(-c), depending + // on what clipping values are used, to get most benefits. + int64_t lo = std::numeric_limits<int64_t>::min(); + int64_t hi = std::numeric_limits<int64_t>::max(); + HInstruction* clippee = FindClippee(instruction, &lo, &hi); + HInstruction* a = nullptr; + HInstruction* b = nullptr; + HInstruction* r = nullptr; + HInstruction* s = nullptr; + bool is_unsigned = false; + bool is_add = true; + int64_t c = 0; + // First try for saturated addition. + if (IsAddConst2(graph_, clippee, /*out*/ &a, /*out*/ &b, /*out*/ &c) && c == 0 && + IsNarrowerOperands(a, b, type, &r, &s, &is_unsigned) && + IsSaturatedAdd(r, s, type, lo, hi, is_unsigned)) { + is_add = true; + } else { + // Then try again for saturated subtraction. + a = b = r = s = nullptr; + if (IsSubConst2(graph_, clippee, /*out*/ &a, /*out*/ &b) && + IsNarrowerOperands(a, b, type, &r, &s, &is_unsigned) && + IsSaturatedSub(r, type, lo, hi, is_unsigned)) { + is_add = false; + } else { + return false; + } + } + // Accept saturation idiom for vectorizable operands. + DCHECK(r != nullptr && s != nullptr); + if (generate_code && vector_mode_ != kVector) { // de-idiom + r = instruction->InputAt(0); + s = instruction->InputAt(1); + restrictions &= ~(kNoHiBits | kNoMinMax); // allow narrow MIN/MAX in seq + } + if (VectorizeUse(node, r, generate_code, type, restrictions) && + VectorizeUse(node, s, generate_code, type, restrictions)) { + if (generate_code) { + if (vector_mode_ == kVector) { + DataType::Type vtype = HVecOperation::ToProperType(type, is_unsigned); + HInstruction* op1 = vector_map_->Get(r); + HInstruction* op2 = vector_map_->Get(s); + vector_map_->Put(instruction, is_add + ? reinterpret_cast<HInstruction*>(new (global_allocator_) HVecSaturationAdd( + global_allocator_, op1, op2, vtype, vector_length_, kNoDexPc)) + : reinterpret_cast<HInstruction*>(new (global_allocator_) HVecSaturationSub( + global_allocator_, op1, op2, vtype, vector_length_, kNoDexPc))); + MaybeRecordStat(stats_, MethodCompilationStat::kLoopVectorizedIdiom); + } else { + GenerateVecOp(instruction, vector_map_->Get(r), vector_map_->Get(s), type); + } + } + return true; + } + return false; +} + // Method recognizes the following idioms: // rounding halving add (a + b + 1) >> 1 for unsigned/signed operands a, b // truncated halving add (a + b) >> 1 for unsigned/signed operands a, b @@ -1924,8 +2200,7 @@ bool HLoopOptimization::VectorizeHalvingAddIdiom(LoopNode* node, HInstruction* a = nullptr; HInstruction* b = nullptr; int64_t c = 0; - if (IsAddConst(instruction->InputAt(0), /*out*/ &a, /*out*/ &b, /*out*/ &c)) { - DCHECK(a != nullptr && b != nullptr); + if (IsAddConst2(graph_, instruction->InputAt(0), /*out*/ &a, /*out*/ &b, /*out*/ &c)) { // Accept c == 1 (rounded) or c == 0 (not rounded). bool is_rounded = false; if (c == 1) { @@ -1947,8 +2222,7 @@ bool HLoopOptimization::VectorizeHalvingAddIdiom(LoopNode* node, } // Accept recognized halving add for vectorizable operands. Vectorized code uses the // shorthand idiomatic operation. Sequential code uses the original scalar expressions. - DCHECK(r != nullptr); - DCHECK(s != nullptr); + DCHECK(r != nullptr && s != nullptr); if (generate_code && vector_mode_ != kVector) { // de-idiom r = instruction->InputAt(0); s = instruction->InputAt(1); @@ -1998,21 +2272,11 @@ bool HLoopOptimization::VectorizeSADIdiom(LoopNode* node, HInstruction* v = instruction->InputAt(1); HInstruction* a = nullptr; HInstruction* b = nullptr; - if (v->IsInvokeStaticOrDirect() && - (v->AsInvokeStaticOrDirect()->GetIntrinsic() == Intrinsics::kMathAbsInt || - v->AsInvokeStaticOrDirect()->GetIntrinsic() == Intrinsics::kMathAbsLong)) { - HInstruction* x = v->InputAt(0); - if (x->GetType() == reduction_type) { - int64_t c = 0; - if (x->IsSub()) { - a = x->InputAt(0); - b = x->InputAt(1); - } else if (IsAddConst(x, /*out*/ &a, /*out*/ &c)) { - b = graph_->GetConstant(reduction_type, -c); // hidden SUB! - } - } - } - if (a == nullptr || b == nullptr) { + if (v->IsAbs() && + v->GetType() == reduction_type && + IsSubConst2(graph_, v->InputAt(0), /*out*/ &a, /*out*/ &b)) { + DCHECK(a != nullptr && b != nullptr); + } else { return false; } // Accept same-type or consistent sign extension for narrower-type on operands a and b. @@ -2045,8 +2309,7 @@ bool HLoopOptimization::VectorizeSADIdiom(LoopNode* node, } // Accept SAD idiom for vectorizable operands. Vectorized code uses the shorthand // idiomatic operation. Sequential code uses the original scalar expressions. - DCHECK(r != nullptr); - DCHECK(s != nullptr); + DCHECK(r != nullptr && s != nullptr); if (generate_code && vector_mode_ != kVector) { // de-idiom r = s = v->InputAt(0); } @@ -2054,14 +2317,13 @@ bool HLoopOptimization::VectorizeSADIdiom(LoopNode* node, VectorizeUse(node, r, generate_code, sub_type, restrictions) && VectorizeUse(node, s, generate_code, sub_type, restrictions)) { if (generate_code) { - reduction_type = HVecOperation::ToProperType(reduction_type, is_unsigned); if (vector_mode_ == kVector) { vector_map_->Put(instruction, new (global_allocator_) HVecSADAccumulate( global_allocator_, vector_map_->Get(q), vector_map_->Get(r), vector_map_->Get(s), - reduction_type, + HVecOperation::ToProperType(reduction_type, is_unsigned), GetOtherVL(reduction_type, sub_type, vector_length_), kNoDexPc)); MaybeRecordStat(stats_, MethodCompilationStat::kLoopVectorizedIdiom); @@ -2134,41 +2396,6 @@ bool HLoopOptimization::IsVectorizationProfitable(int64_t trip_count) { return true; } -static constexpr uint32_t ARM64_SIMD_MAXIMUM_UNROLL_FACTOR = 8; -static constexpr uint32_t ARM64_SIMD_HEURISTIC_MAX_BODY_SIZE = 50; - -uint32_t HLoopOptimization::GetUnrollingFactor(HBasicBlock* block, int64_t trip_count) { - uint32_t max_peel = MaxNumberPeeled(); - switch (compiler_driver_->GetInstructionSet()) { - case InstructionSet::kArm64: { - // Don't unroll with insufficient iterations. - // TODO: Unroll loops with unknown trip count. - DCHECK_NE(vector_length_, 0u); - if (trip_count < (2 * vector_length_ + max_peel)) { - return kNoUnrollingFactor; - } - // Don't unroll for large loop body size. - uint32_t instruction_count = block->GetInstructions().CountSize(); - if (instruction_count >= ARM64_SIMD_HEURISTIC_MAX_BODY_SIZE) { - return kNoUnrollingFactor; - } - // Find a beneficial unroll factor with the following restrictions: - // - At least one iteration of the transformed loop should be executed. - // - The loop body shouldn't be "too big" (heuristic). - uint32_t uf1 = ARM64_SIMD_HEURISTIC_MAX_BODY_SIZE / instruction_count; - uint32_t uf2 = (trip_count - max_peel) / vector_length_; - uint32_t unroll_factor = - TruncToPowerOfTwo(std::min({uf1, uf2, ARM64_SIMD_MAXIMUM_UNROLL_FACTOR})); - DCHECK_GE(unroll_factor, 1u); - return unroll_factor; - } - case InstructionSet::kX86: - case InstructionSet::kX86_64: - default: - return kNoUnrollingFactor; - } -} - // // Helpers. // diff --git a/compiler/optimizing/loop_optimization.h b/compiler/optimizing/loop_optimization.h index a707ad13580..f9a31a34d40 100644 --- a/compiler/optimizing/loop_optimization.h +++ b/compiler/optimizing/loop_optimization.h @@ -20,12 +20,15 @@ #include "base/scoped_arena_allocator.h" #include "base/scoped_arena_containers.h" #include "induction_var_range.h" +#include "loop_analysis.h" #include "nodes.h" #include "optimization.h" +#include "superblock_cloner.h" namespace art { class CompilerDriver; +class ArchDefaultLoopHelper; /** * Loop optimizations. Builds a loop hierarchy and applies optimizations to @@ -80,6 +83,7 @@ class HLoopOptimization : public HOptimization { kNoReduction = 1 << 10, // no reduction kNoSAD = 1 << 11, // no sum of absolute differences (SAD) kNoWideSAD = 1 << 12, // no sum of absolute differences (SAD) with operand widening + kNoSaturation = 1 << 13, // no saturation arithmetic }; /* @@ -134,10 +138,21 @@ class HLoopOptimization : public HOptimization { void SimplifyInduction(LoopNode* node); void SimplifyBlocks(LoopNode* node); - // Performs optimizations specific to inner loop (empty loop removal, + // Performs optimizations specific to inner loop with finite header logic (empty loop removal, // unrolling, vectorization). Returns true if anything changed. + bool TryOptimizeInnerLoopFinite(LoopNode* node); + + // Performs optimizations specific to inner loop. Returns true if anything changed. bool OptimizeInnerLoop(LoopNode* node); + // Tries to apply loop unrolling for branch penalty reduction and better instruction scheduling + // opportunities. Returns whether transformation happened. + bool TryUnrollingForBranchPenaltyReduction(LoopNode* loop_node); + + // Tries to apply loop peeling for loop invariant exits elimination. Returns whether + // transformation happened. + bool TryPeelingForLoopInvariantExitsElimination(LoopNode* loop_node); + // // Vectorization analysis and synthesis. // @@ -177,6 +192,11 @@ class HLoopOptimization : public HOptimization { bool is_unsigned = false); // Vectorization idioms. + bool VectorizeSaturationIdiom(LoopNode* node, + HInstruction* instruction, + bool generate_code, + DataType::Type type, + uint64_t restrictions); bool VectorizeHalvingAddIdiom(LoopNode* node, HInstruction* instruction, bool generate_code, @@ -197,7 +217,6 @@ class HLoopOptimization : public HOptimization { const ArrayReference* peeling_candidate); uint32_t MaxNumberPeeled(); bool IsVectorizationProfitable(int64_t trip_count); - uint32_t GetUnrollingFactor(HBasicBlock* block, int64_t trip_count); // // Helpers. @@ -291,6 +310,9 @@ class HLoopOptimization : public HOptimization { HBasicBlock* vector_body_; // body of the new loop HInstruction* vector_index_; // normalized index of the new loop + // Helper for target-specific behaviour for loop optimizations. + ArchDefaultLoopHelper* arch_loop_helper_; + friend class LoopOptimizationTest; DISALLOW_COPY_AND_ASSIGN(HLoopOptimization); diff --git a/compiler/optimizing/loop_optimization_test.cc b/compiler/optimizing/loop_optimization_test.cc index db8368986c9..c21bd65d97e 100644 --- a/compiler/optimizing/loop_optimization_test.cc +++ b/compiler/optimizing/loop_optimization_test.cc @@ -227,11 +227,14 @@ TEST_F(LoopOptimizationTest, SimplifyLoopReoderPredecessors) { graph_->ClearDominanceInformation(); graph_->BuildDominatorTree(); + // BuildDominatorTree inserts a block beetween loop header and entry block. + EXPECT_EQ(header->GetPredecessors()[0]->GetSinglePredecessor(), entry_block_); + // Check that after optimizations in BuildDominatorTree()/SimplifyCFG() phi inputs // are still mapped correctly to the block predecessors. for (size_t i = 0, e = phi->InputCount(); i < e; i++) { HInstruction* input = phi->InputAt(i); - ASSERT_TRUE(input->GetBlock()->Dominates(header->GetPredecessors()[i])); + EXPECT_TRUE(input->GetBlock()->Dominates(header->GetPredecessors()[i])); } } diff --git a/compiler/optimizing/nodes.cc b/compiler/optimizing/nodes.cc index f6ba19f22a8..f784f8f7f35 100644 --- a/compiler/optimizing/nodes.cc +++ b/compiler/optimizing/nodes.cc @@ -2891,6 +2891,8 @@ std::ostream& operator<<(std::ostream& os, HInvokeStaticOrDirect::MethodLoadKind return os << "BootImageLinkTimePcRelative"; case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress: return os << "DirectAddress"; + case HInvokeStaticOrDirect::MethodLoadKind::kBootImageRelRo: + return os << "BootImageRelRo"; case HInvokeStaticOrDirect::MethodLoadKind::kBssEntry: return os << "BssEntry"; case HInvokeStaticOrDirect::MethodLoadKind::kRuntimeCall: @@ -2925,7 +2927,7 @@ bool HLoadClass::InstructionDataEquals(const HInstruction* other) const { } switch (GetLoadKind()) { case LoadKind::kBootImageAddress: - case LoadKind::kBootImageClassTable: + case LoadKind::kBootImageRelRo: case LoadKind::kJitTableAddress: { ScopedObjectAccess soa(Thread::Current()); return GetClass().Get() == other_load_class->GetClass().Get(); @@ -2944,8 +2946,8 @@ std::ostream& operator<<(std::ostream& os, HLoadClass::LoadKind rhs) { return os << "BootImageLinkTimePcRelative"; case HLoadClass::LoadKind::kBootImageAddress: return os << "BootImageAddress"; - case HLoadClass::LoadKind::kBootImageClassTable: - return os << "BootImageClassTable"; + case HLoadClass::LoadKind::kBootImageRelRo: + return os << "BootImageRelRo"; case HLoadClass::LoadKind::kBssEntry: return os << "BssEntry"; case HLoadClass::LoadKind::kJitTableAddress: @@ -2968,7 +2970,7 @@ bool HLoadString::InstructionDataEquals(const HInstruction* other) const { } switch (GetLoadKind()) { case LoadKind::kBootImageAddress: - case LoadKind::kBootImageInternTable: + case LoadKind::kBootImageRelRo: case LoadKind::kJitTableAddress: { ScopedObjectAccess soa(Thread::Current()); return GetString().Get() == other_load_string->GetString().Get(); @@ -2984,8 +2986,8 @@ std::ostream& operator<<(std::ostream& os, HLoadString::LoadKind rhs) { return os << "BootImageLinkTimePcRelative"; case HLoadString::LoadKind::kBootImageAddress: return os << "BootImageAddress"; - case HLoadString::LoadKind::kBootImageInternTable: - return os << "BootImageInternTable"; + case HLoadString::LoadKind::kBootImageRelRo: + return os << "BootImageRelRo"; case HLoadString::LoadKind::kBssEntry: return os << "BssEntry"; case HLoadString::LoadKind::kJitTableAddress: @@ -3101,6 +3103,8 @@ std::ostream& operator<<(std::ostream& os, TypeCheckKind rhs) { return os << "array_object_check"; case TypeCheckKind::kArrayCheck: return os << "array_check"; + case TypeCheckKind::kBitstringCheck: + return os << "bitstring_check"; default: LOG(FATAL) << "Unknown TypeCheckKind: " << static_cast<int>(rhs); UNREACHABLE(); diff --git a/compiler/optimizing/nodes.h b/compiler/optimizing/nodes.h index fe992a7f399..79d733060b3 100644 --- a/compiler/optimizing/nodes.h +++ b/compiler/optimizing/nodes.h @@ -1338,6 +1338,7 @@ class HLoopInformationOutwardIterator : public ValueObject { #define FOR_EACH_CONCRETE_INSTRUCTION_COMMON(M) \ M(Above, Condition) \ M(AboveOrEqual, Condition) \ + M(Abs, UnaryOperation) \ M(Add, BinaryOperation) \ M(And, BinaryOperation) \ M(ArrayGet, Instruction) \ @@ -1383,7 +1384,9 @@ class HLoopInformationOutwardIterator : public ValueObject { M(LoadException, Instruction) \ M(LoadString, Instruction) \ M(LongConstant, Constant) \ + M(Max, Instruction) \ M(MemoryBarrier, Instruction) \ + M(Min, BinaryOperation) \ M(MonitorOperation, Instruction) \ M(Mul, BinaryOperation) \ M(NativeDebugInfo, Instruction) \ @@ -1437,6 +1440,8 @@ class HLoopInformationOutwardIterator : public ValueObject { M(VecAndNot, VecBinaryOperation) \ M(VecOr, VecBinaryOperation) \ M(VecXor, VecBinaryOperation) \ + M(VecSaturationAdd, VecBinaryOperation) \ + M(VecSaturationSub, VecBinaryOperation) \ M(VecShl, VecBinaryOperation) \ M(VecShr, VecBinaryOperation) \ M(VecUShr, VecBinaryOperation) \ @@ -4428,6 +4433,10 @@ class HInvokeStaticOrDirect FINAL : public HInvoke { // Used for app->boot calls with non-relocatable image and for JIT-compiled calls. kDirectAddress, + // Load from an entry in the .data.bimg.rel.ro using a PC-relative load. + // Used for app->boot calls with relocatable image. + kBootImageRelRo, + // Load from an entry in the .bss section using a PC-relative load. // Used for classes outside boot image when .bss is accessible with a PC-relative load. kBssEntry, @@ -4560,6 +4569,7 @@ class HInvokeStaticOrDirect FINAL : public HInvoke { bool HasMethodAddress() const { return GetMethodLoadKind() == MethodLoadKind::kDirectAddress; } bool HasPcRelativeMethodLoadKind() const { return GetMethodLoadKind() == MethodLoadKind::kBootImageLinkTimePcRelative || + GetMethodLoadKind() == MethodLoadKind::kBootImageRelRo || GetMethodLoadKind() == MethodLoadKind::kBssEntry; } bool HasCurrentMethodInput() const { @@ -5016,6 +5026,117 @@ class HRem FINAL : public HBinaryOperation { DEFAULT_COPY_CONSTRUCTOR(Rem); }; +class HMin FINAL : public HBinaryOperation { + public: + HMin(DataType::Type result_type, + HInstruction* left, + HInstruction* right, + uint32_t dex_pc) + : HBinaryOperation(kMin, result_type, left, right, SideEffects::None(), dex_pc) {} + + bool IsCommutative() const OVERRIDE { return true; } + + // Evaluation for integral values. + template <typename T> static T ComputeIntegral(T x, T y) { + return (x <= y) ? x : y; + } + + HConstant* Evaluate(HIntConstant* x, HIntConstant* y) const OVERRIDE { + return GetBlock()->GetGraph()->GetIntConstant( + ComputeIntegral(x->GetValue(), y->GetValue()), GetDexPc()); + } + HConstant* Evaluate(HLongConstant* x, HLongConstant* y) const OVERRIDE { + return GetBlock()->GetGraph()->GetLongConstant( + ComputeIntegral(x->GetValue(), y->GetValue()), GetDexPc()); + } + // TODO: Evaluation for floating-point values. + HConstant* Evaluate(HFloatConstant* x ATTRIBUTE_UNUSED, + HFloatConstant* y ATTRIBUTE_UNUSED) const OVERRIDE { return nullptr; } + HConstant* Evaluate(HDoubleConstant* x ATTRIBUTE_UNUSED, + HDoubleConstant* y ATTRIBUTE_UNUSED) const OVERRIDE { return nullptr; } + + DECLARE_INSTRUCTION(Min); + + protected: + DEFAULT_COPY_CONSTRUCTOR(Min); +}; + +class HMax FINAL : public HBinaryOperation { + public: + HMax(DataType::Type result_type, + HInstruction* left, + HInstruction* right, + uint32_t dex_pc) + : HBinaryOperation(kMax, result_type, left, right, SideEffects::None(), dex_pc) {} + + bool IsCommutative() const OVERRIDE { return true; } + + // Evaluation for integral values. + template <typename T> static T ComputeIntegral(T x, T y) { + return (x >= y) ? x : y; + } + + HConstant* Evaluate(HIntConstant* x, HIntConstant* y) const OVERRIDE { + return GetBlock()->GetGraph()->GetIntConstant( + ComputeIntegral(x->GetValue(), y->GetValue()), GetDexPc()); + } + HConstant* Evaluate(HLongConstant* x, HLongConstant* y) const OVERRIDE { + return GetBlock()->GetGraph()->GetLongConstant( + ComputeIntegral(x->GetValue(), y->GetValue()), GetDexPc()); + } + // TODO: Evaluation for floating-point values. + HConstant* Evaluate(HFloatConstant* x ATTRIBUTE_UNUSED, + HFloatConstant* y ATTRIBUTE_UNUSED) const OVERRIDE { return nullptr; } + HConstant* Evaluate(HDoubleConstant* x ATTRIBUTE_UNUSED, + HDoubleConstant* y ATTRIBUTE_UNUSED) const OVERRIDE { return nullptr; } + + DECLARE_INSTRUCTION(Max); + + protected: + DEFAULT_COPY_CONSTRUCTOR(Max); +}; + +class HAbs FINAL : public HUnaryOperation { + public: + HAbs(DataType::Type result_type, HInstruction* input, uint32_t dex_pc = kNoDexPc) + : HUnaryOperation(kAbs, result_type, input, dex_pc) {} + + // Evaluation for integral values. + template <typename T> static T ComputeIntegral(T x) { + return x < 0 ? -x : x; + } + + // Evaluation for floating-point values. + // Note, as a "quality of implementation", rather than pure "spec compliance", + // we require that Math.abs() clears the sign bit (but changes nothing else) + // for all floating-point numbers, including NaN (signaling NaN may become quiet though). + // http://b/30758343 + template <typename T, typename S> static T ComputeFP(T x) { + S bits = bit_cast<S, T>(x); + return bit_cast<T, S>(bits & std::numeric_limits<S>::max()); + } + + HConstant* Evaluate(HIntConstant* x) const OVERRIDE { + return GetBlock()->GetGraph()->GetIntConstant(ComputeIntegral(x->GetValue()), GetDexPc()); + } + HConstant* Evaluate(HLongConstant* x) const OVERRIDE { + return GetBlock()->GetGraph()->GetLongConstant(ComputeIntegral(x->GetValue()), GetDexPc()); + } + HConstant* Evaluate(HFloatConstant* x) const OVERRIDE { + return GetBlock()->GetGraph()->GetFloatConstant( + ComputeFP<float, int32_t>(x->GetValue()), GetDexPc()); + } + HConstant* Evaluate(HDoubleConstant* x) const OVERRIDE { + return GetBlock()->GetGraph()->GetDoubleConstant( + ComputeFP<double, int64_t>(x->GetValue()), GetDexPc()); + } + + DECLARE_INSTRUCTION(Abs); + + protected: + DEFAULT_COPY_CONSTRUCTOR(Abs); +}; + class HDivZeroCheck FINAL : public HExpression<1> { public: // `HDivZeroCheck` can trigger GC, as it may call the `ArithmeticException` @@ -6025,12 +6146,12 @@ class HLoadClass FINAL : public HInstruction { kBootImageLinkTimePcRelative, // Use a known boot image Class* address, embedded in the code by the codegen. - // Used for boot image classes referenced by apps in AOT- and JIT-compiled code. + // Used for boot image classes referenced by apps in JIT- and AOT-compiled code (non-PIC). kBootImageAddress, - // Use a PC-relative load from a boot image ClassTable mmapped into the .bss - // of the oat file. - kBootImageClassTable, + // Load from an entry in the .data.bimg.rel.ro using a PC-relative load. + // Used for boot image classes referenced by apps in AOT-compiled code (PIC). + kBootImageRelRo, // Load from an entry in the .bss section using a PC-relative load. // Used for classes outside boot image when .bss is accessible with a PC-relative load. @@ -6057,8 +6178,7 @@ class HLoadClass FINAL : public HInstruction { special_input_(HUserRecord<HInstruction*>(current_method)), type_index_(type_index), dex_file_(dex_file), - klass_(klass), - loaded_class_rti_(ReferenceTypeInfo::CreateInvalid()) { + klass_(klass) { // Referrers class should not need access check. We never inline unverified // methods so we can't possibly end up in this situation. DCHECK(!is_referrers_class || !needs_access_check); @@ -6068,6 +6188,7 @@ class HLoadClass FINAL : public HInstruction { SetPackedFlag<kFlagNeedsAccessCheck>(needs_access_check); SetPackedFlag<kFlagIsInBootImage>(false); SetPackedFlag<kFlagGenerateClInitCheck>(false); + SetPackedFlag<kFlagValidLoadedClassRTI>(false); } bool IsClonable() const OVERRIDE { return true; } @@ -6078,6 +6199,12 @@ class HLoadClass FINAL : public HInstruction { return GetPackedField<LoadKindField>(); } + bool HasPcRelativeLoadKind() const { + return GetLoadKind() == LoadKind::kBootImageLinkTimePcRelative || + GetLoadKind() == LoadKind::kBootImageRelRo || + GetLoadKind() == LoadKind::kBssEntry; + } + bool CanBeMoved() const OVERRIDE { return true; } bool InstructionDataEquals(const HInstruction* other) const; @@ -6116,13 +6243,18 @@ class HLoadClass FINAL : public HInstruction { } ReferenceTypeInfo GetLoadedClassRTI() { - return loaded_class_rti_; + if (GetPackedFlag<kFlagValidLoadedClassRTI>()) { + // Note: The is_exact flag from the return value should not be used. + return ReferenceTypeInfo::CreateUnchecked(klass_, /* is_exact */ true); + } else { + return ReferenceTypeInfo::CreateInvalid(); + } } - void SetLoadedClassRTI(ReferenceTypeInfo rti) { - // Make sure we only set exact types (the loaded class should never be merged). - DCHECK(rti.IsExact()); - loaded_class_rti_ = rti; + // Loaded class RTI is marked as valid by RTP if the klass_ is admissible. + void SetValidLoadedClassRTI() REQUIRES_SHARED(Locks::mutator_lock_) { + DCHECK(klass_ != nullptr); + SetPackedFlag<kFlagValidLoadedClassRTI>(true); } dex::TypeIndex GetTypeIndex() const { return type_index_; } @@ -6175,14 +6307,14 @@ class HLoadClass FINAL : public HInstruction { static constexpr size_t kFieldLoadKind = kFlagGenerateClInitCheck + 1; static constexpr size_t kFieldLoadKindSize = MinimumBitsToStore(static_cast<size_t>(LoadKind::kLast)); - static constexpr size_t kNumberOfLoadClassPackedBits = kFieldLoadKind + kFieldLoadKindSize; + static constexpr size_t kFlagValidLoadedClassRTI = kFieldLoadKind + kFieldLoadKindSize; + static constexpr size_t kNumberOfLoadClassPackedBits = kFlagValidLoadedClassRTI + 1; static_assert(kNumberOfLoadClassPackedBits < kMaxNumberOfPackedBits, "Too many packed fields."); using LoadKindField = BitField<LoadKind, kFieldLoadKind, kFieldLoadKindSize>; static bool HasTypeReference(LoadKind load_kind) { return load_kind == LoadKind::kReferrersClass || load_kind == LoadKind::kBootImageLinkTimePcRelative || - load_kind == LoadKind::kBootImageClassTable || load_kind == LoadKind::kBssEntry || load_kind == LoadKind::kRuntimeCall; } @@ -6203,8 +6335,6 @@ class HLoadClass FINAL : public HInstruction { const DexFile& dex_file_; Handle<mirror::Class> klass_; - - ReferenceTypeInfo loaded_class_rti_; }; std::ostream& operator<<(std::ostream& os, HLoadClass::LoadKind rhs); @@ -6228,7 +6358,7 @@ inline void HLoadClass::AddSpecialInput(HInstruction* special_input) { // including literal pool loads, which are PC-relative too. DCHECK(GetLoadKind() == LoadKind::kBootImageLinkTimePcRelative || GetLoadKind() == LoadKind::kBootImageAddress || - GetLoadKind() == LoadKind::kBootImageClassTable || + GetLoadKind() == LoadKind::kBootImageRelRo || GetLoadKind() == LoadKind::kBssEntry) << GetLoadKind(); DCHECK(special_input_.GetInstruction() == nullptr); special_input_ = HUserRecord<HInstruction*>(special_input); @@ -6244,12 +6374,12 @@ class HLoadString FINAL : public HInstruction { kBootImageLinkTimePcRelative, // Use a known boot image String* address, embedded in the code by the codegen. - // Used for boot image strings referenced by apps in AOT- and JIT-compiled code. + // Used for boot image strings referenced by apps in JIT- and AOT-compiled code (non-PIC). kBootImageAddress, - // Use a PC-relative load from a boot image InternTable mmapped into the .bss - // of the oat file. - kBootImageInternTable, + // Load from an entry in the .data.bimg.rel.ro using a PC-relative load. + // Used for boot image strings referenced by apps in AOT-compiled code (PIC). + kBootImageRelRo, // Load from an entry in the .bss section using a PC-relative load. // Used for strings outside boot image when .bss is accessible with a PC-relative load. @@ -6284,6 +6414,12 @@ class HLoadString FINAL : public HInstruction { return GetPackedField<LoadKindField>(); } + bool HasPcRelativeLoadKind() const { + return GetLoadKind() == LoadKind::kBootImageLinkTimePcRelative || + GetLoadKind() == LoadKind::kBootImageRelRo || + GetLoadKind() == LoadKind::kBssEntry; + } + const DexFile& GetDexFile() const { return dex_file_; } @@ -6312,7 +6448,7 @@ class HLoadString FINAL : public HInstruction { LoadKind load_kind = GetLoadKind(); if (load_kind == LoadKind::kBootImageLinkTimePcRelative || load_kind == LoadKind::kBootImageAddress || - load_kind == LoadKind::kBootImageInternTable || + load_kind == LoadKind::kBootImageRelRo || load_kind == LoadKind::kJitTableAddress) { return false; } @@ -6390,7 +6526,7 @@ inline void HLoadString::AddSpecialInput(HInstruction* special_input) { // including literal pool loads, which are PC-relative too. DCHECK(GetLoadKind() == LoadKind::kBootImageLinkTimePcRelative || GetLoadKind() == LoadKind::kBootImageAddress || - GetLoadKind() == LoadKind::kBootImageInternTable || + GetLoadKind() == LoadKind::kBootImageRelRo || GetLoadKind() == LoadKind::kBssEntry) << GetLoadKind(); // HLoadString::GetInputRecords() returns an empty array at this point, // so use the GetInputRecords() from the base class to set the input record. @@ -6750,72 +6886,159 @@ enum class TypeCheckKind { kInterfaceCheck, // No optimization yet when checking against an interface. kArrayObjectCheck, // Can just check if the array is not primitive. kArrayCheck, // No optimization yet when checking against a generic array. + kBitstringCheck, // Compare the type check bitstring. kLast = kArrayCheck }; std::ostream& operator<<(std::ostream& os, TypeCheckKind rhs); -class HInstanceOf FINAL : public HExpression<2> { +// Note: HTypeCheckInstruction is just a helper class, not an abstract instruction with an +// `IsTypeCheckInstruction()`. (New virtual methods in the HInstruction class have a high cost.) +class HTypeCheckInstruction : public HVariableInputSizeInstruction { public: - HInstanceOf(HInstruction* object, - HLoadClass* target_class, - TypeCheckKind check_kind, - uint32_t dex_pc) - : HExpression(kInstanceOf, - DataType::Type::kBool, - SideEffectsForArchRuntimeCalls(check_kind), - dex_pc) { + HTypeCheckInstruction(InstructionKind kind, + HInstruction* object, + HInstruction* target_class_or_null, + TypeCheckKind check_kind, + Handle<mirror::Class> klass, + uint32_t dex_pc, + ArenaAllocator* allocator, + HIntConstant* bitstring_path_to_root, + HIntConstant* bitstring_mask, + SideEffects side_effects) + : HVariableInputSizeInstruction( + kind, + side_effects, + dex_pc, + allocator, + /* number_of_inputs */ check_kind == TypeCheckKind::kBitstringCheck ? 4u : 2u, + kArenaAllocTypeCheckInputs), + klass_(klass) { SetPackedField<TypeCheckKindField>(check_kind); SetPackedFlag<kFlagMustDoNullCheck>(true); + SetPackedFlag<kFlagValidTargetClassRTI>(false); SetRawInputAt(0, object); - SetRawInputAt(1, target_class); + SetRawInputAt(1, target_class_or_null); + DCHECK_EQ(check_kind == TypeCheckKind::kBitstringCheck, bitstring_path_to_root != nullptr); + DCHECK_EQ(check_kind == TypeCheckKind::kBitstringCheck, bitstring_mask != nullptr); + if (check_kind == TypeCheckKind::kBitstringCheck) { + DCHECK(target_class_or_null->IsNullConstant()); + SetRawInputAt(2, bitstring_path_to_root); + SetRawInputAt(3, bitstring_mask); + } else { + DCHECK(target_class_or_null->IsLoadClass()); + } } HLoadClass* GetTargetClass() const { + DCHECK_NE(GetTypeCheckKind(), TypeCheckKind::kBitstringCheck); HInstruction* load_class = InputAt(1); DCHECK(load_class->IsLoadClass()); return load_class->AsLoadClass(); } - bool IsClonable() const OVERRIDE { return true; } - bool CanBeMoved() const OVERRIDE { return true; } + uint32_t GetBitstringPathToRoot() const { + DCHECK_EQ(GetTypeCheckKind(), TypeCheckKind::kBitstringCheck); + HInstruction* path_to_root = InputAt(2); + DCHECK(path_to_root->IsIntConstant()); + return static_cast<uint32_t>(path_to_root->AsIntConstant()->GetValue()); + } - bool InstructionDataEquals(const HInstruction* other ATTRIBUTE_UNUSED) const OVERRIDE { - return true; + uint32_t GetBitstringMask() const { + DCHECK_EQ(GetTypeCheckKind(), TypeCheckKind::kBitstringCheck); + HInstruction* mask = InputAt(3); + DCHECK(mask->IsIntConstant()); + return static_cast<uint32_t>(mask->AsIntConstant()->GetValue()); } - bool NeedsEnvironment() const OVERRIDE { - return CanCallRuntime(GetTypeCheckKind()); + bool IsClonable() const OVERRIDE { return true; } + bool CanBeMoved() const OVERRIDE { return true; } + + bool InstructionDataEquals(const HInstruction* other) const OVERRIDE { + DCHECK(other->IsInstanceOf() || other->IsCheckCast()) << other->DebugName(); + return GetPackedFields() == down_cast<const HTypeCheckInstruction*>(other)->GetPackedFields(); } - // Used only in code generation. bool MustDoNullCheck() const { return GetPackedFlag<kFlagMustDoNullCheck>(); } void ClearMustDoNullCheck() { SetPackedFlag<kFlagMustDoNullCheck>(false); } TypeCheckKind GetTypeCheckKind() const { return GetPackedField<TypeCheckKindField>(); } bool IsExactCheck() const { return GetTypeCheckKind() == TypeCheckKind::kExactCheck; } - static bool CanCallRuntime(TypeCheckKind check_kind) { - // Mips currently does runtime calls for any other checks. - return check_kind != TypeCheckKind::kExactCheck; + ReferenceTypeInfo GetTargetClassRTI() { + if (GetPackedFlag<kFlagValidTargetClassRTI>()) { + // Note: The is_exact flag from the return value should not be used. + return ReferenceTypeInfo::CreateUnchecked(klass_, /* is_exact */ true); + } else { + return ReferenceTypeInfo::CreateInvalid(); + } } - static SideEffects SideEffectsForArchRuntimeCalls(TypeCheckKind check_kind) { - return CanCallRuntime(check_kind) ? SideEffects::CanTriggerGC() : SideEffects::None(); + // Target class RTI is marked as valid by RTP if the klass_ is admissible. + void SetValidTargetClassRTI() REQUIRES_SHARED(Locks::mutator_lock_) { + DCHECK(klass_ != nullptr); + SetPackedFlag<kFlagValidTargetClassRTI>(true); } - DECLARE_INSTRUCTION(InstanceOf); + Handle<mirror::Class> GetClass() const { + return klass_; + } protected: - DEFAULT_COPY_CONSTRUCTOR(InstanceOf); + DEFAULT_COPY_CONSTRUCTOR(TypeCheckInstruction); private: - static constexpr size_t kFieldTypeCheckKind = kNumberOfExpressionPackedBits; + static constexpr size_t kFieldTypeCheckKind = kNumberOfGenericPackedBits; static constexpr size_t kFieldTypeCheckKindSize = MinimumBitsToStore(static_cast<size_t>(TypeCheckKind::kLast)); static constexpr size_t kFlagMustDoNullCheck = kFieldTypeCheckKind + kFieldTypeCheckKindSize; - static constexpr size_t kNumberOfInstanceOfPackedBits = kFlagMustDoNullCheck + 1; + static constexpr size_t kFlagValidTargetClassRTI = kFlagMustDoNullCheck + 1; + static constexpr size_t kNumberOfInstanceOfPackedBits = kFlagValidTargetClassRTI + 1; static_assert(kNumberOfInstanceOfPackedBits <= kMaxNumberOfPackedBits, "Too many packed fields."); using TypeCheckKindField = BitField<TypeCheckKind, kFieldTypeCheckKind, kFieldTypeCheckKindSize>; + + Handle<mirror::Class> klass_; +}; + +class HInstanceOf FINAL : public HTypeCheckInstruction { + public: + HInstanceOf(HInstruction* object, + HInstruction* target_class_or_null, + TypeCheckKind check_kind, + Handle<mirror::Class> klass, + uint32_t dex_pc, + ArenaAllocator* allocator, + HIntConstant* bitstring_path_to_root, + HIntConstant* bitstring_mask) + : HTypeCheckInstruction(kInstanceOf, + object, + target_class_or_null, + check_kind, + klass, + dex_pc, + allocator, + bitstring_path_to_root, + bitstring_mask, + SideEffectsForArchRuntimeCalls(check_kind)) {} + + DataType::Type GetType() const OVERRIDE { return DataType::Type::kBool; } + + bool NeedsEnvironment() const OVERRIDE { + return CanCallRuntime(GetTypeCheckKind()); + } + + static bool CanCallRuntime(TypeCheckKind check_kind) { + // Mips currently does runtime calls for any other checks. + return check_kind != TypeCheckKind::kExactCheck; + } + + static SideEffects SideEffectsForArchRuntimeCalls(TypeCheckKind check_kind) { + return CanCallRuntime(check_kind) ? SideEffects::CanTriggerGC() : SideEffects::None(); + } + + DECLARE_INSTRUCTION(InstanceOf); + + protected: + DEFAULT_COPY_CONSTRUCTOR(InstanceOf); }; class HBoundType FINAL : public HExpression<1> { @@ -6865,31 +7088,26 @@ class HBoundType FINAL : public HExpression<1> { ReferenceTypeInfo upper_bound_; }; -class HCheckCast FINAL : public HTemplateInstruction<2> { +class HCheckCast FINAL : public HTypeCheckInstruction { public: HCheckCast(HInstruction* object, - HLoadClass* target_class, + HInstruction* target_class_or_null, TypeCheckKind check_kind, - uint32_t dex_pc) - : HTemplateInstruction(kCheckCast, SideEffects::CanTriggerGC(), dex_pc) { - SetPackedField<TypeCheckKindField>(check_kind); - SetPackedFlag<kFlagMustDoNullCheck>(true); - SetRawInputAt(0, object); - SetRawInputAt(1, target_class); - } - - HLoadClass* GetTargetClass() const { - HInstruction* load_class = InputAt(1); - DCHECK(load_class->IsLoadClass()); - return load_class->AsLoadClass(); - } - - bool IsClonable() const OVERRIDE { return true; } - bool CanBeMoved() const OVERRIDE { return true; } - - bool InstructionDataEquals(const HInstruction* other ATTRIBUTE_UNUSED) const OVERRIDE { - return true; - } + Handle<mirror::Class> klass, + uint32_t dex_pc, + ArenaAllocator* allocator, + HIntConstant* bitstring_path_to_root, + HIntConstant* bitstring_mask) + : HTypeCheckInstruction(kCheckCast, + object, + target_class_or_null, + check_kind, + klass, + dex_pc, + allocator, + bitstring_path_to_root, + bitstring_mask, + SideEffects::CanTriggerGC()) {} bool NeedsEnvironment() const OVERRIDE { // Instruction may throw a CheckCastError. @@ -6898,24 +7116,10 @@ class HCheckCast FINAL : public HTemplateInstruction<2> { bool CanThrow() const OVERRIDE { return true; } - bool MustDoNullCheck() const { return GetPackedFlag<kFlagMustDoNullCheck>(); } - void ClearMustDoNullCheck() { SetPackedFlag<kFlagMustDoNullCheck>(false); } - TypeCheckKind GetTypeCheckKind() const { return GetPackedField<TypeCheckKindField>(); } - bool IsExactCheck() const { return GetTypeCheckKind() == TypeCheckKind::kExactCheck; } - DECLARE_INSTRUCTION(CheckCast); protected: DEFAULT_COPY_CONSTRUCTOR(CheckCast); - - private: - static constexpr size_t kFieldTypeCheckKind = kNumberOfGenericPackedBits; - static constexpr size_t kFieldTypeCheckKindSize = - MinimumBitsToStore(static_cast<size_t>(TypeCheckKind::kLast)); - static constexpr size_t kFlagMustDoNullCheck = kFieldTypeCheckKind + kFieldTypeCheckKindSize; - static constexpr size_t kNumberOfCheckCastPackedBits = kFlagMustDoNullCheck + 1; - static_assert(kNumberOfCheckCastPackedBits <= kMaxNumberOfPackedBits, "Too many packed fields."); - using TypeCheckKindField = BitField<TypeCheckKind, kFieldTypeCheckKind, kFieldTypeCheckKindSize>; }; /** diff --git a/compiler/optimizing/nodes_vector.h b/compiler/optimizing/nodes_vector.h index d1eaf5c3666..1a484e1944f 100644 --- a/compiler/optimizing/nodes_vector.h +++ b/compiler/optimizing/nodes_vector.h @@ -328,7 +328,7 @@ class HVecReplicateScalar FINAL : public HVecUnaryOperation { uint32_t dex_pc) : HVecUnaryOperation( kVecReplicateScalar, allocator, scalar, packed_type, vector_length, dex_pc) { - DCHECK(!scalar->IsVecOperation()); + DCHECK(!ReturnsSIMDValue(scalar)); } // A replicate needs to stay in place, since SIMD registers are not @@ -533,6 +533,31 @@ class HVecAdd FINAL : public HVecBinaryOperation { DEFAULT_COPY_CONSTRUCTOR(VecAdd); }; +// Adds every component in the two vectors using saturation arithmetic, +// viz. [ x1, .. , xn ] + [ y1, .. , yn ] = [ x1 +_sat y1, .. , xn +_sat yn ] +// for either both signed or both unsigned operands x, y (reflected in packed_type). +class HVecSaturationAdd FINAL : public HVecBinaryOperation { + public: + HVecSaturationAdd(ArenaAllocator* allocator, + HInstruction* left, + HInstruction* right, + DataType::Type packed_type, + size_t vector_length, + uint32_t dex_pc) + : HVecBinaryOperation( + kVecSaturationAdd, allocator, left, right, packed_type, vector_length, dex_pc) { + DCHECK(HasConsistentPackedTypes(left, packed_type)); + DCHECK(HasConsistentPackedTypes(right, packed_type)); + } + + bool CanBeMoved() const OVERRIDE { return true; } + + DECLARE_INSTRUCTION(VecSaturationAdd); + + protected: + DEFAULT_COPY_CONSTRUCTOR(VecSaturationAdd); +}; + // Performs halving add on every component in the two vectors, viz. // rounded [ x1, .. , xn ] hradd [ y1, .. , yn ] = [ (x1 + y1 + 1) >> 1, .. , (xn + yn + 1) >> 1 ] // truncated [ x1, .. , xn ] hadd [ y1, .. , yn ] = [ (x1 + y1) >> 1, .. , (xn + yn ) >> 1 ] @@ -598,6 +623,31 @@ class HVecSub FINAL : public HVecBinaryOperation { DEFAULT_COPY_CONSTRUCTOR(VecSub); }; +// Subtracts every component in the two vectors using saturation arithmetic, +// viz. [ x1, .. , xn ] + [ y1, .. , yn ] = [ x1 -_sat y1, .. , xn -_sat yn ] +// for either both signed or both unsigned operands x, y (reflected in packed_type). +class HVecSaturationSub FINAL : public HVecBinaryOperation { + public: + HVecSaturationSub(ArenaAllocator* allocator, + HInstruction* left, + HInstruction* right, + DataType::Type packed_type, + size_t vector_length, + uint32_t dex_pc) + : HVecBinaryOperation( + kVecSaturationSub, allocator, left, right, packed_type, vector_length, dex_pc) { + DCHECK(HasConsistentPackedTypes(left, packed_type)); + DCHECK(HasConsistentPackedTypes(right, packed_type)); + } + + bool CanBeMoved() const OVERRIDE { return true; } + + DECLARE_INSTRUCTION(VecSaturationSub); + + protected: + DEFAULT_COPY_CONSTRUCTOR(VecSaturationSub); +}; + // Multiplies every component in the two vectors, // viz. [ x1, .. , xn ] * [ y1, .. , yn ] = [ x1 * y1, .. , xn * yn ]. class HVecMul FINAL : public HVecBinaryOperation { diff --git a/compiler/optimizing/optimizing_cfi_test.cc b/compiler/optimizing/optimizing_cfi_test.cc index d20b681b49a..2e189fdd141 100644 --- a/compiler/optimizing/optimizing_cfi_test.cc +++ b/compiler/optimizing/optimizing_cfi_test.cc @@ -105,15 +105,15 @@ class OptimizingCFITest : public CFITest, public OptimizingUnitTestHelper { const std::vector<uint8_t>& expected_asm, const std::vector<uint8_t>& expected_cfi) { // Get the outputs. - const std::vector<uint8_t>& actual_asm = code_allocator_.GetMemory(); + ArrayRef<const uint8_t> actual_asm = code_allocator_.GetMemory(); Assembler* opt_asm = code_gen_->GetAssembler(); - const std::vector<uint8_t>& actual_cfi = *(opt_asm->cfi().data()); + ArrayRef<const uint8_t> actual_cfi(*(opt_asm->cfi().data())); if (kGenerateExpected) { GenerateExpected(stdout, isa, isa_str, actual_asm, actual_cfi); } else { - EXPECT_EQ(expected_asm, actual_asm); - EXPECT_EQ(expected_cfi, actual_cfi); + EXPECT_EQ(ArrayRef<const uint8_t>(expected_asm), actual_asm); + EXPECT_EQ(ArrayRef<const uint8_t>(expected_cfi), actual_cfi); } } @@ -140,7 +140,7 @@ class OptimizingCFITest : public CFITest, public OptimizingUnitTestHelper { return memory_.data(); } - const std::vector<uint8_t>& GetMemory() { return memory_; } + ArrayRef<const uint8_t> GetMemory() const OVERRIDE { return ArrayRef<const uint8_t>(memory_); } private: std::vector<uint8_t> memory_; diff --git a/compiler/optimizing/optimizing_compiler.cc b/compiler/optimizing/optimizing_compiler.cc index e42dfc10ba5..cadefc3b015 100644 --- a/compiler/optimizing/optimizing_compiler.cc +++ b/compiler/optimizing/optimizing_compiler.cc @@ -75,22 +75,18 @@ static constexpr const char* kPassNameSeparator = "$"; class CodeVectorAllocator FINAL : public CodeAllocator { public: explicit CodeVectorAllocator(ArenaAllocator* allocator) - : memory_(allocator->Adapter(kArenaAllocCodeBuffer)), - size_(0) {} + : memory_(allocator->Adapter(kArenaAllocCodeBuffer)) {} virtual uint8_t* Allocate(size_t size) { - size_ = size; memory_.resize(size); return &memory_[0]; } - size_t GetSize() const { return size_; } - const ArenaVector<uint8_t>& GetMemory() const { return memory_; } + ArrayRef<const uint8_t> GetMemory() const OVERRIDE { return ArrayRef<const uint8_t>(memory_); } uint8_t* GetData() { return memory_.data(); } private: ArenaVector<uint8_t> memory_; - size_t size_; DISALLOW_COPY_AND_ASSIGN(CodeVectorAllocator); }; @@ -647,15 +643,13 @@ void OptimizingCompiler::RunOptimizations(HGraph* graph, MaybeRunInliner(graph, codegen, dex_compilation_unit, pass_observer, handles); OptimizationDef optimizations2[] = { - // SelectGenerator depends on the InstructionSimplifier removing - // redundant suspend checks to recognize empty blocks. + OptDef(OptimizationPass::kSideEffectsAnalysis, "side_effects$before_gvn"), + OptDef(OptimizationPass::kGlobalValueNumbering), OptDef(OptimizationPass::kSelectGenerator), - // TODO: if we don't inline we can also skip fold2. OptDef(OptimizationPass::kConstantFolding, "constant_folding$after_inlining"), OptDef(OptimizationPass::kInstructionSimplifier, "instruction_simplifier$after_inlining"), OptDef(OptimizationPass::kDeadCodeElimination, "dead_code_elimination$after_inlining"), - OptDef(OptimizationPass::kSideEffectsAnalysis, "side_effects$before_gvn"), - OptDef(OptimizationPass::kGlobalValueNumbering), + OptDef(OptimizationPass::kSideEffectsAnalysis, "side_effects$before_licm"), OptDef(OptimizationPass::kInvariantCodeMotion), OptDef(OptimizationPass::kInductionVarAnalysis), OptDef(OptimizationPass::kBoundsCheckElimination), @@ -719,7 +713,7 @@ CompiledMethod* OptimizingCompiler::Emit(ArenaAllocator* allocator, CompiledMethod* compiled_method = CompiledMethod::SwapAllocCompiledMethod( GetCompilerDriver(), codegen->GetInstructionSet(), - ArrayRef<const uint8_t>(code_allocator->GetMemory()), + code_allocator->GetMemory(), // Follow Quick's behavior and set the frame size to zero if it is // considered "empty" (see the definition of // art::CodeGenerator::HasEmptyFrame). @@ -731,6 +725,16 @@ CompiledMethod* OptimizingCompiler::Emit(ArenaAllocator* allocator, ArrayRef<const uint8_t>(*codegen->GetAssembler()->cfi().data()), ArrayRef<const linker::LinkerPatch>(linker_patches)); + CompiledMethodStorage* storage = GetCompilerDriver()->GetCompiledMethodStorage(); + for (const linker::LinkerPatch& patch : linker_patches) { + if (codegen->NeedsThunkCode(patch) && storage->GetThunkCode(patch).empty()) { + ArenaVector<uint8_t> code(allocator->Adapter()); + std::string debug_name; + codegen->EmitThunkCode(patch, &code, &debug_name); + storage->SetThunkCode(patch, ArrayRef<const uint8_t>(code), debug_name); + } + } + return compiled_method; } @@ -1339,7 +1343,7 @@ bool OptimizingCompiler::JitCompile(Thread* self, codegen->GetCoreSpillMask(), codegen->GetFpuSpillMask(), code_allocator.GetMemory().data(), - code_allocator.GetSize(), + code_allocator.GetMemory().size(), data_size, osr, roots, @@ -1369,7 +1373,7 @@ bool OptimizingCompiler::JitCompile(Thread* self, info.is_optimized = true; info.is_code_address_text_relative = false; info.code_address = code_address; - info.code_size = code_allocator.GetSize(); + info.code_size = code_allocator.GetMemory().size(); info.frame_size_in_bytes = method_header->GetFrameSizeInBytes(); info.code_info = stack_map_size == 0 ? nullptr : stack_map_data; info.cfi = ArrayRef<const uint8_t>(*codegen->GetAssembler()->cfi().data()); @@ -1378,7 +1382,7 @@ bool OptimizingCompiler::JitCompile(Thread* self, Runtime::Current()->GetJit()->AddMemoryUsage(method, allocator.BytesUsed()); if (jit_logger != nullptr) { - jit_logger->WriteLog(code, code_allocator.GetSize(), method); + jit_logger->WriteLog(code, code_allocator.GetMemory().size(), method); } if (kArenaAllocatorCountAllocations) { diff --git a/compiler/optimizing/optimizing_compiler_stats.h b/compiler/optimizing/optimizing_compiler_stats.h index 00194ff1fe0..9a26f2f6c40 100644 --- a/compiler/optimizing/optimizing_compiler_stats.h +++ b/compiler/optimizing/optimizing_compiler_stats.h @@ -99,6 +99,7 @@ enum class MethodCompilationStat { kConstructorFenceRemovedLSE, kConstructorFenceRemovedPFRA, kConstructorFenceRemovedCFRE, + kBitstringTypeCheck, kJitOutOfMemoryForCommit, kLastStat }; @@ -124,11 +125,6 @@ class OptimizingCompilerStats { } void Log() const { - if (!kIsDebugBuild && !VLOG_IS_ON(compiler)) { - // Log only in debug builds or if the compiler is verbose. - return; - } - uint32_t compiled_intrinsics = GetStat(MethodCompilationStat::kCompiledIntrinsic); uint32_t compiled_native_stubs = GetStat(MethodCompilationStat::kCompiledNativeStub); uint32_t bytecode_attempts = diff --git a/compiler/optimizing/optimizing_unit_test.h b/compiler/optimizing/optimizing_unit_test.h index 6dcbadba6ed..a9bc5664c09 100644 --- a/compiler/optimizing/optimizing_unit_test.h +++ b/compiler/optimizing/optimizing_unit_test.h @@ -20,6 +20,7 @@ #include <memory> #include <vector> +#include "base/malloc_arena_pool.h" #include "base/scoped_arena_allocator.h" #include "builder.h" #include "common_compiler_test.h" @@ -97,7 +98,7 @@ class ArenaPoolAndAllocator { ScopedArenaAllocator* GetScopedAllocator() { return &scoped_allocator_; } private: - ArenaPool pool_; + MallocArenaPool pool_; ArenaAllocator allocator_; ArenaStack arena_stack_; ScopedArenaAllocator scoped_allocator_; diff --git a/compiler/optimizing/parallel_move_test.cc b/compiler/optimizing/parallel_move_test.cc index cb87cabe1cd..be352011668 100644 --- a/compiler/optimizing/parallel_move_test.cc +++ b/compiler/optimizing/parallel_move_test.cc @@ -15,6 +15,7 @@ */ #include "base/arena_allocator.h" +#include "base/malloc_arena_pool.h" #include "nodes.h" #include "parallel_move_resolver.h" @@ -180,7 +181,7 @@ TYPED_TEST_CASE(ParallelMoveTest, ParallelMoveResolverTestTypes); TYPED_TEST(ParallelMoveTest, Dependency) { - ArenaPool pool; + MallocArenaPool pool; ArenaAllocator allocator(&pool); { @@ -207,7 +208,7 @@ TYPED_TEST(ParallelMoveTest, Dependency) { } TYPED_TEST(ParallelMoveTest, Cycle) { - ArenaPool pool; + MallocArenaPool pool; ArenaAllocator allocator(&pool); { @@ -257,7 +258,7 @@ TYPED_TEST(ParallelMoveTest, Cycle) { } TYPED_TEST(ParallelMoveTest, ConstantLast) { - ArenaPool pool; + MallocArenaPool pool; ArenaAllocator allocator(&pool); TypeParam resolver(&allocator); HParallelMove* moves = new (&allocator) HParallelMove(&allocator); @@ -276,7 +277,7 @@ TYPED_TEST(ParallelMoveTest, ConstantLast) { } TYPED_TEST(ParallelMoveTest, Pairs) { - ArenaPool pool; + MallocArenaPool pool; ArenaAllocator allocator(&pool); { @@ -453,7 +454,7 @@ TYPED_TEST(ParallelMoveTest, Pairs) { } TYPED_TEST(ParallelMoveTest, MultiCycles) { - ArenaPool pool; + MallocArenaPool pool; ArenaAllocator allocator(&pool); { @@ -551,7 +552,7 @@ TYPED_TEST(ParallelMoveTest, MultiCycles) { // Test that we do 64bits moves before 32bits moves. TYPED_TEST(ParallelMoveTest, CyclesWith64BitsMoves) { - ArenaPool pool; + MallocArenaPool pool; ArenaAllocator allocator(&pool); { @@ -610,7 +611,7 @@ TYPED_TEST(ParallelMoveTest, CyclesWith64BitsMoves) { } TYPED_TEST(ParallelMoveTest, CyclesWith64BitsMoves2) { - ArenaPool pool; + MallocArenaPool pool; ArenaAllocator allocator(&pool); { diff --git a/compiler/optimizing/pc_relative_fixups_mips.cc b/compiler/optimizing/pc_relative_fixups_mips.cc index 9d5358514ee..01022542062 100644 --- a/compiler/optimizing/pc_relative_fixups_mips.cc +++ b/compiler/optimizing/pc_relative_fixups_mips.cc @@ -75,7 +75,7 @@ class PCRelativeHandlerVisitor : public HGraphVisitor { switch (load_kind) { case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: case HLoadClass::LoadKind::kBootImageAddress: - case HLoadClass::LoadKind::kBootImageClassTable: + case HLoadClass::LoadKind::kBootImageRelRo: case HLoadClass::LoadKind::kBssEntry: // Add a base register for PC-relative literals on R2. InitializePCRelativeBasePointer(); @@ -91,7 +91,7 @@ class PCRelativeHandlerVisitor : public HGraphVisitor { switch (load_kind) { case HLoadString::LoadKind::kBootImageLinkTimePcRelative: case HLoadString::LoadKind::kBootImageAddress: - case HLoadString::LoadKind::kBootImageInternTable: + case HLoadString::LoadKind::kBootImageRelRo: case HLoadString::LoadKind::kBssEntry: // Add a base register for PC-relative literals on R2. InitializePCRelativeBasePointer(); diff --git a/compiler/optimizing/pc_relative_fixups_x86.cc b/compiler/optimizing/pc_relative_fixups_x86.cc index f92f4b274ae..647336b6b93 100644 --- a/compiler/optimizing/pc_relative_fixups_x86.cc +++ b/compiler/optimizing/pc_relative_fixups_x86.cc @@ -81,20 +81,14 @@ class PCRelativeHandlerVisitor : public HGraphVisitor { } void VisitLoadClass(HLoadClass* load_class) OVERRIDE { - HLoadClass::LoadKind load_kind = load_class->GetLoadKind(); - if (load_kind == HLoadClass::LoadKind::kBootImageLinkTimePcRelative || - load_kind == HLoadClass::LoadKind::kBootImageClassTable || - load_kind == HLoadClass::LoadKind::kBssEntry) { + if (load_class->HasPcRelativeLoadKind()) { HX86ComputeBaseMethodAddress* method_address = GetPCRelativeBasePointer(load_class); load_class->AddSpecialInput(method_address); } } void VisitLoadString(HLoadString* load_string) OVERRIDE { - HLoadString::LoadKind load_kind = load_string->GetLoadKind(); - if (load_kind == HLoadString::LoadKind::kBootImageLinkTimePcRelative || - load_kind == HLoadString::LoadKind::kBootImageInternTable || - load_kind == HLoadString::LoadKind::kBssEntry) { + if (load_string->HasPcRelativeLoadKind()) { HX86ComputeBaseMethodAddress* method_address = GetPCRelativeBasePointer(load_string); load_string->AddSpecialInput(method_address); } @@ -238,6 +232,9 @@ class PCRelativeHandlerVisitor : public HGraphVisitor { case Intrinsics::kMathMaxFloatFloat: case Intrinsics::kMathMinDoubleDouble: case Intrinsics::kMathMinFloatFloat: + LOG(FATAL) << "Unreachable min/max/abs: intrinsics should have been lowered " + "to IR nodes by instruction simplifier"; + UNREACHABLE(); case Intrinsics::kMathRoundFloat: if (!base_added) { DCHECK(invoke_static_or_direct != nullptr); diff --git a/compiler/optimizing/prepare_for_register_allocation.cc b/compiler/optimizing/prepare_for_register_allocation.cc index f843c008d8b..59733397bfe 100644 --- a/compiler/optimizing/prepare_for_register_allocation.cc +++ b/compiler/optimizing/prepare_for_register_allocation.cc @@ -34,6 +34,20 @@ void PrepareForRegisterAllocation::Run() { } } +void PrepareForRegisterAllocation::VisitCheckCast(HCheckCast* check_cast) { + // Record only those bitstring type checks that make it to the codegen stage. + if (check_cast->GetTypeCheckKind() == TypeCheckKind::kBitstringCheck) { + MaybeRecordStat(stats_, MethodCompilationStat::kBitstringTypeCheck); + } +} + +void PrepareForRegisterAllocation::VisitInstanceOf(HInstanceOf* instance_of) { + // Record only those bitstring type checks that make it to the codegen stage. + if (instance_of->GetTypeCheckKind() == TypeCheckKind::kBitstringCheck) { + MaybeRecordStat(stats_, MethodCompilationStat::kBitstringTypeCheck); + } +} + void PrepareForRegisterAllocation::VisitNullCheck(HNullCheck* check) { check->ReplaceWith(check->InputAt(0)); } diff --git a/compiler/optimizing/prepare_for_register_allocation.h b/compiler/optimizing/prepare_for_register_allocation.h index 2c64f016c17..f6e4d3ef99b 100644 --- a/compiler/optimizing/prepare_for_register_allocation.h +++ b/compiler/optimizing/prepare_for_register_allocation.h @@ -40,6 +40,8 @@ class PrepareForRegisterAllocation : public HGraphDelegateVisitor { "prepare_for_register_allocation"; private: + void VisitCheckCast(HCheckCast* check_cast) OVERRIDE; + void VisitInstanceOf(HInstanceOf* instance_of) OVERRIDE; void VisitNullCheck(HNullCheck* check) OVERRIDE; void VisitDivZeroCheck(HDivZeroCheck* check) OVERRIDE; void VisitBoundsCheck(HBoundsCheck* check) OVERRIDE; diff --git a/compiler/optimizing/reference_type_propagation.cc b/compiler/optimizing/reference_type_propagation.cc index 67a61fc01de..4030883a57e 100644 --- a/compiler/optimizing/reference_type_propagation.cc +++ b/compiler/optimizing/reference_type_propagation.cc @@ -87,6 +87,7 @@ class ReferenceTypePropagation::RTPVisitor : public HGraphDelegateVisitor { void VisitDeoptimize(HDeoptimize* deopt) OVERRIDE; void VisitNewInstance(HNewInstance* new_instance) OVERRIDE; void VisitLoadClass(HLoadClass* load_class) OVERRIDE; + void VisitInstanceOf(HInstanceOf* load_class) OVERRIDE; void VisitClinitCheck(HClinitCheck* clinit_check) OVERRIDE; void VisitLoadString(HLoadString* instr) OVERRIDE; void VisitLoadException(HLoadException* instr) OVERRIDE; @@ -171,6 +172,12 @@ void ReferenceTypePropagation::ValidateTypes() { << "NullCheck " << instr->GetReferenceTypeInfo() << "Input(0) " << instr->InputAt(0)->GetReferenceTypeInfo(); } + } else if (instr->IsInstanceOf()) { + HInstanceOf* iof = instr->AsInstanceOf(); + DCHECK(!iof->GetTargetClassRTI().IsValid() || iof->GetTargetClassRTI().IsExact()); + } else if (instr->IsCheckCast()) { + HCheckCast* check = instr->AsCheckCast(); + DCHECK(!check->GetTargetClassRTI().IsValid() || check->GetTargetClassRTI().IsExact()); } } } @@ -499,8 +506,7 @@ void ReferenceTypePropagation::RTPVisitor::BoundTypeForIfInstanceOf(HBasicBlock* return; } - HLoadClass* load_class = instanceOf->InputAt(1)->AsLoadClass(); - ReferenceTypeInfo class_rti = load_class->GetLoadedClassRTI(); + ReferenceTypeInfo class_rti = instanceOf->GetTargetClassRTI(); if (!class_rti.IsValid()) { // He have loaded an unresolved class. Don't bother bounding the type. return; @@ -643,15 +649,20 @@ void ReferenceTypePropagation::RTPVisitor::VisitUnresolvedStaticFieldGet( void ReferenceTypePropagation::RTPVisitor::VisitLoadClass(HLoadClass* instr) { ScopedObjectAccess soa(Thread::Current()); - Handle<mirror::Class> resolved_class = instr->GetClass(); - if (IsAdmissible(resolved_class.Get())) { - instr->SetLoadedClassRTI(ReferenceTypeInfo::Create( - resolved_class, /* is_exact */ true)); + if (IsAdmissible(instr->GetClass().Get())) { + instr->SetValidLoadedClassRTI(); } instr->SetReferenceTypeInfo( ReferenceTypeInfo::Create(handle_cache_->GetClassClassHandle(), /* is_exact */ true)); } +void ReferenceTypePropagation::RTPVisitor::VisitInstanceOf(HInstanceOf* instr) { + ScopedObjectAccess soa(Thread::Current()); + if (IsAdmissible(instr->GetClass().Get())) { + instr->SetValidTargetClassRTI(); + } +} + void ReferenceTypePropagation::RTPVisitor::VisitClinitCheck(HClinitCheck* instr) { instr->SetReferenceTypeInfo(instr->InputAt(0)->GetReferenceTypeInfo()); } @@ -719,8 +730,6 @@ void ReferenceTypePropagation::RTPVisitor::VisitBoundType(HBoundType* instr) { } void ReferenceTypePropagation::RTPVisitor::VisitCheckCast(HCheckCast* check_cast) { - HLoadClass* load_class = check_cast->InputAt(1)->AsLoadClass(); - ReferenceTypeInfo class_rti = load_class->GetLoadedClassRTI(); HBoundType* bound_type = check_cast->GetNext()->AsBoundType(); if (bound_type == nullptr || bound_type->GetUpperBound().IsValid()) { // The next instruction is not an uninitialized BoundType. This must be @@ -729,12 +738,14 @@ void ReferenceTypePropagation::RTPVisitor::VisitCheckCast(HCheckCast* check_cast } DCHECK_EQ(bound_type->InputAt(0), check_cast->InputAt(0)); - if (class_rti.IsValid()) { + ScopedObjectAccess soa(Thread::Current()); + Handle<mirror::Class> klass = check_cast->GetClass(); + if (IsAdmissible(klass.Get())) { DCHECK(is_first_run_); - ScopedObjectAccess soa(Thread::Current()); + check_cast->SetValidTargetClassRTI(); // This is the first run of RTP and class is resolved. - bool is_exact = class_rti.GetTypeHandle()->CannotBeAssignedFromOtherTypes(); - bound_type->SetUpperBound(ReferenceTypeInfo::Create(class_rti.GetTypeHandle(), is_exact), + bool is_exact = klass->CannotBeAssignedFromOtherTypes(); + bound_type->SetUpperBound(ReferenceTypeInfo::Create(klass, is_exact), /* CheckCast succeeds for nulls. */ true); } else { // This is the first run of RTP and class is unresolved. Remove the binding. diff --git a/compiler/optimizing/scheduler.cc b/compiler/optimizing/scheduler.cc index bb28d50b569..bca538fb170 100644 --- a/compiler/optimizing/scheduler.cc +++ b/compiler/optimizing/scheduler.cc @@ -667,7 +667,8 @@ bool HScheduler::IsSchedulable(const HInstruction* instruction) const { // HUnaryOperation (or HBinaryOperation), check in debug mode that we have // the exhaustive lists here. if (instruction->IsUnaryOperation()) { - DCHECK(instruction->IsBooleanNot() || + DCHECK(instruction->IsAbs() || + instruction->IsBooleanNot() || instruction->IsNot() || instruction->IsNeg()) << "unexpected instruction " << instruction->DebugName(); return true; @@ -678,6 +679,8 @@ bool HScheduler::IsSchedulable(const HInstruction* instruction) const { instruction->IsCompare() || instruction->IsCondition() || instruction->IsDiv() || + instruction->IsMin() || + instruction->IsMax() || instruction->IsMul() || instruction->IsOr() || instruction->IsRem() || diff --git a/compiler/optimizing/select_generator.cc b/compiler/optimizing/select_generator.cc index 66e51421ca3..f9acf5aa9a9 100644 --- a/compiler/optimizing/select_generator.cc +++ b/compiler/optimizing/select_generator.cc @@ -16,6 +16,7 @@ #include "select_generator.h" +#include "base/scoped_arena_containers.h" #include "reference_type_propagation.h" namespace art { @@ -43,12 +44,16 @@ static bool IsSimpleBlock(HBasicBlock* block) { for (HInstructionIterator it(block->GetInstructions()); !it.Done(); it.Advance()) { HInstruction* instruction = it.Current(); if (instruction->IsControlFlow()) { - if (num_instructions > kMaxInstructionsInBranch) { - return false; - } return instruction->IsGoto() || instruction->IsReturn(); } else if (instruction->CanBeMoved() && !instruction->HasSideEffects()) { - num_instructions++; + if (instruction->IsSelect() && + instruction->AsSelect()->GetCondition()->GetBlock() == block) { + // Count one HCondition and HSelect in the same block as a single instruction. + // This enables finding nested selects. + continue; + } else if (++num_instructions > kMaxInstructionsInBranch) { + return false; // bail as soon as we exceed number of allowed instructions + } } else { return false; } @@ -86,9 +91,13 @@ static HPhi* GetSingleChangedPhi(HBasicBlock* block, size_t index1, size_t index } void HSelectGenerator::Run() { + // Select cache with local allocator. + ScopedArenaAllocator allocator(graph_->GetArenaStack()); + ScopedArenaSafeMap<HInstruction*, HSelect*> cache( + std::less<HInstruction*>(), allocator.Adapter(kArenaAllocSelectGenerator)); + // Iterate in post order in the unlikely case that removing one occurrence of // the selection pattern empties a branch block of another occurrence. - // Otherwise the order does not matter. for (HBasicBlock* block : graph_->GetPostOrder()) { if (!block->EndsWithIf()) continue; @@ -97,6 +106,7 @@ void HSelectGenerator::Run() { HBasicBlock* true_block = if_instruction->IfTrueSuccessor(); HBasicBlock* false_block = if_instruction->IfFalseSuccessor(); DCHECK_NE(true_block, false_block); + if (!IsSimpleBlock(true_block) || !IsSimpleBlock(false_block) || !BlocksMergeTogether(true_block, false_block)) { @@ -107,10 +117,10 @@ void HSelectGenerator::Run() { // If the branches are not empty, move instructions in front of the If. // TODO(dbrazdil): This puts an instruction between If and its condition. // Implement moving of conditions to first users if possible. - if (!true_block->IsSingleGoto() && !true_block->IsSingleReturn()) { + while (!true_block->IsSingleGoto() && !true_block->IsSingleReturn()) { true_block->GetFirstInstruction()->MoveBefore(if_instruction); } - if (!false_block->IsSingleGoto() && !false_block->IsSingleReturn()) { + while (!false_block->IsSingleGoto() && !false_block->IsSingleReturn()) { false_block->GetFirstInstruction()->MoveBefore(if_instruction); } DCHECK(true_block->IsSingleGoto() || true_block->IsSingleReturn()); @@ -138,7 +148,8 @@ void HSelectGenerator::Run() { DCHECK(both_successors_return || phi != nullptr); // Create the Select instruction and insert it in front of the If. - HSelect* select = new (graph_->GetAllocator()) HSelect(if_instruction->InputAt(0), + HInstruction* condition = if_instruction->InputAt(0); + HSelect* select = new (graph_->GetAllocator()) HSelect(condition, true_value, false_value, if_instruction->GetDexPc()); @@ -175,6 +186,26 @@ void HSelectGenerator::Run() { MaybeRecordStat(stats_, MethodCompilationStat::kSelectGenerated); + // Very simple way of finding common subexpressions in the generated HSelect statements + // (since this runs after GVN). Lookup by condition, and reuse latest one if possible + // (due to post order, latest select is most likely replacement). If needed, we could + // improve this by e.g. using the operands in the map as well. + auto it = cache.find(condition); + if (it == cache.end()) { + cache.Put(condition, select); + } else { + // Found cached value. See if latest can replace cached in the HIR. + HSelect* cached = it->second; + DCHECK_EQ(cached->GetCondition(), select->GetCondition()); + if (cached->GetTrueValue() == select->GetTrueValue() && + cached->GetFalseValue() == select->GetFalseValue() && + select->StrictlyDominates(cached)) { + cached->ReplaceWith(select); + cached->GetBlock()->RemoveInstruction(cached); + } + it->second = select; // always cache latest + } + // No need to update dominance information, as we are simplifying // a simple diamond shape, where the join block is merged with the // entry block. Any following blocks would have had the join block diff --git a/compiler/optimizing/sharpening.cc b/compiler/optimizing/sharpening.cc index 1e49411c72d..70b45763af7 100644 --- a/compiler/optimizing/sharpening.cc +++ b/compiler/optimizing/sharpening.cc @@ -125,8 +125,12 @@ void HSharpening::SharpenInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke, BootImageAOTCanEmbedMethod(callee, compiler_driver)) { method_load_kind = HInvokeStaticOrDirect::MethodLoadKind::kBootImageLinkTimePcRelative; code_ptr_location = HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod; + } else if (IsInBootImage(callee)) { + // Use PC-relative access to the .data.bimg.rel.ro methods array. + method_load_kind = HInvokeStaticOrDirect::MethodLoadKind::kBootImageRelRo; + code_ptr_location = HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod; } else { - // Use PC-relative access to the .bss methods arrays. + // Use PC-relative access to the .bss methods array. method_load_kind = HInvokeStaticOrDirect::MethodLoadKind::kBssEntry; code_ptr_location = HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod; } @@ -207,7 +211,7 @@ HLoadClass::LoadKind HSharpening::ComputeLoadClassKind( } else if (is_in_boot_image) { // AOT app compilation, boot image class. if (codegen->GetCompilerOptions().GetCompilePic()) { - desired_load_kind = HLoadClass::LoadKind::kBootImageClassTable; + desired_load_kind = HLoadClass::LoadKind::kBootImageRelRo; } else { desired_load_kind = HLoadClass::LoadKind::kBootImageAddress; } @@ -236,6 +240,75 @@ HLoadClass::LoadKind HSharpening::ComputeLoadClassKind( return load_kind; } +static inline bool CanUseTypeCheckBitstring(ObjPtr<mirror::Class> klass, + CodeGenerator* codegen, + CompilerDriver* compiler_driver) + REQUIRES_SHARED(Locks::mutator_lock_) { + DCHECK(!klass->IsProxyClass()); + DCHECK(!klass->IsArrayClass()); + + if (Runtime::Current()->UseJitCompilation()) { + // If we're JITting, try to assign a type check bitstring (fall through). + } else if (codegen->GetCompilerOptions().IsBootImage()) { + const char* descriptor = klass->GetDexFile().StringByTypeIdx(klass->GetDexTypeIndex()); + if (!compiler_driver->IsImageClass(descriptor)) { + return false; + } + // If the target is a boot image class, try to assign a type check bitstring (fall through). + // (If --force-determinism, this was already done; repeating is OK and yields the same result.) + } else { + // TODO: Use the bitstring also for AOT app compilation if the target class has a bitstring + // already assigned in the boot image. + return false; + } + + // Try to assign a type check bitstring. + MutexLock subtype_check_lock(Thread::Current(), *Locks::subtype_check_lock_); + if ((false) && // FIXME: Inliner does not respect compiler_driver->IsClassToCompile() + // and we're hitting an unassigned bitstring in dex2oat_image_test. b/26687569 + kIsDebugBuild && + codegen->GetCompilerOptions().IsBootImage() && + codegen->GetCompilerOptions().IsForceDeterminism()) { + SubtypeCheckInfo::State old_state = SubtypeCheck<ObjPtr<mirror::Class>>::GetState(klass); + CHECK(old_state == SubtypeCheckInfo::kAssigned || old_state == SubtypeCheckInfo::kOverflowed) + << klass->PrettyDescriptor() << "/" << old_state + << " in " << codegen->GetGraph()->PrettyMethod(); + } + SubtypeCheckInfo::State state = SubtypeCheck<ObjPtr<mirror::Class>>::EnsureAssigned(klass); + return state == SubtypeCheckInfo::kAssigned; +} + +TypeCheckKind HSharpening::ComputeTypeCheckKind(ObjPtr<mirror::Class> klass, + CodeGenerator* codegen, + CompilerDriver* compiler_driver, + bool needs_access_check) { + if (klass == nullptr) { + return TypeCheckKind::kUnresolvedCheck; + } else if (klass->IsInterface()) { + return TypeCheckKind::kInterfaceCheck; + } else if (klass->IsArrayClass()) { + if (klass->GetComponentType()->IsObjectClass()) { + return TypeCheckKind::kArrayObjectCheck; + } else if (klass->CannotBeAssignedFromOtherTypes()) { + return TypeCheckKind::kExactCheck; + } else { + return TypeCheckKind::kArrayCheck; + } + } else if (klass->IsFinal()) { // TODO: Consider using bitstring for final classes. + return TypeCheckKind::kExactCheck; + } else if (kBitstringSubtypeCheckEnabled && + !needs_access_check && + CanUseTypeCheckBitstring(klass, codegen, compiler_driver)) { + // TODO: We should not need the `!needs_access_check` check but getting rid of that + // requires rewriting some optimizations in instruction simplifier. + return TypeCheckKind::kBitstringCheck; + } else if (klass->IsAbstract()) { + return TypeCheckKind::kAbstractClassCheck; + } else { + return TypeCheckKind::kClassHierarchyCheck; + } +} + void HSharpening::ProcessLoadString( HLoadString* load_string, CodeGenerator* codegen, @@ -288,7 +361,7 @@ void HSharpening::ProcessLoadString( string = class_linker->LookupString(string_index, dex_cache.Get()); if (string != nullptr && runtime->GetHeap()->ObjectIsInBootImageSpace(string)) { if (codegen->GetCompilerOptions().GetCompilePic()) { - desired_load_kind = HLoadString::LoadKind::kBootImageInternTable; + desired_load_kind = HLoadString::LoadKind::kBootImageRelRo; } else { desired_load_kind = HLoadString::LoadKind::kBootImageAddress; } diff --git a/compiler/optimizing/sharpening.h b/compiler/optimizing/sharpening.h index 6df7d6d91ed..fa3e948eeb5 100644 --- a/compiler/optimizing/sharpening.h +++ b/compiler/optimizing/sharpening.h @@ -44,12 +44,10 @@ class HSharpening : public HOptimization { static constexpr const char* kSharpeningPassName = "sharpening"; - // Used by the builder. - static void ProcessLoadString(HLoadString* load_string, - CodeGenerator* codegen, - CompilerDriver* compiler_driver, - const DexCompilationUnit& dex_compilation_unit, - VariableSizedHandleScope* handles); + // Used by Sharpening and InstructionSimplifier. + static void SharpenInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke, + CodeGenerator* codegen, + CompilerDriver* compiler_driver); // Used by the builder and the inliner. static HLoadClass::LoadKind ComputeLoadClassKind(HLoadClass* load_class, @@ -58,10 +56,19 @@ class HSharpening : public HOptimization { const DexCompilationUnit& dex_compilation_unit) REQUIRES_SHARED(Locks::mutator_lock_); - // Used by Sharpening and InstructionSimplifier. - static void SharpenInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke, - CodeGenerator* codegen, - CompilerDriver* compiler_driver); + // Used by the builder. + static TypeCheckKind ComputeTypeCheckKind(ObjPtr<mirror::Class> klass, + CodeGenerator* codegen, + CompilerDriver* compiler_driver, + bool needs_access_check) + REQUIRES_SHARED(Locks::mutator_lock_); + + // Used by the builder. + static void ProcessLoadString(HLoadString* load_string, + CodeGenerator* codegen, + CompilerDriver* compiler_driver, + const DexCompilationUnit& dex_compilation_unit, + VariableSizedHandleScope* handles); private: CodeGenerator* codegen_; diff --git a/compiler/optimizing/stack_map_stream.h b/compiler/optimizing/stack_map_stream.h index 579aabdb5f5..268e9bd6e0c 100644 --- a/compiler/optimizing/stack_map_stream.h +++ b/compiler/optimizing/stack_map_stream.h @@ -19,9 +19,9 @@ #include "base/bit_vector-inl.h" #include "base/hash_map.h" +#include "base/memory_region.h" #include "base/scoped_arena_containers.h" #include "base/value_object.h" -#include "memory_region.h" #include "method_info.h" #include "nodes.h" #include "stack_map.h" diff --git a/compiler/optimizing/stack_map_test.cc b/compiler/optimizing/stack_map_test.cc index 7e517f34850..e36c5926622 100644 --- a/compiler/optimizing/stack_map_test.cc +++ b/compiler/optimizing/stack_map_test.cc @@ -18,6 +18,7 @@ #include "art_method.h" #include "base/arena_bit_vector.h" +#include "base/malloc_arena_pool.h" #include "stack_map_stream.h" #include "gtest/gtest.h" @@ -46,7 +47,7 @@ static bool CheckStackMask( using Kind = DexRegisterLocation::Kind; TEST(StackMapTest, Test1) { - ArenaPool pool; + MallocArenaPool pool; ArenaStack arena_stack(&pool); ScopedArenaAllocator allocator(&arena_stack); StackMapStream stream(&allocator, kRuntimeISA); @@ -128,7 +129,7 @@ TEST(StackMapTest, Test1) { } TEST(StackMapTest, Test2) { - ArenaPool pool; + MallocArenaPool pool; ArenaStack arena_stack(&pool); ScopedArenaAllocator allocator(&arena_stack); StackMapStream stream(&allocator, kRuntimeISA); @@ -413,7 +414,7 @@ TEST(StackMapTest, Test2) { } TEST(StackMapTest, TestDeduplicateInlineInfoDexRegisterMap) { - ArenaPool pool; + MallocArenaPool pool; ArenaStack arena_stack(&pool); ScopedArenaAllocator allocator(&arena_stack); StackMapStream stream(&allocator, kRuntimeISA); @@ -508,7 +509,7 @@ TEST(StackMapTest, TestDeduplicateInlineInfoDexRegisterMap) { } TEST(StackMapTest, TestNonLiveDexRegisters) { - ArenaPool pool; + MallocArenaPool pool; ArenaStack arena_stack(&pool); ScopedArenaAllocator allocator(&arena_stack); StackMapStream stream(&allocator, kRuntimeISA); @@ -588,7 +589,7 @@ TEST(StackMapTest, TestNonLiveDexRegisters) { // StackMap::kNoDexRegisterMapSmallEncoding, and ensure we do // not treat it as kNoDexRegisterMap. TEST(StackMapTest, DexRegisterMapOffsetOverflow) { - ArenaPool pool; + MallocArenaPool pool; ArenaStack arena_stack(&pool); ScopedArenaAllocator allocator(&arena_stack); StackMapStream stream(&allocator, kRuntimeISA); @@ -652,7 +653,7 @@ TEST(StackMapTest, DexRegisterMapOffsetOverflow) { } TEST(StackMapTest, TestShareDexRegisterMap) { - ArenaPool pool; + MallocArenaPool pool; ArenaStack arena_stack(&pool); ScopedArenaAllocator allocator(&arena_stack); StackMapStream stream(&allocator, kRuntimeISA); @@ -711,7 +712,7 @@ TEST(StackMapTest, TestShareDexRegisterMap) { } TEST(StackMapTest, TestNoDexRegisterMap) { - ArenaPool pool; + MallocArenaPool pool; ArenaStack arena_stack(&pool); ScopedArenaAllocator allocator(&arena_stack); StackMapStream stream(&allocator, kRuntimeISA); @@ -761,7 +762,7 @@ TEST(StackMapTest, TestNoDexRegisterMap) { } TEST(StackMapTest, InlineTest) { - ArenaPool pool; + MallocArenaPool pool; ArenaStack arena_stack(&pool); ScopedArenaAllocator allocator(&arena_stack); StackMapStream stream(&allocator, kRuntimeISA); @@ -949,7 +950,7 @@ TEST(StackMapTest, CodeOffsetTest) { } TEST(StackMapTest, TestDeduplicateStackMask) { - ArenaPool pool; + MallocArenaPool pool; ArenaStack arena_stack(&pool); ScopedArenaAllocator allocator(&arena_stack); StackMapStream stream(&allocator, kRuntimeISA); @@ -978,7 +979,7 @@ TEST(StackMapTest, TestDeduplicateStackMask) { } TEST(StackMapTest, TestInvokeInfo) { - ArenaPool pool; + MallocArenaPool pool; ArenaStack arena_stack(&pool); ScopedArenaAllocator allocator(&arena_stack); StackMapStream stream(&allocator, kRuntimeISA); diff --git a/compiler/optimizing/superblock_cloner.cc b/compiler/optimizing/superblock_cloner.cc index a7c23bef7e3..fad7729956b 100644 --- a/compiler/optimizing/superblock_cloner.cc +++ b/compiler/optimizing/superblock_cloner.cc @@ -70,20 +70,18 @@ static bool ArePhiInputsTheSame(const HPhi* phi) { return true; } -// Returns a common predecessor of loop1 and loop2 in the loop tree or nullptr if it is the whole -// graph. -static HLoopInformation* FindCommonLoop(HLoopInformation* loop1, HLoopInformation* loop2) { - if (loop1 != nullptr || loop2 != nullptr) { - return nullptr; +// Returns whether two Edge sets are equal (ArenaHashSet doesn't have "Equal" method). +static bool EdgeHashSetsEqual(const HEdgeSet* set1, const HEdgeSet* set2) { + if (set1->Size() != set2->Size()) { + return false; } - if (loop1->IsIn(*loop2)) { - return loop2; - } else if (loop2->IsIn(*loop1)) { - return loop1; + for (auto e : *set1) { + if (set2->Find(e) == set2->end()) { + return false; + } } - HBasicBlock* block = CommonDominator::ForPair(loop1->GetHeader(), loop2->GetHeader()); - return block->GetLoopInformation(); + return true; } // Calls HGraph::OrderLoopHeaderPredecessors for each loop in the graph. @@ -95,6 +93,21 @@ static void OrderLoopsHeadersPredecessors(HGraph* graph) { } } +// Performs DFS on the subgraph (specified by 'bb_set') starting from the specified block; while +// traversing the function removes basic blocks from the bb_set (instead of traditional DFS +// 'marking'). So what is left in the 'bb_set' after the traversal is not reachable from the start +// block. +static void TraverseSubgraphForConnectivity(HBasicBlock* block, HBasicBlockSet* bb_set) { + DCHECK(bb_set->IsBitSet(block->GetBlockId())); + bb_set->ClearBit(block->GetBlockId()); + + for (HBasicBlock* succ : block->GetSuccessors()) { + if (bb_set->IsBitSet(succ->GetBlockId())) { + TraverseSubgraphForConnectivity(succ, bb_set); + } + } +} + // // Helpers for CloneBasicBlock. // @@ -268,7 +281,6 @@ void SuperblockCloner::FindBackEdgesLocal(HBasicBlock* entry_block, ArenaBitVect } void SuperblockCloner::RecalculateBackEdgesInfo(ArenaBitVector* outer_loop_bb_set) { - // TODO: DCHECK that after the transformation the graph is connected. HBasicBlock* block_entry = nullptr; if (outer_loop_ == nullptr) { @@ -424,6 +436,11 @@ void SuperblockCloner::FindAndSetLocalAreaForAdjustments() { outer_loop_ = nullptr; break; } + if (outer_loop_ == nullptr) { + // We should not use the initial outer_loop_ value 'nullptr' when finding the most outer + // common loop. + outer_loop_ = loop_exit_loop_info; + } outer_loop_ = FindCommonLoop(outer_loop_, loop_exit_loop_info); } @@ -507,6 +524,34 @@ void SuperblockCloner::ResolveDataFlow() { // Debug and logging methods. // +// Debug function to dump graph' BasicBlocks info. +void DumpBB(HGraph* graph) { + for (HBasicBlock* bb : graph->GetBlocks()) { + if (bb == nullptr) { + continue; + } + std::cout << bb->GetBlockId(); + std::cout << " <- "; + for (HBasicBlock* pred : bb->GetPredecessors()) { + std::cout << pred->GetBlockId() << " "; + } + std::cout << " -> "; + for (HBasicBlock* succ : bb->GetSuccessors()) { + std::cout << succ->GetBlockId() << " "; + } + + if (bb->GetDominator()) { + std::cout << " dom " << bb->GetDominator()->GetBlockId(); + } + + if (bb->GetLoopInformation()) { + std::cout << "\tloop: " << bb->GetLoopInformation()->GetHeader()->GetBlockId(); + } + + std::cout << std::endl; + } +} + void SuperblockCloner::CheckInstructionInputsRemapping(HInstruction* orig_instr) { DCHECK(!orig_instr->IsPhi()); HInstruction* copy_instr = GetInstrCopy(orig_instr); @@ -542,6 +587,82 @@ void SuperblockCloner::CheckInstructionInputsRemapping(HInstruction* orig_instr) } } +bool SuperblockCloner::CheckRemappingInfoIsValid() { + for (HEdge edge : *remap_orig_internal_) { + if (!IsEdgeValid(edge, graph_) || + !IsInOrigBBSet(edge.GetFrom()) || + !IsInOrigBBSet(edge.GetTo())) { + return false; + } + } + + for (auto edge : *remap_copy_internal_) { + if (!IsEdgeValid(edge, graph_) || + !IsInOrigBBSet(edge.GetFrom()) || + !IsInOrigBBSet(edge.GetTo())) { + return false; + } + } + + for (auto edge : *remap_incoming_) { + if (!IsEdgeValid(edge, graph_) || + IsInOrigBBSet(edge.GetFrom()) || + !IsInOrigBBSet(edge.GetTo())) { + return false; + } + } + + return true; +} + +void SuperblockCloner::VerifyGraph() { + for (auto it : *hir_map_) { + HInstruction* orig_instr = it.first; + HInstruction* copy_instr = it.second; + if (!orig_instr->IsPhi() && !orig_instr->IsSuspendCheck()) { + DCHECK(it.first->GetBlock() != nullptr); + } + if (!copy_instr->IsPhi() && !copy_instr->IsSuspendCheck()) { + DCHECK(it.second->GetBlock() != nullptr); + } + } + + GraphChecker checker(graph_); + checker.Run(); + if (!checker.IsValid()) { + for (const std::string& error : checker.GetErrors()) { + std::cout << error << std::endl; + } + LOG(FATAL) << "GraphChecker failed: superblock cloner\n"; + } +} + +void DumpBBSet(const ArenaBitVector* set) { + for (uint32_t idx : set->Indexes()) { + std::cout << idx << "\n"; + } +} + +void SuperblockCloner::DumpInputSets() { + std::cout << graph_->PrettyMethod() << "\n"; + std::cout << "orig_bb_set:\n"; + for (uint32_t idx : orig_bb_set_.Indexes()) { + std::cout << idx << "\n"; + } + std::cout << "remap_orig_internal:\n"; + for (HEdge e : *remap_orig_internal_) { + std::cout << e << "\n"; + } + std::cout << "remap_copy_internal:\n"; + for (auto e : *remap_copy_internal_) { + std::cout << e << "\n"; + } + std::cout << "remap_incoming:\n"; + for (auto e : *remap_incoming_) { + std::cout << e << "\n"; + } +} + // // Public methods. // @@ -569,6 +690,7 @@ void SuperblockCloner::SetSuccessorRemappingInfo(const HEdgeSet* remap_orig_inte remap_orig_internal_ = remap_orig_internal; remap_copy_internal_ = remap_copy_internal; remap_incoming_ = remap_incoming; + DCHECK(CheckRemappingInfoIsValid()); } bool SuperblockCloner::IsSubgraphClonable() const { @@ -602,6 +724,63 @@ bool SuperblockCloner::IsSubgraphClonable() const { return true; } +bool SuperblockCloner::IsFastCase() const { + // Check that loop unrolling/loop peeling is being conducted. + // Check that all the basic blocks belong to the same loop. + bool flag = false; + HLoopInformation* common_loop_info = nullptr; + for (uint32_t idx : orig_bb_set_.Indexes()) { + HBasicBlock* block = GetBlockById(idx); + HLoopInformation* block_loop_info = block->GetLoopInformation(); + if (!flag) { + common_loop_info = block_loop_info; + } else { + if (block_loop_info != common_loop_info) { + return false; + } + } + } + + // Check that orig_bb_set_ corresponds to loop peeling/unrolling. + if (common_loop_info == nullptr || !orig_bb_set_.SameBitsSet(&common_loop_info->GetBlocks())) { + return false; + } + + bool peeling_or_unrolling = false; + HEdgeSet remap_orig_internal(graph_->GetAllocator()->Adapter(kArenaAllocSuperblockCloner)); + HEdgeSet remap_copy_internal(graph_->GetAllocator()->Adapter(kArenaAllocSuperblockCloner)); + HEdgeSet remap_incoming(graph_->GetAllocator()->Adapter(kArenaAllocSuperblockCloner)); + + + // Check whether remapping info corresponds to loop unrolling. + CollectRemappingInfoForPeelUnroll(/* to_unroll*/ true, + common_loop_info, + &remap_orig_internal, + &remap_copy_internal, + &remap_incoming); + + peeling_or_unrolling |= EdgeHashSetsEqual(&remap_orig_internal, remap_orig_internal_) && + EdgeHashSetsEqual(&remap_copy_internal, remap_copy_internal_) && + EdgeHashSetsEqual(&remap_incoming, remap_incoming_); + + remap_orig_internal.Clear(); + remap_copy_internal.Clear(); + remap_incoming.Clear(); + + // Check whether remapping info corresponds to loop peeling. + CollectRemappingInfoForPeelUnroll(/* to_unroll*/ false, + common_loop_info, + &remap_orig_internal, + &remap_copy_internal, + &remap_incoming); + + peeling_or_unrolling |= EdgeHashSetsEqual(&remap_orig_internal, remap_orig_internal_) && + EdgeHashSetsEqual(&remap_copy_internal, remap_copy_internal_) && + EdgeHashSetsEqual(&remap_incoming, remap_incoming_); + + return peeling_or_unrolling; +} + void SuperblockCloner::Run() { DCHECK(bb_map_ != nullptr); DCHECK(hir_map_ != nullptr); @@ -609,6 +788,11 @@ void SuperblockCloner::Run() { remap_copy_internal_ != nullptr && remap_incoming_ != nullptr); DCHECK(IsSubgraphClonable()); + DCHECK(IsFastCase()); + + if (kSuperblockClonerLogging) { + DumpInputSets(); + } // Find an area in the graph for which control flow information should be adjusted. FindAndSetLocalAreaForAdjustments(); @@ -618,6 +802,19 @@ void SuperblockCloner::Run() { // Connect the blocks together/remap successors and fix phis which are directly affected my the // remapping. RemapEdgesSuccessors(); + + // Check that the subgraph is connected. + if (kIsDebugBuild) { + HBasicBlockSet work_set(arena_, orig_bb_set_.GetSizeOf(), true, kArenaAllocSuperblockCloner); + + // Add original and copy blocks of the subgraph to the work set. + for (auto iter : *bb_map_) { + work_set.SetBit(iter.first->GetBlockId()); // Original block. + work_set.SetBit(iter.second->GetBlockId()); // Copy block. + } + CHECK(IsSubgraphConnected(&work_set, graph_)); + } + // Recalculate dominance and backedge information which is required by the next stage. AdjustControlFlowInfo(); // Fix data flow of the graph. @@ -650,6 +847,10 @@ void SuperblockCloner::CleanUp() { } } } + + if (kIsDebugBuild) { + VerifyGraph(); + } } HBasicBlock* SuperblockCloner::CloneBasicBlock(const HBasicBlock* orig_block) { @@ -701,4 +902,127 @@ void SuperblockCloner::CloneBasicBlocks() { } } +// +// Stand-alone methods. +// + +void CollectRemappingInfoForPeelUnroll(bool to_unroll, + HLoopInformation* loop_info, + HEdgeSet* remap_orig_internal, + HEdgeSet* remap_copy_internal, + HEdgeSet* remap_incoming) { + DCHECK(loop_info != nullptr); + HBasicBlock* loop_header = loop_info->GetHeader(); + // Set up remap_orig_internal edges set - set is empty. + // Set up remap_copy_internal edges set. + for (HBasicBlock* back_edge_block : loop_info->GetBackEdges()) { + HEdge e = HEdge(back_edge_block, loop_header); + if (to_unroll) { + remap_orig_internal->Insert(e); + remap_copy_internal->Insert(e); + } else { + remap_copy_internal->Insert(e); + } + } + + // Set up remap_incoming edges set. + if (!to_unroll) { + remap_incoming->Insert(HEdge(loop_info->GetPreHeader(), loop_header)); + } +} + +bool IsSubgraphConnected(SuperblockCloner::HBasicBlockSet* work_set, HGraph* graph) { + ArenaVector<HBasicBlock*> entry_blocks( + graph->GetAllocator()->Adapter(kArenaAllocSuperblockCloner)); + + // Find subgraph entry blocks. + for (uint32_t orig_block_id : work_set->Indexes()) { + HBasicBlock* block = graph->GetBlocks()[orig_block_id]; + for (HBasicBlock* pred : block->GetPredecessors()) { + if (!work_set->IsBitSet(pred->GetBlockId())) { + entry_blocks.push_back(block); + break; + } + } + } + + for (HBasicBlock* entry_block : entry_blocks) { + if (work_set->IsBitSet(entry_block->GetBlockId())) { + TraverseSubgraphForConnectivity(entry_block, work_set); + } + } + + // Return whether there are unvisited - unreachable - blocks. + return work_set->NumSetBits() == 0; +} + +HLoopInformation* FindCommonLoop(HLoopInformation* loop1, HLoopInformation* loop2) { + if (loop1 == nullptr || loop2 == nullptr) { + return nullptr; + } + + if (loop1->IsIn(*loop2)) { + return loop2; + } + + HLoopInformation* current = loop1; + while (current != nullptr && !loop2->IsIn(*current)) { + current = current->GetPreHeader()->GetLoopInformation(); + } + + return current; +} + +bool PeelUnrollHelper::IsLoopClonable(HLoopInformation* loop_info) { + PeelUnrollHelper helper(loop_info, nullptr, nullptr); + return helper.IsLoopClonable(); +} + +HBasicBlock* PeelUnrollHelper::DoPeelUnrollImpl(bool to_unroll) { + // For now do peeling only for natural loops. + DCHECK(!loop_info_->IsIrreducible()); + + HBasicBlock* loop_header = loop_info_->GetHeader(); + // Check that loop info is up-to-date. + DCHECK(loop_info_ == loop_header->GetLoopInformation()); + + HGraph* graph = loop_header->GetGraph(); + ArenaAllocator allocator(graph->GetAllocator()->GetArenaPool()); + + HEdgeSet remap_orig_internal(graph->GetAllocator()->Adapter(kArenaAllocSuperblockCloner)); + HEdgeSet remap_copy_internal(graph->GetAllocator()->Adapter(kArenaAllocSuperblockCloner)); + HEdgeSet remap_incoming(graph->GetAllocator()->Adapter(kArenaAllocSuperblockCloner)); + + CollectRemappingInfoForPeelUnroll(to_unroll, + loop_info_, + &remap_orig_internal, + &remap_copy_internal, + &remap_incoming); + + cloner_.SetSuccessorRemappingInfo(&remap_orig_internal, &remap_copy_internal, &remap_incoming); + cloner_.Run(); + cloner_.CleanUp(); + + // Check that loop info is preserved. + DCHECK(loop_info_ == loop_header->GetLoopInformation()); + + return loop_header; +} + +PeelUnrollSimpleHelper::PeelUnrollSimpleHelper(HLoopInformation* info) + : bb_map_(std::less<HBasicBlock*>(), + info->GetHeader()->GetGraph()->GetAllocator()->Adapter(kArenaAllocSuperblockCloner)), + hir_map_(std::less<HInstruction*>(), + info->GetHeader()->GetGraph()->GetAllocator()->Adapter(kArenaAllocSuperblockCloner)), + helper_(info, &bb_map_, &hir_map_) {} + } // namespace art + +namespace std { + +ostream& operator<<(ostream& os, const art::HEdge& e) { + e.Dump(os); + return os; +} + +} // namespace std diff --git a/compiler/optimizing/superblock_cloner.h b/compiler/optimizing/superblock_cloner.h index 23de6926735..e0931674cb3 100644 --- a/compiler/optimizing/superblock_cloner.h +++ b/compiler/optimizing/superblock_cloner.h @@ -25,7 +25,6 @@ namespace art { static const bool kSuperblockClonerLogging = false; -static const bool kSuperblockClonerVerify = false; // Represents an edge between two HBasicBlocks. // @@ -152,6 +151,15 @@ class SuperblockCloner : public ValueObject { // TODO: Start from small range of graph patterns then extend it. bool IsSubgraphClonable() const; + // Returns whether selected subgraph satisfies the criteria for fast data flow resolution + // when iterative DF algorithm is not required and dominators/instructions inputs can be + // trivially adjusted. + // + // TODO: formally describe the criteria. + // + // Loop peeling and unrolling satisfy the criteria. + bool IsFastCase() const; + // Runs the copy algorithm according to the description. void Run(); @@ -202,11 +210,17 @@ class SuperblockCloner : public ValueObject { return IsInOrigBBSet(block->GetBlockId()); } + // Returns the area (the most outer loop) in the graph for which control flow (back edges, loops, + // dominators) needs to be adjusted. + HLoopInformation* GetRegionToBeAdjusted() const { + return outer_loop_; + } + private: // Fills the 'exits' vector with the subgraph exits. void SearchForSubgraphExits(ArenaVector<HBasicBlock*>* exits); - // Finds and records information about the area in the graph for which control-flow (back edges, + // Finds and records information about the area in the graph for which control flow (back edges, // loops, dominators) needs to be adjusted. void FindAndSetLocalAreaForAdjustments(); @@ -217,7 +231,7 @@ class SuperblockCloner : public ValueObject { // phis' nor instructions' inputs values are resolved. void RemapEdgesSuccessors(); - // Adjusts control-flow (back edges, loops, dominators) for the local area defined by + // Adjusts control flow (back edges, loops, dominators) for the local area defined by // FindAndSetLocalAreaForAdjustments. void AdjustControlFlowInfo(); @@ -272,6 +286,9 @@ class SuperblockCloner : public ValueObject { // Debug and logging methods. // void CheckInstructionInputsRemapping(HInstruction* orig_instr); + bool CheckRemappingInfoIsValid(); + void VerifyGraph(); + void DumpInputSets(); HBasicBlock* GetBlockById(uint32_t block_id) const { DCHECK(block_id < graph_->GetBlocks().size()); @@ -295,15 +312,97 @@ class SuperblockCloner : public ValueObject { HBasicBlockMap* bb_map_; // Correspondence map for instructions: (original HInstruction, copy HInstruction). HInstructionMap* hir_map_; - // Area in the graph for which control-flow (back edges, loops, dominators) needs to be adjusted. + // Area in the graph for which control flow (back edges, loops, dominators) needs to be adjusted. HLoopInformation* outer_loop_; HBasicBlockSet outer_loop_bb_set_; ART_FRIEND_TEST(SuperblockClonerTest, AdjustControlFlowInfo); + ART_FRIEND_TEST(SuperblockClonerTest, IsGraphConnected); DISALLOW_COPY_AND_ASSIGN(SuperblockCloner); }; +// Helper class to perform loop peeling/unrolling. +// +// This helper should be used when correspondence map between original and copied +// basic blocks/instructions are demanded. +class PeelUnrollHelper : public ValueObject { + public: + explicit PeelUnrollHelper(HLoopInformation* info, + SuperblockCloner::HBasicBlockMap* bb_map, + SuperblockCloner::HInstructionMap* hir_map) : + loop_info_(info), + cloner_(info->GetHeader()->GetGraph(), &info->GetBlocks(), bb_map, hir_map) { + // For now do peeling/unrolling only for natural loops. + DCHECK(!info->IsIrreducible()); + } + + // Returns whether the loop can be peeled/unrolled (static function). + static bool IsLoopClonable(HLoopInformation* loop_info); + + // Returns whether the loop can be peeled/unrolled. + bool IsLoopClonable() const { return cloner_.IsSubgraphClonable(); } + + HBasicBlock* DoPeeling() { return DoPeelUnrollImpl(/* to_unroll */ false); } + HBasicBlock* DoUnrolling() { return DoPeelUnrollImpl(/* to_unroll */ true); } + HLoopInformation* GetRegionToBeAdjusted() const { return cloner_.GetRegionToBeAdjusted(); } + + protected: + // Applies loop peeling/unrolling for the loop specified by 'loop_info'. + // + // Depending on 'do_unroll' either unrolls loop by 2 or peels one iteration from it. + HBasicBlock* DoPeelUnrollImpl(bool to_unroll); + + private: + HLoopInformation* loop_info_; + SuperblockCloner cloner_; + + DISALLOW_COPY_AND_ASSIGN(PeelUnrollHelper); +}; + +// Helper class to perform loop peeling/unrolling. +// +// This helper should be used when there is no need to get correspondence information between +// original and copied basic blocks/instructions. +class PeelUnrollSimpleHelper : public ValueObject { + public: + explicit PeelUnrollSimpleHelper(HLoopInformation* info); + bool IsLoopClonable() const { return helper_.IsLoopClonable(); } + HBasicBlock* DoPeeling() { return helper_.DoPeeling(); } + HBasicBlock* DoUnrolling() { return helper_.DoUnrolling(); } + HLoopInformation* GetRegionToBeAdjusted() const { return helper_.GetRegionToBeAdjusted(); } + + const SuperblockCloner::HBasicBlockMap* GetBasicBlockMap() const { return &bb_map_; } + const SuperblockCloner::HInstructionMap* GetInstructionMap() const { return &hir_map_; } + + private: + SuperblockCloner::HBasicBlockMap bb_map_; + SuperblockCloner::HInstructionMap hir_map_; + PeelUnrollHelper helper_; + + DISALLOW_COPY_AND_ASSIGN(PeelUnrollSimpleHelper); +}; + +// Collects edge remapping info for loop peeling/unrolling for the loop specified by loop info. +void CollectRemappingInfoForPeelUnroll(bool to_unroll, + HLoopInformation* loop_info, + SuperblockCloner::HEdgeSet* remap_orig_internal, + SuperblockCloner::HEdgeSet* remap_copy_internal, + SuperblockCloner::HEdgeSet* remap_incoming); + +// Returns whether blocks from 'work_set' are reachable from the rest of the graph. +// +// Returns whether such a set 'outer_entries' of basic blocks exists that: +// - each block from 'outer_entries' is not from 'work_set'. +// - each block from 'work_set' is reachable from at least one block from 'outer_entries'. +// +// After the function returns work_set contains only blocks from the original 'work_set' +// which are unreachable from the rest of the graph. +bool IsSubgraphConnected(SuperblockCloner::HBasicBlockSet* work_set, HGraph* graph); + +// Returns a common predecessor of loop1 and loop2 in the loop tree or nullptr if it is the whole +// graph. +HLoopInformation* FindCommonLoop(HLoopInformation* loop1, HLoopInformation* loop2); } // namespace art namespace std { @@ -312,11 +411,12 @@ template <> struct hash<art::HEdge> { size_t operator()(art::HEdge const& x) const noexcept { // Use Cantor pairing function as the hash function. - uint32_t a = x.GetFrom(); - uint32_t b = x.GetTo(); + size_t a = x.GetFrom(); + size_t b = x.GetTo(); return (a + b) * (a + b + 1) / 2 + b; } }; +ostream& operator<<(ostream& os, const art::HEdge& e); } // namespace std diff --git a/compiler/optimizing/superblock_cloner_test.cc b/compiler/optimizing/superblock_cloner_test.cc index f1b7bffdf5f..df2e517afff 100644 --- a/compiler/optimizing/superblock_cloner_test.cc +++ b/compiler/optimizing/superblock_cloner_test.cc @@ -25,52 +25,65 @@ namespace art { using HBasicBlockMap = SuperblockCloner::HBasicBlockMap; using HInstructionMap = SuperblockCloner::HInstructionMap; +using HBasicBlockSet = SuperblockCloner::HBasicBlockSet; +using HEdgeSet = SuperblockCloner::HEdgeSet; // This class provides methods and helpers for testing various cloning and copying routines: // individual instruction cloning and cloning of the more coarse-grain structures. class SuperblockClonerTest : public OptimizingUnitTest { public: - SuperblockClonerTest() - : graph_(CreateGraph()), entry_block_(nullptr), exit_block_(nullptr), parameter_(nullptr) {} + SuperblockClonerTest() : graph_(CreateGraph()), + entry_block_(nullptr), + return_block_(nullptr), + exit_block_(nullptr), + parameter_(nullptr) {} - void CreateBasicLoopControlFlow(/* out */ HBasicBlock** header_p, - /* out */ HBasicBlock** body_p) { + void InitGraph() { entry_block_ = new (GetAllocator()) HBasicBlock(graph_); graph_->AddBlock(entry_block_); graph_->SetEntryBlock(entry_block_); + return_block_ = new (GetAllocator()) HBasicBlock(graph_); + graph_->AddBlock(return_block_); + + exit_block_ = new (GetAllocator()) HBasicBlock(graph_); + graph_->AddBlock(exit_block_); + graph_->SetExitBlock(exit_block_); + + entry_block_->AddSuccessor(return_block_); + return_block_->AddSuccessor(exit_block_); + + parameter_ = new (GetAllocator()) HParameterValue(graph_->GetDexFile(), + dex::TypeIndex(0), + 0, + DataType::Type::kInt32); + entry_block_->AddInstruction(parameter_); + return_block_->AddInstruction(new (GetAllocator()) HReturnVoid()); + exit_block_->AddInstruction(new (GetAllocator()) HExit()); + } + + void CreateBasicLoopControlFlow(HBasicBlock* position, + HBasicBlock* successor, + /* out */ HBasicBlock** header_p, + /* out */ HBasicBlock** body_p) { HBasicBlock* loop_preheader = new (GetAllocator()) HBasicBlock(graph_); HBasicBlock* loop_header = new (GetAllocator()) HBasicBlock(graph_); HBasicBlock* loop_body = new (GetAllocator()) HBasicBlock(graph_); - HBasicBlock* loop_exit = new (GetAllocator()) HBasicBlock(graph_); graph_->AddBlock(loop_preheader); graph_->AddBlock(loop_header); graph_->AddBlock(loop_body); - graph_->AddBlock(loop_exit); - exit_block_ = new (GetAllocator()) HBasicBlock(graph_); - graph_->AddBlock(exit_block_); - graph_->SetExitBlock(exit_block_); + position->ReplaceSuccessor(successor, loop_preheader); - entry_block_->AddSuccessor(loop_preheader); loop_preheader->AddSuccessor(loop_header); // Loop exit first to have a proper exit condition/target for HIf. - loop_header->AddSuccessor(loop_exit); + loop_header->AddSuccessor(successor); loop_header->AddSuccessor(loop_body); loop_body->AddSuccessor(loop_header); - loop_exit->AddSuccessor(exit_block_); *header_p = loop_header; *body_p = loop_body; - - parameter_ = new (GetAllocator()) HParameterValue(graph_->GetDexFile(), - dex::TypeIndex(0), - 0, - DataType::Type::kInt32); - entry_block_->AddInstruction(parameter_); - loop_exit->AddInstruction(new (GetAllocator()) HReturnVoid()); - exit_block_->AddInstruction(new (GetAllocator()) HExit()); } void CreateBasicLoopDataFlow(HBasicBlock* loop_header, HBasicBlock* loop_body) { @@ -84,11 +97,12 @@ class SuperblockClonerTest : public OptimizingUnitTest { // Header block. HPhi* phi = new (GetAllocator()) HPhi(GetAllocator(), 0, 0, DataType::Type::kInt32); HInstruction* suspend_check = new (GetAllocator()) HSuspendCheck(); + HInstruction* loop_check = new (GetAllocator()) HGreaterThanOrEqual(phi, const_128); loop_header->AddPhi(phi); loop_header->AddInstruction(suspend_check); - loop_header->AddInstruction(new (GetAllocator()) HGreaterThanOrEqual(phi, const_128)); - loop_header->AddInstruction(new (GetAllocator()) HIf(parameter_)); + loop_header->AddInstruction(loop_check); + loop_header->AddInstruction(new (GetAllocator()) HIf(loop_check)); // Loop body block. HInstruction* null_check = new (GetAllocator()) HNullCheck(parameter_, dex_pc); @@ -97,8 +111,8 @@ class SuperblockClonerTest : public OptimizingUnitTest { HInstruction* array_get = new (GetAllocator()) HArrayGet(null_check, bounds_check, DataType::Type::kInt32, dex_pc); HInstruction* add = new (GetAllocator()) HAdd(DataType::Type::kInt32, array_get, const_1); - HInstruction* array_set = - new (GetAllocator()) HArraySet(null_check, bounds_check, add, DataType::Type::kInt32, dex_pc); + HInstruction* array_set = new (GetAllocator()) HArraySet( + null_check, bounds_check, add, DataType::Type::kInt32, dex_pc); HInstruction* induction_inc = new (GetAllocator()) HAdd(DataType::Type::kInt32, phi, const_1); loop_body->AddInstruction(null_check); @@ -153,6 +167,7 @@ class SuperblockClonerTest : public OptimizingUnitTest { HGraph* graph_; HBasicBlock* entry_block_; + HBasicBlock* return_block_; HBasicBlock* exit_block_; HInstruction* parameter_; @@ -162,10 +177,11 @@ TEST_F(SuperblockClonerTest, IndividualInstrCloner) { HBasicBlock* header = nullptr; HBasicBlock* loop_body = nullptr; - CreateBasicLoopControlFlow(&header, &loop_body); + InitGraph(); + CreateBasicLoopControlFlow(entry_block_, return_block_, &header, &loop_body); CreateBasicLoopDataFlow(header, loop_body); graph_->BuildDominatorTree(); - ASSERT_TRUE(CheckGraph()); + EXPECT_TRUE(CheckGraph()); HSuspendCheck* old_suspend_check = header->GetLoopInformation()->GetSuspendCheck(); CloneAndReplaceInstructionVisitor visitor(graph_); @@ -193,7 +209,8 @@ TEST_F(SuperblockClonerTest, CloneBasicBlocks) { HBasicBlock* loop_body = nullptr; ArenaAllocator* arena = graph_->GetAllocator(); - CreateBasicLoopControlFlow(&header, &loop_body); + InitGraph(); + CreateBasicLoopControlFlow(entry_block_, return_block_, &header, &loop_body); CreateBasicLoopDataFlow(header, loop_body); graph_->BuildDominatorTree(); ASSERT_TRUE(CheckGraph()); @@ -272,7 +289,8 @@ TEST_F(SuperblockClonerTest, AdjustControlFlowInfo) { HBasicBlock* loop_body = nullptr; ArenaAllocator* arena = graph_->GetAllocator(); - CreateBasicLoopControlFlow(&header, &loop_body); + InitGraph(); + CreateBasicLoopControlFlow(entry_block_, return_block_, &header, &loop_body); CreateBasicLoopDataFlow(header, loop_body); graph_->BuildDominatorTree(); ASSERT_TRUE(CheckGraph()); @@ -303,4 +321,487 @@ TEST_F(SuperblockClonerTest, AdjustControlFlowInfo) { EXPECT_TRUE(loop_info->IsBackEdge(*loop_body)); } +// Tests IsSubgraphConnected function for negative case. +TEST_F(SuperblockClonerTest, IsGraphConnected) { + HBasicBlock* header = nullptr; + HBasicBlock* loop_body = nullptr; + ArenaAllocator* arena = graph_->GetAllocator(); + + InitGraph(); + CreateBasicLoopControlFlow(entry_block_, return_block_, &header, &loop_body); + CreateBasicLoopDataFlow(header, loop_body); + HBasicBlock* unreachable_block = new (GetAllocator()) HBasicBlock(graph_); + graph_->AddBlock(unreachable_block); + + HBasicBlockSet bb_set( + arena, graph_->GetBlocks().size(), false, kArenaAllocSuperblockCloner); + bb_set.SetBit(header->GetBlockId()); + bb_set.SetBit(loop_body->GetBlockId()); + bb_set.SetBit(unreachable_block->GetBlockId()); + + EXPECT_FALSE(IsSubgraphConnected(&bb_set, graph_)); + EXPECT_EQ(bb_set.NumSetBits(), 1u); + EXPECT_TRUE(bb_set.IsBitSet(unreachable_block->GetBlockId())); +} + +// Tests SuperblockCloner for loop peeling case. +// +// Control Flow of the example (ignoring critical edges splitting). +// +// Before After +// +// |B| |B| +// | | +// v v +// |1| |1| +// | | +// v v +// |2|<-\ (6) |2A| +// / \ / / \ +// v v/ / v +// |4| |3| / |3A| (7) +// | / / +// v | v +// |E| \ |2|<-\ +// \ / \ / +// v v / +// |4| |3| +// | +// v +// |E| +TEST_F(SuperblockClonerTest, LoopPeeling) { + HBasicBlock* header = nullptr; + HBasicBlock* loop_body = nullptr; + + InitGraph(); + CreateBasicLoopControlFlow(entry_block_, return_block_, &header, &loop_body); + CreateBasicLoopDataFlow(header, loop_body); + graph_->BuildDominatorTree(); + EXPECT_TRUE(CheckGraph()); + + HBasicBlockMap bb_map( + std::less<HBasicBlock*>(), graph_->GetAllocator()->Adapter(kArenaAllocSuperblockCloner)); + HInstructionMap hir_map( + std::less<HInstruction*>(), graph_->GetAllocator()->Adapter(kArenaAllocSuperblockCloner)); + + HLoopInformation* loop_info = header->GetLoopInformation(); + PeelUnrollHelper helper(loop_info, &bb_map, &hir_map); + EXPECT_TRUE(helper.IsLoopClonable()); + HBasicBlock* new_header = helper.DoPeeling(); + HLoopInformation* new_loop_info = new_header->GetLoopInformation(); + + EXPECT_TRUE(CheckGraph()); + + // Check loop body successors. + EXPECT_EQ(loop_body->GetSingleSuccessor(), header); + EXPECT_EQ(bb_map.Get(loop_body)->GetSingleSuccessor(), header); + + // Check loop structure. + EXPECT_EQ(header, new_header); + EXPECT_EQ(new_loop_info->GetHeader(), header); + EXPECT_EQ(new_loop_info->GetBackEdges().size(), 1u); + EXPECT_EQ(new_loop_info->GetBackEdges()[0], loop_body); +} + +// Tests SuperblockCloner for loop unrolling case. +// +// Control Flow of the example (ignoring critical edges splitting). +// +// Before After +// +// |B| |B| +// | | +// v v +// |1| |1| +// | | +// v v +// |2|<-\ (6) |2A|<-\ +// / \ / / \ \ +// v v/ / v \ +// |4| |3| /(7)|3A| | +// | / / / +// v | v / +// |E| \ |2| / +// \ / \ / +// v v/ +// |4| |3| +// | +// v +// |E| +TEST_F(SuperblockClonerTest, LoopUnrolling) { + HBasicBlock* header = nullptr; + HBasicBlock* loop_body = nullptr; + + InitGraph(); + CreateBasicLoopControlFlow(entry_block_, return_block_, &header, &loop_body); + CreateBasicLoopDataFlow(header, loop_body); + graph_->BuildDominatorTree(); + EXPECT_TRUE(CheckGraph()); + + HBasicBlockMap bb_map( + std::less<HBasicBlock*>(), graph_->GetAllocator()->Adapter(kArenaAllocSuperblockCloner)); + HInstructionMap hir_map( + std::less<HInstruction*>(), graph_->GetAllocator()->Adapter(kArenaAllocSuperblockCloner)); + + HLoopInformation* loop_info = header->GetLoopInformation(); + PeelUnrollHelper helper(loop_info, &bb_map, &hir_map); + EXPECT_TRUE(helper.IsLoopClonable()); + HBasicBlock* new_header = helper.DoUnrolling(); + + EXPECT_TRUE(CheckGraph()); + + // Check loop body successors. + EXPECT_EQ(loop_body->GetSingleSuccessor(), bb_map.Get(header)); + EXPECT_EQ(bb_map.Get(loop_body)->GetSingleSuccessor(), header); + + // Check loop structure. + EXPECT_EQ(header, new_header); + EXPECT_EQ(loop_info, new_header->GetLoopInformation()); + EXPECT_EQ(loop_info->GetHeader(), new_header); + EXPECT_EQ(loop_info->GetBackEdges().size(), 1u); + EXPECT_EQ(loop_info->GetBackEdges()[0], bb_map.Get(loop_body)); +} + +// Checks that loop unrolling works fine for a loop with multiple back edges. Tests that after +// the transformation the loop has a single preheader. +TEST_F(SuperblockClonerTest, LoopPeelingMultipleBackEdges) { + HBasicBlock* header = nullptr; + HBasicBlock* loop_body = nullptr; + + InitGraph(); + CreateBasicLoopControlFlow(entry_block_, return_block_, &header, &loop_body); + CreateBasicLoopDataFlow(header, loop_body); + + // Transform a basic loop to have multiple back edges. + HBasicBlock* latch = header->GetSuccessors()[1]; + HBasicBlock* if_block = new (GetAllocator()) HBasicBlock(graph_); + HBasicBlock* temp1 = new (GetAllocator()) HBasicBlock(graph_); + graph_->AddBlock(if_block); + graph_->AddBlock(temp1); + header->ReplaceSuccessor(latch, if_block); + if_block->AddSuccessor(latch); + if_block->AddSuccessor(temp1); + temp1->AddSuccessor(header); + + if_block->AddInstruction(new (GetAllocator()) HIf(parameter_)); + + HInstructionIterator it(header->GetPhis()); + DCHECK(!it.Done()); + HPhi* loop_phi = it.Current()->AsPhi(); + HInstruction* temp_add = new (GetAllocator()) HAdd(DataType::Type::kInt32, + loop_phi, + graph_->GetIntConstant(2)); + temp1->AddInstruction(temp_add); + temp1->AddInstruction(new (GetAllocator()) HGoto()); + loop_phi->AddInput(temp_add); + + graph_->BuildDominatorTree(); + EXPECT_TRUE(CheckGraph()); + + HLoopInformation* loop_info = header->GetLoopInformation(); + PeelUnrollSimpleHelper helper(loop_info); + HBasicBlock* new_header = helper.DoPeeling(); + EXPECT_EQ(header, new_header); + + EXPECT_TRUE(CheckGraph()); + EXPECT_EQ(header->GetPredecessors().size(), 3u); +} + +static void CheckLoopStructureForLoopPeelingNested(HBasicBlock* loop1_header, + HBasicBlock* loop2_header, + HBasicBlock* loop3_header) { + EXPECT_EQ(loop1_header->GetLoopInformation()->GetHeader(), loop1_header); + EXPECT_EQ(loop2_header->GetLoopInformation()->GetHeader(), loop2_header); + EXPECT_EQ(loop3_header->GetLoopInformation()->GetHeader(), loop3_header); + EXPECT_EQ(loop1_header->GetLoopInformation()->GetPreHeader()->GetLoopInformation(), nullptr); + EXPECT_EQ(loop2_header->GetLoopInformation()->GetPreHeader()->GetLoopInformation(), nullptr); + EXPECT_EQ(loop3_header->GetLoopInformation()->GetPreHeader()->GetLoopInformation()->GetHeader(), + loop2_header); +} + +TEST_F(SuperblockClonerTest, LoopPeelingNested) { + HBasicBlock* header = nullptr; + HBasicBlock* loop_body = nullptr; + + InitGraph(); + + // Create the following nested structure of loops + // Headers: 1 2 3 + // [ ], [ [ ] ] + CreateBasicLoopControlFlow(entry_block_, return_block_, &header, &loop_body); + CreateBasicLoopDataFlow(header, loop_body); + HBasicBlock* loop1_header = header; + + CreateBasicLoopControlFlow(header, return_block_, &header, &loop_body); + CreateBasicLoopDataFlow(header, loop_body); + HBasicBlock* loop2_header = header; + + CreateBasicLoopControlFlow(header, header->GetSuccessors()[1], &header, &loop_body); + CreateBasicLoopDataFlow(header, loop_body); + HBasicBlock* loop3_header = header; + + graph_->BuildDominatorTree(); + EXPECT_TRUE(CheckGraph()); + + HLoopInformation* loop2_info_before = loop2_header->GetLoopInformation(); + HLoopInformation* loop3_info_before = loop3_header->GetLoopInformation(); + + // Check nested loops structure. + CheckLoopStructureForLoopPeelingNested(loop1_header, loop2_header, loop3_header); + PeelUnrollSimpleHelper helper(loop1_header->GetLoopInformation()); + helper.DoPeeling(); + // Check that nested loops structure has not changed after the transformation. + CheckLoopStructureForLoopPeelingNested(loop1_header, loop2_header, loop3_header); + + // Test that the loop info is preserved. + EXPECT_EQ(loop2_info_before, loop2_header->GetLoopInformation()); + EXPECT_EQ(loop3_info_before, loop3_header->GetLoopInformation()); + + EXPECT_EQ(loop3_info_before->GetPreHeader()->GetLoopInformation(), loop2_info_before); + EXPECT_EQ(loop2_info_before->GetPreHeader()->GetLoopInformation(), nullptr); + + EXPECT_EQ(helper.GetRegionToBeAdjusted(), nullptr); + + EXPECT_TRUE(CheckGraph()); +} + +// Checks that the loop population is correctly propagated after an inner loop is peeled. +TEST_F(SuperblockClonerTest, OuterLoopPopulationAfterInnerPeeled) { + HBasicBlock* header = nullptr; + HBasicBlock* loop_body = nullptr; + + InitGraph(); + + // Create the following nested structure of loops + // Headers: 1 2 3 4 + // [ [ [ ] ] ], [ ] + CreateBasicLoopControlFlow(entry_block_, return_block_, &header, &loop_body); + CreateBasicLoopDataFlow(header, loop_body); + HBasicBlock* loop1_header = header; + + CreateBasicLoopControlFlow(header, header->GetSuccessors()[1], &header, &loop_body); + CreateBasicLoopDataFlow(header, loop_body); + HBasicBlock* loop2_header = header; + + CreateBasicLoopControlFlow(header, header->GetSuccessors()[1], &header, &loop_body); + CreateBasicLoopDataFlow(header, loop_body); + HBasicBlock* loop3_header = header; + + CreateBasicLoopControlFlow(loop1_header, return_block_, &header, &loop_body); + CreateBasicLoopDataFlow(header, loop_body); + HBasicBlock* loop4_header = header; + + graph_->BuildDominatorTree(); + EXPECT_TRUE(CheckGraph()); + + PeelUnrollSimpleHelper helper(loop3_header->GetLoopInformation()); + helper.DoPeeling(); + HLoopInformation* loop1 = loop1_header->GetLoopInformation(); + HLoopInformation* loop2 = loop2_header->GetLoopInformation(); + HLoopInformation* loop3 = loop3_header->GetLoopInformation(); + HLoopInformation* loop4 = loop4_header->GetLoopInformation(); + + EXPECT_TRUE(loop1->Contains(*loop2_header)); + EXPECT_TRUE(loop1->Contains(*loop3_header)); + EXPECT_TRUE(loop1->Contains(*loop3_header->GetLoopInformation()->GetPreHeader())); + + // Check that loop4 info has not been touched after local run of AnalyzeLoops. + EXPECT_EQ(loop4, loop4_header->GetLoopInformation()); + + EXPECT_TRUE(loop1->IsIn(*loop1)); + EXPECT_TRUE(loop2->IsIn(*loop1)); + EXPECT_TRUE(loop3->IsIn(*loop1)); + EXPECT_TRUE(loop3->IsIn(*loop2)); + EXPECT_TRUE(!loop4->IsIn(*loop1)); + + EXPECT_EQ(loop4->GetPreHeader()->GetLoopInformation(), nullptr); + + EXPECT_EQ(helper.GetRegionToBeAdjusted(), loop2); + + EXPECT_TRUE(CheckGraph()); +} + +// Checks the case when inner loop have an exit not to its immediate outer_loop but some other loop +// in the hierarchy. Loop population information must be valid after loop peeling. +TEST_F(SuperblockClonerTest, NestedCaseExitToOutermost) { + HBasicBlock* header = nullptr; + HBasicBlock* loop_body = nullptr; + + InitGraph(); + + // Create the following nested structure of loops then peel loop3. + // Headers: 1 2 3 + // [ [ [ ] ] ] + CreateBasicLoopControlFlow(entry_block_, return_block_, &header, &loop_body); + CreateBasicLoopDataFlow(header, loop_body); + HBasicBlock* loop1_header = header; + HBasicBlock* loop_body1 = loop_body; + + CreateBasicLoopControlFlow(header, header->GetSuccessors()[1], &header, &loop_body); + CreateBasicLoopDataFlow(header, loop_body); + + CreateBasicLoopControlFlow(header, header->GetSuccessors()[1], &header, &loop_body); + CreateBasicLoopDataFlow(header, loop_body); + HBasicBlock* loop3_header = header; + HBasicBlock* loop_body3 = loop_body; + + // Change the loop3 - insert an exit which leads to loop1. + HBasicBlock* loop3_extra_if_block = new (GetAllocator()) HBasicBlock(graph_); + graph_->AddBlock(loop3_extra_if_block); + loop3_extra_if_block->AddInstruction(new (GetAllocator()) HIf(parameter_)); + + loop3_header->ReplaceSuccessor(loop_body3, loop3_extra_if_block); + loop3_extra_if_block->AddSuccessor(loop_body1); // Long exit. + loop3_extra_if_block->AddSuccessor(loop_body3); + + graph_->BuildDominatorTree(); + EXPECT_TRUE(CheckGraph()); + + HBasicBlock* loop3_long_exit = loop3_extra_if_block->GetSuccessors()[0]; + EXPECT_TRUE(loop1_header->GetLoopInformation()->Contains(*loop3_long_exit)); + + PeelUnrollSimpleHelper helper(loop3_header->GetLoopInformation()); + helper.DoPeeling(); + + HLoopInformation* loop1 = loop1_header->GetLoopInformation(); + // Check that after the transformation the local area for CF adjustments has been chosen + // correctly and loop population has been updated. + loop3_long_exit = loop3_extra_if_block->GetSuccessors()[0]; + EXPECT_TRUE(loop1->Contains(*loop3_long_exit)); + + EXPECT_EQ(helper.GetRegionToBeAdjusted(), loop1); + + EXPECT_TRUE(loop1->Contains(*loop3_header)); + EXPECT_TRUE(loop1->Contains(*loop3_header->GetLoopInformation()->GetPreHeader())); + + EXPECT_TRUE(CheckGraph()); +} + +TEST_F(SuperblockClonerTest, FastCaseCheck) { + HBasicBlock* header = nullptr; + HBasicBlock* loop_body = nullptr; + ArenaAllocator* arena = graph_->GetAllocator(); + + InitGraph(); + CreateBasicLoopControlFlow(entry_block_, return_block_, &header, &loop_body); + CreateBasicLoopDataFlow(header, loop_body); + graph_->BuildDominatorTree(); + + HLoopInformation* loop_info = header->GetLoopInformation(); + + ArenaBitVector orig_bb_set( + arena, graph_->GetBlocks().size(), false, kArenaAllocSuperblockCloner); + orig_bb_set.Union(&loop_info->GetBlocks()); + + HEdgeSet remap_orig_internal(graph_->GetAllocator()->Adapter(kArenaAllocSuperblockCloner)); + HEdgeSet remap_copy_internal(graph_->GetAllocator()->Adapter(kArenaAllocSuperblockCloner)); + HEdgeSet remap_incoming(graph_->GetAllocator()->Adapter(kArenaAllocSuperblockCloner)); + + CollectRemappingInfoForPeelUnroll(true, + loop_info, + &remap_orig_internal, + &remap_copy_internal, + &remap_incoming); + + // Insert some extra nodes and edges. + HBasicBlock* preheader = loop_info->GetPreHeader(); + orig_bb_set.SetBit(preheader->GetBlockId()); + + // Adjust incoming edges. + remap_incoming.Clear(); + remap_incoming.Insert(HEdge(preheader->GetSinglePredecessor(), preheader)); + + HBasicBlockMap bb_map(std::less<HBasicBlock*>(), arena->Adapter(kArenaAllocSuperblockCloner)); + HInstructionMap hir_map(std::less<HInstruction*>(), arena->Adapter(kArenaAllocSuperblockCloner)); + + SuperblockCloner cloner(graph_, + &orig_bb_set, + &bb_map, + &hir_map); + cloner.SetSuccessorRemappingInfo(&remap_orig_internal, &remap_copy_internal, &remap_incoming); + + EXPECT_FALSE(cloner.IsFastCase()); +} + +// Helper for FindCommonLoop which also check that FindCommonLoop is symmetric. +static HLoopInformation* FindCommonLoopCheck(HLoopInformation* loop1, HLoopInformation* loop2) { + HLoopInformation* common_loop12 = FindCommonLoop(loop1, loop2); + HLoopInformation* common_loop21 = FindCommonLoop(loop2, loop1); + EXPECT_EQ(common_loop21, common_loop12); + return common_loop12; +} + +// Tests FindCommonLoop function on a loop nest. +TEST_F(SuperblockClonerTest, FindCommonLoop) { + HBasicBlock* header = nullptr; + HBasicBlock* loop_body = nullptr; + + InitGraph(); + + // Create the following nested structure of loops + // Headers: 1 2 3 4 5 + // [ [ [ ] ], [ ] ], [ ] + CreateBasicLoopControlFlow(entry_block_, return_block_, &header, &loop_body); + CreateBasicLoopDataFlow(header, loop_body); + HBasicBlock* loop1_header = header; + + CreateBasicLoopControlFlow(header, header->GetSuccessors()[1], &header, &loop_body); + CreateBasicLoopDataFlow(header, loop_body); + HBasicBlock* loop2_header = header; + + CreateBasicLoopControlFlow(header, header->GetSuccessors()[1], &header, &loop_body); + CreateBasicLoopDataFlow(header, loop_body); + HBasicBlock* loop3_header = header; + + CreateBasicLoopControlFlow(loop2_header, loop2_header->GetSuccessors()[0], &header, &loop_body); + CreateBasicLoopDataFlow(header, loop_body); + HBasicBlock* loop4_header = header; + + CreateBasicLoopControlFlow(loop1_header, return_block_, &header, &loop_body); + CreateBasicLoopDataFlow(header, loop_body); + HBasicBlock* loop5_header = header; + + graph_->BuildDominatorTree(); + EXPECT_TRUE(CheckGraph()); + + HLoopInformation* loop1 = loop1_header->GetLoopInformation(); + HLoopInformation* loop2 = loop2_header->GetLoopInformation(); + HLoopInformation* loop3 = loop3_header->GetLoopInformation(); + HLoopInformation* loop4 = loop4_header->GetLoopInformation(); + HLoopInformation* loop5 = loop5_header->GetLoopInformation(); + + EXPECT_TRUE(loop1->IsIn(*loop1)); + EXPECT_TRUE(loop2->IsIn(*loop1)); + EXPECT_TRUE(loop3->IsIn(*loop1)); + EXPECT_TRUE(loop3->IsIn(*loop2)); + EXPECT_TRUE(loop4->IsIn(*loop1)); + + EXPECT_FALSE(loop5->IsIn(*loop1)); + EXPECT_FALSE(loop4->IsIn(*loop2)); + EXPECT_FALSE(loop4->IsIn(*loop3)); + + EXPECT_EQ(loop1->GetPreHeader()->GetLoopInformation(), nullptr); + EXPECT_EQ(loop4->GetPreHeader()->GetLoopInformation(), loop1); + + EXPECT_EQ(FindCommonLoopCheck(nullptr, nullptr), nullptr); + EXPECT_EQ(FindCommonLoopCheck(loop2, nullptr), nullptr); + + EXPECT_EQ(FindCommonLoopCheck(loop1, loop1), loop1); + EXPECT_EQ(FindCommonLoopCheck(loop1, loop2), loop1); + EXPECT_EQ(FindCommonLoopCheck(loop1, loop3), loop1); + EXPECT_EQ(FindCommonLoopCheck(loop1, loop4), loop1); + EXPECT_EQ(FindCommonLoopCheck(loop1, loop5), nullptr); + + EXPECT_EQ(FindCommonLoopCheck(loop2, loop3), loop2); + EXPECT_EQ(FindCommonLoopCheck(loop2, loop4), loop1); + EXPECT_EQ(FindCommonLoopCheck(loop2, loop5), nullptr); + + EXPECT_EQ(FindCommonLoopCheck(loop3, loop4), loop1); + EXPECT_EQ(FindCommonLoopCheck(loop3, loop5), nullptr); + + EXPECT_EQ(FindCommonLoopCheck(loop4, loop5), nullptr); + + EXPECT_EQ(FindCommonLoopCheck(loop5, loop5), loop5); +} + } // namespace art diff --git a/compiler/trampolines/trampoline_compiler.cc b/compiler/trampolines/trampoline_compiler.cc index 921d4018492..57360e74a33 100644 --- a/compiler/trampolines/trampoline_compiler.cc +++ b/compiler/trampolines/trampoline_compiler.cc @@ -17,6 +17,7 @@ #include "trampoline_compiler.h" #include "base/arena_allocator.h" +#include "base/malloc_arena_pool.h" #include "jni_env_ext.h" #ifdef ART_ENABLE_CODEGEN_arm @@ -243,7 +244,7 @@ static std::unique_ptr<const std::vector<uint8_t>> CreateTrampoline(ArenaAllocat std::unique_ptr<const std::vector<uint8_t>> CreateTrampoline64(InstructionSet isa, EntryPointCallingConvention abi, ThreadOffset64 offset) { - ArenaPool pool; + MallocArenaPool pool; ArenaAllocator allocator(&pool); switch (isa) { #ifdef ART_ENABLE_CODEGEN_arm64 @@ -269,7 +270,7 @@ std::unique_ptr<const std::vector<uint8_t>> CreateTrampoline64(InstructionSet is std::unique_ptr<const std::vector<uint8_t>> CreateTrampoline32(InstructionSet isa, EntryPointCallingConvention abi, ThreadOffset32 offset) { - ArenaPool pool; + MallocArenaPool pool; ArenaAllocator allocator(&pool); switch (isa) { #ifdef ART_ENABLE_CODEGEN_arm diff --git a/compiler/utils/assembler.cc b/compiler/utils/assembler.cc index 944c64b5918..421c1b60895 100644 --- a/compiler/utils/assembler.cc +++ b/compiler/utils/assembler.cc @@ -20,8 +20,8 @@ #include <vector> #include "base/casts.h" +#include "base/memory_region.h" #include "globals.h" -#include "memory_region.h" namespace art { diff --git a/compiler/utils/assembler.h b/compiler/utils/assembler.h index 5b0cd6baa8d..379a6396eb6 100644 --- a/compiler/utils/assembler.h +++ b/compiler/utils/assembler.h @@ -29,10 +29,10 @@ #include "base/array_ref.h" #include "base/enums.h" #include "base/macros.h" +#include "base/memory_region.h" #include "debug/dwarf/debug_frame_opcode_writer.h" #include "label.h" #include "managed_register.h" -#include "memory_region.h" #include "mips/constants_mips.h" #include "offsets.h" #include "x86/constants_x86.h" diff --git a/compiler/utils/assembler_test.h b/compiler/utils/assembler_test.h index 0cb8bbb2d54..7c800b355fe 100644 --- a/compiler/utils/assembler_test.h +++ b/compiler/utils/assembler_test.h @@ -26,6 +26,7 @@ #include <fstream> #include <iterator> +#include "base/malloc_arena_pool.h" #include "assembler_test_base.h" #include "common_runtime_test.h" // For ScratchFile @@ -1606,7 +1607,7 @@ class AssemblerTest : public testing::Test { static constexpr size_t kWarnManyCombinationsThreshold = 500; - ArenaPool pool_; + MallocArenaPool pool_; std::unique_ptr<ArenaAllocator> allocator_; std::unique_ptr<Ass> assembler_; std::unique_ptr<AssemblerTestInfrastructure> test_helper_; diff --git a/compiler/utils/assembler_thumb_test.cc b/compiler/utils/assembler_thumb_test.cc index 655d17d4fbf..053e202523a 100644 --- a/compiler/utils/assembler_thumb_test.cc +++ b/compiler/utils/assembler_thumb_test.cc @@ -27,6 +27,7 @@ #include "utils/arm/jni_macro_assembler_arm_vixl.h" #include "base/hex_dump.h" +#include "base/malloc_arena_pool.h" #include "common_runtime_test.h" namespace art { @@ -169,7 +170,7 @@ class ArmVIXLAssemblerTest : public ::testing::Test { public: ArmVIXLAssemblerTest() : pool(), allocator(&pool), assembler(&allocator) { } - ArenaPool pool; + MallocArenaPool pool; ArenaAllocator allocator; ArmVIXLJNIMacroAssembler assembler; }; diff --git a/compiler/utils/atomic_dex_ref_map-inl.h b/compiler/utils/atomic_dex_ref_map-inl.h index ce3302bb628..9915498acca 100644 --- a/compiler/utils/atomic_dex_ref_map-inl.h +++ b/compiler/utils/atomic_dex_ref_map-inl.h @@ -70,7 +70,7 @@ inline bool AtomicDexRefMap<DexFileReferenceType, Value>::Get(const DexFileRefer if (array == nullptr) { return false; } - *out = (*array)[ref.index].LoadRelaxed(); + *out = (*array)[ref.index].load(std::memory_order_relaxed); return true; } @@ -81,7 +81,7 @@ inline bool AtomicDexRefMap<DexFileReferenceType, Value>::Remove(const DexFileRe if (array == nullptr) { return false; } - *out = (*array)[ref.index].ExchangeSequentiallyConsistent(nullptr); + *out = (*array)[ref.index].exchange(nullptr, std::memory_order_seq_cst); return true; } @@ -120,7 +120,7 @@ inline void AtomicDexRefMap<DexFileReferenceType, Value>::Visit(const Visitor& v const DexFile* dex_file = pair.first; const ElementArray& elements = pair.second; for (size_t i = 0; i < elements.size(); ++i) { - visitor(DexFileReference(dex_file, i), elements[i].LoadRelaxed()); + visitor(DexFileReference(dex_file, i), elements[i].load(std::memory_order_relaxed)); } } } @@ -129,7 +129,7 @@ template <typename DexFileReferenceType, typename Value> inline void AtomicDexRefMap<DexFileReferenceType, Value>::ClearEntries() { for (auto& it : arrays_) { for (auto& element : it.second) { - element.StoreRelaxed(nullptr); + element.store(nullptr, std::memory_order_relaxed); } } } diff --git a/compiler/utils/jni_macro_assembler.cc b/compiler/utils/jni_macro_assembler.cc index 3f7691b6a86..0c34aa4f1dc 100644 --- a/compiler/utils/jni_macro_assembler.cc +++ b/compiler/utils/jni_macro_assembler.cc @@ -38,8 +38,8 @@ #include "x86_64/jni_macro_assembler_x86_64.h" #endif #include "base/casts.h" +#include "base/memory_region.h" #include "globals.h" -#include "memory_region.h" namespace art { diff --git a/compiler/utils/jni_macro_assembler_test.h b/compiler/utils/jni_macro_assembler_test.h index 1aefc84c78c..b70c18b3e2f 100644 --- a/compiler/utils/jni_macro_assembler_test.h +++ b/compiler/utils/jni_macro_assembler_test.h @@ -20,6 +20,7 @@ #include "jni_macro_assembler.h" #include "assembler_test_base.h" +#include "base/malloc_arena_pool.h" #include "common_runtime_test.h" // For ScratchFile #include <sys/stat.h> @@ -139,7 +140,7 @@ class JNIMacroAssemblerTest : public testing::Test { test_helper_->Driver(*data, assembly_text, test_name); } - ArenaPool pool_; + MallocArenaPool pool_; std::unique_ptr<ArenaAllocator> allocator_; std::unique_ptr<Ass> assembler_; std::unique_ptr<AssemblerTestInfrastructure> test_helper_; diff --git a/compiler/utils/mips/assembler_mips.cc b/compiler/utils/mips/assembler_mips.cc index 2218ef9af29..dce5b95fec3 100644 --- a/compiler/utils/mips/assembler_mips.cc +++ b/compiler/utils/mips/assembler_mips.cc @@ -18,9 +18,9 @@ #include "base/bit_utils.h" #include "base/casts.h" +#include "base/memory_region.h" #include "entrypoints/quick/quick_entrypoints.h" #include "entrypoints/quick/quick_entrypoints_enum.h" -#include "memory_region.h" #include "thread.h" namespace art { @@ -2793,6 +2793,26 @@ void MipsAssembler::Hadd_uD(VectorRegister wd, VectorRegister ws, VectorRegister DsFsmInstr(EmitMsa3R(0x5, 0x3, wt, ws, wd, 0x15)).FprOuts(wd).FprIns(ws, wt); } +void MipsAssembler::PcntB(VectorRegister wd, VectorRegister ws) { + CHECK(HasMsa()); + DsFsmInstr(EmitMsa2R(0xc1, 0x0, ws, wd, 0x1e)).FprOuts(wd).FprIns(ws); +} + +void MipsAssembler::PcntH(VectorRegister wd, VectorRegister ws) { + CHECK(HasMsa()); + DsFsmInstr(EmitMsa2R(0xc1, 0x1, ws, wd, 0x1e)).FprOuts(wd).FprIns(ws); +} + +void MipsAssembler::PcntW(VectorRegister wd, VectorRegister ws) { + CHECK(HasMsa()); + DsFsmInstr(EmitMsa2R(0xc1, 0x2, ws, wd, 0x1e)).FprOuts(wd).FprIns(ws); +} + +void MipsAssembler::PcntD(VectorRegister wd, VectorRegister ws) { + CHECK(HasMsa()); + DsFsmInstr(EmitMsa2R(0xc1, 0x3, ws, wd, 0x1e)).FprOuts(wd).FprIns(ws); +} + void MipsAssembler::ReplicateFPToVectorRegister(VectorRegister dst, FRegister src, bool is_double) { diff --git a/compiler/utils/mips/assembler_mips.h b/compiler/utils/mips/assembler_mips.h index 7de8e2e3665..c6ce62b4f4a 100644 --- a/compiler/utils/mips/assembler_mips.h +++ b/compiler/utils/mips/assembler_mips.h @@ -756,6 +756,11 @@ class MipsAssembler FINAL : public Assembler, public JNIMacroAssembler<PointerSi void Hadd_uW(VectorRegister wd, VectorRegister ws, VectorRegister wt); void Hadd_uD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void PcntB(VectorRegister wd, VectorRegister ws); + void PcntH(VectorRegister wd, VectorRegister ws); + void PcntW(VectorRegister wd, VectorRegister ws); + void PcntD(VectorRegister wd, VectorRegister ws); + // Helper for replicating floating point value in all destination elements. void ReplicateFPToVectorRegister(VectorRegister dst, FRegister src, bool is_double); diff --git a/compiler/utils/mips/assembler_mips32r6_test.cc b/compiler/utils/mips/assembler_mips32r6_test.cc index 937ee25bcb1..691c33f3e7a 100644 --- a/compiler/utils/mips/assembler_mips32r6_test.cc +++ b/compiler/utils/mips/assembler_mips32r6_test.cc @@ -2277,6 +2277,22 @@ TEST_F(AssemblerMIPS32r6Test, FillW) { DriverStr(RepeatVR(&mips::MipsAssembler::FillW, "fill.w ${reg1}, ${reg2}"), "fill.w"); } +TEST_F(AssemblerMIPS32r6Test, PcntB) { + DriverStr(RepeatVV(&mips::MipsAssembler::PcntB, "pcnt.b ${reg1}, ${reg2}"), "pcnt.b"); +} + +TEST_F(AssemblerMIPS32r6Test, PcntH) { + DriverStr(RepeatVV(&mips::MipsAssembler::PcntH, "pcnt.h ${reg1}, ${reg2}"), "pcnt.h"); +} + +TEST_F(AssemblerMIPS32r6Test, PcntW) { + DriverStr(RepeatVV(&mips::MipsAssembler::PcntW, "pcnt.w ${reg1}, ${reg2}"), "pcnt.w"); +} + +TEST_F(AssemblerMIPS32r6Test, PcntD) { + DriverStr(RepeatVV(&mips::MipsAssembler::PcntD, "pcnt.d ${reg1}, ${reg2}"), "pcnt.d"); +} + TEST_F(AssemblerMIPS32r6Test, LdiB) { DriverStr(RepeatVIb(&mips::MipsAssembler::LdiB, -8, "ldi.b ${reg}, {imm}"), "ldi.b"); } diff --git a/compiler/utils/mips64/assembler_mips64.cc b/compiler/utils/mips64/assembler_mips64.cc index e1b0e75108b..bb1bb82fa5d 100644 --- a/compiler/utils/mips64/assembler_mips64.cc +++ b/compiler/utils/mips64/assembler_mips64.cc @@ -18,9 +18,9 @@ #include "base/bit_utils.h" #include "base/casts.h" +#include "base/memory_region.h" #include "entrypoints/quick/quick_entrypoints.h" #include "entrypoints/quick/quick_entrypoints_enum.h" -#include "memory_region.h" #include "thread.h" namespace art { @@ -2279,6 +2279,26 @@ void Mips64Assembler::Hadd_uD(VectorRegister wd, VectorRegister ws, VectorRegist EmitMsa3R(0x5, 0x3, wt, ws, wd, 0x15); } +void Mips64Assembler::PcntB(VectorRegister wd, VectorRegister ws) { + CHECK(HasMsa()); + EmitMsa2R(0xc1, 0x0, ws, wd, 0x1e); +} + +void Mips64Assembler::PcntH(VectorRegister wd, VectorRegister ws) { + CHECK(HasMsa()); + EmitMsa2R(0xc1, 0x1, ws, wd, 0x1e); +} + +void Mips64Assembler::PcntW(VectorRegister wd, VectorRegister ws) { + CHECK(HasMsa()); + EmitMsa2R(0xc1, 0x2, ws, wd, 0x1e); +} + +void Mips64Assembler::PcntD(VectorRegister wd, VectorRegister ws) { + CHECK(HasMsa()); + EmitMsa2R(0xc1, 0x3, ws, wd, 0x1e); +} + void Mips64Assembler::ReplicateFPToVectorRegister(VectorRegister dst, FpuRegister src, bool is_double) { diff --git a/compiler/utils/mips64/assembler_mips64.h b/compiler/utils/mips64/assembler_mips64.h index 7a61f39e64a..542dbafc87d 100644 --- a/compiler/utils/mips64/assembler_mips64.h +++ b/compiler/utils/mips64/assembler_mips64.h @@ -863,6 +863,11 @@ class Mips64Assembler FINAL : public Assembler, public JNIMacroAssembler<Pointer void Hadd_uW(VectorRegister wd, VectorRegister ws, VectorRegister wt); void Hadd_uD(VectorRegister wd, VectorRegister ws, VectorRegister wt); + void PcntB(VectorRegister wd, VectorRegister ws); + void PcntH(VectorRegister wd, VectorRegister ws); + void PcntW(VectorRegister wd, VectorRegister ws); + void PcntD(VectorRegister wd, VectorRegister ws); + // Helper for replicating floating point value in all destination elements. void ReplicateFPToVectorRegister(VectorRegister dst, FpuRegister src, bool is_double); diff --git a/compiler/utils/mips64/assembler_mips64_test.cc b/compiler/utils/mips64/assembler_mips64_test.cc index b0e1d91c3f8..fb5f12be936 100644 --- a/compiler/utils/mips64/assembler_mips64_test.cc +++ b/compiler/utils/mips64/assembler_mips64_test.cc @@ -3529,6 +3529,22 @@ TEST_F(AssemblerMIPS64Test, FillD) { DriverStr(RepeatVR(&mips64::Mips64Assembler::FillD, "fill.d ${reg1}, ${reg2}"), "fill.d"); } +TEST_F(AssemblerMIPS64Test, PcntB) { + DriverStr(RepeatVV(&mips64::Mips64Assembler::PcntB, "pcnt.b ${reg1}, ${reg2}"), "pcnt.b"); +} + +TEST_F(AssemblerMIPS64Test, PcntH) { + DriverStr(RepeatVV(&mips64::Mips64Assembler::PcntH, "pcnt.h ${reg1}, ${reg2}"), "pcnt.h"); +} + +TEST_F(AssemblerMIPS64Test, PcntW) { + DriverStr(RepeatVV(&mips64::Mips64Assembler::PcntW, "pcnt.w ${reg1}, ${reg2}"), "pcnt.w"); +} + +TEST_F(AssemblerMIPS64Test, PcntD) { + DriverStr(RepeatVV(&mips64::Mips64Assembler::PcntD, "pcnt.d ${reg1}, ${reg2}"), "pcnt.d"); +} + TEST_F(AssemblerMIPS64Test, LdiB) { DriverStr(RepeatVIb(&mips64::Mips64Assembler::LdiB, -8, "ldi.b ${reg}, {imm}"), "ldi.b"); } diff --git a/compiler/utils/x86/assembler_x86.cc b/compiler/utils/x86/assembler_x86.cc index ea160c8993c..86f9010ea32 100644 --- a/compiler/utils/x86/assembler_x86.cc +++ b/compiler/utils/x86/assembler_x86.cc @@ -17,8 +17,8 @@ #include "assembler_x86.h" #include "base/casts.h" +#include "base/memory_region.h" #include "entrypoints/quick/quick_entrypoints.h" -#include "memory_region.h" #include "thread.h" namespace art { @@ -913,6 +913,78 @@ void X86Assembler::psubq(XmmRegister dst, XmmRegister src) { } +void X86Assembler::paddusb(XmmRegister dst, XmmRegister src) { + AssemblerBuffer::EnsureCapacity ensured(&buffer_); + EmitUint8(0x66); + EmitUint8(0x0F); + EmitUint8(0xDC); + EmitXmmRegisterOperand(dst, src); +} + + +void X86Assembler::paddsb(XmmRegister dst, XmmRegister src) { + AssemblerBuffer::EnsureCapacity ensured(&buffer_); + EmitUint8(0x66); + EmitUint8(0x0F); + EmitUint8(0xEC); + EmitXmmRegisterOperand(dst, src); +} + + +void X86Assembler::paddusw(XmmRegister dst, XmmRegister src) { + AssemblerBuffer::EnsureCapacity ensured(&buffer_); + EmitUint8(0x66); + EmitUint8(0x0F); + EmitUint8(0xDD); + EmitXmmRegisterOperand(dst, src); +} + + +void X86Assembler::paddsw(XmmRegister dst, XmmRegister src) { + AssemblerBuffer::EnsureCapacity ensured(&buffer_); + EmitUint8(0x66); + EmitUint8(0x0F); + EmitUint8(0xED); + EmitXmmRegisterOperand(dst, src); +} + + +void X86Assembler::psubusb(XmmRegister dst, XmmRegister src) { + AssemblerBuffer::EnsureCapacity ensured(&buffer_); + EmitUint8(0x66); + EmitUint8(0x0F); + EmitUint8(0xD8); + EmitXmmRegisterOperand(dst, src); +} + + +void X86Assembler::psubsb(XmmRegister dst, XmmRegister src) { + AssemblerBuffer::EnsureCapacity ensured(&buffer_); + EmitUint8(0x66); + EmitUint8(0x0F); + EmitUint8(0xE8); + EmitXmmRegisterOperand(dst, src); +} + + +void X86Assembler::psubusw(XmmRegister dst, XmmRegister src) { + AssemblerBuffer::EnsureCapacity ensured(&buffer_); + EmitUint8(0x66); + EmitUint8(0x0F); + EmitUint8(0xD9); + EmitXmmRegisterOperand(dst, src); +} + + +void X86Assembler::psubsw(XmmRegister dst, XmmRegister src) { + AssemblerBuffer::EnsureCapacity ensured(&buffer_); + EmitUint8(0x66); + EmitUint8(0x0F); + EmitUint8(0xE9); + EmitXmmRegisterOperand(dst, src); +} + + void X86Assembler::cvtsi2ss(XmmRegister dst, Register src) { AssemblerBuffer::EnsureCapacity ensured(&buffer_); EmitUint8(0xF3); diff --git a/compiler/utils/x86/assembler_x86.h b/compiler/utils/x86/assembler_x86.h index a0856770836..22eaedce612 100644 --- a/compiler/utils/x86/assembler_x86.h +++ b/compiler/utils/x86/assembler_x86.h @@ -449,6 +449,15 @@ class X86Assembler FINAL : public Assembler { void paddq(XmmRegister dst, XmmRegister src); void psubq(XmmRegister dst, XmmRegister src); + void paddusb(XmmRegister dst, XmmRegister src); + void paddsb(XmmRegister dst, XmmRegister src); + void paddusw(XmmRegister dst, XmmRegister src); + void paddsw(XmmRegister dst, XmmRegister src); + void psubusb(XmmRegister dst, XmmRegister src); + void psubsb(XmmRegister dst, XmmRegister src); + void psubusw(XmmRegister dst, XmmRegister src); + void psubsw(XmmRegister dst, XmmRegister src); + void cvtsi2ss(XmmRegister dst, Register src); void cvtsi2sd(XmmRegister dst, Register src); diff --git a/compiler/utils/x86/assembler_x86_test.cc b/compiler/utils/x86/assembler_x86_test.cc index 2fd1b271828..cd007b32d41 100644 --- a/compiler/utils/x86/assembler_x86_test.cc +++ b/compiler/utils/x86/assembler_x86_test.cc @@ -17,13 +17,14 @@ #include "assembler_x86.h" #include "base/arena_allocator.h" +#include "base/malloc_arena_pool.h" #include "base/stl_util.h" #include "utils/assembler_test.h" namespace art { TEST(AssemblerX86, CreateBuffer) { - ArenaPool pool; + MallocArenaPool pool; ArenaAllocator allocator(&pool); AssemblerBuffer buffer(&allocator); AssemblerBuffer::EnsureCapacity ensured(&buffer); @@ -600,6 +601,38 @@ TEST_F(AssemblerX86Test, PSubQ) { DriverStr(RepeatFF(&x86::X86Assembler::psubq, "psubq %{reg2}, %{reg1}"), "psubq"); } +TEST_F(AssemblerX86Test, PAddUSB) { + DriverStr(RepeatFF(&x86::X86Assembler::paddusb, "paddusb %{reg2}, %{reg1}"), "paddusb"); +} + +TEST_F(AssemblerX86Test, PAddSB) { + DriverStr(RepeatFF(&x86::X86Assembler::paddsb, "paddsb %{reg2}, %{reg1}"), "paddsb"); +} + +TEST_F(AssemblerX86Test, PAddUSW) { + DriverStr(RepeatFF(&x86::X86Assembler::paddusw, "paddusw %{reg2}, %{reg1}"), "paddusw"); +} + +TEST_F(AssemblerX86Test, PAddSW) { + DriverStr(RepeatFF(&x86::X86Assembler::psubsw, "psubsw %{reg2}, %{reg1}"), "psubsw"); +} + +TEST_F(AssemblerX86Test, PSubUSB) { + DriverStr(RepeatFF(&x86::X86Assembler::psubusb, "psubusb %{reg2}, %{reg1}"), "psubusb"); +} + +TEST_F(AssemblerX86Test, PSubSB) { + DriverStr(RepeatFF(&x86::X86Assembler::psubsb, "psubsb %{reg2}, %{reg1}"), "psubsb"); +} + +TEST_F(AssemblerX86Test, PSubUSW) { + DriverStr(RepeatFF(&x86::X86Assembler::psubusw, "psubusw %{reg2}, %{reg1}"), "psubusw"); +} + +TEST_F(AssemblerX86Test, PSubSW) { + DriverStr(RepeatFF(&x86::X86Assembler::psubsw, "psubsw %{reg2}, %{reg1}"), "psubsw"); +} + TEST_F(AssemblerX86Test, XorPD) { DriverStr(RepeatFF(&x86::X86Assembler::xorpd, "xorpd %{reg2}, %{reg1}"), "xorpd"); } diff --git a/compiler/utils/x86_64/assembler_x86_64.cc b/compiler/utils/x86_64/assembler_x86_64.cc index ff5a357c5e3..bd31561937d 100644 --- a/compiler/utils/x86_64/assembler_x86_64.cc +++ b/compiler/utils/x86_64/assembler_x86_64.cc @@ -17,8 +17,8 @@ #include "assembler_x86_64.h" #include "base/casts.h" +#include "base/memory_region.h" #include "entrypoints/quick/quick_entrypoints.h" -#include "memory_region.h" #include "thread.h" namespace art { @@ -1011,6 +1011,86 @@ void X86_64Assembler::psubq(XmmRegister dst, XmmRegister src) { } +void X86_64Assembler::paddusb(XmmRegister dst, XmmRegister src) { + AssemblerBuffer::EnsureCapacity ensured(&buffer_); + EmitUint8(0x66); + EmitOptionalRex32(dst, src); + EmitUint8(0x0F); + EmitUint8(0xDC); + EmitXmmRegisterOperand(dst.LowBits(), src); +} + + +void X86_64Assembler::paddsb(XmmRegister dst, XmmRegister src) { + AssemblerBuffer::EnsureCapacity ensured(&buffer_); + EmitUint8(0x66); + EmitOptionalRex32(dst, src); + EmitUint8(0x0F); + EmitUint8(0xEC); + EmitXmmRegisterOperand(dst.LowBits(), src); +} + + +void X86_64Assembler::paddusw(XmmRegister dst, XmmRegister src) { + AssemblerBuffer::EnsureCapacity ensured(&buffer_); + EmitUint8(0x66); + EmitOptionalRex32(dst, src); + EmitUint8(0x0F); + EmitUint8(0xDD); + EmitXmmRegisterOperand(dst.LowBits(), src); +} + + +void X86_64Assembler::paddsw(XmmRegister dst, XmmRegister src) { + AssemblerBuffer::EnsureCapacity ensured(&buffer_); + EmitUint8(0x66); + EmitOptionalRex32(dst, src); + EmitUint8(0x0F); + EmitUint8(0xED); + EmitXmmRegisterOperand(dst.LowBits(), src); +} + + +void X86_64Assembler::psubusb(XmmRegister dst, XmmRegister src) { + AssemblerBuffer::EnsureCapacity ensured(&buffer_); + EmitUint8(0x66); + EmitOptionalRex32(dst, src); + EmitUint8(0x0F); + EmitUint8(0xD8); + EmitXmmRegisterOperand(dst.LowBits(), src); +} + + +void X86_64Assembler::psubsb(XmmRegister dst, XmmRegister src) { + AssemblerBuffer::EnsureCapacity ensured(&buffer_); + EmitUint8(0x66); + EmitOptionalRex32(dst, src); + EmitUint8(0x0F); + EmitUint8(0xE8); + EmitXmmRegisterOperand(dst.LowBits(), src); +} + + +void X86_64Assembler::psubusw(XmmRegister dst, XmmRegister src) { + AssemblerBuffer::EnsureCapacity ensured(&buffer_); + EmitUint8(0x66); + EmitOptionalRex32(dst, src); + EmitUint8(0x0F); + EmitUint8(0xD9); + EmitXmmRegisterOperand(dst.LowBits(), src); +} + + +void X86_64Assembler::psubsw(XmmRegister dst, XmmRegister src) { + AssemblerBuffer::EnsureCapacity ensured(&buffer_); + EmitUint8(0x66); + EmitOptionalRex32(dst, src); + EmitUint8(0x0F); + EmitUint8(0xE9); + EmitXmmRegisterOperand(dst.LowBits(), src); +} + + void X86_64Assembler::cvtsi2ss(XmmRegister dst, CpuRegister src) { cvtsi2ss(dst, src, false); } diff --git a/compiler/utils/x86_64/assembler_x86_64.h b/compiler/utils/x86_64/assembler_x86_64.h index 7a5fdb502f6..ab761fb1fc1 100644 --- a/compiler/utils/x86_64/assembler_x86_64.h +++ b/compiler/utils/x86_64/assembler_x86_64.h @@ -485,6 +485,15 @@ class X86_64Assembler FINAL : public Assembler { void paddq(XmmRegister dst, XmmRegister src); void psubq(XmmRegister dst, XmmRegister src); + void paddusb(XmmRegister dst, XmmRegister src); + void paddsb(XmmRegister dst, XmmRegister src); + void paddusw(XmmRegister dst, XmmRegister src); + void paddsw(XmmRegister dst, XmmRegister src); + void psubusb(XmmRegister dst, XmmRegister src); + void psubsb(XmmRegister dst, XmmRegister src); + void psubusw(XmmRegister dst, XmmRegister src); + void psubsw(XmmRegister dst, XmmRegister src); + void cvtsi2ss(XmmRegister dst, CpuRegister src); // Note: this is the r/m32 version. void cvtsi2ss(XmmRegister dst, CpuRegister src, bool is64bit); void cvtsi2ss(XmmRegister dst, const Address& src, bool is64bit); diff --git a/compiler/utils/x86_64/assembler_x86_64_test.cc b/compiler/utils/x86_64/assembler_x86_64_test.cc index 6b1e53c35ab..0589df55d23 100644 --- a/compiler/utils/x86_64/assembler_x86_64_test.cc +++ b/compiler/utils/x86_64/assembler_x86_64_test.cc @@ -21,6 +21,7 @@ #include <random> #include "base/bit_utils.h" +#include "base/malloc_arena_pool.h" #include "base/stl_util.h" #include "jni_macro_assembler_x86_64.h" #include "utils/assembler_test.h" @@ -29,7 +30,7 @@ namespace art { TEST(AssemblerX86_64, CreateBuffer) { - ArenaPool pool; + MallocArenaPool pool; ArenaAllocator allocator(&pool); AssemblerBuffer buffer(&allocator); AssemblerBuffer::EnsureCapacity ensured(&buffer); @@ -1282,6 +1283,38 @@ TEST_F(AssemblerX86_64Test, Psubq) { DriverStr(RepeatFF(&x86_64::X86_64Assembler::psubq, "psubq %{reg2}, %{reg1}"), "psubq"); } +TEST_F(AssemblerX86_64Test, Paddusb) { + DriverStr(RepeatFF(&x86_64::X86_64Assembler::paddusb, "paddusb %{reg2}, %{reg1}"), "paddusb"); +} + +TEST_F(AssemblerX86_64Test, Paddsb) { + DriverStr(RepeatFF(&x86_64::X86_64Assembler::paddsb, "paddsb %{reg2}, %{reg1}"), "paddsb"); +} + +TEST_F(AssemblerX86_64Test, Paddusw) { + DriverStr(RepeatFF(&x86_64::X86_64Assembler::paddusw, "paddusw %{reg2}, %{reg1}"), "paddusw"); +} + +TEST_F(AssemblerX86_64Test, Paddsw) { + DriverStr(RepeatFF(&x86_64::X86_64Assembler::paddsw, "paddsw %{reg2}, %{reg1}"), "paddsw"); +} + +TEST_F(AssemblerX86_64Test, Psubusb) { + DriverStr(RepeatFF(&x86_64::X86_64Assembler::psubusb, "psubusb %{reg2}, %{reg1}"), "psubusb"); +} + +TEST_F(AssemblerX86_64Test, Psubsb) { + DriverStr(RepeatFF(&x86_64::X86_64Assembler::psubsb, "psubsb %{reg2}, %{reg1}"), "psubsb"); +} + +TEST_F(AssemblerX86_64Test, Psubusw) { + DriverStr(RepeatFF(&x86_64::X86_64Assembler::psubusw, "psubusw %{reg2}, %{reg1}"), "psubusw"); +} + +TEST_F(AssemblerX86_64Test, Psubsw) { + DriverStr(RepeatFF(&x86_64::X86_64Assembler::psubsw, "psubsw %{reg2}, %{reg1}"), "psubsw"); +} + TEST_F(AssemblerX86_64Test, Cvtsi2ss) { DriverStr(RepeatFr(&x86_64::X86_64Assembler::cvtsi2ss, "cvtsi2ss %{reg2}, %{reg1}"), "cvtsi2ss"); } diff --git a/compiler/utils/x86_64/jni_macro_assembler_x86_64.cc b/compiler/utils/x86_64/jni_macro_assembler_x86_64.cc index 5766f9d44b9..9486cb44c5b 100644 --- a/compiler/utils/x86_64/jni_macro_assembler_x86_64.cc +++ b/compiler/utils/x86_64/jni_macro_assembler_x86_64.cc @@ -17,8 +17,8 @@ #include "jni_macro_assembler_x86_64.h" #include "base/casts.h" +#include "base/memory_region.h" #include "entrypoints/quick/quick_entrypoints.h" -#include "memory_region.h" #include "thread.h" namespace art { diff --git a/compiler/verifier_deps_test.cc b/compiler/verifier_deps_test.cc index 76448d819c2..553d131e2f5 100644 --- a/compiler/verifier_deps_test.cc +++ b/compiler/verifier_deps_test.cc @@ -18,6 +18,7 @@ #include "verifier/verifier_deps.h" #include "art_method-inl.h" +#include "base/indenter.h" #include "class_linker.h" #include "common_compiler_test.h" #include "compiler_callbacks.h" @@ -28,7 +29,6 @@ #include "driver/compiler_driver-inl.h" #include "driver/compiler_options.h" #include "handle_scope-inl.h" -#include "indenter.h" #include "mirror/class_loader.h" #include "runtime.h" #include "scoped_thread_state_change-inl.h" diff --git a/dex2oat/Android.bp b/dex2oat/Android.bp index b158231bc3f..49b65fd35eb 100644 --- a/dex2oat/Android.bp +++ b/dex2oat/Android.bp @@ -24,7 +24,44 @@ art_cc_defaults { "linker/image_writer.cc", "linker/multi_oat_relative_patcher.cc", "linker/oat_writer.cc", + "linker/relative_patcher.cc", ], + + codegen: { + arm: { + srcs: [ + "linker/arm/relative_patcher_arm_base.cc", + "linker/arm/relative_patcher_thumb2.cc", + ], + }, + arm64: { + srcs: [ + "linker/arm64/relative_patcher_arm64.cc", + ], + }, + mips: { + srcs: [ + "linker/mips/relative_patcher_mips.cc", + ], + }, + mips64: { + srcs: [ + "linker/mips64/relative_patcher_mips64.cc", + ], + }, + x86: { + srcs: [ + "linker/x86/relative_patcher_x86.cc", + "linker/x86/relative_patcher_x86_base.cc", + ], + }, + x86_64: { + srcs: [ + "linker/x86_64/relative_patcher_x86_64.cc", + ], + }, + }, + target: { android: { // For atrace. @@ -245,6 +282,41 @@ art_cc_test { "linker/multi_oat_relative_patcher_test.cc", "linker/oat_writer_test.cc", ], + + codegen: { + arm: { + srcs: [ + "linker/arm/relative_patcher_thumb2_test.cc", + ], + }, + arm64: { + srcs: [ + "linker/arm64/relative_patcher_arm64_test.cc", + ], + }, + mips: { + srcs: [ + "linker/mips/relative_patcher_mips_test.cc", + "linker/mips/relative_patcher_mips32r6_test.cc", + ], + }, + mips64: { + srcs: [ + "linker/mips64/relative_patcher_mips64_test.cc", + ], + }, + x86: { + srcs: [ + "linker/x86/relative_patcher_x86_test.cc", + ], + }, + x86_64: { + srcs: [ + "linker/x86_64/relative_patcher_x86_64_test.cc", + ], + }, + }, + header_libs: ["dex2oat_headers"], include_dirs: [ "external/zlib", diff --git a/dex2oat/dex2oat.cc b/dex2oat/dex2oat.cc index fe927bbc1c2..3fe9c477d5b 100644 --- a/dex2oat/dex2oat.cc +++ b/dex2oat/dex2oat.cc @@ -55,6 +55,7 @@ #include "base/timing_logger.h" #include "base/unix_file/fd_file.h" #include "base/utils.h" +#include "base/zip_archive.h" #include "class_linker.h" #include "class_loader_context.h" #include "cmdline_parser.h" @@ -98,7 +99,6 @@ #include "vdex_file.h" #include "verifier/verifier_deps.h" #include "well_known_classes.h" -#include "zip_archive.h" namespace art { @@ -629,10 +629,6 @@ class Dex2Oat FINAL { image_classes_zip_filename_(nullptr), image_classes_filename_(nullptr), image_storage_mode_(ImageHeader::kStorageModeUncompressed), - compiled_classes_zip_filename_(nullptr), - compiled_classes_filename_(nullptr), - compiled_methods_zip_filename_(nullptr), - compiled_methods_filename_(nullptr), passes_to_run_filename_(nullptr), dirty_image_objects_filename_(nullptr), multi_image_(false), @@ -756,7 +752,7 @@ class Dex2Oat FINAL { } if ((output_vdex_fd_ == -1) != (oat_fd_ == -1)) { - Usage("VDEX and OAT output must be specified either with one --oat-filename " + Usage("VDEX and OAT output must be specified either with one --oat-file " "or with --oat-fd and --output-vdex-fd file descriptors"); } @@ -818,18 +814,6 @@ class Dex2Oat FINAL { Usage("--image-classes-zip should be used with --image-classes"); } - if (compiled_classes_filename_ != nullptr && !IsBootImage()) { - Usage("--compiled-classes should only be used with --image"); - } - - if (compiled_classes_filename_ != nullptr && !boot_image_filename_.empty()) { - Usage("--compiled-classes should not be used with --boot-image"); - } - - if (compiled_classes_zip_filename_ != nullptr && compiled_classes_filename_ == nullptr) { - Usage("--compiled-classes-zip should be used with --compiled-classes"); - } - if (dex_filenames_.empty() && zip_fd_ == -1) { Usage("Input must be supplied with either --dex-file or --zip-fd"); } @@ -873,9 +857,7 @@ class Dex2Oat FINAL { } if (have_profile_file || have_profile_fd) { - if (compiled_classes_filename_ != nullptr || - compiled_classes_zip_filename_ != nullptr || - image_classes_filename_ != nullptr || + if (image_classes_filename_ != nullptr || image_classes_zip_filename_ != nullptr) { Usage("Profile based image creation is not supported with image or compiled classes"); } @@ -1210,10 +1192,6 @@ class Dex2Oat FINAL { AssignIfExists(args, M::Threads, &thread_count_); AssignIfExists(args, M::ImageClasses, &image_classes_filename_); AssignIfExists(args, M::ImageClassesZip, &image_classes_zip_filename_); - AssignIfExists(args, M::CompiledClasses, &compiled_classes_filename_); - AssignIfExists(args, M::CompiledClassesZip, &compiled_classes_zip_filename_); - AssignIfExists(args, M::CompiledMethods, &compiled_methods_filename_); - AssignIfExists(args, M::CompiledMethodsZip, &compiled_methods_zip_filename_); AssignIfExists(args, M::Passes, &passes_to_run_filename_); AssignIfExists(args, M::BootImage, &parser_options->boot_image_filename); AssignIfExists(args, M::AndroidRoot, &android_root_); @@ -1534,8 +1512,7 @@ class Dex2Oat FINAL { dex2oat::ReturnCode Setup() { TimingLogger::ScopedTiming t("dex2oat Setup", timings_); - if (!PrepareImageClasses() || !PrepareCompiledClasses() || !PrepareCompiledMethods() || - !PrepareDirtyObjects()) { + if (!PrepareImageClasses() || !PrepareDirtyObjects()) { return dex2oat::ReturnCode::kOther; } @@ -1871,8 +1848,6 @@ class Dex2Oat FINAL { instruction_set_, instruction_set_features_.get(), image_classes_.release(), - compiled_classes_.release(), - compiled_methods_.release(), thread_count_, swap_fd_, profile_compilation_info_.get())); @@ -2054,7 +2029,7 @@ class Dex2Oat FINAL { // We need to prepare method offsets in the image address space for direct method patching. TimingLogger::ScopedTiming t2("dex2oat Prepare image address space", timings_); - if (!image_writer_->PrepareImageAddressSpace()) { + if (!image_writer_->PrepareImageAddressSpace(timings_)) { LOG(ERROR) << "Failed to prepare image address space."; return false; } @@ -2098,17 +2073,17 @@ class Dex2Oat FINAL { { TimingLogger::ScopedTiming t2("dex2oat Write ELF", timings_); - linker::MultiOatRelativePatcher patcher(instruction_set_, instruction_set_features_.get()); + linker::MultiOatRelativePatcher patcher(instruction_set_, + instruction_set_features_.get(), + driver_->GetCompiledMethodStorage()); for (size_t i = 0, size = oat_files_.size(); i != size; ++i) { std::unique_ptr<linker::ElfWriter>& elf_writer = elf_writers_[i]; std::unique_ptr<linker::OatWriter>& oat_writer = oat_writers_[i]; oat_writer->PrepareLayout(&patcher); - - size_t rodata_size = oat_writer->GetOatHeader().GetExecutableOffset(); - size_t text_size = oat_writer->GetOatSize() - rodata_size; - elf_writer->PrepareDynamicSection(rodata_size, - text_size, + elf_writer->PrepareDynamicSection(oat_writer->GetOatHeader().GetExecutableOffset(), + oat_writer->GetCodeSize(), + oat_writer->GetDataBimgRelRoSize(), oat_writer->GetBssSize(), oat_writer->GetBssMethodsOffset(), oat_writer->GetBssRootsOffset(), @@ -2158,6 +2133,16 @@ class Dex2Oat FINAL { } elf_writer->EndText(text); + if (oat_writer->GetDataBimgRelRoSize() != 0u) { + linker::OutputStream* data_bimg_rel_ro = elf_writer->StartDataBimgRelRo(); + if (!oat_writer->WriteDataBimgRelRo(data_bimg_rel_ro)) { + LOG(ERROR) << "Failed to write .data.bimg.rel.ro section to the ELF file " + << oat_file->GetPath(); + return false; + } + elf_writer->EndDataBimgRelRo(data_bimg_rel_ro); + } + if (!oat_writer->WriteHeader(elf_writer->GetStream(), image_file_location_oat_checksum_, image_file_location_oat_data_begin_, @@ -2407,21 +2392,6 @@ class Dex2Oat FINAL { return true; } - bool PrepareCompiledClasses() { - // If --compiled-classes was specified, calculate the full list of classes to compile in the - // image. - if (compiled_classes_filename_ != nullptr) { - compiled_classes_ = - ReadClasses(compiled_classes_zip_filename_, compiled_classes_filename_, "compiled"); - if (compiled_classes_ == nullptr) { - return false; - } - } else { - compiled_classes_.reset(nullptr); // By default compile everything. - } - return true; - } - static std::unique_ptr<std::unordered_set<std::string>> ReadClasses(const char* zip_filename, const char* classes_filename, const char* tag) { @@ -2439,32 +2409,6 @@ class Dex2Oat FINAL { return classes; } - bool PrepareCompiledMethods() { - // If --compiled-methods was specified, read the methods to compile from the given file(s). - if (compiled_methods_filename_ != nullptr) { - std::string error_msg; - if (compiled_methods_zip_filename_ != nullptr) { - compiled_methods_.reset(ReadCommentedInputFromZip<std::unordered_set<std::string>>( - compiled_methods_zip_filename_, - compiled_methods_filename_, - nullptr, // No post-processing. - &error_msg)); - } else { - compiled_methods_.reset(ReadCommentedInputFromFile<std::unordered_set<std::string>>( - compiled_methods_filename_, - nullptr)); // No post-processing. - } - if (compiled_methods_.get() == nullptr) { - LOG(ERROR) << "Failed to create list of compiled methods from '" - << compiled_methods_filename_ << "': " << error_msg; - return false; - } - } else { - compiled_methods_.reset(nullptr); // By default compile everything. - } - return true; - } - bool PrepareDirtyObjects() { if (dirty_image_objects_filename_ != nullptr) { dirty_image_objects_.reset(ReadCommentedInputFromFile<std::unordered_set<std::string>>( @@ -2905,10 +2849,6 @@ class Dex2Oat FINAL { const char* image_classes_zip_filename_; const char* image_classes_filename_; ImageHeader::StorageMode image_storage_mode_; - const char* compiled_classes_zip_filename_; - const char* compiled_classes_filename_; - const char* compiled_methods_zip_filename_; - const char* compiled_methods_filename_; const char* passes_to_run_filename_; const char* dirty_image_objects_filename_; std::unique_ptr<std::unordered_set<std::string>> image_classes_; diff --git a/dex2oat/dex2oat_image_test.cc b/dex2oat/dex2oat_image_test.cc index 6f1224916f5..11c0c950606 100644 --- a/dex2oat/dex2oat_image_test.cc +++ b/dex2oat/dex2oat_image_test.cc @@ -239,9 +239,7 @@ TEST_F(Dex2oatImageTest, TestModesAndFilters) { ImageSizes base_sizes = CompileImageAndGetSizes({}); ImageSizes image_classes_sizes; ImageSizes compiled_classes_sizes; - ImageSizes compiled_all_classes_sizes; ImageSizes compiled_methods_sizes; - ImageSizes compiled_all_methods_sizes; ImageSizes profile_sizes; std::cout << "Base compile sizes " << base_sizes << std::endl; // Test image classes @@ -257,65 +255,28 @@ TEST_F(Dex2oatImageTest, TestModesAndFilters) { // Sanity check that dex is the same size. EXPECT_EQ(image_classes_sizes.vdex_size, base_sizes.vdex_size); } - // Test compiled classes with all the classes. - { - ScratchFile classes; - // Only compile every even class. - GenerateClasses(classes.GetFile(), /*frequency*/ 1u); - compiled_all_classes_sizes = CompileImageAndGetSizes( - {"--compiled-classes=" + classes.GetFilename()}); - classes.Close(); - std::cout << "Compiled all classes sizes " << compiled_all_classes_sizes << std::endl; - // Check that oat size is smaller since we didn't compile everything. - EXPECT_EQ(compiled_all_classes_sizes.art_size, base_sizes.art_size); - // TODO(mathieuc): Find a reliable way to check compiled code. - // EXPECT_EQ(compiled_all_classes_sizes.oat_size, base_sizes.oat_size); - EXPECT_EQ(compiled_all_classes_sizes.vdex_size, base_sizes.vdex_size); - } // Test compiled classes. { ScratchFile classes; // Only compile every even class. GenerateClasses(classes.GetFile(), /*frequency*/ 2u); compiled_classes_sizes = CompileImageAndGetSizes( - {"--image-classes=" + classes.GetFilename(), - "--compiled-classes=" + classes.GetFilename()}); + {"--image-classes=" + classes.GetFilename()}); classes.Close(); std::cout << "Compiled classes sizes " << compiled_classes_sizes << std::endl; - // Check that oat size is smaller since we didn't compile everything. - // TODO(mathieuc): Find a reliable way to check compiled code. - // EXPECT_LT(compiled_classes_sizes.oat_size, base_sizes.oat_size); // Art file should be smaller than image classes version since we included fewer classes in the // list. EXPECT_LT(compiled_classes_sizes.art_size, image_classes_sizes.art_size); } - // Test compiled methods. - { - ScratchFile methods; - // Only compile every even class. - GenerateMethods(methods.GetFile(), /*frequency*/ 1u); - compiled_all_methods_sizes = CompileImageAndGetSizes( - {"--compiled-methods=" + methods.GetFilename()}); - methods.Close(); - std::cout << "Compiled all methods sizes " << compiled_all_methods_sizes << std::endl; - EXPECT_EQ(compiled_all_classes_sizes.art_size, base_sizes.art_size); - // TODO(mathieuc): Find a reliable way to check compiled code. b/63746626 - // EXPECT_EQ(compiled_all_classes_sizes.oat_size, base_sizes.oat_size); - EXPECT_EQ(compiled_all_classes_sizes.vdex_size, base_sizes.vdex_size); - } static size_t kMethodFrequency = 3; static size_t kTypeFrequency = 4; // Test compiling fewer methods and classes. { - ScratchFile methods; ScratchFile classes; // Only compile every even class. - GenerateMethods(methods.GetFile(), kMethodFrequency); GenerateClasses(classes.GetFile(), kTypeFrequency); compiled_methods_sizes = CompileImageAndGetSizes( - {"--image-classes=" + classes.GetFilename(), - "--compiled-methods=" + methods.GetFilename()}); - methods.Close(); + {"--image-classes=" + classes.GetFilename()}); classes.Close(); std::cout << "Compiled fewer methods sizes " << compiled_methods_sizes << std::endl; } diff --git a/dex2oat/dex2oat_options.cc b/dex2oat/dex2oat_options.cc index 5843691a242..dbb00c22e94 100644 --- a/dex2oat/dex2oat_options.cc +++ b/dex2oat/dex2oat_options.cc @@ -157,19 +157,7 @@ static void AddSwapMappings(Builder& builder) { static void AddCompilerMappings(Builder& builder) { builder. - Define("--compiled-classes=_") - .WithType<std::string>() - .IntoKey(M::CompiledClasses) - .Define("--compiled-classes-zip=_") - .WithType<std::string>() - .IntoKey(M::CompiledClassesZip) - .Define("--compiled-methods=_") - .WithType<std::string>() - .IntoKey(M::CompiledMethods) - .Define("--compiled-methods-zip=_") - .WithType<std::string>() - .IntoKey(M::CompiledMethodsZip) - .Define("--run-passes=_") + Define("--run-passes=_") .WithType<std::string>() .IntoKey(M::Passes) .Define("--profile-file=_") diff --git a/dex2oat/dex2oat_options.def b/dex2oat/dex2oat_options.def index 1a913a9bbf3..7be8e565010 100644 --- a/dex2oat/dex2oat_options.def +++ b/dex2oat/dex2oat_options.def @@ -56,10 +56,6 @@ DEX2OAT_OPTIONS_KEY (std::vector<std::string>, ImageFilenames) DEX2OAT_OPTIONS_KEY (std::string, ImageClasses) DEX2OAT_OPTIONS_KEY (std::string, ImageClassesZip) DEX2OAT_OPTIONS_KEY (ImageHeader::StorageMode, ImageFormat) -DEX2OAT_OPTIONS_KEY (std::string, CompiledClasses) -DEX2OAT_OPTIONS_KEY (std::string, CompiledClassesZip) -DEX2OAT_OPTIONS_KEY (std::string, CompiledMethods) -DEX2OAT_OPTIONS_KEY (std::string, CompiledMethodsZip) DEX2OAT_OPTIONS_KEY (std::string, Passes) DEX2OAT_OPTIONS_KEY (std::string, Base) // TODO: Hex string parsing. DEX2OAT_OPTIONS_KEY (std::string, BootImage) diff --git a/compiler/linker/arm/relative_patcher_arm_base.cc b/dex2oat/linker/arm/relative_patcher_arm_base.cc index 6e0286afac1..7cb8ae55c57 100644 --- a/compiler/linker/arm/relative_patcher_arm_base.cc +++ b/dex2oat/linker/arm/relative_patcher_arm_base.cc @@ -30,8 +30,9 @@ namespace linker { class ArmBaseRelativePatcher::ThunkData { public: - ThunkData(std::vector<uint8_t> code, uint32_t max_next_offset) - : code_(std::move(code)), + ThunkData(ArrayRef<const uint8_t> code, const std::string& debug_name, uint32_t max_next_offset) + : code_(code), + debug_name_(debug_name), offsets_(), max_next_offset_(max_next_offset), pending_offset_(0u) { @@ -45,7 +46,11 @@ class ArmBaseRelativePatcher::ThunkData { } ArrayRef<const uint8_t> GetCode() const { - return ArrayRef<const uint8_t>(code_); + return code_; + } + + const std::string& GetDebugName() const { + return debug_name_; } bool NeedsNextThunk() const { @@ -142,10 +147,11 @@ class ArmBaseRelativePatcher::ThunkData { } private: - std::vector<uint8_t> code_; // The code of the thunk. - std::vector<uint32_t> offsets_; // Offsets at which the thunk needs to be written. - uint32_t max_next_offset_; // The maximum offset at which the next thunk can be placed. - uint32_t pending_offset_; // The index of the next offset to write. + const ArrayRef<const uint8_t> code_; // The code of the thunk. + const std::string debug_name_; // The debug name of the thunk. + std::vector<uint32_t> offsets_; // Offsets at which the thunk needs to be written. + uint32_t max_next_offset_; // The maximum offset at which the next thunk can be placed. + uint32_t pending_offset_; // The index of the next offset to write. }; class ArmBaseRelativePatcher::PendingThunkComparator { @@ -239,14 +245,13 @@ std::vector<debug::MethodDebugInfo> ArmBaseRelativePatcher::GenerateThunkDebugIn std::vector<debug::MethodDebugInfo> result; result.reserve(number_of_thunks); for (auto&& entry : thunks_) { - const ThunkKey& key = entry.first; const ThunkData& data = entry.second; size_t start = data.IndexOfFirstThunkAtOrAfter(executable_offset); if (start == data.NumberOfThunks()) { continue; } // Get the base name to use for the first occurrence of the thunk. - std::string base_name = GetThunkDebugName(key); + std::string base_name = data.GetDebugName(); for (size_t i = start, num = data.NumberOfThunks(); i != num; ++i) { debug::MethodDebugInfo info = {}; if (i == 0u) { @@ -267,9 +272,11 @@ std::vector<debug::MethodDebugInfo> ArmBaseRelativePatcher::GenerateThunkDebugIn return result; } -ArmBaseRelativePatcher::ArmBaseRelativePatcher(RelativePatcherTargetProvider* provider, +ArmBaseRelativePatcher::ArmBaseRelativePatcher(RelativePatcherThunkProvider* thunk_provider, + RelativePatcherTargetProvider* target_provider, InstructionSet instruction_set) - : provider_(provider), + : thunk_provider_(thunk_provider), + target_provider_(target_provider), instruction_set_(instruction_set), thunks_(), unprocessed_method_call_patches_(), @@ -398,7 +405,7 @@ void ArmBaseRelativePatcher::ProcessPatches(const CompiledMethod* compiled_metho unprocessed_method_call_patches_.emplace_back(patch_offset, patch.TargetMethod()); if (method_call_thunk_ == nullptr) { uint32_t max_next_offset = CalculateMaxNextOffset(patch_offset, key); - auto it = thunks_.Put(key, ThunkData(CompileThunk(key), max_next_offset)); + auto it = thunks_.Put(key, ThunkDataForPatch(patch, max_next_offset)); method_call_thunk_ = &it->second; AddUnreservedThunk(method_call_thunk_); } else { @@ -409,7 +416,7 @@ void ArmBaseRelativePatcher::ProcessPatches(const CompiledMethod* compiled_metho auto lb = thunks_.lower_bound(key); if (lb == thunks_.end() || thunks_.key_comp()(key, lb->first)) { uint32_t max_next_offset = CalculateMaxNextOffset(patch_offset, key); - auto it = thunks_.PutBefore(lb, key, ThunkData(CompileThunk(key), max_next_offset)); + auto it = thunks_.PutBefore(lb, key, ThunkDataForPatch(patch, max_next_offset)); AddUnreservedThunk(&it->second); } else { old_data = &lb->second; @@ -477,7 +484,7 @@ void ArmBaseRelativePatcher::ResolveMethodCalls(uint32_t quick_code_offset, break; } } else { - auto result = provider_->FindMethodOffset(target_method); + auto result = target_provider_->FindMethodOffset(target_method); if (!result.first) { break; } @@ -518,5 +525,14 @@ inline uint32_t ArmBaseRelativePatcher::CalculateMaxNextOffset(uint32_t patch_of GetInstructionSetAlignment(instruction_set_)); } +inline ArmBaseRelativePatcher::ThunkData ArmBaseRelativePatcher::ThunkDataForPatch( + const LinkerPatch& patch, uint32_t max_next_offset) { + ArrayRef<const uint8_t> code; + std::string debug_name; + thunk_provider_->GetThunkCode(patch, &code, &debug_name); + DCHECK(!code.empty()); + return ThunkData(code, debug_name, max_next_offset); +} + } // namespace linker } // namespace art diff --git a/compiler/linker/arm/relative_patcher_arm_base.h b/dex2oat/linker/arm/relative_patcher_arm_base.h index ee09bf96b3d..f5a1395bdd9 100644 --- a/compiler/linker/arm/relative_patcher_arm_base.h +++ b/dex2oat/linker/arm/relative_patcher_arm_base.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef ART_COMPILER_LINKER_ARM_RELATIVE_PATCHER_ARM_BASE_H_ -#define ART_COMPILER_LINKER_ARM_RELATIVE_PATCHER_ARM_BASE_H_ +#ifndef ART_DEX2OAT_LINKER_ARM_RELATIVE_PATCHER_ARM_BASE_H_ +#define ART_DEX2OAT_LINKER_ARM_RELATIVE_PATCHER_ARM_BASE_H_ #include <deque> #include <vector> @@ -37,7 +37,8 @@ class ArmBaseRelativePatcher : public RelativePatcher { std::vector<debug::MethodDebugInfo> GenerateThunkDebugInfo(uint32_t executable_offset) OVERRIDE; protected: - ArmBaseRelativePatcher(RelativePatcherTargetProvider* provider, + ArmBaseRelativePatcher(RelativePatcherThunkProvider* thunk_provider, + RelativePatcherTargetProvider* target_provider, InstructionSet instruction_set); ~ArmBaseRelativePatcher(); @@ -94,8 +95,6 @@ class ArmBaseRelativePatcher : public RelativePatcher { uint32_t CalculateMethodCallDisplacement(uint32_t patch_offset, uint32_t target_offset); - virtual std::vector<uint8_t> CompileThunk(const ThunkKey& key) = 0; - virtual std::string GetThunkDebugName(const ThunkKey& key) = 0; virtual uint32_t MaxPositiveDisplacement(const ThunkKey& key) = 0; virtual uint32_t MaxNegativeDisplacement(const ThunkKey& key) = 0; @@ -108,8 +107,10 @@ class ArmBaseRelativePatcher : public RelativePatcher { void ResolveMethodCalls(uint32_t quick_code_offset, MethodReference method_ref); uint32_t CalculateMaxNextOffset(uint32_t patch_offset, const ThunkKey& key); + ThunkData ThunkDataForPatch(const LinkerPatch& patch, uint32_t max_next_offset); - RelativePatcherTargetProvider* const provider_; + RelativePatcherThunkProvider* const thunk_provider_; + RelativePatcherTargetProvider* const target_provider_; const InstructionSet instruction_set_; // The data for all thunks. @@ -154,4 +155,4 @@ class ArmBaseRelativePatcher : public RelativePatcher { } // namespace linker } // namespace art -#endif // ART_COMPILER_LINKER_ARM_RELATIVE_PATCHER_ARM_BASE_H_ +#endif // ART_DEX2OAT_LINKER_ARM_RELATIVE_PATCHER_ARM_BASE_H_ diff --git a/dex2oat/linker/arm/relative_patcher_thumb2.cc b/dex2oat/linker/arm/relative_patcher_thumb2.cc new file mode 100644 index 00000000000..697fb09f739 --- /dev/null +++ b/dex2oat/linker/arm/relative_patcher_thumb2.cc @@ -0,0 +1,186 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "linker/arm/relative_patcher_thumb2.h" + +#include <sstream> + +#include "arch/arm/asm_support_arm.h" +#include "art_method.h" +#include "base/bit_utils.h" +#include "base/malloc_arena_pool.h" +#include "compiled_method.h" +#include "entrypoints/quick/quick_entrypoints_enum.h" +#include "linker/linker_patch.h" +#include "lock_word.h" +#include "mirror/array-inl.h" +#include "mirror/object.h" +#include "read_barrier.h" +#include "utils/arm/assembler_arm_vixl.h" + +namespace art { +namespace linker { + +// PC displacement from patch location; Thumb2 PC is always at instruction address + 4. +static constexpr int32_t kPcDisplacement = 4; + +// Maximum positive and negative displacement for method call measured from the patch location. +// (Signed 25 bit displacement with the last bit 0 has range [-2^24, 2^24-2] measured from +// the Thumb2 PC pointing right after the BL, i.e. 4 bytes later than the patch location.) +constexpr uint32_t kMaxMethodCallPositiveDisplacement = (1u << 24) - 2 + kPcDisplacement; +constexpr uint32_t kMaxMethodCallNegativeDisplacement = (1u << 24) - kPcDisplacement; + +// Maximum positive and negative displacement for a conditional branch measured from the patch +// location. (Signed 21 bit displacement with the last bit 0 has range [-2^20, 2^20-2] measured +// from the Thumb2 PC pointing right after the B.cond, i.e. 4 bytes later than the patch location.) +constexpr uint32_t kMaxBcondPositiveDisplacement = (1u << 20) - 2u + kPcDisplacement; +constexpr uint32_t kMaxBcondNegativeDisplacement = (1u << 20) - kPcDisplacement; + +Thumb2RelativePatcher::Thumb2RelativePatcher(RelativePatcherThunkProvider* thunk_provider, + RelativePatcherTargetProvider* target_provider) + : ArmBaseRelativePatcher(thunk_provider, target_provider, InstructionSet::kThumb2) { +} + +void Thumb2RelativePatcher::PatchCall(std::vector<uint8_t>* code, + uint32_t literal_offset, + uint32_t patch_offset, + uint32_t target_offset) { + DCHECK_LE(literal_offset + 4u, code->size()); + DCHECK_EQ(literal_offset & 1u, 0u); + DCHECK_EQ(patch_offset & 1u, 0u); + DCHECK_EQ(target_offset & 1u, 1u); // Thumb2 mode bit. + uint32_t displacement = CalculateMethodCallDisplacement(patch_offset, target_offset & ~1u); + displacement -= kPcDisplacement; // The base PC is at the end of the 4-byte patch. + DCHECK_EQ(displacement & 1u, 0u); + DCHECK((displacement >> 24) == 0u || (displacement >> 24) == 255u); // 25-bit signed. + uint32_t signbit = (displacement >> 31) & 0x1; + uint32_t i1 = (displacement >> 23) & 0x1; + uint32_t i2 = (displacement >> 22) & 0x1; + uint32_t imm10 = (displacement >> 12) & 0x03ff; + uint32_t imm11 = (displacement >> 1) & 0x07ff; + uint32_t j1 = i1 ^ (signbit ^ 1); + uint32_t j2 = i2 ^ (signbit ^ 1); + uint32_t value = (signbit << 26) | (j1 << 13) | (j2 << 11) | (imm10 << 16) | imm11; + value |= 0xf000d000; // BL + + // Check that we're just overwriting an existing BL. + DCHECK_EQ(GetInsn32(code, literal_offset) & 0xf800d000, 0xf000d000); + // Write the new BL. + SetInsn32(code, literal_offset, value); +} + +void Thumb2RelativePatcher::PatchPcRelativeReference(std::vector<uint8_t>* code, + const LinkerPatch& patch, + uint32_t patch_offset, + uint32_t target_offset) { + uint32_t literal_offset = patch.LiteralOffset(); + uint32_t pc_literal_offset = patch.PcInsnOffset(); + uint32_t pc_base = patch_offset + (pc_literal_offset - literal_offset) + 4u /* PC adjustment */; + uint32_t diff = target_offset - pc_base; + + uint32_t insn = GetInsn32(code, literal_offset); + DCHECK_EQ(insn & 0xff7ff0ffu, 0xf2400000u); // MOVW/MOVT, unpatched (imm16 == 0). + uint32_t diff16 = ((insn & 0x00800000u) != 0u) ? (diff >> 16) : (diff & 0xffffu); + uint32_t imm4 = (diff16 >> 12) & 0xfu; + uint32_t imm = (diff16 >> 11) & 0x1u; + uint32_t imm3 = (diff16 >> 8) & 0x7u; + uint32_t imm8 = diff16 & 0xffu; + insn = (insn & 0xfbf08f00u) | (imm << 26) | (imm4 << 16) | (imm3 << 12) | imm8; + SetInsn32(code, literal_offset, insn); +} + +void Thumb2RelativePatcher::PatchBakerReadBarrierBranch(std::vector<uint8_t>* code, + const LinkerPatch& patch, + uint32_t patch_offset) { + DCHECK_ALIGNED(patch_offset, 2u); + uint32_t literal_offset = patch.LiteralOffset(); + DCHECK_ALIGNED(literal_offset, 2u); + DCHECK_LT(literal_offset, code->size()); + uint32_t insn = GetInsn32(code, literal_offset); + DCHECK_EQ(insn, 0xf0408000); // BNE +0 (unpatched) + ThunkKey key = GetBakerThunkKey(patch); + uint32_t target_offset = GetThunkTargetOffset(key, patch_offset); + DCHECK_ALIGNED(target_offset, 4u); + uint32_t disp = target_offset - (patch_offset + kPcDisplacement); + DCHECK((disp >> 20) == 0u || (disp >> 20) == 0xfffu); // 21-bit signed. + insn |= ((disp << (26 - 20)) & 0x04000000u) | // Shift bit 20 to 26, "S". + ((disp >> (19 - 11)) & 0x00000800u) | // Shift bit 19 to 13, "J1". + ((disp >> (18 - 13)) & 0x00002000u) | // Shift bit 18 to 11, "J2". + ((disp << (16 - 12)) & 0x003f0000u) | // Shift bits 12-17 to 16-25, "imm6". + ((disp >> (1 - 0)) & 0x000007ffu); // Shift bits 1-12 to 0-11, "imm11". + SetInsn32(code, literal_offset, insn); +} + +uint32_t Thumb2RelativePatcher::MaxPositiveDisplacement(const ThunkKey& key) { + switch (key.GetType()) { + case ThunkType::kMethodCall: + return kMaxMethodCallPositiveDisplacement; + case ThunkType::kBakerReadBarrier: + return kMaxBcondPositiveDisplacement; + } +} + +uint32_t Thumb2RelativePatcher::MaxNegativeDisplacement(const ThunkKey& key) { + switch (key.GetType()) { + case ThunkType::kMethodCall: + return kMaxMethodCallNegativeDisplacement; + case ThunkType::kBakerReadBarrier: + return kMaxBcondNegativeDisplacement; + } +} + +void Thumb2RelativePatcher::SetInsn32(std::vector<uint8_t>* code, uint32_t offset, uint32_t value) { + DCHECK_LE(offset + 4u, code->size()); + DCHECK_ALIGNED(offset, 2u); + uint8_t* addr = &(*code)[offset]; + addr[0] = (value >> 16) & 0xff; + addr[1] = (value >> 24) & 0xff; + addr[2] = (value >> 0) & 0xff; + addr[3] = (value >> 8) & 0xff; +} + +uint32_t Thumb2RelativePatcher::GetInsn32(ArrayRef<const uint8_t> code, uint32_t offset) { + DCHECK_LE(offset + 4u, code.size()); + DCHECK_ALIGNED(offset, 2u); + const uint8_t* addr = &code[offset]; + return + (static_cast<uint32_t>(addr[0]) << 16) + + (static_cast<uint32_t>(addr[1]) << 24) + + (static_cast<uint32_t>(addr[2]) << 0)+ + (static_cast<uint32_t>(addr[3]) << 8); +} + +template <typename Vector> +uint32_t Thumb2RelativePatcher::GetInsn32(Vector* code, uint32_t offset) { + static_assert(std::is_same<typename Vector::value_type, uint8_t>::value, "Invalid value type"); + return GetInsn32(ArrayRef<const uint8_t>(*code), offset); +} + +uint32_t Thumb2RelativePatcher::GetInsn16(ArrayRef<const uint8_t> code, uint32_t offset) { + DCHECK_LE(offset + 2u, code.size()); + DCHECK_ALIGNED(offset, 2u); + const uint8_t* addr = &code[offset]; + return (static_cast<uint32_t>(addr[0]) << 0) + (static_cast<uint32_t>(addr[1]) << 8); +} + +template <typename Vector> +uint32_t Thumb2RelativePatcher::GetInsn16(Vector* code, uint32_t offset) { + static_assert(std::is_same<typename Vector::value_type, uint8_t>::value, "Invalid value type"); + return GetInsn16(ArrayRef<const uint8_t>(*code), offset); +} + +} // namespace linker +} // namespace art diff --git a/dex2oat/linker/arm/relative_patcher_thumb2.h b/dex2oat/linker/arm/relative_patcher_thumb2.h new file mode 100644 index 00000000000..3a429284665 --- /dev/null +++ b/dex2oat/linker/arm/relative_patcher_thumb2.h @@ -0,0 +1,73 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_DEX2OAT_LINKER_ARM_RELATIVE_PATCHER_THUMB2_H_ +#define ART_DEX2OAT_LINKER_ARM_RELATIVE_PATCHER_THUMB2_H_ + +#include "arch/arm/registers_arm.h" +#include "base/array_ref.h" +#include "linker/arm/relative_patcher_arm_base.h" + +namespace art { + +namespace arm { +class ArmVIXLAssembler; +} // namespace arm + +namespace linker { + +class Thumb2RelativePatcher FINAL : public ArmBaseRelativePatcher { + public: + explicit Thumb2RelativePatcher(RelativePatcherThunkProvider* thunk_provider, + RelativePatcherTargetProvider* target_provider); + + void PatchCall(std::vector<uint8_t>* code, + uint32_t literal_offset, + uint32_t patch_offset, + uint32_t target_offset) OVERRIDE; + void PatchPcRelativeReference(std::vector<uint8_t>* code, + const LinkerPatch& patch, + uint32_t patch_offset, + uint32_t target_offset) OVERRIDE; + void PatchBakerReadBarrierBranch(std::vector<uint8_t>* code, + const LinkerPatch& patch, + uint32_t patch_offset) OVERRIDE; + + protected: + uint32_t MaxPositiveDisplacement(const ThunkKey& key) OVERRIDE; + uint32_t MaxNegativeDisplacement(const ThunkKey& key) OVERRIDE; + + private: + void SetInsn32(std::vector<uint8_t>* code, uint32_t offset, uint32_t value); + static uint32_t GetInsn32(ArrayRef<const uint8_t> code, uint32_t offset); + + template <typename Vector> + static uint32_t GetInsn32(Vector* code, uint32_t offset); + + static uint32_t GetInsn16(ArrayRef<const uint8_t> code, uint32_t offset); + + template <typename Vector> + static uint32_t GetInsn16(Vector* code, uint32_t offset); + + friend class Thumb2RelativePatcherTest; + + DISALLOW_COPY_AND_ASSIGN(Thumb2RelativePatcher); +}; + +} // namespace linker +} // namespace art + +#endif // ART_DEX2OAT_LINKER_ARM_RELATIVE_PATCHER_THUMB2_H_ diff --git a/compiler/linker/arm/relative_patcher_thumb2_test.cc b/dex2oat/linker/arm/relative_patcher_thumb2_test.cc index 2c22a352c23..e7b11bd16b4 100644 --- a/compiler/linker/arm/relative_patcher_thumb2_test.cc +++ b/dex2oat/linker/arm/relative_patcher_thumb2_test.cc @@ -16,12 +16,15 @@ #include "linker/arm/relative_patcher_thumb2.h" +#include "arch/arm/instruction_set_features_arm.h" #include "base/casts.h" #include "linker/relative_patcher_test.h" #include "lock_word.h" #include "mirror/array-inl.h" #include "mirror/object.h" #include "oat_quick_method_header.h" +#include "optimizing/code_generator_arm_vixl.h" +#include "optimizing/optimizing_unit_test.h" namespace art { namespace linker { @@ -189,9 +192,42 @@ class Thumb2RelativePatcherTest : public RelativePatcherTest { return result.second - 1 /* thumb mode */; } + std::vector<uint8_t> CompileThunk(const LinkerPatch& patch, + /*out*/ std::string* debug_name = nullptr) { + OptimizingUnitTestHelper helper; + HGraph* graph = helper.CreateGraph(); + std::string error_msg; + ArmFeaturesUniquePtr features = + ArmInstructionSetFeatures::FromVariant("default", &error_msg); + CompilerOptions options; + arm::CodeGeneratorARMVIXL codegen(graph, *features, options); + ArenaVector<uint8_t> code(helper.GetAllocator()->Adapter()); + codegen.EmitThunkCode(patch, &code, debug_name); + return std::vector<uint8_t>(code.begin(), code.end()); + } + + void AddCompiledMethod( + MethodReference method_ref, + const ArrayRef<const uint8_t>& code, + const ArrayRef<const LinkerPatch>& patches = ArrayRef<const LinkerPatch>()) { + RelativePatcherTest::AddCompiledMethod(method_ref, code, patches); + + // Make sure the ThunkProvider has all the necessary thunks. + for (const LinkerPatch& patch : patches) { + if (patch.GetType() == LinkerPatch::Type::kBakerReadBarrierBranch || + patch.GetType() == LinkerPatch::Type::kCallRelative) { + std::string debug_name; + std::vector<uint8_t> thunk_code = CompileThunk(patch, &debug_name); + thunk_provider_.SetThunkCode(patch, ArrayRef<const uint8_t>(thunk_code), debug_name); + } + } + } + std::vector<uint8_t> CompileMethodCallThunk() { - ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetMethodCallKey(); - return static_cast<Thumb2RelativePatcher*>(patcher_.get())->CompileThunk(key); + LinkerPatch patch = LinkerPatch::RelativeCodePatch(/* literal_offset */ 0u, + /* target_dex_file*/ nullptr, + /* target_method_idx */ 0u); + return CompileThunk(patch); } uint32_t MethodCallThunkSize() { @@ -228,27 +264,38 @@ class Thumb2RelativePatcherTest : public RelativePatcherTest { void TestStringReference(uint32_t string_offset); void CheckPcRelativePatch(const ArrayRef<const LinkerPatch>& patches, uint32_t target_offset); + static uint32_t EncodeBakerReadBarrierFieldData(uint32_t base_reg, + uint32_t holder_reg, + bool narrow) { + return arm::CodeGeneratorARMVIXL::EncodeBakerReadBarrierFieldData(base_reg, holder_reg, narrow); + } + + static uint32_t EncodeBakerReadBarrierArrayData(uint32_t base_reg) { + return arm::CodeGeneratorARMVIXL::EncodeBakerReadBarrierArrayData(base_reg); + } + + static uint32_t EncodeBakerReadBarrierGcRootData(uint32_t root_reg, bool narrow) { + return arm::CodeGeneratorARMVIXL::EncodeBakerReadBarrierGcRootData(root_reg, narrow); + } + std::vector<uint8_t> CompileBakerOffsetThunk(uint32_t base_reg, uint32_t holder_reg, bool narrow) { const LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch( - 0u, Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData(base_reg, holder_reg, narrow)); - ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetBakerThunkKey(patch); - return down_cast<Thumb2RelativePatcher*>(patcher_.get())->CompileThunk(key); + /* literal_offset */ 0u, EncodeBakerReadBarrierFieldData(base_reg, holder_reg, narrow)); + return CompileThunk(patch); } std::vector<uint8_t> CompileBakerArrayThunk(uint32_t base_reg) { LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch( - 0u, Thumb2RelativePatcher::EncodeBakerReadBarrierArrayData(base_reg)); - ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetBakerThunkKey(patch); - return down_cast<Thumb2RelativePatcher*>(patcher_.get())->CompileThunk(key); + /* literal_offset */ 0u, EncodeBakerReadBarrierArrayData(base_reg)); + return CompileThunk(patch); } std::vector<uint8_t> CompileBakerGcRootThunk(uint32_t root_reg, bool narrow) { LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch( - 0u, Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData(root_reg, narrow)); - ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetBakerThunkKey(patch); - return down_cast<Thumb2RelativePatcher*>(patcher_.get())->CompileThunk(key); + /* literal_offset */ 0u, EncodeBakerReadBarrierGcRootData(root_reg, narrow)); + return CompileThunk(patch); } uint32_t GetOutputInsn32(uint32_t offset) { @@ -594,7 +641,7 @@ void Thumb2RelativePatcherTest::TestBakerFieldWide(uint32_t offset, uint32_t ref const std::vector<uint8_t> raw_code = RawCode({kBneWPlus0, ldr}); ASSERT_EQ(kMethodCodeSize, raw_code.size()); ArrayRef<const uint8_t> code(raw_code); - uint32_t encoded_data = Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData( + uint32_t encoded_data = EncodeBakerReadBarrierFieldData( base_reg, holder_reg, /* narrow */ false); const LinkerPatch patches[] = { LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset, encoded_data), @@ -696,7 +743,7 @@ void Thumb2RelativePatcherTest::TestBakerFieldNarrow(uint32_t offset, uint32_t r const std::vector<uint8_t> raw_code = RawCode({kBneWPlus0, ldr}); ASSERT_EQ(kMethodCodeSize, raw_code.size()); ArrayRef<const uint8_t> code(raw_code); - uint32_t encoded_data = Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData( + uint32_t encoded_data = EncodeBakerReadBarrierFieldData( base_reg, holder_reg, /* narrow */ true); const LinkerPatch patches[] = { LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset, encoded_data), @@ -809,7 +856,7 @@ TEST_F(Thumb2RelativePatcherTest, BakerOffsetThunkInTheMiddle) { constexpr uint32_t kLiteralOffset1 = 6u; const std::vector<uint8_t> raw_code1 = RawCode({kNopWInsn, kNopInsn, kBneWPlus0, kLdrWInsn}); ArrayRef<const uint8_t> code1(raw_code1); - uint32_t encoded_data = Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData( + uint32_t encoded_data = EncodeBakerReadBarrierFieldData( /* base_reg */ 0, /* holder_reg */ 0, /* narrow */ false); const LinkerPatch patches1[] = { LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data), @@ -877,7 +924,7 @@ TEST_F(Thumb2RelativePatcherTest, BakerOffsetThunkBeforeFiller) { constexpr uint32_t kLiteralOffset1 = 4u; const std::vector<uint8_t> raw_code1 = RawCode({kNopWInsn, kBneWPlus0, kLdrWInsn, kNopInsn}); ArrayRef<const uint8_t> code1(raw_code1); - uint32_t encoded_data = Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData( + uint32_t encoded_data = EncodeBakerReadBarrierFieldData( /* base_reg */ 0, /* holder_reg */ 0, /* narrow */ false); const LinkerPatch patches1[] = { LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data), @@ -907,7 +954,7 @@ TEST_F(Thumb2RelativePatcherTest, BakerOffsetThunkInTheMiddleUnreachableFromLast constexpr uint32_t kLiteralOffset1 = 6u; const std::vector<uint8_t> raw_code1 = RawCode({kNopWInsn, kNopInsn, kBneWPlus0, kLdrWInsn}); ArrayRef<const uint8_t> code1(raw_code1); - uint32_t encoded_data = Thumb2RelativePatcher::EncodeBakerReadBarrierFieldData( + uint32_t encoded_data = EncodeBakerReadBarrierFieldData( /* base_reg */ 0, /* holder_reg */ 0, /* narrow */ false); const LinkerPatch patches1[] = { LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data), @@ -993,7 +1040,7 @@ TEST_F(Thumb2RelativePatcherTest, BakerArray) { ArrayRef<const uint8_t> code(raw_code); const LinkerPatch patches[] = { LinkerPatch::BakerReadBarrierBranchPatch( - kLiteralOffset, Thumb2RelativePatcher::EncodeBakerReadBarrierArrayData(base_reg)), + kLiteralOffset, EncodeBakerReadBarrierArrayData(base_reg)), }; AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches)); } @@ -1074,8 +1121,7 @@ TEST_F(Thumb2RelativePatcherTest, BakerGcRootWide) { ArrayRef<const uint8_t> code(raw_code); const LinkerPatch patches[] = { LinkerPatch::BakerReadBarrierBranchPatch( - kLiteralOffset, - Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData(root_reg, /* narrow */ false)), + kLiteralOffset, EncodeBakerReadBarrierGcRootData(root_reg, /* narrow */ false)), }; AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches)); } @@ -1134,8 +1180,7 @@ TEST_F(Thumb2RelativePatcherTest, BakerGcRootNarrow) { ArrayRef<const uint8_t> code(raw_code); const LinkerPatch patches[] = { LinkerPatch::BakerReadBarrierBranchPatch( - kLiteralOffset, - Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData(root_reg, /* narrow */ true)), + kLiteralOffset, EncodeBakerReadBarrierGcRootData(root_reg, /* narrow */ true)), }; AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches)); } @@ -1182,8 +1227,7 @@ TEST_F(Thumb2RelativePatcherTest, BakerGcRootOffsetBits) { patches.reserve(num_patches); const uint32_t ldr = kLdrWInsn | (/* offset */ 8) | (/* base_reg */ 0 << 16) | (/* root_reg */ 0 << 12); - uint32_t encoded_data = - Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData(/* root_reg */ 0, /* narrow */ false); + uint32_t encoded_data = EncodeBakerReadBarrierGcRootData(/* root_reg */ 0, /* narrow */ false); for (size_t i = 0; i != num_patches; ++i) { PushBackInsn(&code, ldr); PushBackInsn(&code, kBneWPlus0); @@ -1264,10 +1308,8 @@ TEST_F(Thumb2RelativePatcherTest, BakerAndMethodCallInteraction) { ldr1, kBneWPlus0, // First GC root LDR with read barrier. ldr2, kBneWPlus0, // Second GC root LDR with read barrier. }); - uint32_t encoded_data1 = - Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData(/* root_reg */ 1, /* narrow */ false); - uint32_t encoded_data2 = - Thumb2RelativePatcher::EncodeBakerReadBarrierGcRootData(/* root_reg */ 2, /* narrow */ false); + uint32_t encoded_data1 = EncodeBakerReadBarrierGcRootData(/* root_reg */ 1, /* narrow */ false); + uint32_t encoded_data2 = EncodeBakerReadBarrierGcRootData(/* root_reg */ 2, /* narrow */ false); const LinkerPatch last_method_patches[] = { LinkerPatch::BakerReadBarrierBranchPatch(kBakerLiteralOffset1, encoded_data1), LinkerPatch::BakerReadBarrierBranchPatch(kBakerLiteralOffset2, encoded_data2), diff --git a/compiler/linker/arm64/relative_patcher_arm64.cc b/dex2oat/linker/arm64/relative_patcher_arm64.cc index 52a07965b92..71d1287c872 100644 --- a/compiler/linker/arm64/relative_patcher_arm64.cc +++ b/dex2oat/linker/arm64/relative_patcher_arm64.cc @@ -20,6 +20,7 @@ #include "arch/arm64/instruction_set_features_arm64.h" #include "art_method.h" #include "base/bit_utils.h" +#include "base/malloc_arena_pool.h" #include "compiled_method-inl.h" #include "driver/compiler_driver.h" #include "entrypoints/quick/quick_entrypoints_enum.h" @@ -60,13 +61,12 @@ inline bool IsAdrpPatch(const LinkerPatch& patch) { case LinkerPatch::Type::kCallRelative: case LinkerPatch::Type::kBakerReadBarrierBranch: return false; + case LinkerPatch::Type::kDataBimgRelRo: case LinkerPatch::Type::kMethodRelative: case LinkerPatch::Type::kMethodBssEntry: case LinkerPatch::Type::kTypeRelative: - case LinkerPatch::Type::kTypeClassTable: case LinkerPatch::Type::kTypeBssEntry: case LinkerPatch::Type::kStringRelative: - case LinkerPatch::Type::kStringInternTable: case LinkerPatch::Type::kStringBssEntry: return patch.LiteralOffset() == patch.PcInsnOffset(); } @@ -83,9 +83,10 @@ inline uint32_t MaxExtraSpace(size_t num_adrp, size_t code_size) { } // anonymous namespace -Arm64RelativePatcher::Arm64RelativePatcher(RelativePatcherTargetProvider* provider, +Arm64RelativePatcher::Arm64RelativePatcher(RelativePatcherThunkProvider* thunk_provider, + RelativePatcherTargetProvider* target_provider, const Arm64InstructionSetFeatures* features) - : ArmBaseRelativePatcher(provider, InstructionSet::kArm64), + : ArmBaseRelativePatcher(thunk_provider, target_provider, InstructionSet::kArm64), fix_cortex_a53_843419_(features->NeedFixCortexA53_843419()), reserved_adrp_thunks_(0u), processed_adrp_thunks_(0u) { @@ -271,10 +272,9 @@ void Arm64RelativePatcher::PatchPcRelativeReference(std::vector<uint8_t>* code, shift = 0u; // No shift for ADD. } else { // LDR/STR 32-bit or 64-bit with imm12 == 0 (unset). - DCHECK(patch.GetType() == LinkerPatch::Type::kMethodBssEntry || - patch.GetType() == LinkerPatch::Type::kTypeClassTable || + DCHECK(patch.GetType() == LinkerPatch::Type::kDataBimgRelRo || + patch.GetType() == LinkerPatch::Type::kMethodBssEntry || patch.GetType() == LinkerPatch::Type::kTypeBssEntry || - patch.GetType() == LinkerPatch::Type::kStringInternTable || patch.GetType() == LinkerPatch::Type::kStringBssEntry) << patch.GetType(); DCHECK_EQ(insn & 0xbfbffc00, 0xb9000000) << std::hex << insn; } @@ -315,44 +315,6 @@ void Arm64RelativePatcher::PatchBakerReadBarrierBranch(std::vector<uint8_t>* cod uint32_t insn = GetInsn(code, literal_offset); DCHECK_EQ(insn & 0xffffffe0u, 0xb5000000); // CBNZ Xt, +0 (unpatched) ThunkKey key = GetBakerThunkKey(patch); - if (kIsDebugBuild) { - const uint32_t encoded_data = key.GetCustomValue1(); - BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data); - // Check that the next instruction matches the expected LDR. - switch (kind) { - case BakerReadBarrierKind::kField: { - DCHECK_GE(code->size() - literal_offset, 8u); - uint32_t next_insn = GetInsn(code, literal_offset + 4u); - // LDR (immediate) with correct base_reg. - CheckValidReg(next_insn & 0x1fu); // Check destination register. - const uint32_t base_reg = BakerReadBarrierFirstRegField::Decode(encoded_data); - CHECK_EQ(next_insn & 0xffc003e0u, 0xb9400000u | (base_reg << 5)); - break; - } - case BakerReadBarrierKind::kArray: { - DCHECK_GE(code->size() - literal_offset, 8u); - uint32_t next_insn = GetInsn(code, literal_offset + 4u); - // LDR (register) with the correct base_reg, size=10 (32-bit), option=011 (extend = LSL), - // and S=1 (shift amount = 2 for 32-bit version), i.e. LDR Wt, [Xn, Xm, LSL #2]. - CheckValidReg(next_insn & 0x1fu); // Check destination register. - const uint32_t base_reg = BakerReadBarrierFirstRegField::Decode(encoded_data); - CHECK_EQ(next_insn & 0xffe0ffe0u, 0xb8607800u | (base_reg << 5)); - CheckValidReg((next_insn >> 16) & 0x1f); // Check index register - break; - } - case BakerReadBarrierKind::kGcRoot: { - DCHECK_GE(literal_offset, 4u); - uint32_t prev_insn = GetInsn(code, literal_offset - 4u); - // LDR (immediate) with correct root_reg. - const uint32_t root_reg = BakerReadBarrierFirstRegField::Decode(encoded_data); - CHECK_EQ(prev_insn & 0xffc0001fu, 0xb9400000u | root_reg); - break; - } - default: - LOG(FATAL) << "Unexpected kind: " << static_cast<uint32_t>(kind); - UNREACHABLE(); - } - } uint32_t target_offset = GetThunkTargetOffset(key, patch_offset); DCHECK_ALIGNED(target_offset, 4u); uint32_t disp = target_offset - patch_offset; @@ -361,216 +323,6 @@ void Arm64RelativePatcher::PatchBakerReadBarrierBranch(std::vector<uint8_t>* cod SetInsn(code, literal_offset, insn); } -#define __ assembler.GetVIXLAssembler()-> - -static void EmitGrayCheckAndFastPath(arm64::Arm64Assembler& assembler, - vixl::aarch64::Register base_reg, - vixl::aarch64::MemOperand& lock_word, - vixl::aarch64::Label* slow_path) { - using namespace vixl::aarch64; // NOLINT(build/namespaces) - // Load the lock word containing the rb_state. - __ Ldr(ip0.W(), lock_word); - // Given the numeric representation, it's enough to check the low bit of the rb_state. - static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); - static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); - __ Tbnz(ip0.W(), LockWord::kReadBarrierStateShift, slow_path); - static_assert( - BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET == BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET, - "Field and array LDR offsets must be the same to reuse the same code."); - // Adjust the return address back to the LDR (1 instruction; 2 for heap poisoning). - static_assert(BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET == (kPoisonHeapReferences ? -8 : -4), - "Field LDR must be 1 instruction (4B) before the return address label; " - " 2 instructions (8B) for heap poisoning."); - __ Add(lr, lr, BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET); - // Introduce a dependency on the lock_word including rb_state, - // to prevent load-load reordering, and without using - // a memory barrier (which would be more expensive). - __ Add(base_reg, base_reg, Operand(ip0, LSR, 32)); - __ Br(lr); // And return back to the function. - // Note: The fake dependency is unnecessary for the slow path. -} - -// Load the read barrier introspection entrypoint in register `entrypoint`. -static void LoadReadBarrierMarkIntrospectionEntrypoint(arm64::Arm64Assembler& assembler, - vixl::aarch64::Register entrypoint) { - using vixl::aarch64::MemOperand; - using vixl::aarch64::ip0; - // Thread Register. - const vixl::aarch64::Register tr = vixl::aarch64::x19; - - // entrypoint = Thread::Current()->pReadBarrierMarkReg16, i.e. pReadBarrierMarkIntrospection. - DCHECK_EQ(ip0.GetCode(), 16u); - const int32_t entry_point_offset = - Thread::ReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(ip0.GetCode()); - __ Ldr(entrypoint, MemOperand(tr, entry_point_offset)); -} - -void Arm64RelativePatcher::CompileBakerReadBarrierThunk(arm64::Arm64Assembler& assembler, - uint32_t encoded_data) { - using namespace vixl::aarch64; // NOLINT(build/namespaces) - BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data); - switch (kind) { - case BakerReadBarrierKind::kField: { - // Check if the holder is gray and, if not, add fake dependency to the base register - // and return to the LDR instruction to load the reference. Otherwise, use introspection - // to load the reference and call the entrypoint (in IP1) that performs further checks - // on the reference and marks it if needed. - auto base_reg = - Register::GetXRegFromCode(BakerReadBarrierFirstRegField::Decode(encoded_data)); - CheckValidReg(base_reg.GetCode()); - auto holder_reg = - Register::GetXRegFromCode(BakerReadBarrierSecondRegField::Decode(encoded_data)); - CheckValidReg(holder_reg.GetCode()); - UseScratchRegisterScope temps(assembler.GetVIXLAssembler()); - temps.Exclude(ip0, ip1); - // If base_reg differs from holder_reg, the offset was too large and we must have - // emitted an explicit null check before the load. Otherwise, we need to null-check - // the holder as we do not necessarily do that check before going to the thunk. - vixl::aarch64::Label throw_npe; - if (holder_reg.Is(base_reg)) { - __ Cbz(holder_reg.W(), &throw_npe); - } - vixl::aarch64::Label slow_path; - MemOperand lock_word(holder_reg, mirror::Object::MonitorOffset().Int32Value()); - EmitGrayCheckAndFastPath(assembler, base_reg, lock_word, &slow_path); - __ Bind(&slow_path); - MemOperand ldr_address(lr, BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET); - __ Ldr(ip0.W(), ldr_address); // Load the LDR (immediate) unsigned offset. - LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ip1); - __ Ubfx(ip0.W(), ip0.W(), 10, 12); // Extract the offset. - __ Ldr(ip0.W(), MemOperand(base_reg, ip0, LSL, 2)); // Load the reference. - // Do not unpoison. With heap poisoning enabled, the entrypoint expects a poisoned reference. - __ Br(ip1); // Jump to the entrypoint. - if (holder_reg.Is(base_reg)) { - // Add null check slow path. The stack map is at the address pointed to by LR. - __ Bind(&throw_npe); - int32_t offset = GetThreadOffset<kArm64PointerSize>(kQuickThrowNullPointer).Int32Value(); - __ Ldr(ip0, MemOperand(/* Thread* */ vixl::aarch64::x19, offset)); - __ Br(ip0); - } - break; - } - case BakerReadBarrierKind::kArray: { - auto base_reg = - Register::GetXRegFromCode(BakerReadBarrierFirstRegField::Decode(encoded_data)); - CheckValidReg(base_reg.GetCode()); - DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data)); - UseScratchRegisterScope temps(assembler.GetVIXLAssembler()); - temps.Exclude(ip0, ip1); - vixl::aarch64::Label slow_path; - int32_t data_offset = - mirror::Array::DataOffset(Primitive::ComponentSize(Primitive::kPrimNot)).Int32Value(); - MemOperand lock_word(base_reg, mirror::Object::MonitorOffset().Int32Value() - data_offset); - DCHECK_LT(lock_word.GetOffset(), 0); - EmitGrayCheckAndFastPath(assembler, base_reg, lock_word, &slow_path); - __ Bind(&slow_path); - MemOperand ldr_address(lr, BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET); - __ Ldr(ip0.W(), ldr_address); // Load the LDR (register) unsigned offset. - LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ip1); - __ Ubfx(ip0, ip0, 16, 6); // Extract the index register, plus 32 (bit 21 is set). - __ Bfi(ip1, ip0, 3, 6); // Insert ip0 to the entrypoint address to create - // a switch case target based on the index register. - __ Mov(ip0, base_reg); // Move the base register to ip0. - __ Br(ip1); // Jump to the entrypoint's array switch case. - break; - } - case BakerReadBarrierKind::kGcRoot: { - // Check if the reference needs to be marked and if so (i.e. not null, not marked yet - // and it does not have a forwarding address), call the correct introspection entrypoint; - // otherwise return the reference (or the extracted forwarding address). - // There is no gray bit check for GC roots. - auto root_reg = - Register::GetWRegFromCode(BakerReadBarrierFirstRegField::Decode(encoded_data)); - CheckValidReg(root_reg.GetCode()); - DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data)); - UseScratchRegisterScope temps(assembler.GetVIXLAssembler()); - temps.Exclude(ip0, ip1); - vixl::aarch64::Label return_label, not_marked, forwarding_address; - __ Cbz(root_reg, &return_label); - MemOperand lock_word(root_reg.X(), mirror::Object::MonitorOffset().Int32Value()); - __ Ldr(ip0.W(), lock_word); - __ Tbz(ip0.W(), LockWord::kMarkBitStateShift, ¬_marked); - __ Bind(&return_label); - __ Br(lr); - __ Bind(¬_marked); - __ Tst(ip0.W(), Operand(ip0.W(), LSL, 1)); - __ B(&forwarding_address, mi); - LoadReadBarrierMarkIntrospectionEntrypoint(assembler, ip1); - // Adjust the art_quick_read_barrier_mark_introspection address in IP1 to - // art_quick_read_barrier_mark_introspection_gc_roots. - __ Add(ip1, ip1, Operand(BAKER_MARK_INTROSPECTION_GC_ROOT_ENTRYPOINT_OFFSET)); - __ Mov(ip0.W(), root_reg); - __ Br(ip1); - __ Bind(&forwarding_address); - __ Lsl(root_reg, ip0.W(), LockWord::kForwardingAddressShift); - __ Br(lr); - break; - } - default: - LOG(FATAL) << "Unexpected kind: " << static_cast<uint32_t>(kind); - UNREACHABLE(); - } -} - -std::vector<uint8_t> Arm64RelativePatcher::CompileThunk(const ThunkKey& key) { - ArenaPool pool; - ArenaAllocator allocator(&pool); - arm64::Arm64Assembler assembler(&allocator); - - switch (key.GetType()) { - case ThunkType::kMethodCall: { - // The thunk just uses the entry point in the ArtMethod. This works even for calls - // to the generic JNI and interpreter trampolines. - Offset offset(ArtMethod::EntryPointFromQuickCompiledCodeOffset( - kArm64PointerSize).Int32Value()); - assembler.JumpTo(ManagedRegister(arm64::X0), offset, ManagedRegister(arm64::IP0)); - break; - } - case ThunkType::kBakerReadBarrier: { - CompileBakerReadBarrierThunk(assembler, key.GetCustomValue1()); - break; - } - } - - // Ensure we emit the literal pool. - assembler.FinalizeCode(); - std::vector<uint8_t> thunk_code(assembler.CodeSize()); - MemoryRegion code(thunk_code.data(), thunk_code.size()); - assembler.FinalizeInstructions(code); - return thunk_code; -} - -std::string Arm64RelativePatcher::GetThunkDebugName(const ThunkKey& key) { - switch (key.GetType()) { - case ThunkType::kMethodCall: - return "MethodCallThunk"; - - case ThunkType::kBakerReadBarrier: { - uint32_t encoded_data = key.GetCustomValue1(); - BakerReadBarrierKind kind = BakerReadBarrierKindField::Decode(encoded_data); - std::ostringstream oss; - oss << "BakerReadBarrierThunk"; - switch (kind) { - case BakerReadBarrierKind::kField: - oss << "Field_r" << BakerReadBarrierFirstRegField::Decode(encoded_data) - << "_r" << BakerReadBarrierSecondRegField::Decode(encoded_data); - break; - case BakerReadBarrierKind::kArray: - oss << "Array_r" << BakerReadBarrierFirstRegField::Decode(encoded_data); - DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data)); - break; - case BakerReadBarrierKind::kGcRoot: - oss << "GcRoot_r" << BakerReadBarrierFirstRegField::Decode(encoded_data); - DCHECK_EQ(kInvalidEncodedReg, BakerReadBarrierSecondRegField::Decode(encoded_data)); - break; - } - return oss.str(); - } - } -} - -#undef __ - uint32_t Arm64RelativePatcher::MaxPositiveDisplacement(const ThunkKey& key) { switch (key.GetType()) { case ThunkType::kMethodCall: diff --git a/compiler/linker/arm64/relative_patcher_arm64.h b/dex2oat/linker/arm64/relative_patcher_arm64.h index 8ba59976e7d..f7f673c1ba3 100644 --- a/compiler/linker/arm64/relative_patcher_arm64.h +++ b/dex2oat/linker/arm64/relative_patcher_arm64.h @@ -14,12 +14,10 @@ * limitations under the License. */ -#ifndef ART_COMPILER_LINKER_ARM64_RELATIVE_PATCHER_ARM64_H_ -#define ART_COMPILER_LINKER_ARM64_RELATIVE_PATCHER_ARM64_H_ +#ifndef ART_DEX2OAT_LINKER_ARM64_RELATIVE_PATCHER_ARM64_H_ +#define ART_DEX2OAT_LINKER_ARM64_RELATIVE_PATCHER_ARM64_H_ #include "base/array_ref.h" -#include "base/bit_field.h" -#include "base/bit_utils.h" #include "linker/arm/relative_patcher_arm_base.h" namespace art { @@ -32,29 +30,8 @@ namespace linker { class Arm64RelativePatcher FINAL : public ArmBaseRelativePatcher { public: - static uint32_t EncodeBakerReadBarrierFieldData(uint32_t base_reg, uint32_t holder_reg) { - CheckValidReg(base_reg); - CheckValidReg(holder_reg); - return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kField) | - BakerReadBarrierFirstRegField::Encode(base_reg) | - BakerReadBarrierSecondRegField::Encode(holder_reg); - } - - static uint32_t EncodeBakerReadBarrierArrayData(uint32_t base_reg) { - CheckValidReg(base_reg); - return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kArray) | - BakerReadBarrierFirstRegField::Encode(base_reg) | - BakerReadBarrierSecondRegField::Encode(kInvalidEncodedReg); - } - - static uint32_t EncodeBakerReadBarrierGcRootData(uint32_t root_reg) { - CheckValidReg(root_reg); - return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kGcRoot) | - BakerReadBarrierFirstRegField::Encode(root_reg) | - BakerReadBarrierSecondRegField::Encode(kInvalidEncodedReg); - } - - Arm64RelativePatcher(RelativePatcherTargetProvider* provider, + Arm64RelativePatcher(RelativePatcherThunkProvider* thunk_provider, + RelativePatcherTargetProvider* target_provider, const Arm64InstructionSetFeatures* features); uint32_t ReserveSpace(uint32_t offset, @@ -75,37 +52,10 @@ class Arm64RelativePatcher FINAL : public ArmBaseRelativePatcher { uint32_t patch_offset) OVERRIDE; protected: - std::vector<uint8_t> CompileThunk(const ThunkKey& key) OVERRIDE; - std::string GetThunkDebugName(const ThunkKey& key) OVERRIDE; uint32_t MaxPositiveDisplacement(const ThunkKey& key) OVERRIDE; uint32_t MaxNegativeDisplacement(const ThunkKey& key) OVERRIDE; private: - static constexpr uint32_t kInvalidEncodedReg = /* sp/zr is invalid */ 31u; - - enum class BakerReadBarrierKind : uint8_t { - kField, // Field get or array get with constant offset (i.e. constant index). - kArray, // Array get with index in register. - kGcRoot, // GC root load. - kLast = kGcRoot - }; - - static constexpr size_t kBitsForBakerReadBarrierKind = - MinimumBitsToStore(static_cast<size_t>(BakerReadBarrierKind::kLast)); - static constexpr size_t kBitsForRegister = 5u; - using BakerReadBarrierKindField = - BitField<BakerReadBarrierKind, 0, kBitsForBakerReadBarrierKind>; - using BakerReadBarrierFirstRegField = - BitField<uint32_t, kBitsForBakerReadBarrierKind, kBitsForRegister>; - using BakerReadBarrierSecondRegField = - BitField<uint32_t, kBitsForBakerReadBarrierKind + kBitsForRegister, kBitsForRegister>; - - static void CheckValidReg(uint32_t reg) { - DCHECK(reg < 30u && reg != 16u && reg != 17u) << reg; - } - - void CompileBakerReadBarrierThunk(arm64::Arm64Assembler& assembler, uint32_t encoded_data); - static uint32_t PatchAdrp(uint32_t adrp, uint32_t disp); static bool NeedsErratum843419Thunk(ArrayRef<const uint8_t> code, uint32_t literal_offset, @@ -131,4 +81,4 @@ class Arm64RelativePatcher FINAL : public ArmBaseRelativePatcher { } // namespace linker } // namespace art -#endif // ART_COMPILER_LINKER_ARM64_RELATIVE_PATCHER_ARM64_H_ +#endif // ART_DEX2OAT_LINKER_ARM64_RELATIVE_PATCHER_ARM64_H_ diff --git a/compiler/linker/arm64/relative_patcher_arm64_test.cc b/dex2oat/linker/arm64/relative_patcher_arm64_test.cc index 05459a2a823..393733dd0cc 100644 --- a/compiler/linker/arm64/relative_patcher_arm64_test.cc +++ b/dex2oat/linker/arm64/relative_patcher_arm64_test.cc @@ -16,12 +16,15 @@ #include "linker/arm64/relative_patcher_arm64.h" +#include "arch/arm64/instruction_set_features_arm64.h" #include "base/casts.h" #include "linker/relative_patcher_test.h" #include "lock_word.h" #include "mirror/array-inl.h" #include "mirror/object.h" #include "oat_quick_method_header.h" +#include "optimizing/code_generator_arm64.h" +#include "optimizing/optimizing_unit_test.h" namespace art { namespace linker { @@ -168,9 +171,42 @@ class Arm64RelativePatcherTest : public RelativePatcherTest { return result.second; } + std::vector<uint8_t> CompileThunk(const LinkerPatch& patch, + /*out*/ std::string* debug_name = nullptr) { + OptimizingUnitTestHelper helper; + HGraph* graph = helper.CreateGraph(); + std::string error_msg; + Arm64FeaturesUniquePtr features = + Arm64InstructionSetFeatures::FromVariant("default", &error_msg); + CompilerOptions options; + arm64::CodeGeneratorARM64 codegen(graph, *features, options); + ArenaVector<uint8_t> code(helper.GetAllocator()->Adapter()); + codegen.EmitThunkCode(patch, &code, debug_name); + return std::vector<uint8_t>(code.begin(), code.end()); + } + + void AddCompiledMethod( + MethodReference method_ref, + const ArrayRef<const uint8_t>& code, + const ArrayRef<const LinkerPatch>& patches = ArrayRef<const LinkerPatch>()) { + RelativePatcherTest::AddCompiledMethod(method_ref, code, patches); + + // Make sure the ThunkProvider has all the necessary thunks. + for (const LinkerPatch& patch : patches) { + if (patch.GetType() == LinkerPatch::Type::kBakerReadBarrierBranch || + patch.GetType() == LinkerPatch::Type::kCallRelative) { + std::string debug_name; + std::vector<uint8_t> thunk_code = CompileThunk(patch, &debug_name); + thunk_provider_.SetThunkCode(patch, ArrayRef<const uint8_t>(thunk_code), debug_name); + } + } + } + std::vector<uint8_t> CompileMethodCallThunk() { - ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetMethodCallKey(); - return down_cast<Arm64RelativePatcher*>(patcher_.get())->CompileThunk(key); + LinkerPatch patch = LinkerPatch::RelativeCodePatch(/* literal_offset */ 0u, + /* target_dex_file*/ nullptr, + /* target_method_idx */ 0u); + return CompileThunk(patch); } uint32_t MethodCallThunkSize() { @@ -475,25 +511,34 @@ class Arm64RelativePatcherTest : public RelativePatcherTest { TestAdrpInsn2Add(insn2, adrp_offset, has_thunk, string_offset); } + static uint32_t EncodeBakerReadBarrierFieldData(uint32_t base_reg, uint32_t holder_reg) { + return arm64::CodeGeneratorARM64::EncodeBakerReadBarrierFieldData(base_reg, holder_reg); + } + + static uint32_t EncodeBakerReadBarrierArrayData(uint32_t base_reg) { + return arm64::CodeGeneratorARM64::EncodeBakerReadBarrierArrayData(base_reg); + } + + static uint32_t EncodeBakerReadBarrierGcRootData(uint32_t root_reg) { + return arm64::CodeGeneratorARM64::EncodeBakerReadBarrierGcRootData(root_reg); + } + std::vector<uint8_t> CompileBakerOffsetThunk(uint32_t base_reg, uint32_t holder_reg) { const LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch( - 0u, Arm64RelativePatcher::EncodeBakerReadBarrierFieldData(base_reg, holder_reg)); - ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetBakerThunkKey(patch); - return down_cast<Arm64RelativePatcher*>(patcher_.get())->CompileThunk(key); + /* literal_offset */ 0u, EncodeBakerReadBarrierFieldData(base_reg, holder_reg)); + return CompileThunk(patch); } std::vector<uint8_t> CompileBakerArrayThunk(uint32_t base_reg) { LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch( - 0u, Arm64RelativePatcher::EncodeBakerReadBarrierArrayData(base_reg)); - ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetBakerThunkKey(patch); - return down_cast<Arm64RelativePatcher*>(patcher_.get())->CompileThunk(key); + /* literal_offset */ 0u, EncodeBakerReadBarrierArrayData(base_reg)); + return CompileThunk(patch); } std::vector<uint8_t> CompileBakerGcRootThunk(uint32_t root_reg) { LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch( - 0u, Arm64RelativePatcher::EncodeBakerReadBarrierGcRootData(root_reg)); - ArmBaseRelativePatcher::ThunkKey key = ArmBaseRelativePatcher::GetBakerThunkKey(patch); - return down_cast<Arm64RelativePatcher*>(patcher_.get())->CompileThunk(key); + /* literal_offset */ 0u, EncodeBakerReadBarrierGcRootData(root_reg)); + return CompileThunk(patch); } uint32_t GetOutputInsn(uint32_t offset) { @@ -919,8 +964,7 @@ void Arm64RelativePatcherTest::TestBakerField(uint32_t offset, uint32_t ref_reg) const std::vector<uint8_t> raw_code = RawCode({kCbnzIP1Plus0Insn, ldr}); ASSERT_EQ(kMethodCodeSize, raw_code.size()); ArrayRef<const uint8_t> code(raw_code); - uint32_t encoded_data = - Arm64RelativePatcher::EncodeBakerReadBarrierFieldData(base_reg, holder_reg); + uint32_t encoded_data = EncodeBakerReadBarrierFieldData(base_reg, holder_reg); const LinkerPatch patches[] = { LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset, encoded_data), }; @@ -1005,8 +1049,7 @@ TEST_F(Arm64RelativePatcherTestDefault, BakerOffsetThunkInTheMiddle) { constexpr uint32_t kLiteralOffset1 = 4; const std::vector<uint8_t> raw_code1 = RawCode({kNopInsn, kCbnzIP1Plus0Insn, kLdrWInsn}); ArrayRef<const uint8_t> code1(raw_code1); - uint32_t encoded_data = - Arm64RelativePatcher::EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0); + uint32_t encoded_data = EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0); const LinkerPatch patches1[] = { LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data), }; @@ -1066,8 +1109,7 @@ TEST_F(Arm64RelativePatcherTestDefault, BakerOffsetThunkBeforeFiller) { constexpr uint32_t kLiteralOffset1 = 0; const std::vector<uint8_t> raw_code1 = RawCode({kCbnzIP1Plus0Insn, kLdrWInsn, kNopInsn}); ArrayRef<const uint8_t> code1(raw_code1); - uint32_t encoded_data = - Arm64RelativePatcher::EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0); + uint32_t encoded_data = EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0); const LinkerPatch patches1[] = { LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data), }; @@ -1096,8 +1138,7 @@ TEST_F(Arm64RelativePatcherTestDefault, BakerOffsetThunkInTheMiddleUnreachableFr constexpr uint32_t kLiteralOffset1 = 4; const std::vector<uint8_t> raw_code1 = RawCode({kNopInsn, kCbnzIP1Plus0Insn, kLdrWInsn}); ArrayRef<const uint8_t> code1(raw_code1); - uint32_t encoded_data = - Arm64RelativePatcher::EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0); + uint32_t encoded_data = EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0); const LinkerPatch patches1[] = { LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data), }; @@ -1170,7 +1211,7 @@ TEST_F(Arm64RelativePatcherTestDefault, BakerArray) { ArrayRef<const uint8_t> code(raw_code); const LinkerPatch patches[] = { LinkerPatch::BakerReadBarrierBranchPatch( - kLiteralOffset, Arm64RelativePatcher::EncodeBakerReadBarrierArrayData(base_reg)), + kLiteralOffset, EncodeBakerReadBarrierArrayData(base_reg)), }; AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches)); } @@ -1247,7 +1288,7 @@ TEST_F(Arm64RelativePatcherTestDefault, BakerGcRoot) { ArrayRef<const uint8_t> code(raw_code); const LinkerPatch patches[] = { LinkerPatch::BakerReadBarrierBranchPatch( - kLiteralOffset, Arm64RelativePatcher::EncodeBakerReadBarrierGcRootData(root_reg)), + kLiteralOffset, EncodeBakerReadBarrierGcRootData(root_reg)), }; AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches)); } @@ -1343,8 +1384,8 @@ TEST_F(Arm64RelativePatcherTestDefault, BakerAndMethodCallInteraction) { kNopInsn, kNopInsn, // Padding before second GC root read barrier. ldr2, kCbnzIP1Plus0Insn, // Second GC root LDR with read barrier. }); - uint32_t encoded_data1 = Arm64RelativePatcher::EncodeBakerReadBarrierGcRootData(/* root_reg */ 1); - uint32_t encoded_data2 = Arm64RelativePatcher::EncodeBakerReadBarrierGcRootData(/* root_reg */ 2); + uint32_t encoded_data1 = EncodeBakerReadBarrierGcRootData(/* root_reg */ 1); + uint32_t encoded_data2 = EncodeBakerReadBarrierGcRootData(/* root_reg */ 2); const LinkerPatch last_method_patches[] = { LinkerPatch::BakerReadBarrierBranchPatch(kBakerLiteralOffset1, encoded_data1), LinkerPatch::BakerReadBarrierBranchPatch(kBakerLiteralOffset2, encoded_data2), diff --git a/dex2oat/linker/elf_writer.h b/dex2oat/linker/elf_writer.h index bcf2cd7d4b9..cd8cf4c54ea 100644 --- a/dex2oat/linker/elf_writer.h +++ b/dex2oat/linker/elf_writer.h @@ -63,6 +63,7 @@ class ElfWriter { // This method must be called before calling GetLoadedSize(). virtual void PrepareDynamicSection(size_t rodata_size, size_t text_size, + size_t data_bimg_rel_ro_size, size_t bss_size, size_t bss_methods_offset, size_t bss_roots_offset, @@ -72,6 +73,8 @@ class ElfWriter { virtual void EndRoData(OutputStream* rodata) = 0; virtual OutputStream* StartText() = 0; virtual void EndText(OutputStream* text) = 0; + virtual OutputStream* StartDataBimgRelRo() = 0; + virtual void EndDataBimgRelRo(OutputStream* data_bimg_rel_ro) = 0; virtual void WriteDynamicSection() = 0; virtual void WriteDebugInfo(const debug::DebugInfo& debug_info) = 0; virtual bool End() = 0; diff --git a/dex2oat/linker/elf_writer_quick.cc b/dex2oat/linker/elf_writer_quick.cc index 07b02f10335..4ab20123760 100644 --- a/dex2oat/linker/elf_writer_quick.cc +++ b/dex2oat/linker/elf_writer_quick.cc @@ -105,6 +105,7 @@ class ElfWriterQuick FINAL : public ElfWriter { void Start() OVERRIDE; void PrepareDynamicSection(size_t rodata_size, size_t text_size, + size_t data_bimg_rel_ro_size, size_t bss_size, size_t bss_methods_offset, size_t bss_roots_offset, @@ -114,6 +115,8 @@ class ElfWriterQuick FINAL : public ElfWriter { void EndRoData(OutputStream* rodata) OVERRIDE; OutputStream* StartText() OVERRIDE; void EndText(OutputStream* text) OVERRIDE; + OutputStream* StartDataBimgRelRo() OVERRIDE; + void EndDataBimgRelRo(OutputStream* data_bimg_rel_ro) OVERRIDE; void WriteDynamicSection() OVERRIDE; void WriteDebugInfo(const debug::DebugInfo& debug_info) OVERRIDE; bool End() OVERRIDE; @@ -131,6 +134,7 @@ class ElfWriterQuick FINAL : public ElfWriter { File* const elf_file_; size_t rodata_size_; size_t text_size_; + size_t data_bimg_rel_ro_size_; size_t bss_size_; size_t dex_section_size_; std::unique_ptr<BufferedOutputStream> output_stream_; @@ -171,6 +175,7 @@ ElfWriterQuick<ElfTypes>::ElfWriterQuick(InstructionSet instruction_set, elf_file_(elf_file), rodata_size_(0u), text_size_(0u), + data_bimg_rel_ro_size_(0u), bss_size_(0u), dex_section_size_(0u), output_stream_( @@ -192,6 +197,7 @@ void ElfWriterQuick<ElfTypes>::Start() { template <typename ElfTypes> void ElfWriterQuick<ElfTypes>::PrepareDynamicSection(size_t rodata_size, size_t text_size, + size_t data_bimg_rel_ro_size, size_t bss_size, size_t bss_methods_offset, size_t bss_roots_offset, @@ -200,6 +206,8 @@ void ElfWriterQuick<ElfTypes>::PrepareDynamicSection(size_t rodata_size, rodata_size_ = rodata_size; DCHECK_EQ(text_size_, 0u); text_size_ = text_size; + DCHECK_EQ(data_bimg_rel_ro_size_, 0u); + data_bimg_rel_ro_size_ = data_bimg_rel_ro_size; DCHECK_EQ(bss_size_, 0u); bss_size_ = bss_size; DCHECK_EQ(dex_section_size_, 0u); @@ -207,6 +215,7 @@ void ElfWriterQuick<ElfTypes>::PrepareDynamicSection(size_t rodata_size, builder_->PrepareDynamicSection(elf_file_->GetPath(), rodata_size_, text_size_, + data_bimg_rel_ro_size_, bss_size_, bss_methods_offset, bss_roots_offset, @@ -240,6 +249,19 @@ void ElfWriterQuick<ElfTypes>::EndText(OutputStream* text) { } template <typename ElfTypes> +OutputStream* ElfWriterQuick<ElfTypes>::StartDataBimgRelRo() { + auto* data_bimg_rel_ro = builder_->GetDataBimgRelRo(); + data_bimg_rel_ro->Start(); + return data_bimg_rel_ro; +} + +template <typename ElfTypes> +void ElfWriterQuick<ElfTypes>::EndDataBimgRelRo(OutputStream* data_bimg_rel_ro) { + CHECK_EQ(builder_->GetDataBimgRelRo(), data_bimg_rel_ro); + builder_->GetDataBimgRelRo()->End(); +} + +template <typename ElfTypes> void ElfWriterQuick<ElfTypes>::WriteDynamicSection() { if (builder_->GetIsa() == InstructionSet::kMips || builder_->GetIsa() == InstructionSet::kMips64) { diff --git a/dex2oat/linker/image_test.h b/dex2oat/linker/image_test.h index 319c5fb6758..a95252d3ed2 100644 --- a/dex2oat/linker/image_test.h +++ b/dex2oat/linker/image_test.h @@ -293,13 +293,14 @@ inline void CompilationHelper::Compile(CompilerDriver* driver, ASSERT_TRUE(cur_opened_dex_files.empty()); } } - bool image_space_ok = writer->PrepareImageAddressSpace(); + bool image_space_ok = writer->PrepareImageAddressSpace(&timings); ASSERT_TRUE(image_space_ok); DCHECK_EQ(vdex_files.size(), oat_files.size()); for (size_t i = 0, size = oat_files.size(); i != size; ++i) { MultiOatRelativePatcher patcher(driver->GetInstructionSet(), - driver->GetInstructionSetFeatures()); + driver->GetInstructionSetFeatures(), + driver->GetCompiledMethodStorage()); OatWriter* const oat_writer = oat_writers[i].get(); ElfWriter* const elf_writer = elf_writers[i].get(); std::vector<const DexFile*> cur_dex_files(1u, class_path[i]); @@ -313,10 +314,9 @@ inline void CompilationHelper::Compile(CompilerDriver* driver, oat_writer->WriteChecksumsAndVdexHeader(vdex_out.get()); oat_writer->PrepareLayout(&patcher); - size_t rodata_size = oat_writer->GetOatHeader().GetExecutableOffset(); - size_t text_size = oat_writer->GetOatSize() - rodata_size; - elf_writer->PrepareDynamicSection(rodata_size, - text_size, + elf_writer->PrepareDynamicSection(oat_writer->GetOatHeader().GetExecutableOffset(), + oat_writer->GetCodeSize(), + oat_writer->GetDataBimgRelRoSize(), oat_writer->GetBssSize(), oat_writer->GetBssMethodsOffset(), oat_writer->GetBssRootsOffset(), @@ -336,6 +336,13 @@ inline void CompilationHelper::Compile(CompilerDriver* driver, ASSERT_TRUE(text_ok); elf_writer->EndText(text); + if (oat_writer->GetDataBimgRelRoSize() != 0u) { + OutputStream* data_bimg_rel_ro = elf_writer->StartDataBimgRelRo(); + bool data_bimg_rel_ro_ok = oat_writer->WriteDataBimgRelRo(data_bimg_rel_ro); + ASSERT_TRUE(data_bimg_rel_ro_ok); + elf_writer->EndDataBimgRelRo(data_bimg_rel_ro); + } + bool header_ok = oat_writer->WriteHeader(elf_writer->GetStream(), 0u, 0u, 0u); ASSERT_TRUE(header_ok); diff --git a/dex2oat/linker/image_writer.cc b/dex2oat/linker/image_writer.cc index a2ba816f6cc..c7a30a06ede 100644 --- a/dex2oat/linker/image_writer.cc +++ b/dex2oat/linker/image_writer.cc @@ -133,23 +133,31 @@ static void ClearDexFileCookies() REQUIRES_SHARED(Locks::mutator_lock_) { Runtime::Current()->GetHeap()->VisitObjects(visitor); } -bool ImageWriter::PrepareImageAddressSpace() { +bool ImageWriter::PrepareImageAddressSpace(TimingLogger* timings) { target_ptr_size_ = InstructionSetPointerSize(compiler_driver_.GetInstructionSet()); gc::Heap* const heap = Runtime::Current()->GetHeap(); { ScopedObjectAccess soa(Thread::Current()); - PruneNonImageClasses(); // Remove junk + { + TimingLogger::ScopedTiming t("PruneNonImageClasses", timings); + PruneNonImageClasses(); // Remove junk + } if (compile_app_image_) { + TimingLogger::ScopedTiming t("ClearDexFileCookies", timings); // Clear dex file cookies for app images to enable app image determinism. This is required // since the cookie field contains long pointers to DexFiles which are not deterministic. // b/34090128 ClearDexFileCookies(); } else { + TimingLogger::ScopedTiming t("ComputeLazyFieldsForImageClasses", timings); // Avoid for app image since this may increase RAM and image size. ComputeLazyFieldsForImageClasses(); // Add useful information } } - heap->CollectGarbage(/* clear_soft_references */ false); // Remove garbage. + { + TimingLogger::ScopedTiming t("CollectGarbage", timings); + heap->CollectGarbage(/* clear_soft_references */ false); // Remove garbage. + } if (kIsDebugBuild) { ScopedObjectAccess soa(Thread::Current()); @@ -157,12 +165,14 @@ bool ImageWriter::PrepareImageAddressSpace() { } { + TimingLogger::ScopedTiming t("CalculateNewObjectOffsets", timings); ScopedObjectAccess soa(Thread::Current()); CalculateNewObjectOffsets(); } // This needs to happen after CalculateNewObjectOffsets since it relies on intern_table_bytes_ and // bin size sums being calculated. + TimingLogger::ScopedTiming t("AllocMemory", timings); if (!AllocMemory()) { return false; } @@ -692,7 +702,7 @@ bool ImageWriter::AllocMemory() { for (ImageInfo& image_info : image_infos_) { ImageSection unused_sections[ImageHeader::kSectionCount]; const size_t length = RoundUp( - image_info.CreateImageSections(unused_sections, compile_app_image_), kPageSize); + image_info.CreateImageSections(unused_sections), kPageSize); std::string error_msg; image_info.image_.reset(MemMap::MapAnonymous("image writer image", @@ -1842,7 +1852,7 @@ void ImageWriter::CalculateNewObjectOffsets() { image_info.image_offset_ = image_offset; ImageSection unused_sections[ImageHeader::kSectionCount]; image_info.image_size_ = - RoundUp(image_info.CreateImageSections(unused_sections, compile_app_image_), kPageSize); + RoundUp(image_info.CreateImageSections(unused_sections), kPageSize); // There should be no gaps until the next image. image_offset += image_info.image_size_; } @@ -1873,8 +1883,7 @@ void ImageWriter::CalculateNewObjectOffsets() { } } -size_t ImageWriter::ImageInfo::CreateImageSections(ImageSection* out_sections, - bool app_image) const { +size_t ImageWriter::ImageInfo::CreateImageSections(ImageSection* out_sections) const { DCHECK(out_sections != nullptr); // Do not round up any sections here that are represented by the bins since it will break @@ -1912,13 +1921,8 @@ size_t ImageWriter::ImageInfo::CreateImageSections(ImageSection* out_sections, ImageSection* dex_cache_arrays_section = &out_sections[ImageHeader::kSectionDexCacheArrays]; *dex_cache_arrays_section = ImageSection(GetBinSlotOffset(Bin::kDexCacheArray), GetBinSlotSize(Bin::kDexCacheArray)); - // For boot image, round up to the page boundary to separate the interned strings and - // class table from the modifiable data. We shall mprotect() these pages read-only when - // we load the boot image. This is more than sufficient for the string table alignment, - // namely sizeof(uint64_t). See HashSet::WriteToMemory. - static_assert(IsAligned<sizeof(uint64_t)>(kPageSize), "String table alignment check."); - size_t cur_pos = - RoundUp(dex_cache_arrays_section->End(), app_image ? sizeof(uint64_t) : kPageSize); + // Round up to the alignment the string table expects. See HashSet::WriteToMemory. + size_t cur_pos = RoundUp(dex_cache_arrays_section->End(), sizeof(uint64_t)); // Calculate the size of the interned strings. ImageSection* interned_strings_section = &out_sections[ImageHeader::kSectionInternedStrings]; *interned_strings_section = ImageSection(cur_pos, intern_table_bytes_); @@ -1941,7 +1945,7 @@ void ImageWriter::CreateHeader(size_t oat_index) { // Create the image sections. ImageSection sections[ImageHeader::kSectionCount]; - const size_t image_end = image_info.CreateImageSections(sections, compile_app_image_); + const size_t image_end = image_info.CreateImageSections(sections); // Finally bitmap section. const size_t bitmap_bytes = image_info.image_bitmap_->Size(); diff --git a/dex2oat/linker/image_writer.h b/dex2oat/linker/image_writer.h index 36bbb477867..960d6986895 100644 --- a/dex2oat/linker/image_writer.h +++ b/dex2oat/linker/image_writer.h @@ -33,6 +33,7 @@ #include "base/enums.h" #include "base/length_prefixed_array.h" #include "base/macros.h" +#include "base/mem_map.h" #include "base/os.h" #include "base/safe_map.h" #include "base/utils.h" @@ -41,7 +42,6 @@ #include "image.h" #include "intern_table.h" #include "lock_word.h" -#include "mem_map.h" #include "mirror/dex_cache.h" #include "oat_file.h" #include "obj_ptr.h" @@ -64,6 +64,7 @@ class ClassLoader; class ClassLoaderVisitor; class ImTable; class ImtConflictTable; +class TimingLogger; static constexpr int kInvalidFd = -1; @@ -81,7 +82,7 @@ class ImageWriter FINAL { const std::unordered_map<const DexFile*, size_t>& dex_file_oat_index_map, const std::unordered_set<std::string>* dirty_image_objects); - bool PrepareImageAddressSpace(); + bool PrepareImageAddressSpace(TimingLogger* timings); bool IsImageAddressSpaceReady() const { DCHECK(!image_infos_.empty()); @@ -267,7 +268,7 @@ class ImageWriter FINAL { // Create the image sections into the out sections variable, returns the size of the image // excluding the bitmap. - size_t CreateImageSections(ImageSection* out_sections, bool app_image) const; + size_t CreateImageSections(ImageSection* out_sections) const; size_t GetStubOffset(StubType stub_type) const { DCHECK_LT(static_cast<size_t>(stub_type), kNumberOfStubTypes); diff --git a/compiler/linker/mips/relative_patcher_mips.cc b/dex2oat/linker/mips/relative_patcher_mips.cc index 69e0846cb7e..69e0846cb7e 100644 --- a/compiler/linker/mips/relative_patcher_mips.cc +++ b/dex2oat/linker/mips/relative_patcher_mips.cc diff --git a/compiler/linker/mips/relative_patcher_mips.h b/dex2oat/linker/mips/relative_patcher_mips.h index 5714a7d1b0c..d3a4c5a14f2 100644 --- a/compiler/linker/mips/relative_patcher_mips.h +++ b/dex2oat/linker/mips/relative_patcher_mips.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef ART_COMPILER_LINKER_MIPS_RELATIVE_PATCHER_MIPS_H_ -#define ART_COMPILER_LINKER_MIPS_RELATIVE_PATCHER_MIPS_H_ +#ifndef ART_DEX2OAT_LINKER_MIPS_RELATIVE_PATCHER_MIPS_H_ +#define ART_DEX2OAT_LINKER_MIPS_RELATIVE_PATCHER_MIPS_H_ #include "arch/mips/instruction_set_features_mips.h" #include "linker/relative_patcher.h" @@ -55,4 +55,4 @@ class MipsRelativePatcher FINAL : public RelativePatcher { } // namespace linker } // namespace art -#endif // ART_COMPILER_LINKER_MIPS_RELATIVE_PATCHER_MIPS_H_ +#endif // ART_DEX2OAT_LINKER_MIPS_RELATIVE_PATCHER_MIPS_H_ diff --git a/compiler/linker/mips/relative_patcher_mips32r6_test.cc b/dex2oat/linker/mips/relative_patcher_mips32r6_test.cc index 629fdd535de..629fdd535de 100644 --- a/compiler/linker/mips/relative_patcher_mips32r6_test.cc +++ b/dex2oat/linker/mips/relative_patcher_mips32r6_test.cc diff --git a/compiler/linker/mips/relative_patcher_mips_test.cc b/dex2oat/linker/mips/relative_patcher_mips_test.cc index d876c76daae..d876c76daae 100644 --- a/compiler/linker/mips/relative_patcher_mips_test.cc +++ b/dex2oat/linker/mips/relative_patcher_mips_test.cc diff --git a/compiler/linker/mips64/relative_patcher_mips64.cc b/dex2oat/linker/mips64/relative_patcher_mips64.cc index aae5746278d..aae5746278d 100644 --- a/compiler/linker/mips64/relative_patcher_mips64.cc +++ b/dex2oat/linker/mips64/relative_patcher_mips64.cc diff --git a/compiler/linker/mips64/relative_patcher_mips64.h b/dex2oat/linker/mips64/relative_patcher_mips64.h index 183bbedb396..9f5a1254080 100644 --- a/compiler/linker/mips64/relative_patcher_mips64.h +++ b/dex2oat/linker/mips64/relative_patcher_mips64.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef ART_COMPILER_LINKER_MIPS64_RELATIVE_PATCHER_MIPS64_H_ -#define ART_COMPILER_LINKER_MIPS64_RELATIVE_PATCHER_MIPS64_H_ +#ifndef ART_DEX2OAT_LINKER_MIPS64_RELATIVE_PATCHER_MIPS64_H_ +#define ART_DEX2OAT_LINKER_MIPS64_RELATIVE_PATCHER_MIPS64_H_ #include "linker/relative_patcher.h" @@ -51,4 +51,4 @@ class Mips64RelativePatcher FINAL : public RelativePatcher { } // namespace linker } // namespace art -#endif // ART_COMPILER_LINKER_MIPS64_RELATIVE_PATCHER_MIPS64_H_ +#endif // ART_DEX2OAT_LINKER_MIPS64_RELATIVE_PATCHER_MIPS64_H_ diff --git a/compiler/linker/mips64/relative_patcher_mips64_test.cc b/dex2oat/linker/mips64/relative_patcher_mips64_test.cc index a02f5005e8c..a02f5005e8c 100644 --- a/compiler/linker/mips64/relative_patcher_mips64_test.cc +++ b/dex2oat/linker/mips64/relative_patcher_mips64_test.cc diff --git a/dex2oat/linker/multi_oat_relative_patcher.cc b/dex2oat/linker/multi_oat_relative_patcher.cc index 1abaf7dfd18..1449d478f99 100644 --- a/dex2oat/linker/multi_oat_relative_patcher.cc +++ b/dex2oat/linker/multi_oat_relative_patcher.cc @@ -20,14 +20,28 @@ #include "base/bit_utils.h" #include "globals.h" +#include "driver/compiled_method_storage.h" namespace art { namespace linker { +void MultiOatRelativePatcher::ThunkProvider::GetThunkCode(const LinkerPatch& patch, + /*out*/ ArrayRef<const uint8_t>* code, + /*out*/ std::string* debug_name) { + *code = storage_->GetThunkCode(patch, debug_name); + DCHECK(!code->empty()); +} + + MultiOatRelativePatcher::MultiOatRelativePatcher(InstructionSet instruction_set, - const InstructionSetFeatures* features) - : method_offset_map_(), - relative_patcher_(RelativePatcher::Create(instruction_set, features, &method_offset_map_)), + const InstructionSetFeatures* features, + CompiledMethodStorage* storage) + : thunk_provider_(storage), + method_offset_map_(), + relative_patcher_(RelativePatcher::Create(instruction_set, + features, + &thunk_provider_, + &method_offset_map_)), adjustment_(0u), instruction_set_(instruction_set), start_size_code_alignment_(0u), diff --git a/dex2oat/linker/multi_oat_relative_patcher.h b/dex2oat/linker/multi_oat_relative_patcher.h index bd33b953187..60fcfe8b581 100644 --- a/dex2oat/linker/multi_oat_relative_patcher.h +++ b/dex2oat/linker/multi_oat_relative_patcher.h @@ -26,6 +26,7 @@ namespace art { class CompiledMethod; +class CompiledMethodStorage; class InstructionSetFeatures; namespace linker { @@ -38,7 +39,9 @@ class MultiOatRelativePatcher FINAL { public: using const_iterator = SafeMap<MethodReference, uint32_t>::const_iterator; - MultiOatRelativePatcher(InstructionSet instruction_set, const InstructionSetFeatures* features); + MultiOatRelativePatcher(InstructionSet instruction_set, + const InstructionSetFeatures* features, + CompiledMethodStorage* storage); // Mark the start of a new oat file (for statistics retrieval) and set the // adjustment for a new oat file to apply to all relative offsets that are @@ -129,6 +132,19 @@ class MultiOatRelativePatcher FINAL { uint32_t MiscThunksSize() const; private: + class ThunkProvider : public RelativePatcherThunkProvider { + public: + explicit ThunkProvider(CompiledMethodStorage* storage) + : storage_(storage) {} + + void GetThunkCode(const LinkerPatch& patch, + /*out*/ ArrayRef<const uint8_t>* code, + /*out*/ std::string* debug_name) OVERRIDE; + + private: + CompiledMethodStorage* storage_; + }; + // Map method reference to assigned offset. // Wrap the map in a class implementing RelativePatcherTargetProvider. class MethodOffsetMap : public RelativePatcherTargetProvider { @@ -137,6 +153,7 @@ class MultiOatRelativePatcher FINAL { SafeMap<MethodReference, uint32_t> map; }; + ThunkProvider thunk_provider_; MethodOffsetMap method_offset_map_; std::unique_ptr<RelativePatcher> relative_patcher_; uint32_t adjustment_; diff --git a/dex2oat/linker/multi_oat_relative_patcher_test.cc b/dex2oat/linker/multi_oat_relative_patcher_test.cc index ca9c5f1e847..05fe36a5904 100644 --- a/dex2oat/linker/multi_oat_relative_patcher_test.cc +++ b/dex2oat/linker/multi_oat_relative_patcher_test.cc @@ -122,7 +122,7 @@ class MultiOatRelativePatcherTest : public testing::Test { MultiOatRelativePatcherTest() : instruction_set_features_(InstructionSetFeatures::FromCppDefines()), - patcher_(kRuntimeISA, instruction_set_features_.get()) { + patcher_(kRuntimeISA, instruction_set_features_.get(), /* storage */ nullptr) { std::unique_ptr<MockPatcher> mock(new MockPatcher()); mock_ = mock.get(); patcher_.relative_patcher_ = std::move(mock); diff --git a/dex2oat/linker/oat_writer.cc b/dex2oat/linker/oat_writer.cc index c635d59cd70..7078b05a1ba 100644 --- a/dex2oat/linker/oat_writer.cc +++ b/dex2oat/linker/oat_writer.cc @@ -31,6 +31,7 @@ #include "base/safe_map.h" #include "base/stl_util.h" #include "base/unix_file/fd_file.h" +#include "base/zip_archive.h" #include "class_linker.h" #include "class_table-inl.h" #include "compiled_method-inl.h" @@ -40,6 +41,7 @@ #include "dex/dex_file_loader.h" #include "dex/dex_file_types.h" #include "dex/standard_dex_file.h" +#include "dex/type_lookup_table.h" #include "dex/verification_results.h" #include "dex_container.h" #include "dexlayout.h" @@ -63,11 +65,9 @@ #include "oat_quick_method_header.h" #include "quicken_info.h" #include "scoped_thread_state_change-inl.h" -#include "type_lookup_table.h" #include "utils/dex_cache_arrays_layout-inl.h" #include "vdex_file.h" #include "verifier/verifier_deps.h" -#include "zip_archive.h" namespace art { namespace linker { @@ -375,16 +375,19 @@ OatWriter::OatWriter(bool compiling_boot_image, vdex_dex_shared_data_offset_(0u), vdex_verifier_deps_offset_(0u), vdex_quickening_info_offset_(0u), + code_size_(0u), oat_size_(0u), + data_bimg_rel_ro_start_(0u), + data_bimg_rel_ro_size_(0u), bss_start_(0u), bss_size_(0u), bss_methods_offset_(0u), bss_roots_offset_(0u), + data_bimg_rel_ro_entries_(), bss_method_entry_references_(), bss_method_entries_(), bss_type_entries_(), bss_string_entries_(), - map_boot_image_tables_to_bss_(false), oat_data_offset_(0u), oat_header_(nullptr), size_vdex_header_(0), @@ -409,6 +412,8 @@ OatWriter::OatWriter(bool compiling_boot_image, size_method_header_(0), size_code_(0), size_code_alignment_(0), + size_data_bimg_rel_ro_(0), + size_data_bimg_rel_ro_alignment_(0), size_relative_call_thunks_(0), size_misc_thunks_(0), size_vmap_table_(0), @@ -744,8 +749,13 @@ void OatWriter::PrepareLayout(MultiOatRelativePatcher* relative_patcher) { { TimingLogger::ScopedTiming split("InitOatCodeDexFiles", timings_); offset = InitOatCodeDexFiles(offset); + code_size_ = offset - GetOatHeader().GetExecutableOffset(); } - oat_size_ = offset; + { + TimingLogger::ScopedTiming split("InitDataBimgRelRoLayout", timings_); + offset = InitDataBimgRelRoLayout(offset); + } + oat_size_ = offset; // .bss does not count towards oat_size_. bss_start_ = (bss_size_ != 0u) ? RoundUp(oat_size_, kPageSize) : 0u; CHECK_EQ(dex_files_->size(), oat_dex_files_.size()); @@ -852,7 +862,10 @@ class OatWriter::InitBssLayoutMethodVisitor : public DexMethodVisitor { MethodReference(dex_file_, it.GetMemberIndex())); if (HasCompiledCode(compiled_method)) { for (const LinkerPatch& patch : compiled_method->GetPatches()) { - if (patch.GetType() == LinkerPatch::Type::kMethodBssEntry) { + if (patch.GetType() == LinkerPatch::Type::kDataBimgRelRo) { + writer_->data_bimg_rel_ro_entries_.Overwrite(patch.BootImageOffset(), + /* placeholder */ 0u); + } else if (patch.GetType() == LinkerPatch::Type::kMethodBssEntry) { MethodReference target_method = patch.TargetMethod(); AddBssReference(target_method, target_method.dex_file->NumMethodIds(), @@ -870,9 +883,6 @@ class OatWriter::InitBssLayoutMethodVisitor : public DexMethodVisitor { target_string.dex_file->NumStringIds(), &writer_->bss_string_entry_references_); writer_->bss_string_entries_.Overwrite(target_string, /* placeholder */ 0u); - } else if (patch.GetType() == LinkerPatch::Type::kStringInternTable || - patch.GetType() == LinkerPatch::Type::kTypeClassTable) { - writer_->map_boot_image_tables_to_bss_ = true; } } } else { @@ -1783,6 +1793,16 @@ class OatWriter::WriteCodeMethodVisitor : public OrderedMethodVisitor { for (const LinkerPatch& patch : compiled_method->GetPatches()) { uint32_t literal_offset = patch.LiteralOffset(); switch (patch.GetType()) { + case LinkerPatch::Type::kDataBimgRelRo: { + uint32_t target_offset = + writer_->data_bimg_rel_ro_start_ + + writer_->data_bimg_rel_ro_entries_.Get(patch.BootImageOffset()); + writer_->relative_patcher_->PatchPcRelativeReference(&patched_code_, + patch, + offset_ + literal_offset, + target_offset); + break; + } case LinkerPatch::Type::kMethodBssEntry: { uint32_t target_offset = writer_->bss_start_ + writer_->bss_method_entries_.Get(patch.TargetMethod()); @@ -1809,14 +1829,6 @@ class OatWriter::WriteCodeMethodVisitor : public OrderedMethodVisitor { target_offset); break; } - case LinkerPatch::Type::kStringInternTable: { - uint32_t target_offset = GetInternTableEntryOffset(patch); - writer_->relative_patcher_->PatchPcRelativeReference(&patched_code_, - patch, - offset_ + literal_offset, - target_offset); - break; - } case LinkerPatch::Type::kStringBssEntry: { StringReference ref(patch.TargetStringDexFile(), patch.TargetStringIndex()); uint32_t target_offset = @@ -1835,14 +1847,6 @@ class OatWriter::WriteCodeMethodVisitor : public OrderedMethodVisitor { target_offset); break; } - case LinkerPatch::Type::kTypeClassTable: { - uint32_t target_offset = GetClassTableEntryOffset(patch); - writer_->relative_patcher_->PatchPcRelativeReference(&patched_code_, - patch, - offset_ + literal_offset, - target_offset); - break; - } case LinkerPatch::Type::kTypeBssEntry: { TypeReference ref(patch.TargetTypeDexFile(), patch.TargetTypeIndex()); uint32_t target_offset = writer_->bss_start_ + writer_->bss_type_entries_.Get(ref); @@ -2044,42 +2048,6 @@ class OatWriter::WriteCodeMethodVisitor : public OrderedMethodVisitor { data[2] = (address >> 16) & 0xffu; data[3] = (address >> 24) & 0xffu; } - - // Calculate the offset of the InternTable slot (GcRoot<String>) when mmapped to the .bss. - uint32_t GetInternTableEntryOffset(const LinkerPatch& patch) - REQUIRES_SHARED(Locks::mutator_lock_) { - DCHECK(!writer_->HasBootImage()); - const uint8_t* string_root = writer_->LookupBootImageInternTableSlot( - *patch.TargetStringDexFile(), patch.TargetStringIndex()); - DCHECK(string_root != nullptr); - return GetBootImageTableEntryOffset(string_root); - } - - // Calculate the offset of the ClassTable::TableSlot when mmapped to the .bss. - uint32_t GetClassTableEntryOffset(const LinkerPatch& patch) - REQUIRES_SHARED(Locks::mutator_lock_) { - DCHECK(!writer_->HasBootImage()); - const uint8_t* table_slot = - writer_->LookupBootImageClassTableSlot(*patch.TargetTypeDexFile(), patch.TargetTypeIndex()); - DCHECK(table_slot != nullptr); - return GetBootImageTableEntryOffset(table_slot); - } - - uint32_t GetBootImageTableEntryOffset(const uint8_t* raw_root) { - uint32_t base_offset = writer_->bss_start_; - for (gc::space::ImageSpace* space : Runtime::Current()->GetHeap()->GetBootImageSpaces()) { - const uint8_t* const_tables_begin = - space->Begin() + space->GetImageHeader().GetBootImageConstantTablesOffset(); - size_t offset = static_cast<size_t>(raw_root - const_tables_begin); - if (offset < space->GetImageHeader().GetBootImageConstantTablesSize()) { - DCHECK_LE(base_offset + offset, writer_->bss_start_ + writer_->bss_methods_offset_); - return base_offset + offset; - } - base_offset += space->GetImageHeader().GetBootImageConstantTablesSize(); - } - LOG(FATAL) << "Didn't find boot image string in boot image intern tables!"; - UNREACHABLE(); - } }; class OatWriter::WriteMapMethodVisitor : public OatDexMethodVisitor { @@ -2517,6 +2485,25 @@ size_t OatWriter::InitOatCodeDexFiles(size_t offset) { return offset; } +size_t OatWriter::InitDataBimgRelRoLayout(size_t offset) { + DCHECK_EQ(data_bimg_rel_ro_size_, 0u); + if (data_bimg_rel_ro_entries_.empty()) { + // Nothing to put to the .data.bimg.rel.ro section. + return offset; + } + + data_bimg_rel_ro_start_ = RoundUp(offset, kPageSize); + + for (auto& entry : data_bimg_rel_ro_entries_) { + size_t& entry_offset = entry.second; + entry_offset = data_bimg_rel_ro_size_; + data_bimg_rel_ro_size_ += sizeof(uint32_t); + } + + offset = data_bimg_rel_ro_start_ + data_bimg_rel_ro_size_; + return offset; +} + void OatWriter::InitBssLayout(InstructionSet instruction_set) { { InitBssLayoutMethodVisitor visitor(this); @@ -2526,25 +2513,16 @@ void OatWriter::InitBssLayout(InstructionSet instruction_set) { DCHECK_EQ(bss_size_, 0u); if (HasBootImage()) { - DCHECK(!map_boot_image_tables_to_bss_); DCHECK(bss_string_entries_.empty()); } - if (!map_boot_image_tables_to_bss_ && - bss_method_entries_.empty() && + if (bss_method_entries_.empty() && bss_type_entries_.empty() && bss_string_entries_.empty()) { // Nothing to put to the .bss section. return; } - // Allocate space for boot image tables in the .bss section. PointerSize pointer_size = GetInstructionSetPointerSize(instruction_set); - if (map_boot_image_tables_to_bss_) { - for (gc::space::ImageSpace* space : Runtime::Current()->GetHeap()->GetBootImageSpaces()) { - bss_size_ += space->GetImageHeader().GetBootImageConstantTablesSize(); - } - } - bss_methods_offset_ = bss_size_; // Prepare offsets for .bss ArtMethod entries. @@ -2912,6 +2890,49 @@ bool OatWriter::WriteCode(OutputStream* out) { return false; } + if (data_bimg_rel_ro_size_ != 0u) { + write_state_ = WriteState::kWriteDataBimgRelRo; + } else { + if (!CheckOatSize(out, file_offset, relative_offset)) { + return false; + } + write_state_ = WriteState::kWriteHeader; + } + return true; +} + +bool OatWriter::WriteDataBimgRelRo(OutputStream* out) { + CHECK(write_state_ == WriteState::kWriteDataBimgRelRo); + + // Wrap out to update checksum with each write. + ChecksumUpdatingOutputStream checksum_updating_out(out, oat_header_.get()); + out = &checksum_updating_out; + + const size_t file_offset = oat_data_offset_; + size_t relative_offset = data_bimg_rel_ro_start_; + + // Record the padding before the .data.bimg.rel.ro section. + // Do not write anything, this zero-filled part was skipped (Seek()) when starting the section. + size_t code_end = GetOatHeader().GetExecutableOffset() + code_size_; + DCHECK_EQ(RoundUp(code_end, kPageSize), relative_offset); + size_t padding_size = relative_offset - code_end; + DCHECK_EQ(size_data_bimg_rel_ro_alignment_, 0u); + size_data_bimg_rel_ro_alignment_ = padding_size; + + relative_offset = WriteDataBimgRelRo(out, file_offset, relative_offset); + if (relative_offset == 0) { + LOG(ERROR) << "Failed to write boot image relocations to " << out->GetLocation(); + return false; + } + + if (!CheckOatSize(out, file_offset, relative_offset)) { + return false; + } + write_state_ = WriteState::kWriteHeader; + return true; +} + +bool OatWriter::CheckOatSize(OutputStream* out, size_t file_offset, size_t relative_offset) { const off_t oat_end_file_offset = out->Seek(0, kSeekCurrent); if (oat_end_file_offset == static_cast<off_t>(-1)) { LOG(ERROR) << "Failed to get oat end file offset in " << out->GetLocation(); @@ -2946,6 +2967,8 @@ bool OatWriter::WriteCode(OutputStream* out) { DO_STAT(size_method_header_); DO_STAT(size_code_); DO_STAT(size_code_alignment_); + DO_STAT(size_data_bimg_rel_ro_); + DO_STAT(size_data_bimg_rel_ro_alignment_); DO_STAT(size_relative_call_thunks_); DO_STAT(size_misc_thunks_); DO_STAT(size_vmap_table_); @@ -3323,6 +3346,32 @@ size_t OatWriter::WriteCodeDexFiles(OutputStream* out, return relative_offset; } +size_t OatWriter::WriteDataBimgRelRo(OutputStream* out, + size_t file_offset, + size_t relative_offset) { + if (data_bimg_rel_ro_entries_.empty()) { + return relative_offset; + } + + // Write the entire .data.bimg.rel.ro with a single WriteFully(). + std::vector<uint32_t> data; + data.reserve(data_bimg_rel_ro_entries_.size()); + for (const auto& entry : data_bimg_rel_ro_entries_) { + uint32_t boot_image_offset = entry.first; + data.push_back(boot_image_offset); + } + DCHECK_EQ(data.size(), data_bimg_rel_ro_entries_.size()); + DCHECK_OFFSET(); + if (!out->WriteFully(data.data(), data.size() * sizeof(data[0]))) { + PLOG(ERROR) << "Failed to write .data.bimg.rel.ro in " << out->GetLocation(); + return 0u; + } + DCHECK_EQ(size_data_bimg_rel_ro_, 0u); + size_data_bimg_rel_ro_ = data.size() * sizeof(data[0]); + relative_offset += size_data_bimg_rel_ro_; + return relative_offset; +} + bool OatWriter::RecordOatDataOffset(OutputStream* out) { // Get the elf file offset of the oat file. const off_t raw_file_offset = out->Seek(0, kSeekCurrent); @@ -3349,7 +3398,7 @@ bool OatWriter::WriteDexFiles(OutputStream* out, break; } ZipEntry* entry = oat_dex_file.source_.GetZipEntry(); - if (!entry->IsUncompressed() || !entry->IsAlignedToDexHeader()) { + if (!entry->IsUncompressed() || !entry->IsAlignedTo(alignof(DexFile::Header))) { extract_dex_files_into_vdex_ = true; break; } @@ -4376,42 +4425,6 @@ bool OatWriter::OatClass::Write(OatWriter* oat_writer, OutputStream* out) const return true; } -const uint8_t* OatWriter::LookupBootImageInternTableSlot(const DexFile& dex_file, - dex::StringIndex string_idx) - NO_THREAD_SAFETY_ANALYSIS { // Single-threaded OatWriter can avoid locking. - uint32_t utf16_length; - const char* utf8_data = dex_file.StringDataAndUtf16LengthByIdx(string_idx, &utf16_length); - DCHECK_EQ(utf16_length, CountModifiedUtf8Chars(utf8_data)); - InternTable::Utf8String string(utf16_length, - utf8_data, - ComputeUtf16HashFromModifiedUtf8(utf8_data, utf16_length)); - const InternTable* intern_table = Runtime::Current()->GetClassLinker()->intern_table_; - for (const InternTable::Table::UnorderedSet& table : intern_table->strong_interns_.tables_) { - auto it = table.Find(string); - if (it != table.end()) { - return reinterpret_cast<const uint8_t*>(std::addressof(*it)); - } - } - LOG(FATAL) << "Did not find boot image string " << utf8_data; - UNREACHABLE(); -} - -const uint8_t* OatWriter::LookupBootImageClassTableSlot(const DexFile& dex_file, - dex::TypeIndex type_idx) - NO_THREAD_SAFETY_ANALYSIS { // Single-threaded OatWriter can avoid locking. - const char* descriptor = dex_file.StringByTypeIdx(type_idx); - ClassTable::DescriptorHashPair pair(descriptor, ComputeModifiedUtf8Hash(descriptor)); - ClassTable* table = Runtime::Current()->GetClassLinker()->boot_class_table_.get(); - for (const ClassTable::ClassSet& class_set : table->classes_) { - auto it = class_set.Find(pair); - if (it != class_set.end()) { - return reinterpret_cast<const uint8_t*>(std::addressof(*it)); - } - } - LOG(FATAL) << "Did not find boot image class " << descriptor; - UNREACHABLE(); -} - debug::DebugInfo OatWriter::GetDebugInfo() const { debug::DebugInfo debug_info{}; debug_info.compiled_methods = ArrayRef<const debug::MethodDebugInfo>(method_info_); diff --git a/dex2oat/linker/oat_writer.h b/dex2oat/linker/oat_writer.h index 2dae811e04c..619743ef144 100644 --- a/dex2oat/linker/oat_writer.h +++ b/dex2oat/linker/oat_writer.h @@ -25,6 +25,7 @@ #include "base/array_ref.h" #include "base/dchecked_vector.h" #include "base/os.h" +#include "base/mem_map.h" #include "base/safe_map.h" #include "compiler.h" #include "debug/debug_info.h" @@ -33,7 +34,6 @@ #include "dex/string_reference.h" #include "dex/type_reference.h" #include "linker/relative_patcher.h" // For RelativePatcherTargetProvider. -#include "mem_map.h" #include "mirror/class.h" #include "oat.h" @@ -137,6 +137,7 @@ class OatWriter { // - PrepareLayout(), // - WriteRodata(), // - WriteCode(), + // - WriteDataBimgRelRo() iff GetDataBimgRelRoSize() != 0, // - WriteHeader(). // Add dex file source(s) from a file, either a plain dex file or @@ -197,6 +198,10 @@ class OatWriter { bool WriteRodata(OutputStream* out); // Write the code to the .text section. bool WriteCode(OutputStream* out); + // Write the boot image relocation data to the .data.bimg.rel.ro section. + bool WriteDataBimgRelRo(OutputStream* out); + // Check the size of the written oat file. + bool CheckOatSize(OutputStream* out, size_t file_offset, size_t relative_offset); // Write the oat header. This finalizes the oat file. bool WriteHeader(OutputStream* out, uint32_t image_file_location_oat_checksum, @@ -218,10 +223,18 @@ class OatWriter { return *oat_header_; } + size_t GetCodeSize() const { + return code_size_; + } + size_t GetOatSize() const { return oat_size_; } + size_t GetDataBimgRelRoSize() const { + return data_bimg_rel_ro_size_; + } + size_t GetBssSize() const { return bss_size_; } @@ -323,6 +336,7 @@ class OatWriter { size_t InitOatDexFiles(size_t offset); size_t InitOatCode(size_t offset); size_t InitOatCodeDexFiles(size_t offset); + size_t InitDataBimgRelRoLayout(size_t offset); void InitBssLayout(InstructionSet instruction_set); size_t WriteClassOffsets(OutputStream* out, size_t file_offset, size_t relative_offset); @@ -332,6 +346,7 @@ class OatWriter { size_t WriteOatDexFiles(OutputStream* out, size_t file_offset, size_t relative_offset); size_t WriteCode(OutputStream* out, size_t file_offset, size_t relative_offset); size_t WriteCodeDexFiles(OutputStream* out, size_t file_offset, size_t relative_offset); + size_t WriteDataBimgRelRo(OutputStream* out, size_t file_offset, size_t relative_offset); bool RecordOatDataOffset(OutputStream* out); bool WriteTypeLookupTables(OutputStream* oat_rodata, @@ -349,17 +364,12 @@ class OatWriter { return dex_files_ != nullptr && extract_dex_files_into_vdex_; } - // Find the address of the GcRoot<String> in the InternTable for a boot image string. - const uint8_t* LookupBootImageInternTableSlot(const DexFile& dex_file, - dex::StringIndex string_idx); - // Find the address of the ClassTable::TableSlot for a boot image class. - const uint8_t* LookupBootImageClassTableSlot(const DexFile& dex_file, dex::TypeIndex type_idx); - enum class WriteState { kAddingDexFileSources, kPrepareLayout, kWriteRoData, kWriteText, + kWriteDataBimgRelRo, kWriteHeader, kDone }; @@ -401,9 +411,18 @@ class OatWriter { // Offset of section holding quickening info inside Vdex. size_t vdex_quickening_info_offset_; + // Size of the .text segment. + size_t code_size_; + // Size required for Oat data structures. size_t oat_size_; + // The start of the required .data.bimg.rel.ro section. + size_t data_bimg_rel_ro_start_; + + // The size of the required .data.bimg.rel.ro section holding the boot image relocations. + size_t data_bimg_rel_ro_size_; + // The start of the required .bss section. size_t bss_start_; @@ -416,6 +435,10 @@ class OatWriter { // The offset of the GC roots in .bss section. size_t bss_roots_offset_; + // Map for allocating .data.bimg.rel.ro entries. Indexed by the boot image offset of the + // relocation. The value is the assigned offset within the .data.bimg.rel.ro section. + SafeMap<uint32_t, size_t> data_bimg_rel_ro_entries_; + // Map for recording references to ArtMethod entries in .bss. SafeMap<const DexFile*, BitVector> bss_method_entry_references_; @@ -440,10 +463,6 @@ class OatWriter { // is the target offset for patching, starting at `bss_start_ + bss_roots_offset_`. SafeMap<StringReference, size_t, StringReferenceValueComparator> bss_string_entries_; - // Whether boot image tables should be mapped to the .bss. This is needed for compiled - // code that reads from these tables with PC-relative instructions. - bool map_boot_image_tables_to_bss_; - // Offset of the oat data from the start of the mmapped region of the elf file. size_t oat_data_offset_; @@ -484,6 +503,8 @@ class OatWriter { uint32_t size_method_header_; uint32_t size_code_; uint32_t size_code_alignment_; + uint32_t size_data_bimg_rel_ro_; + uint32_t size_data_bimg_rel_ro_alignment_; uint32_t size_relative_call_thunks_; uint32_t size_misc_thunks_; uint32_t size_vmap_table_; diff --git a/dex2oat/linker/oat_writer_test.cc b/dex2oat/linker/oat_writer_test.cc index eb4ef4501d4..37e69f7706e 100644 --- a/dex2oat/linker/oat_writer_test.cc +++ b/dex2oat/linker/oat_writer_test.cc @@ -110,8 +110,6 @@ class OatTest : public CommonCompilerTest { insn_set, insn_features_.get(), /* image_classes */ nullptr, - /* compiled_classes */ nullptr, - /* compiled_methods */ nullptr, /* thread_count */ 2, /* swap_fd */ -1, /* profile_compilation_info */ nullptr)); @@ -213,13 +211,13 @@ class OatTest : public CommonCompilerTest { class_linker->RegisterDexFile(*dex_file, nullptr); } MultiOatRelativePatcher patcher(compiler_driver_->GetInstructionSet(), - instruction_set_features_.get()); + instruction_set_features_.get(), + compiler_driver_->GetCompiledMethodStorage()); oat_writer.Initialize(compiler_driver_.get(), nullptr, dex_files); oat_writer.PrepareLayout(&patcher); - size_t rodata_size = oat_writer.GetOatHeader().GetExecutableOffset(); - size_t text_size = oat_writer.GetOatSize() - rodata_size; - elf_writer->PrepareDynamicSection(rodata_size, - text_size, + elf_writer->PrepareDynamicSection(oat_writer.GetOatHeader().GetExecutableOffset(), + oat_writer.GetCodeSize(), + oat_writer.GetDataBimgRelRoSize(), oat_writer.GetBssSize(), oat_writer.GetBssMethodsOffset(), oat_writer.GetBssRootsOffset(), @@ -248,6 +246,14 @@ class OatTest : public CommonCompilerTest { } elf_writer->EndText(text); + if (oat_writer.GetDataBimgRelRoSize() != 0u) { + OutputStream* data_bimg_rel_ro = elf_writer->StartDataBimgRelRo(); + if (!oat_writer.WriteDataBimgRelRo(data_bimg_rel_ro)) { + return false; + } + elf_writer->EndDataBimgRelRo(data_bimg_rel_ro); + } + if (!oat_writer.WriteHeader(elf_writer->GetStream(), 42U, 4096U, 0)) { return false; } @@ -407,7 +413,7 @@ TEST_F(OatTest, WriteRead) { compiler_driver_->CompileAll(class_loader, class_linker->GetBootClassPath(), &timings2); } - ScratchFile tmp_oat, tmp_vdex(tmp_oat, ".vdex"); + ScratchFile tmp_base, tmp_oat(tmp_base, ".oat"), tmp_vdex(tmp_base, ".vdex"); SafeMap<std::string, std::string> key_value_store; key_value_store.Put(OatHeader::kImageLocationKey, "lue.art"); bool success = WriteElf(tmp_vdex.GetFile(), @@ -544,10 +550,14 @@ TEST_F(OatTest, EmptyTextSection) { compiler_driver_->SetDexFilesForOatFile(dex_files); compiler_driver_->CompileAll(class_loader, dex_files, &timings); - ScratchFile tmp_oat, tmp_vdex(tmp_oat, ".vdex"); + ScratchFile tmp_base, tmp_oat(tmp_base, ".oat"), tmp_vdex(tmp_base, ".vdex"); SafeMap<std::string, std::string> key_value_store; key_value_store.Put(OatHeader::kImageLocationKey, "test.art"); - bool success = WriteElf(tmp_vdex.GetFile(), tmp_oat.GetFile(), dex_files, key_value_store, false); + bool success = WriteElf(tmp_vdex.GetFile(), + tmp_oat.GetFile(), + dex_files, + key_value_store, + /* verify */ false); ASSERT_TRUE(success); std::unique_ptr<OatFile> oat_file(OatFile::Open(tmp_oat.GetFilename(), @@ -606,13 +616,13 @@ void OatTest::TestDexFileInput(bool verify, bool low_4gb, bool use_profile) { ASSERT_TRUE(success); input_filenames.push_back(dex_file2.GetFilename().c_str()); - ScratchFile oat_file, vdex_file(oat_file, ".vdex"); + ScratchFile tmp_base, tmp_oat(tmp_base, ".oat"), tmp_vdex(tmp_base, ".vdex"); SafeMap<std::string, std::string> key_value_store; key_value_store.Put(OatHeader::kImageLocationKey, "test.art"); std::unique_ptr<ProfileCompilationInfo> profile_compilation_info(use_profile ? new ProfileCompilationInfo() : nullptr); - success = WriteElf(vdex_file.GetFile(), - oat_file.GetFile(), + success = WriteElf(tmp_vdex.GetFile(), + tmp_oat.GetFile(), input_filenames, key_value_store, verify, @@ -627,19 +637,19 @@ void OatTest::TestDexFileInput(bool verify, bool low_4gb, bool use_profile) { ASSERT_TRUE(success); std::string error_msg; - std::unique_ptr<OatFile> opened_oat_file(OatFile::Open(oat_file.GetFilename(), - oat_file.GetFilename(), + std::unique_ptr<OatFile> opened_oat_file(OatFile::Open(tmp_oat.GetFilename(), + tmp_oat.GetFilename(), nullptr, nullptr, false, low_4gb, nullptr, &error_msg)); + ASSERT_TRUE(opened_oat_file != nullptr) << error_msg; if (low_4gb) { uintptr_t begin = reinterpret_cast<uintptr_t>(opened_oat_file->Begin()); EXPECT_EQ(begin, static_cast<uint32_t>(begin)); } - ASSERT_TRUE(opened_oat_file != nullptr); ASSERT_EQ(2u, opened_oat_file->GetOatDexFiles().size()); std::unique_ptr<const DexFile> opened_dex_file1 = opened_oat_file->GetOatDexFiles()[0]->OpenDexFile(&error_msg); @@ -671,7 +681,7 @@ void OatTest::TestDexFileInput(bool verify, bool low_4gb, bool use_profile) { ASSERT_EQ(vdex_header.GetQuickeningInfoSize(), 0u); } - int64_t actual_vdex_size = vdex_file.GetFile()->GetLength(); + int64_t actual_vdex_size = tmp_vdex.GetFile()->GetLength(); ASSERT_GE(actual_vdex_size, 0); ASSERT_EQ((uint64_t) actual_vdex_size, opened_oat_file->GetVdexFile()->GetComputedFileSize()); } @@ -743,9 +753,13 @@ void OatTest::TestZipFileInput(bool verify) { // Test using the AddDexFileSource() interface with the zip file. std::vector<const char*> input_filenames = { zip_file.GetFilename().c_str() }; - ScratchFile oat_file, vdex_file(oat_file, ".vdex"); - success = WriteElf(vdex_file.GetFile(), oat_file.GetFile(), input_filenames, - key_value_store, verify, /*profile_compilation_info*/nullptr); + ScratchFile tmp_base, tmp_oat(tmp_base, ".oat"), tmp_vdex(tmp_base, ".vdex"); + success = WriteElf(tmp_vdex.GetFile(), + tmp_oat.GetFile(), + input_filenames, + key_value_store, + verify, + /* profile_compilation_info */ nullptr); if (verify) { ASSERT_FALSE(success); @@ -753,15 +767,15 @@ void OatTest::TestZipFileInput(bool verify) { ASSERT_TRUE(success); std::string error_msg; - std::unique_ptr<OatFile> opened_oat_file(OatFile::Open(oat_file.GetFilename(), - oat_file.GetFilename(), + std::unique_ptr<OatFile> opened_oat_file(OatFile::Open(tmp_oat.GetFilename(), + tmp_oat.GetFilename(), nullptr, nullptr, false, /*low_4gb*/false, nullptr, &error_msg)); - ASSERT_TRUE(opened_oat_file != nullptr); + ASSERT_TRUE(opened_oat_file != nullptr) << error_msg; ASSERT_EQ(2u, opened_oat_file->GetOatDexFiles().size()); std::unique_ptr<const DexFile> opened_dex_file1 = opened_oat_file->GetOatDexFiles()[0]->OpenDexFile(&error_msg); @@ -789,9 +803,9 @@ void OatTest::TestZipFileInput(bool verify) { File zip_fd(dup(zip_file.GetFd()), /* check_usage */ false); ASSERT_NE(-1, zip_fd.Fd()); - ScratchFile oat_file, vdex_file(oat_file, ".vdex"); - success = WriteElf(vdex_file.GetFile(), - oat_file.GetFile(), + ScratchFile tmp_base, tmp_oat(tmp_base, ".oat"), tmp_vdex(tmp_base, ".vdex"); + success = WriteElf(tmp_vdex.GetFile(), + tmp_oat.GetFile(), std::move(zip_fd), zip_file.GetFilename().c_str(), key_value_store, @@ -802,15 +816,15 @@ void OatTest::TestZipFileInput(bool verify) { ASSERT_TRUE(success); std::string error_msg; - std::unique_ptr<OatFile> opened_oat_file(OatFile::Open(oat_file.GetFilename(), - oat_file.GetFilename(), + std::unique_ptr<OatFile> opened_oat_file(OatFile::Open(tmp_oat.GetFilename(), + tmp_oat.GetFilename(), nullptr, nullptr, false, /*low_4gb*/false, nullptr, &error_msg)); - ASSERT_TRUE(opened_oat_file != nullptr); + ASSERT_TRUE(opened_oat_file != nullptr) << error_msg; ASSERT_EQ(2u, opened_oat_file->GetOatDexFiles().size()); std::unique_ptr<const DexFile> opened_dex_file1 = opened_oat_file->GetOatDexFiles()[0]->OpenDexFile(&error_msg); @@ -855,8 +869,12 @@ void OatTest::TestZipFileInputWithEmptyDex() { std::vector<const char*> input_filenames = { zip_file.GetFilename().c_str() }; ScratchFile oat_file, vdex_file(oat_file, ".vdex"); std::unique_ptr<ProfileCompilationInfo> profile_compilation_info(new ProfileCompilationInfo()); - success = WriteElf(vdex_file.GetFile(), oat_file.GetFile(), input_filenames, - key_value_store, /*verify*/false, profile_compilation_info.get()); + success = WriteElf(vdex_file.GetFile(), + oat_file.GetFile(), + input_filenames, + key_value_store, + /* verify */ false, + profile_compilation_info.get()); ASSERT_FALSE(success); } diff --git a/compiler/linker/relative_patcher.cc b/dex2oat/linker/relative_patcher.cc index 13877f8f128..b6135c9b5fc 100644 --- a/compiler/linker/relative_patcher.cc +++ b/dex2oat/linker/relative_patcher.cc @@ -35,7 +35,7 @@ #ifdef ART_ENABLE_CODEGEN_x86_64 #include "linker/x86_64/relative_patcher_x86_64.h" #endif -#include "output_stream.h" +#include "linker/output_stream.h" namespace art { namespace linker { @@ -43,7 +43,8 @@ namespace linker { std::unique_ptr<RelativePatcher> RelativePatcher::Create( InstructionSet instruction_set, const InstructionSetFeatures* features, - RelativePatcherTargetProvider* provider) { + RelativePatcherThunkProvider* thunk_provider, + RelativePatcherTargetProvider* target_provider) { class RelativePatcherNone FINAL : public RelativePatcher { public: RelativePatcherNone() { } @@ -92,7 +93,8 @@ std::unique_ptr<RelativePatcher> RelativePatcher::Create( }; UNUSED(features); - UNUSED(provider); + UNUSED(thunk_provider); + UNUSED(target_provider); switch (instruction_set) { #ifdef ART_ENABLE_CODEGEN_x86 case InstructionSet::kX86: @@ -106,12 +108,15 @@ std::unique_ptr<RelativePatcher> RelativePatcher::Create( case InstructionSet::kArm: // Fall through: we generate Thumb2 code for "arm". case InstructionSet::kThumb2: - return std::unique_ptr<RelativePatcher>(new Thumb2RelativePatcher(provider)); + return std::unique_ptr<RelativePatcher>( + new Thumb2RelativePatcher(thunk_provider, target_provider)); #endif #ifdef ART_ENABLE_CODEGEN_arm64 case InstructionSet::kArm64: return std::unique_ptr<RelativePatcher>( - new Arm64RelativePatcher(provider, features->AsArm64InstructionSetFeatures())); + new Arm64RelativePatcher(thunk_provider, + target_provider, + features->AsArm64InstructionSetFeatures())); #endif #ifdef ART_ENABLE_CODEGEN_mips case InstructionSet::kMips: diff --git a/compiler/linker/relative_patcher.h b/dex2oat/linker/relative_patcher.h index b58e3dffbd6..d80eaf94f7d 100644 --- a/compiler/linker/relative_patcher.h +++ b/dex2oat/linker/relative_patcher.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef ART_COMPILER_LINKER_RELATIVE_PATCHER_H_ -#define ART_COMPILER_LINKER_RELATIVE_PATCHER_H_ +#ifndef ART_DEX2OAT_LINKER_RELATIVE_PATCHER_H_ +#define ART_DEX2OAT_LINKER_RELATIVE_PATCHER_H_ #include <vector> @@ -39,6 +39,27 @@ class LinkerPatch; class OutputStream; /** + * @class RelativePatcherThunkProvider + * @brief Interface for providing method offsets for relative call targets. + */ +class RelativePatcherThunkProvider { + public: + /** + * Get the code and debug name of a thunk needed by the given linker patch. + * + * @param patch The patch for which we need to retrieve the thunk code. + * @param code A variable to receive the code of the thunk. This code must not be empty. + * @param debug_name A variable to receive the debug name of the thunk. + */ + virtual void GetThunkCode(const LinkerPatch& patch, + /*out*/ ArrayRef<const uint8_t>* code, + /*out*/ std::string* debug_name) = 0; + + protected: + virtual ~RelativePatcherThunkProvider() { } +}; + +/** * @class RelativePatcherTargetProvider * @brief Interface for providing method offsets for relative call targets. */ @@ -70,8 +91,10 @@ class RelativePatcherTargetProvider { class RelativePatcher { public: static std::unique_ptr<RelativePatcher> Create( - InstructionSet instruction_set, const InstructionSetFeatures* features, - RelativePatcherTargetProvider* provider); + InstructionSet instruction_set, + const InstructionSetFeatures* features, + RelativePatcherThunkProvider* thunk_provider, + RelativePatcherTargetProvider* target_provider); virtual ~RelativePatcher() { } @@ -144,4 +167,4 @@ class RelativePatcher { } // namespace linker } // namespace art -#endif // ART_COMPILER_LINKER_RELATIVE_PATCHER_H_ +#endif // ART_DEX2OAT_LINKER_RELATIVE_PATCHER_H_ diff --git a/compiler/linker/relative_patcher_test.h b/dex2oat/linker/relative_patcher_test.h index d21f2795b98..9cd51e9b46a 100644 --- a/compiler/linker/relative_patcher_test.h +++ b/dex2oat/linker/relative_patcher_test.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef ART_COMPILER_LINKER_RELATIVE_PATCHER_TEST_H_ -#define ART_COMPILER_LINKER_RELATIVE_PATCHER_TEST_H_ +#ifndef ART_DEX2OAT_LINKER_RELATIVE_PATCHER_TEST_H_ +#define ART_DEX2OAT_LINKER_RELATIVE_PATCHER_TEST_H_ #include "arch/instruction_set.h" #include "arch/instruction_set_features.h" @@ -30,9 +30,9 @@ #include "globals.h" #include "gtest/gtest.h" #include "linker/relative_patcher.h" +#include "linker/vector_output_stream.h" #include "oat.h" #include "oat_quick_method_header.h" -#include "vector_output_stream.h" namespace art { namespace linker { @@ -49,8 +49,6 @@ class RelativePatcherTest : public testing::Test { instruction_set, /* instruction_set_features*/ nullptr, /* image_classes */ nullptr, - /* compiled_classes */ nullptr, - /* compiled_methods */ nullptr, /* thread_count */ 1u, /* swap_fd */ -1, /* profile_compilation_info */ nullptr), @@ -58,7 +56,10 @@ class RelativePatcherTest : public testing::Test { instruction_set_(instruction_set), features_(InstructionSetFeatures::FromVariant(instruction_set, variant, &error_msg_)), method_offset_map_(), - patcher_(RelativePatcher::Create(instruction_set, features_.get(), &method_offset_map_)), + patcher_(RelativePatcher::Create(instruction_set, + features_.get(), + &thunk_provider_, + &method_offset_map_)), bss_begin_(0u), compiled_method_refs_(), compiled_methods_(), @@ -248,6 +249,72 @@ class RelativePatcherTest : public testing::Test { LOG(ERROR) << " " << diff_indicator_str; } + class ThunkProvider : public RelativePatcherThunkProvider { + public: + ThunkProvider() {} + + void SetThunkCode(const LinkerPatch& patch, + ArrayRef<const uint8_t> code, + const std::string& debug_name) { + thunk_map_.emplace(ThunkKey(patch), ThunkValue(code, debug_name)); + } + + void GetThunkCode(const LinkerPatch& patch, + /*out*/ ArrayRef<const uint8_t>* code, + /*out*/ std::string* debug_name) OVERRIDE { + auto it = thunk_map_.find(ThunkKey(patch)); + CHECK(it != thunk_map_.end()); + const ThunkValue& value = it->second; + CHECK(code != nullptr); + *code = value.GetCode(); + CHECK(debug_name != nullptr); + *debug_name = value.GetDebugName(); + } + + private: + class ThunkKey { + public: + explicit ThunkKey(const LinkerPatch& patch) + : type_(patch.GetType()), + custom_value1_(patch.GetType() == LinkerPatch::Type::kBakerReadBarrierBranch + ? patch.GetBakerCustomValue1() : 0u), + custom_value2_(patch.GetType() == LinkerPatch::Type::kBakerReadBarrierBranch + ? patch.GetBakerCustomValue2() : 0u) { + CHECK(patch.GetType() == LinkerPatch::Type::kBakerReadBarrierBranch || + patch.GetType() == LinkerPatch::Type::kCallRelative); + } + + bool operator<(const ThunkKey& other) const { + if (custom_value1_ != other.custom_value1_) { + return custom_value1_ < other.custom_value1_; + } + if (custom_value2_ != other.custom_value2_) { + return custom_value2_ < other.custom_value2_; + } + return type_ < other.type_; + } + + private: + const LinkerPatch::Type type_; + const uint32_t custom_value1_; + const uint32_t custom_value2_; + }; + + class ThunkValue { + public: + ThunkValue(ArrayRef<const uint8_t> code, const std::string& debug_name) + : code_(code.begin(), code.end()), debug_name_(debug_name) {} + ArrayRef<const uint8_t> GetCode() const { return ArrayRef<const uint8_t>(code_); } + const std::string& GetDebugName() const { return debug_name_; } + + private: + const std::vector<uint8_t> code_; + const std::string debug_name_; + }; + + std::map<ThunkKey, ThunkValue> thunk_map_; + }; + // Map method reference to assinged offset. // Wrap the map in a class implementing RelativePatcherTargetProvider. class MethodOffsetMap FINAL : public RelativePatcherTargetProvider { @@ -272,6 +339,7 @@ class RelativePatcherTest : public testing::Test { std::string error_msg_; InstructionSet instruction_set_; std::unique_ptr<const InstructionSetFeatures> features_; + ThunkProvider thunk_provider_; MethodOffsetMap method_offset_map_; std::unique_ptr<RelativePatcher> patcher_; uint32_t bss_begin_; @@ -286,4 +354,4 @@ class RelativePatcherTest : public testing::Test { } // namespace linker } // namespace art -#endif // ART_COMPILER_LINKER_RELATIVE_PATCHER_TEST_H_ +#endif // ART_DEX2OAT_LINKER_RELATIVE_PATCHER_TEST_H_ diff --git a/compiler/linker/x86/relative_patcher_x86.cc b/dex2oat/linker/x86/relative_patcher_x86.cc index cdd2cef13ab..cdd2cef13ab 100644 --- a/compiler/linker/x86/relative_patcher_x86.cc +++ b/dex2oat/linker/x86/relative_patcher_x86.cc diff --git a/compiler/linker/x86/relative_patcher_x86.h b/dex2oat/linker/x86/relative_patcher_x86.h index 63a83387223..e723580dae9 100644 --- a/compiler/linker/x86/relative_patcher_x86.h +++ b/dex2oat/linker/x86/relative_patcher_x86.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef ART_COMPILER_LINKER_X86_RELATIVE_PATCHER_X86_H_ -#define ART_COMPILER_LINKER_X86_RELATIVE_PATCHER_X86_H_ +#ifndef ART_DEX2OAT_LINKER_X86_RELATIVE_PATCHER_X86_H_ +#define ART_DEX2OAT_LINKER_X86_RELATIVE_PATCHER_X86_H_ #include "linker/x86/relative_patcher_x86_base.h" @@ -38,4 +38,4 @@ class X86RelativePatcher FINAL : public X86BaseRelativePatcher { } // namespace linker } // namespace art -#endif // ART_COMPILER_LINKER_X86_RELATIVE_PATCHER_X86_H_ +#endif // ART_DEX2OAT_LINKER_X86_RELATIVE_PATCHER_X86_H_ diff --git a/compiler/linker/x86/relative_patcher_x86_base.cc b/dex2oat/linker/x86/relative_patcher_x86_base.cc index 6a9690d7681..6a9690d7681 100644 --- a/compiler/linker/x86/relative_patcher_x86_base.cc +++ b/dex2oat/linker/x86/relative_patcher_x86_base.cc diff --git a/compiler/linker/x86/relative_patcher_x86_base.h b/dex2oat/linker/x86/relative_patcher_x86_base.h index 6097345657d..4cc7b07d2da 100644 --- a/compiler/linker/x86/relative_patcher_x86_base.h +++ b/dex2oat/linker/x86/relative_patcher_x86_base.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef ART_COMPILER_LINKER_X86_RELATIVE_PATCHER_X86_BASE_H_ -#define ART_COMPILER_LINKER_X86_RELATIVE_PATCHER_X86_BASE_H_ +#ifndef ART_DEX2OAT_LINKER_X86_RELATIVE_PATCHER_X86_BASE_H_ +#define ART_DEX2OAT_LINKER_X86_RELATIVE_PATCHER_X86_BASE_H_ #include "linker/relative_patcher.h" @@ -50,4 +50,4 @@ class X86BaseRelativePatcher : public RelativePatcher { } // namespace linker } // namespace art -#endif // ART_COMPILER_LINKER_X86_RELATIVE_PATCHER_X86_BASE_H_ +#endif // ART_DEX2OAT_LINKER_X86_RELATIVE_PATCHER_X86_BASE_H_ diff --git a/compiler/linker/x86/relative_patcher_x86_test.cc b/dex2oat/linker/x86/relative_patcher_x86_test.cc index b855dec91db..b855dec91db 100644 --- a/compiler/linker/x86/relative_patcher_x86_test.cc +++ b/dex2oat/linker/x86/relative_patcher_x86_test.cc diff --git a/compiler/linker/x86_64/relative_patcher_x86_64.cc b/dex2oat/linker/x86_64/relative_patcher_x86_64.cc index 96335649990..96335649990 100644 --- a/compiler/linker/x86_64/relative_patcher_x86_64.cc +++ b/dex2oat/linker/x86_64/relative_patcher_x86_64.cc diff --git a/compiler/linker/x86_64/relative_patcher_x86_64.h b/dex2oat/linker/x86_64/relative_patcher_x86_64.h index 4f3ec498cb8..a31e1ebfbb4 100644 --- a/compiler/linker/x86_64/relative_patcher_x86_64.h +++ b/dex2oat/linker/x86_64/relative_patcher_x86_64.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef ART_COMPILER_LINKER_X86_64_RELATIVE_PATCHER_X86_64_H_ -#define ART_COMPILER_LINKER_X86_64_RELATIVE_PATCHER_X86_64_H_ +#ifndef ART_DEX2OAT_LINKER_X86_64_RELATIVE_PATCHER_X86_64_H_ +#define ART_DEX2OAT_LINKER_X86_64_RELATIVE_PATCHER_X86_64_H_ #include "linker/x86/relative_patcher_x86_base.h" @@ -38,4 +38,4 @@ class X86_64RelativePatcher FINAL : public X86BaseRelativePatcher { } // namespace linker } // namespace art -#endif // ART_COMPILER_LINKER_X86_64_RELATIVE_PATCHER_X86_64_H_ +#endif // ART_DEX2OAT_LINKER_X86_64_RELATIVE_PATCHER_X86_64_H_ diff --git a/compiler/linker/x86_64/relative_patcher_x86_64_test.cc b/dex2oat/linker/x86_64/relative_patcher_x86_64_test.cc index 6baa92de36c..6baa92de36c 100644 --- a/compiler/linker/x86_64/relative_patcher_x86_64_test.cc +++ b/dex2oat/linker/x86_64/relative_patcher_x86_64_test.cc diff --git a/dexdump/dexdump_test.cc b/dexdump/dexdump_test.cc index 63e0e3f20de..3a2d38dec63 100644 --- a/dexdump/dexdump_test.cc +++ b/dexdump/dexdump_test.cc @@ -58,7 +58,7 @@ TEST_F(DexDumpTest, NoInputFileGiven) { TEST_F(DexDumpTest, CantOpenOutput) { std::string error_msg; - ASSERT_FALSE(Exec({"-o", "/joho", dex_file_}, &error_msg)) << error_msg; + ASSERT_FALSE(Exec({"-o", "/non/existent/path", dex_file_}, &error_msg)) << error_msg; } TEST_F(DexDumpTest, BadFlagCombination) { diff --git a/dexlayout/dexlayout.cc b/dexlayout/dexlayout.cc index ec0cbe6a600..7a8c31ba84f 100644 --- a/dexlayout/dexlayout.cc +++ b/dexlayout/dexlayout.cc @@ -34,6 +34,7 @@ #include "android-base/stringprintf.h" #include "base/logging.h" // For VLOG_IS_ON. +#include "base/mem_map.h" #include "base/os.h" #include "base/utils.h" #include "dex/art_dex_file_loader.h" @@ -49,7 +50,6 @@ #include "dex_visualize.h" #include "dex_writer.h" #include "jit/profile_compilation_info.h" -#include "mem_map.h" namespace art { diff --git a/dexlayout/dexlayout_main.cc b/dexlayout/dexlayout_main.cc index f30cfee4ec0..f81d16cbf5c 100644 --- a/dexlayout/dexlayout_main.cc +++ b/dexlayout/dexlayout_main.cc @@ -32,8 +32,8 @@ #include <android-base/logging.h> #include "base/logging.h" // For InitLogging. +#include "base/mem_map.h" #include "jit/profile_compilation_info.h" -#include "mem_map.h" #include "runtime.h" namespace art { diff --git a/dexlist/dexlist_test.cc b/dexlist/dexlist_test.cc index 0b9adbddb8c..68e67134cc5 100644 --- a/dexlist/dexlist_test.cc +++ b/dexlist/dexlist_test.cc @@ -60,7 +60,7 @@ TEST_F(DexListTest, NoInputFileGiven) { TEST_F(DexListTest, CantOpenOutput) { std::string error_msg; - ASSERT_FALSE(Exec({"-o", "/joho", dex_file_}, &error_msg)) << error_msg; + ASSERT_FALSE(Exec({"-o", "/non/existent/path", dex_file_}, &error_msg)) << error_msg; } TEST_F(DexListTest, IllFormedMethod) { diff --git a/disassembler/disassembler_mips.cc b/disassembler/disassembler_mips.cc index b5f5d6ff105..eaf11be7f27 100644 --- a/disassembler/disassembler_mips.cc +++ b/disassembler/disassembler_mips.cc @@ -487,6 +487,7 @@ static const MipsInstruction gMipsInstructions[] = { { kMsaMask | (0xf << 22), kMsa | (0x3 << 22) | 0x19, "copy_u", "yX" }, { kMsaMask | (0xf << 22), kMsa | (0x4 << 22) | 0x19, "insert", "YD" }, { kMsaMask | (0xff << 18), kMsa | (0xc0 << 18) | 0x1e, "fill", "vkD" }, + { kMsaMask | (0xff << 18), kMsa | (0xc1 << 18) | 0x1e, "pcnt", "vkm" }, { kMsaMask | (0x7 << 23), kMsa | (0x6 << 23) | 0x7, "ldi", "kx" }, { kMsaSpecialMask | (0xf << 2), kMsa | (0x8 << 2), "ld", "kw" }, { kMsaSpecialMask | (0xf << 2), kMsa | (0x9 << 2), "st", "kw" }, diff --git a/disassembler/disassembler_x86.cc b/disassembler/disassembler_x86.cc index bbc8e370ea6..dbdde647b2c 100644 --- a/disassembler/disassembler_x86.cc +++ b/disassembler/disassembler_x86.cc @@ -1194,11 +1194,19 @@ DISASSEMBLER_ENTRY(cmp, opcode1 = opcode_tmp.c_str(); } break; + case 0xD8: + case 0xD9: case 0xDA: + case 0xDC: + case 0xDD: case 0xDE: case 0xE0: case 0xE3: + case 0xE8: + case 0xE9: case 0xEA: + case 0xEC: + case 0xED: case 0xEE: if (prefix[2] == 0x66) { src_reg_file = dst_reg_file = SSE; @@ -1207,11 +1215,19 @@ DISASSEMBLER_ENTRY(cmp, src_reg_file = dst_reg_file = MMX; } switch (*instr) { + case 0xD8: opcode1 = "psubusb"; break; + case 0xD9: opcode1 = "psubusw"; break; case 0xDA: opcode1 = "pminub"; break; + case 0xDC: opcode1 = "paddusb"; break; + case 0xDD: opcode1 = "paddusw"; break; case 0xDE: opcode1 = "pmaxub"; break; case 0xE0: opcode1 = "pavgb"; break; case 0xE3: opcode1 = "pavgw"; break; + case 0xE8: opcode1 = "psubsb"; break; + case 0xE9: opcode1 = "psubsw"; break; case 0xEA: opcode1 = "pminsw"; break; + case 0xEC: opcode1 = "paddsb"; break; + case 0xED: opcode1 = "paddsw"; break; case 0xEE: opcode1 = "pmaxsw"; break; } prefix[2] = 0; diff --git a/libartbase/Android.bp b/libartbase/Android.bp index 3c619444770..8cf4d0376de 100644 --- a/libartbase/Android.bp +++ b/libartbase/Android.bp @@ -20,25 +20,54 @@ cc_defaults { host_supported: true, srcs: [ "base/allocator.cc", + "base/arena_allocator.cc", + "base/arena_bit_vector.cc", "base/bit_vector.cc", "base/file_magic.cc", "base/hex_dump.cc", "base/logging.cc", + "base/malloc_arena_pool.cc", + "base/memory_region.cc", + "base/mem_map.cc", "base/os_linux.cc", "base/runtime_debug.cc", "base/safe_copy.cc", + "base/scoped_arena_allocator.cc", "base/scoped_flock.cc", "base/time_utils.cc", "base/unix_file/fd_file.cc", "base/unix_file/random_access_file_utils.cc", "base/utils.cc", + "base/zip_archive.cc", ], + target: { + android: { + static_libs: [ + // ZipArchive support, the order matters here to get all symbols. + "libziparchive", + "libz", + ], + shared_libs: [ + // For android::FileMap used by libziparchive. + "libutils", + ], + }, + host: { + shared_libs: [ + "libziparchive", + "libz", + ], + }, + }, generated_sources: ["art_libartbase_operator_srcs"], cflags: ["-DBUILDING_LIBART=1"], shared_libs: [ + "libbacktrace", + "liblog", + // For ashmem. + "libcutils", // For common macros. "libbase", - "liblog", ], export_include_dirs: ["."], // ART's macros.h depends on libbase's macros.h. @@ -67,7 +96,10 @@ art_cc_library { strip: { keep_symbols: true, }, - shared_libs: ["libbase"], + shared_libs: [ + "libbase", + "libziparchive", + ], export_shared_lib_headers: ["libbase"], } @@ -77,7 +109,10 @@ art_cc_library { "art_debug_defaults", "libartbase_defaults", ], - shared_libs: ["libbase"], + shared_libs: [ + "libbase", + "libziparchive", + ], export_shared_lib_headers: ["libbase"], } @@ -90,6 +125,7 @@ art_cc_test { "art_gtest_defaults", ], srcs: [ + "base/arena_allocator_test.cc", "base/bit_field_test.cc", "base/bit_string_test.cc", "base/bit_struct_test.cc", @@ -98,8 +134,11 @@ art_cc_test { "base/hash_set_test.cc", "base/hex_dump_test.cc", "base/histogram_test.cc", + "base/indenter_test.cc", "base/leb128_test.cc", "base/logging_test.cc", + "base/memory_region_test.cc", + "base/mem_map_test.cc", "base/safe_copy_test.cc", "base/scoped_flock_test.cc", "base/time_utils_test.cc", @@ -108,6 +147,7 @@ art_cc_test { "base/unix_file/fd_file_test.cc", "base/utils_test.cc", "base/variant_map_test.cc", + "base/zip_archive_test.cc", ], shared_libs: [ "libbase", diff --git a/libartbase/base/allocator.cc b/libartbase/base/allocator.cc index a42414507b2..c7be4e01611 100644 --- a/libartbase/base/allocator.cc +++ b/libartbase/base/allocator.cc @@ -76,16 +76,16 @@ namespace TrackedAllocators { // These globals are safe since they don't have any non-trivial destructors. Atomic<size_t> g_bytes_used[kAllocatorTagCount]; -volatile size_t g_max_bytes_used[kAllocatorTagCount]; +Atomic<size_t> g_max_bytes_used[kAllocatorTagCount]; Atomic<uint64_t> g_total_bytes_used[kAllocatorTagCount]; void Dump(std::ostream& os) { if (kEnableTrackingAllocator) { os << "Dumping native memory usage\n"; for (size_t i = 0; i < kAllocatorTagCount; ++i) { - uint64_t bytes_used = g_bytes_used[i].LoadRelaxed(); - uint64_t max_bytes_used = g_max_bytes_used[i]; - uint64_t total_bytes_used = g_total_bytes_used[i].LoadRelaxed(); + uint64_t bytes_used = g_bytes_used[i].load(std::memory_order_relaxed); + uint64_t max_bytes_used = g_max_bytes_used[i].load(std::memory_order_relaxed); + uint64_t total_bytes_used = g_total_bytes_used[i].load(std::memory_order_relaxed); if (total_bytes_used != 0) { os << static_cast<AllocatorTag>(i) << " active=" << bytes_used << " max=" << max_bytes_used << " total=" << total_bytes_used << "\n"; diff --git a/libartbase/base/allocator.h b/libartbase/base/allocator.h index d92fe193e6b..662f78e4489 100644 --- a/libartbase/base/allocator.h +++ b/libartbase/base/allocator.h @@ -71,12 +71,14 @@ std::ostream& operator<<(std::ostream& os, const AllocatorTag& tag); namespace TrackedAllocators { +// We use memory_order_relaxed updates of the following counters. Values are treated as approximate +// wherever concurrent updates are possible. // Running count of number of bytes used for this kind of allocation. Increased by allocations, // decreased by deallocations. extern Atomic<size_t> g_bytes_used[kAllocatorTagCount]; // Largest value of bytes used seen. -extern volatile size_t g_max_bytes_used[kAllocatorTagCount]; +extern Atomic<size_t> g_max_bytes_used[kAllocatorTagCount]; // Total number of bytes allocated of this kind. extern Atomic<uint64_t> g_total_bytes_used[kAllocatorTagCount]; @@ -84,15 +86,17 @@ extern Atomic<uint64_t> g_total_bytes_used[kAllocatorTagCount]; void Dump(std::ostream& os); inline void RegisterAllocation(AllocatorTag tag, size_t bytes) { - g_total_bytes_used[tag].FetchAndAddSequentiallyConsistent(bytes); - size_t new_bytes = g_bytes_used[tag].FetchAndAddSequentiallyConsistent(bytes) + bytes; - if (g_max_bytes_used[tag] < new_bytes) { - g_max_bytes_used[tag] = new_bytes; + g_total_bytes_used[tag].fetch_add(bytes, std::memory_order_relaxed); + size_t new_bytes = g_bytes_used[tag].fetch_add(bytes, std::memory_order_relaxed) + bytes; + size_t max_bytes = g_max_bytes_used[tag].load(std::memory_order_relaxed); + while (max_bytes < new_bytes + && !g_max_bytes_used[tag].compare_exchange_weak(max_bytes /* updated */, new_bytes, + std::memory_order_relaxed)) { } } inline void RegisterFree(AllocatorTag tag, size_t bytes) { - g_bytes_used[tag].FetchAndSubSequentiallyConsistent(bytes); + g_bytes_used[tag].fetch_sub(bytes, std::memory_order_relaxed); } } // namespace TrackedAllocators diff --git a/runtime/base/arena_allocator-inl.h b/libartbase/base/arena_allocator-inl.h index 0e4383741ec..a03e9df3fe4 100644 --- a/runtime/base/arena_allocator-inl.h +++ b/libartbase/base/arena_allocator-inl.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef ART_RUNTIME_BASE_ARENA_ALLOCATOR_INL_H_ -#define ART_RUNTIME_BASE_ARENA_ALLOCATOR_INL_H_ +#ifndef ART_LIBARTBASE_BASE_ARENA_ALLOCATOR_INL_H_ +#define ART_LIBARTBASE_BASE_ARENA_ALLOCATOR_INL_H_ #include "arena_allocator.h" @@ -31,4 +31,4 @@ static constexpr size_t kArenaDefaultSize = kArenaAllocatorPreciseTracking } // namespace arena_allocator } // namespace art -#endif // ART_RUNTIME_BASE_ARENA_ALLOCATOR_INL_H_ +#endif // ART_LIBARTBASE_BASE_ARENA_ALLOCATOR_INL_H_ diff --git a/runtime/base/arena_allocator.cc b/libartbase/base/arena_allocator.cc index 292bde02721..183e5c9f74f 100644 --- a/runtime/base/arena_allocator.cc +++ b/libartbase/base/arena_allocator.cc @@ -25,11 +25,6 @@ #include <android-base/logging.h> -#include "base/systrace.h" -#include "mem_map.h" -#include "mutex.h" -#include "thread-current-inl.h" - namespace art { constexpr size_t kMemoryToolRedZoneBytes = 8; @@ -56,6 +51,7 @@ const char* const ArenaAllocatorStatsImpl<kCount>::kAllocNames[] = { "CtorFenceIns ", "InvokeInputs ", "PhiInputs ", + "TypeCheckIns ", "LoopInfo ", "LIBackEdges ", "TryCatchInf ", @@ -81,6 +77,7 @@ const char* const ArenaAllocatorStatsImpl<kCount>::kAllocNames[] = { "SsaLiveness ", "SsaPhiElim ", "RefTypeProp ", + "SelectGen ", "SideEffects ", "RegAllocator ", "RegAllocVldt ", @@ -189,194 +186,6 @@ void ArenaAllocatorMemoryTool::DoMakeInaccessible(void* ptr, size_t size) { Arena::Arena() : bytes_allocated_(0), memory_(nullptr), size_(0), next_(nullptr) { } -class MallocArena FINAL : public Arena { - public: - explicit MallocArena(size_t size = arena_allocator::kArenaDefaultSize); - virtual ~MallocArena(); - private: - static constexpr size_t RequiredOverallocation() { - return (alignof(std::max_align_t) < ArenaAllocator::kArenaAlignment) - ? ArenaAllocator::kArenaAlignment - alignof(std::max_align_t) - : 0u; - } - - uint8_t* unaligned_memory_; -}; - -MallocArena::MallocArena(size_t size) { - // We need to guarantee kArenaAlignment aligned allocation for the new arena. - // TODO: Use std::aligned_alloc() when it becomes available with C++17. - constexpr size_t overallocation = RequiredOverallocation(); - unaligned_memory_ = reinterpret_cast<uint8_t*>(calloc(1, size + overallocation)); - CHECK(unaligned_memory_ != nullptr); // Abort on OOM. - DCHECK_ALIGNED(unaligned_memory_, alignof(std::max_align_t)); - if (overallocation == 0u) { - memory_ = unaligned_memory_; - } else { - memory_ = AlignUp(unaligned_memory_, ArenaAllocator::kArenaAlignment); - if (UNLIKELY(RUNNING_ON_MEMORY_TOOL > 0)) { - size_t head = memory_ - unaligned_memory_; - size_t tail = overallocation - head; - MEMORY_TOOL_MAKE_NOACCESS(unaligned_memory_, head); - MEMORY_TOOL_MAKE_NOACCESS(memory_ + size, tail); - } - } - DCHECK_ALIGNED(memory_, ArenaAllocator::kArenaAlignment); - size_ = size; -} - -MallocArena::~MallocArena() { - constexpr size_t overallocation = RequiredOverallocation(); - if (overallocation != 0u && UNLIKELY(RUNNING_ON_MEMORY_TOOL > 0)) { - size_t head = memory_ - unaligned_memory_; - size_t tail = overallocation - head; - MEMORY_TOOL_MAKE_UNDEFINED(unaligned_memory_, head); - MEMORY_TOOL_MAKE_UNDEFINED(memory_ + size_, tail); - } - free(reinterpret_cast<void*>(unaligned_memory_)); -} - -class MemMapArena FINAL : public Arena { - public: - MemMapArena(size_t size, bool low_4gb, const char* name); - virtual ~MemMapArena(); - void Release() OVERRIDE; - - private: - std::unique_ptr<MemMap> map_; -}; - -MemMapArena::MemMapArena(size_t size, bool low_4gb, const char* name) { - // Round up to a full page as that's the smallest unit of allocation for mmap() - // and we want to be able to use all memory that we actually allocate. - size = RoundUp(size, kPageSize); - std::string error_msg; - map_.reset(MemMap::MapAnonymous( - name, nullptr, size, PROT_READ | PROT_WRITE, low_4gb, false, &error_msg)); - CHECK(map_.get() != nullptr) << error_msg; - memory_ = map_->Begin(); - static_assert(ArenaAllocator::kArenaAlignment <= kPageSize, - "Arena should not need stronger alignment than kPageSize."); - DCHECK_ALIGNED(memory_, ArenaAllocator::kArenaAlignment); - size_ = map_->Size(); -} - -MemMapArena::~MemMapArena() { - // Destroys MemMap via std::unique_ptr<>. -} - -void MemMapArena::Release() { - if (bytes_allocated_ > 0) { - map_->MadviseDontNeedAndZero(); - bytes_allocated_ = 0; - } -} - -void Arena::Reset() { - if (bytes_allocated_ > 0) { - memset(Begin(), 0, bytes_allocated_); - bytes_allocated_ = 0; - } -} - -ArenaPool::ArenaPool(bool use_malloc, bool low_4gb, const char* name) - : use_malloc_(use_malloc), - lock_("Arena pool lock", kArenaPoolLock), - free_arenas_(nullptr), - low_4gb_(low_4gb), - name_(name) { - if (low_4gb) { - CHECK(!use_malloc) << "low4gb must use map implementation"; - } - if (!use_malloc) { - MemMap::Init(); - } -} - -ArenaPool::~ArenaPool() { - ReclaimMemory(); -} - -void ArenaPool::ReclaimMemory() { - while (free_arenas_ != nullptr) { - Arena* arena = free_arenas_; - free_arenas_ = free_arenas_->next_; - delete arena; - } -} - -void ArenaPool::LockReclaimMemory() { - MutexLock lock(Thread::Current(), lock_); - ReclaimMemory(); -} - -Arena* ArenaPool::AllocArena(size_t size) { - Thread* self = Thread::Current(); - Arena* ret = nullptr; - { - MutexLock lock(self, lock_); - if (free_arenas_ != nullptr && LIKELY(free_arenas_->Size() >= size)) { - ret = free_arenas_; - free_arenas_ = free_arenas_->next_; - } - } - if (ret == nullptr) { - ret = use_malloc_ ? static_cast<Arena*>(new MallocArena(size)) : - new MemMapArena(size, low_4gb_, name_); - } - ret->Reset(); - return ret; -} - -void ArenaPool::TrimMaps() { - if (!use_malloc_) { - ScopedTrace trace(__PRETTY_FUNCTION__); - // Doesn't work for malloc. - MutexLock lock(Thread::Current(), lock_); - for (Arena* arena = free_arenas_; arena != nullptr; arena = arena->next_) { - arena->Release(); - } - } -} - -size_t ArenaPool::GetBytesAllocated() const { - size_t total = 0; - MutexLock lock(Thread::Current(), lock_); - for (Arena* arena = free_arenas_; arena != nullptr; arena = arena->next_) { - total += arena->GetBytesAllocated(); - } - return total; -} - -void ArenaPool::FreeArenaChain(Arena* first) { - if (UNLIKELY(RUNNING_ON_MEMORY_TOOL > 0)) { - for (Arena* arena = first; arena != nullptr; arena = arena->next_) { - MEMORY_TOOL_MAKE_UNDEFINED(arena->memory_, arena->bytes_allocated_); - } - } - - if (arena_allocator::kArenaAllocatorPreciseTracking) { - // Do not reuse arenas when tracking. - while (first != nullptr) { - Arena* next = first->next_; - delete first; - first = next; - } - return; - } - - if (first != nullptr) { - Arena* last = first; - while (last->next_ != nullptr) { - last = last->next_; - } - Thread* self = Thread::Current(); - MutexLock lock(self, lock_); - last->next_ = free_arenas_; - free_arenas_ = first; - } -} - size_t ArenaAllocator::BytesAllocated() const { return ArenaAllocatorStats::BytesAllocated(); } diff --git a/runtime/base/arena_allocator.h b/libartbase/base/arena_allocator.h index c3011091e85..4d535df1d43 100644 --- a/runtime/base/arena_allocator.h +++ b/libartbase/base/arena_allocator.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef ART_RUNTIME_BASE_ARENA_ALLOCATOR_H_ -#define ART_RUNTIME_BASE_ARENA_ALLOCATOR_H_ +#ifndef ART_LIBARTBASE_BASE_ARENA_ALLOCATOR_H_ +#define ART_LIBARTBASE_BASE_ARENA_ALLOCATOR_H_ #include <stddef.h> #include <stdint.h> @@ -25,7 +25,6 @@ #include "base/dchecked_vector.h" #include "base/macros.h" #include "base/memory_tool.h" -#include "mutex.h" namespace art { @@ -62,6 +61,7 @@ enum ArenaAllocKind { kArenaAllocConstructorFenceInputs, kArenaAllocInvokeInputs, kArenaAllocPhiInputs, + kArenaAllocTypeCheckInputs, kArenaAllocLoopInfo, kArenaAllocLoopInfoBackEdges, kArenaAllocTryCatchInfo, @@ -87,6 +87,7 @@ enum ArenaAllocKind { kArenaAllocSsaLiveness, kArenaAllocSsaPhiElimination, kArenaAllocReferenceTypePropagation, + kArenaAllocSelectGenerator, kArenaAllocSideEffectsAnalysis, kArenaAllocRegisterAllocator, kArenaAllocRegisterAllocatorValidate, @@ -235,7 +236,8 @@ class Arena { uint8_t* memory_; size_t size_; Arena* next_; - friend class ArenaPool; + friend class MallocArenaPool; + friend class MemMapArenaPool; friend class ArenaAllocator; friend class ArenaStack; friend class ScopedArenaAllocator; @@ -249,25 +251,20 @@ class Arena { class ArenaPool { public: - explicit ArenaPool(bool use_malloc = true, - bool low_4gb = false, - const char* name = "LinearAlloc"); - ~ArenaPool(); - Arena* AllocArena(size_t size) REQUIRES(!lock_); - void FreeArenaChain(Arena* first) REQUIRES(!lock_); - size_t GetBytesAllocated() const REQUIRES(!lock_); - void ReclaimMemory() NO_THREAD_SAFETY_ANALYSIS; - void LockReclaimMemory() REQUIRES(!lock_); - // Trim the maps in arenas by madvising, used by JIT to reduce memory usage. This only works - // use_malloc is false. - void TrimMaps() REQUIRES(!lock_); + virtual ~ArenaPool() = default; + + virtual Arena* AllocArena(size_t size) = 0; + virtual void FreeArenaChain(Arena* first) = 0; + virtual size_t GetBytesAllocated() const = 0; + virtual void ReclaimMemory() = 0; + virtual void LockReclaimMemory() = 0; + // Trim the maps in arenas by madvising, used by JIT to reduce memory usage. + virtual void TrimMaps() = 0; + + protected: + ArenaPool() = default; private: - const bool use_malloc_; - mutable Mutex lock_ DEFAULT_MUTEX_ACQUIRED_AFTER; - Arena* free_arenas_ GUARDED_BY(lock_); - const bool low_4gb_; - const char* name_; DISALLOW_COPY_AND_ASSIGN(ArenaPool); }; @@ -427,4 +424,4 @@ class MemStats { } // namespace art -#endif // ART_RUNTIME_BASE_ARENA_ALLOCATOR_H_ +#endif // ART_LIBARTBASE_BASE_ARENA_ALLOCATOR_H_ diff --git a/runtime/base/arena_allocator_test.cc b/libartbase/base/arena_allocator_test.cc index 68e26af9f8b..68e99f4be04 100644 --- a/runtime/base/arena_allocator_test.cc +++ b/libartbase/base/arena_allocator_test.cc @@ -16,6 +16,7 @@ #include "base/arena_allocator-inl.h" #include "base/arena_bit_vector.h" +#include "base/malloc_arena_pool.h" #include "base/memory_tool.h" #include "gtest/gtest.h" @@ -33,7 +34,7 @@ class ArenaAllocatorTest : public testing::Test { }; TEST_F(ArenaAllocatorTest, Test) { - ArenaPool pool; + MallocArenaPool pool; ArenaAllocator allocator(&pool); ArenaBitVector bv(&allocator, 10, true); bv.SetBit(5); @@ -44,7 +45,7 @@ TEST_F(ArenaAllocatorTest, Test) { TEST_F(ArenaAllocatorTest, MakeDefined) { // Regression test to make sure we mark the allocated area defined. - ArenaPool pool; + MallocArenaPool pool; static constexpr size_t kSmallArraySize = 10; static constexpr size_t kLargeArraySize = 50; uint32_t* small_array; @@ -71,7 +72,7 @@ TEST_F(ArenaAllocatorTest, LargeAllocations) { } { - ArenaPool pool; + MallocArenaPool pool; ArenaAllocator allocator(&pool); // Note: Leaving some space for memory tool red zones. void* alloc1 = allocator.Alloc(arena_allocator::kArenaDefaultSize * 5 / 8); @@ -80,7 +81,7 @@ TEST_F(ArenaAllocatorTest, LargeAllocations) { ASSERT_EQ(1u, NumberOfArenas(&allocator)); } { - ArenaPool pool; + MallocArenaPool pool; ArenaAllocator allocator(&pool); void* alloc1 = allocator.Alloc(arena_allocator::kArenaDefaultSize * 13 / 16); void* alloc2 = allocator.Alloc(arena_allocator::kArenaDefaultSize * 11 / 16); @@ -92,7 +93,7 @@ TEST_F(ArenaAllocatorTest, LargeAllocations) { ASSERT_EQ(3u, NumberOfArenas(&allocator)); } { - ArenaPool pool; + MallocArenaPool pool; ArenaAllocator allocator(&pool); void* alloc1 = allocator.Alloc(arena_allocator::kArenaDefaultSize * 13 / 16); void* alloc2 = allocator.Alloc(arena_allocator::kArenaDefaultSize * 9 / 16); @@ -105,7 +106,7 @@ TEST_F(ArenaAllocatorTest, LargeAllocations) { ASSERT_EQ(2u, NumberOfArenas(&allocator)); } { - ArenaPool pool; + MallocArenaPool pool; ArenaAllocator allocator(&pool); void* alloc1 = allocator.Alloc(arena_allocator::kArenaDefaultSize * 9 / 16); void* alloc2 = allocator.Alloc(arena_allocator::kArenaDefaultSize * 13 / 16); @@ -118,7 +119,7 @@ TEST_F(ArenaAllocatorTest, LargeAllocations) { ASSERT_EQ(2u, NumberOfArenas(&allocator)); } { - ArenaPool pool; + MallocArenaPool pool; ArenaAllocator allocator(&pool); // Note: Leaving some space for memory tool red zones. for (size_t i = 0; i != 15; ++i) { @@ -133,7 +134,7 @@ TEST_F(ArenaAllocatorTest, LargeAllocations) { } TEST_F(ArenaAllocatorTest, AllocAlignment) { - ArenaPool pool; + MallocArenaPool pool; ArenaAllocator allocator(&pool); for (size_t iterations = 0; iterations <= 10; ++iterations) { for (size_t size = 1; size <= ArenaAllocator::kAlignment + 1; ++size) { @@ -153,7 +154,7 @@ TEST_F(ArenaAllocatorTest, ReallocReuse) { { // Case 1: small aligned allocation, aligned extend inside arena. - ArenaPool pool; + MallocArenaPool pool; ArenaAllocator allocator(&pool); const size_t original_size = ArenaAllocator::kAlignment * 2; @@ -166,7 +167,7 @@ TEST_F(ArenaAllocatorTest, ReallocReuse) { { // Case 2: small aligned allocation, non-aligned extend inside arena. - ArenaPool pool; + MallocArenaPool pool; ArenaAllocator allocator(&pool); const size_t original_size = ArenaAllocator::kAlignment * 2; @@ -179,7 +180,7 @@ TEST_F(ArenaAllocatorTest, ReallocReuse) { { // Case 3: small non-aligned allocation, aligned extend inside arena. - ArenaPool pool; + MallocArenaPool pool; ArenaAllocator allocator(&pool); const size_t original_size = ArenaAllocator::kAlignment * 2 + (ArenaAllocator::kAlignment / 2); @@ -192,7 +193,7 @@ TEST_F(ArenaAllocatorTest, ReallocReuse) { { // Case 4: small non-aligned allocation, aligned non-extend inside arena. - ArenaPool pool; + MallocArenaPool pool; ArenaAllocator allocator(&pool); const size_t original_size = ArenaAllocator::kAlignment * 2 + (ArenaAllocator::kAlignment / 2); @@ -208,7 +209,7 @@ TEST_F(ArenaAllocatorTest, ReallocReuse) { { // Case 5: large allocation, aligned extend into next arena. - ArenaPool pool; + MallocArenaPool pool; ArenaAllocator allocator(&pool); const size_t original_size = arena_allocator::kArenaDefaultSize - @@ -222,7 +223,7 @@ TEST_F(ArenaAllocatorTest, ReallocReuse) { { // Case 6: large allocation, non-aligned extend into next arena. - ArenaPool pool; + MallocArenaPool pool; ArenaAllocator allocator(&pool); const size_t original_size = arena_allocator::kArenaDefaultSize - @@ -241,7 +242,7 @@ TEST_F(ArenaAllocatorTest, ReallocReuse) { TEST_F(ArenaAllocatorTest, ReallocAlignment) { { // Case 1: small aligned allocation, aligned extend inside arena. - ArenaPool pool; + MallocArenaPool pool; ArenaAllocator allocator(&pool); const size_t original_size = ArenaAllocator::kAlignment * 2; @@ -258,7 +259,7 @@ TEST_F(ArenaAllocatorTest, ReallocAlignment) { { // Case 2: small aligned allocation, non-aligned extend inside arena. - ArenaPool pool; + MallocArenaPool pool; ArenaAllocator allocator(&pool); const size_t original_size = ArenaAllocator::kAlignment * 2; @@ -275,7 +276,7 @@ TEST_F(ArenaAllocatorTest, ReallocAlignment) { { // Case 3: small non-aligned allocation, aligned extend inside arena. - ArenaPool pool; + MallocArenaPool pool; ArenaAllocator allocator(&pool); const size_t original_size = ArenaAllocator::kAlignment * 2 + (ArenaAllocator::kAlignment / 2); @@ -292,7 +293,7 @@ TEST_F(ArenaAllocatorTest, ReallocAlignment) { { // Case 4: small non-aligned allocation, aligned non-extend inside arena. - ArenaPool pool; + MallocArenaPool pool; ArenaAllocator allocator(&pool); const size_t original_size = ArenaAllocator::kAlignment * 2 + (ArenaAllocator::kAlignment / 2); @@ -312,7 +313,7 @@ TEST_F(ArenaAllocatorTest, ReallocAlignment) { { // Case 5: large allocation, aligned extend into next arena. - ArenaPool pool; + MallocArenaPool pool; ArenaAllocator allocator(&pool); const size_t original_size = arena_allocator::kArenaDefaultSize - @@ -330,7 +331,7 @@ TEST_F(ArenaAllocatorTest, ReallocAlignment) { { // Case 6: large allocation, non-aligned extend into next arena. - ArenaPool pool; + MallocArenaPool pool; ArenaAllocator allocator(&pool); const size_t original_size = arena_allocator::kArenaDefaultSize - diff --git a/runtime/base/arena_bit_vector.cc b/libartbase/base/arena_bit_vector.cc index 1542e9d5f71..1542e9d5f71 100644 --- a/runtime/base/arena_bit_vector.cc +++ b/libartbase/base/arena_bit_vector.cc diff --git a/runtime/base/arena_bit_vector.h b/libartbase/base/arena_bit_vector.h index ca1d5b1d668..2b2322e74fe 100644 --- a/runtime/base/arena_bit_vector.h +++ b/libartbase/base/arena_bit_vector.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef ART_RUNTIME_BASE_ARENA_BIT_VECTOR_H_ -#define ART_RUNTIME_BASE_ARENA_BIT_VECTOR_H_ +#ifndef ART_LIBARTBASE_BASE_ARENA_BIT_VECTOR_H_ +#define ART_LIBARTBASE_BASE_ARENA_BIT_VECTOR_H_ #include "base/arena_object.h" #include "base/bit_vector.h" @@ -55,4 +55,4 @@ class ArenaBitVector : public BitVector, public ArenaObject<kArenaAllocGrowableB } // namespace art -#endif // ART_RUNTIME_BASE_ARENA_BIT_VECTOR_H_ +#endif // ART_LIBARTBASE_BASE_ARENA_BIT_VECTOR_H_ diff --git a/runtime/base/arena_containers.h b/libartbase/base/arena_containers.h index 4f572120ba5..40cf23c9fb8 100644 --- a/runtime/base/arena_containers.h +++ b/libartbase/base/arena_containers.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef ART_RUNTIME_BASE_ARENA_CONTAINERS_H_ -#define ART_RUNTIME_BASE_ARENA_CONTAINERS_H_ +#ifndef ART_LIBARTBASE_BASE_ARENA_CONTAINERS_H_ +#define ART_LIBARTBASE_BASE_ARENA_CONTAINERS_H_ #include <deque> #include <queue> @@ -241,4 +241,4 @@ inline ArenaAllocatorAdapter<void> ArenaAllocator::Adapter(ArenaAllocKind kind) } // namespace art -#endif // ART_RUNTIME_BASE_ARENA_CONTAINERS_H_ +#endif // ART_LIBARTBASE_BASE_ARENA_CONTAINERS_H_ diff --git a/runtime/base/arena_object.h b/libartbase/base/arena_object.h index d01e346f7ad..de7cb64a75e 100644 --- a/runtime/base/arena_object.h +++ b/libartbase/base/arena_object.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef ART_RUNTIME_BASE_ARENA_OBJECT_H_ -#define ART_RUNTIME_BASE_ARENA_OBJECT_H_ +#ifndef ART_LIBARTBASE_BASE_ARENA_OBJECT_H_ +#define ART_LIBARTBASE_BASE_ARENA_OBJECT_H_ #include <android-base/logging.h> @@ -69,4 +69,4 @@ class DeletableArenaObject { } // namespace art -#endif // ART_RUNTIME_BASE_ARENA_OBJECT_H_ +#endif // ART_LIBARTBASE_BASE_ARENA_OBJECT_H_ diff --git a/libartbase/base/atomic.h b/libartbase/base/atomic.h index fd34cc61433..f736667ca85 100644 --- a/libartbase/base/atomic.h +++ b/libartbase/base/atomic.h @@ -35,94 +35,28 @@ class PACKED(sizeof(T)) Atomic : public std::atomic<T> { explicit Atomic<T>(T value) : std::atomic<T>(value) { } - // Load from memory without ordering or synchronization constraints. - T LoadRelaxed() const { - return this->load(std::memory_order_relaxed); - } - - // Load from memory with acquire ordering. - T LoadAcquire() const { - return this->load(std::memory_order_acquire); - } - - // Word tearing allowed, but may race. - // TODO: Optimize? - // There has been some discussion of eventually disallowing word - // tearing for Java data loads. + // Load data from an atomic variable with Java data memory order semantics. + // + // Promises memory access semantics of ordinary Java data. + // Does not order other memory accesses. + // Long and double accesses may be performed 32 bits at a time. + // There are no "cache coherence" guarantees; e.g. loads from the same location may be reordered. + // In contrast to normal C++ accesses, racing accesses are allowed. T LoadJavaData() const { return this->load(std::memory_order_relaxed); } - // Load from memory with a total ordering. - // Corresponds exactly to a Java volatile load. - T LoadSequentiallyConsistent() const { - return this->load(std::memory_order_seq_cst); - } - - // Store to memory without ordering or synchronization constraints. - void StoreRelaxed(T desired_value) { - this->store(desired_value, std::memory_order_relaxed); - } - - // Word tearing allowed, but may race. + // Store data in an atomic variable with Java data memory ordering semantics. + // + // Promises memory access semantics of ordinary Java data. + // Does not order other memory accesses. + // Long and double accesses may be performed 32 bits at a time. + // There are no "cache coherence" guarantees; e.g. loads from the same location may be reordered. + // In contrast to normal C++ accesses, racing accesses are allowed. void StoreJavaData(T desired_value) { this->store(desired_value, std::memory_order_relaxed); } - // Store to memory with release ordering. - void StoreRelease(T desired_value) { - this->store(desired_value, std::memory_order_release); - } - - // Store to memory with a total ordering. - void StoreSequentiallyConsistent(T desired_value) { - this->store(desired_value, std::memory_order_seq_cst); - } - - // Atomically replace the value with desired_value. - T ExchangeRelaxed(T desired_value) { - return this->exchange(desired_value, std::memory_order_relaxed); - } - - // Atomically replace the value with desired_value. - T ExchangeSequentiallyConsistent(T desired_value) { - return this->exchange(desired_value, std::memory_order_seq_cst); - } - - // Atomically replace the value with desired_value. - T ExchangeAcquire(T desired_value) { - return this->exchange(desired_value, std::memory_order_acquire); - } - - // Atomically replace the value with desired_value. - T ExchangeRelease(T desired_value) { - return this->exchange(desired_value, std::memory_order_release); - } - - // Atomically replace the value with desired_value if it matches the expected_value. - // Participates in total ordering of atomic operations. Returns true on success, false otherwise. - // If the value does not match, updates the expected_value argument with the value that was - // atomically read for the failed comparison. - bool CompareAndExchangeStrongSequentiallyConsistent(T* expected_value, T desired_value) { - return this->compare_exchange_strong(*expected_value, desired_value, std::memory_order_seq_cst); - } - - // Atomically replace the value with desired_value if it matches the expected_value. - // Participates in total ordering of atomic operations. Returns true on success, false otherwise. - // If the value does not match, updates the expected_value argument with the value that was - // atomically read for the failed comparison. - bool CompareAndExchangeStrongAcquire(T* expected_value, T desired_value) { - return this->compare_exchange_strong(*expected_value, desired_value, std::memory_order_acquire); - } - - // Atomically replace the value with desired_value if it matches the expected_value. - // Participates in total ordering of atomic operations. Returns true on success, false otherwise. - // If the value does not match, updates the expected_value argument with the value that was - // atomically read for the failed comparison. - bool CompareAndExchangeStrongRelease(T* expected_value, T desired_value) { - return this->compare_exchange_strong(*expected_value, desired_value, std::memory_order_release); - } - // Atomically replace the value with desired_value if it matches the expected_value. // Participates in total ordering of atomic operations. bool CompareAndSetStrongSequentiallyConsistent(T expected_value, T desired_value) { @@ -166,66 +100,8 @@ class PACKED(sizeof(T)) Atomic : public std::atomic<T> { return this->compare_exchange_weak(expected_value, desired_value, std::memory_order_release); } - T FetchAndAddSequentiallyConsistent(const T value) { - return this->fetch_add(value, std::memory_order_seq_cst); // Return old_value. - } - - T FetchAndAddRelaxed(const T value) { - return this->fetch_add(value, std::memory_order_relaxed); // Return old_value. - } - - T FetchAndAddAcquire(const T value) { - return this->fetch_add(value, std::memory_order_acquire); // Return old_value. - } - - T FetchAndAddRelease(const T value) { - return this->fetch_add(value, std::memory_order_acquire); // Return old_value. - } - - T FetchAndSubSequentiallyConsistent(const T value) { - return this->fetch_sub(value, std::memory_order_seq_cst); // Return old value. - } - - T FetchAndSubRelaxed(const T value) { - return this->fetch_sub(value, std::memory_order_relaxed); // Return old value. - } - - T FetchAndBitwiseAndSequentiallyConsistent(const T value) { - return this->fetch_and(value, std::memory_order_seq_cst); // Return old_value. - } - - T FetchAndBitwiseAndAcquire(const T value) { - return this->fetch_and(value, std::memory_order_acquire); // Return old_value. - } - - T FetchAndBitwiseAndRelease(const T value) { - return this->fetch_and(value, std::memory_order_release); // Return old_value. - } - - T FetchAndBitwiseOrSequentiallyConsistent(const T value) { - return this->fetch_or(value, std::memory_order_seq_cst); // Return old_value. - } - - T FetchAndBitwiseOrAcquire(const T value) { - return this->fetch_or(value, std::memory_order_acquire); // Return old_value. - } - - T FetchAndBitwiseOrRelease(const T value) { - return this->fetch_or(value, std::memory_order_release); // Return old_value. - } - - T FetchAndBitwiseXorSequentiallyConsistent(const T value) { - return this->fetch_xor(value, std::memory_order_seq_cst); // Return old_value. - } - - T FetchAndBitwiseXorAcquire(const T value) { - return this->fetch_xor(value, std::memory_order_acquire); // Return old_value. - } - - T FetchAndBitwiseXorRelease(const T value) { - return this->fetch_xor(value, std::memory_order_release); // Return old_value. - } - + // Returns the address of the current atomic variable. This is only used by futex() which is + // declared to take a volatile address (see base/mutex-inl.h). volatile T* Address() { return reinterpret_cast<T*>(this); } diff --git a/runtime/bit_memory_region.h b/libartbase/base/bit_memory_region.h index 3a696f19694..f3926bc9d81 100644 --- a/runtime/bit_memory_region.h +++ b/libartbase/base/bit_memory_region.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef ART_RUNTIME_BIT_MEMORY_REGION_H_ -#define ART_RUNTIME_BIT_MEMORY_REGION_H_ +#ifndef ART_LIBARTBASE_BASE_BIT_MEMORY_REGION_H_ +#define ART_LIBARTBASE_BASE_BIT_MEMORY_REGION_H_ #include "memory_region.h" @@ -70,4 +70,4 @@ class BitMemoryRegion FINAL : public ValueObject { } // namespace art -#endif // ART_RUNTIME_BIT_MEMORY_REGION_H_ +#endif // ART_LIBARTBASE_BASE_BIT_MEMORY_REGION_H_ diff --git a/runtime/base/dumpable.h b/libartbase/base/dumpable.h index 9ef8d69b489..66213972f78 100644 --- a/runtime/base/dumpable.h +++ b/libartbase/base/dumpable.h @@ -14,13 +14,12 @@ * limitations under the License. */ -#ifndef ART_RUNTIME_BASE_DUMPABLE_H_ -#define ART_RUNTIME_BASE_DUMPABLE_H_ +#ifndef ART_LIBARTBASE_BASE_DUMPABLE_H_ +#define ART_LIBARTBASE_BASE_DUMPABLE_H_ #include <ostream> #include "base/macros.h" -#include "base/mutex.h" namespace art { @@ -51,27 +50,6 @@ std::ostream& operator<<(std::ostream& os, const Dumpable<T>& rhs) { return os; } -template<typename T> -class MutatorLockedDumpable { - public: - explicit MutatorLockedDumpable(T& value) REQUIRES_SHARED(Locks::mutator_lock_) : value_(value) {} - - void Dump(std::ostream& os) const REQUIRES_SHARED(Locks::mutator_lock_) { - value_.Dump(os); - } - - private: - const T& value_; - - DISALLOW_COPY_AND_ASSIGN(MutatorLockedDumpable); -}; - -template<typename T> -std::ostream& operator<<(std::ostream& os, const MutatorLockedDumpable<T>& rhs) - // TODO: should be REQUIRES_SHARED(Locks::mutator_lock_) however annotalysis - // currently fails for this. - NO_THREAD_SAFETY_ANALYSIS; - } // namespace art -#endif // ART_RUNTIME_BASE_DUMPABLE_H_ +#endif // ART_LIBARTBASE_BASE_DUMPABLE_H_ diff --git a/runtime/indenter.h b/libartbase/base/indenter.h index 6361dd2092c..850b7c4f737 100644 --- a/runtime/indenter.h +++ b/libartbase/base/indenter.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef ART_RUNTIME_INDENTER_H_ -#define ART_RUNTIME_INDENTER_H_ +#ifndef ART_LIBARTBASE_BASE_INDENTER_H_ +#define ART_LIBARTBASE_BASE_INDENTER_H_ #include <ostream> #include <streambuf> @@ -160,4 +160,4 @@ class ScopedIndentation { } // namespace art -#endif // ART_RUNTIME_INDENTER_H_ +#endif // ART_LIBARTBASE_BASE_INDENTER_H_ diff --git a/runtime/indenter_test.cc b/libartbase/base/indenter_test.cc index 09c0c54e5af..09c0c54e5af 100644 --- a/runtime/indenter_test.cc +++ b/libartbase/base/indenter_test.cc diff --git a/libartbase/base/logging.cc b/libartbase/base/logging.cc index 37b1f8264a7..fd2cc20af23 100644 --- a/libartbase/base/logging.cc +++ b/libartbase/base/logging.cc @@ -21,6 +21,8 @@ #include <sstream> #include "aborting.h" +#include "base/os.h" +#include "base/unix_file/fd_file.h" // Headers for LogMessage::LogLine. #ifdef ART_TARGET_ANDROID @@ -140,4 +142,57 @@ void LogHelper::LogLineLowStack(const char* file, #endif // ART_TARGET_ANDROID } +bool PrintFileToLog(const std::string& file_name, android::base::LogSeverity level) { + File file(file_name, O_RDONLY, false); + if (!file.IsOpened()) { + return false; + } + + constexpr size_t kBufSize = 256; // Small buffer. Avoid stack overflow and stack size warnings. + char buf[kBufSize + 1]; // +1 for terminator. + size_t filled_to = 0; + while (true) { + DCHECK_LT(filled_to, kBufSize); + int64_t n = TEMP_FAILURE_RETRY(read(file.Fd(), &buf[filled_to], kBufSize - filled_to)); + if (n <= 0) { + // Print the rest of the buffer, if it exists. + if (filled_to > 0) { + buf[filled_to] = 0; + LOG(level) << buf; + } + return n == 0; + } + // Scan for '\n'. + size_t i = filled_to; + bool found_newline = false; + for (; i < filled_to + n; ++i) { + if (buf[i] == '\n') { + // Found a line break, that's something to print now. + buf[i] = 0; + LOG(level) << buf; + // Copy the rest to the front. + if (i + 1 < filled_to + n) { + memmove(&buf[0], &buf[i + 1], filled_to + n - i - 1); + filled_to = filled_to + n - i - 1; + } else { + filled_to = 0; + } + found_newline = true; + break; + } + } + if (found_newline) { + continue; + } else { + filled_to += n; + // Check if we must flush now. + if (filled_to == kBufSize) { + buf[kBufSize] = 0; + LOG(level) << buf; + filled_to = 0; + } + } + } +} + } // namespace art diff --git a/libartbase/base/logging.h b/libartbase/base/logging.h index fd5fc743832..986704ec982 100644 --- a/libartbase/base/logging.h +++ b/libartbase/base/logging.h @@ -98,6 +98,9 @@ class LogHelper { DISALLOW_COPY_AND_ASSIGN(LogHelper); }; +// Copy the contents of file_name to the log stream for level. +bool PrintFileToLog(const std::string& file_name, android::base::LogSeverity level); + // Is verbose logging enabled for the given module? Where the module is defined in LogVerbosity. #define VLOG_IS_ON(module) UNLIKELY(::art::gLogVerbosity.module) diff --git a/libartbase/base/logging_test.cc b/libartbase/base/logging_test.cc index 404e080b03c..1456eb30faa 100644 --- a/libartbase/base/logging_test.cc +++ b/libartbase/base/logging_test.cc @@ -21,7 +21,7 @@ #include "android-base/logging.h" #include "base/bit_utils.h" #include "base/macros.h" -#include "common_runtime_test.h" +#include "gtest/gtest.h" #include "runtime_debug.h" namespace art { @@ -31,9 +31,9 @@ static void SimpleAborter(const char* msg) { _exit(1); } -class LoggingTest : public CommonRuntimeTest { +class LoggingTest : public testing::Test { protected: - void PostRuntimeCreate() OVERRIDE { + LoggingTest() { // In our abort tests we really don't want the runtime to create a real dump. android::base::SetAborter(SimpleAborter); } diff --git a/libartbase/base/macros.h b/libartbase/base/macros.h index 6dd981dc9ef..f26cf0708b2 100644 --- a/libartbase/base/macros.h +++ b/libartbase/base/macros.h @@ -45,8 +45,6 @@ template<typename T> ART_FRIEND_TEST(test_set_name, individual_test) private: \ void* operator new(size_t) = delete // NOLINT -#define SIZEOF_MEMBER(t, f) sizeof((reinterpret_cast<t*>(4096))->f) // NOLINT - #define OFFSETOF_MEMBER(t, f) \ (reinterpret_cast<uintptr_t>(&reinterpret_cast<t*>(16)->f) - static_cast<uintptr_t>(16u)) // NOLINT diff --git a/libartbase/base/malloc_arena_pool.cc b/libartbase/base/malloc_arena_pool.cc new file mode 100644 index 00000000000..7df4aef25be --- /dev/null +++ b/libartbase/base/malloc_arena_pool.cc @@ -0,0 +1,162 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "malloc_arena_pool.h" + +#include <sys/mman.h> + +#include <algorithm> +#include <cstddef> +#include <iomanip> +#include <numeric> + +#include <android-base/logging.h> +#include "base/arena_allocator-inl.h" + +namespace art { + +class MallocArena FINAL : public Arena { + public: + explicit MallocArena(size_t size = arena_allocator::kArenaDefaultSize); + virtual ~MallocArena(); + private: + static constexpr size_t RequiredOverallocation() { + return (alignof(std::max_align_t) < ArenaAllocator::kArenaAlignment) + ? ArenaAllocator::kArenaAlignment - alignof(std::max_align_t) + : 0u; + } + + uint8_t* unaligned_memory_; +}; + +MallocArena::MallocArena(size_t size) { + // We need to guarantee kArenaAlignment aligned allocation for the new arena. + // TODO: Use std::aligned_alloc() when it becomes available with C++17. + constexpr size_t overallocation = RequiredOverallocation(); + unaligned_memory_ = reinterpret_cast<uint8_t*>(calloc(1, size + overallocation)); + CHECK(unaligned_memory_ != nullptr); // Abort on OOM. + DCHECK_ALIGNED(unaligned_memory_, alignof(std::max_align_t)); + if (overallocation == 0u) { + memory_ = unaligned_memory_; + } else { + memory_ = AlignUp(unaligned_memory_, ArenaAllocator::kArenaAlignment); + if (UNLIKELY(RUNNING_ON_MEMORY_TOOL > 0)) { + size_t head = memory_ - unaligned_memory_; + size_t tail = overallocation - head; + MEMORY_TOOL_MAKE_NOACCESS(unaligned_memory_, head); + MEMORY_TOOL_MAKE_NOACCESS(memory_ + size, tail); + } + } + DCHECK_ALIGNED(memory_, ArenaAllocator::kArenaAlignment); + size_ = size; +} + +MallocArena::~MallocArena() { + constexpr size_t overallocation = RequiredOverallocation(); + if (overallocation != 0u && UNLIKELY(RUNNING_ON_MEMORY_TOOL > 0)) { + size_t head = memory_ - unaligned_memory_; + size_t tail = overallocation - head; + MEMORY_TOOL_MAKE_UNDEFINED(unaligned_memory_, head); + MEMORY_TOOL_MAKE_UNDEFINED(memory_ + size_, tail); + } + free(reinterpret_cast<void*>(unaligned_memory_)); +} + +void Arena::Reset() { + if (bytes_allocated_ > 0) { + memset(Begin(), 0, bytes_allocated_); + bytes_allocated_ = 0; + } +} + +MallocArenaPool::MallocArenaPool() : free_arenas_(nullptr) { +} + +MallocArenaPool::~MallocArenaPool() { + ReclaimMemory(); +} + +void MallocArenaPool::ReclaimMemory() { + while (free_arenas_ != nullptr) { + Arena* arena = free_arenas_; + free_arenas_ = free_arenas_->next_; + delete arena; + } +} + +void MallocArenaPool::LockReclaimMemory() { + std::lock_guard<std::mutex> lock(lock_); + ReclaimMemory(); +} + +Arena* MallocArenaPool::AllocArena(size_t size) { + Arena* ret = nullptr; + { + std::lock_guard<std::mutex> lock(lock_); + if (free_arenas_ != nullptr && LIKELY(free_arenas_->Size() >= size)) { + ret = free_arenas_; + free_arenas_ = free_arenas_->next_; + } + } + if (ret == nullptr) { + ret = new MallocArena(size); + } + ret->Reset(); + return ret; +} + +void MallocArenaPool::TrimMaps() { + // Nop, because there is no way to do madvise here. +} + +size_t MallocArenaPool::GetBytesAllocated() const { + size_t total = 0; + std::lock_guard<std::mutex> lock(lock_); + for (Arena* arena = free_arenas_; arena != nullptr; arena = arena->next_) { + total += arena->GetBytesAllocated(); + } + return total; +} + +void MallocArenaPool::FreeArenaChain(Arena* first) { + if (UNLIKELY(RUNNING_ON_MEMORY_TOOL > 0)) { + for (Arena* arena = first; arena != nullptr; arena = arena->next_) { + MEMORY_TOOL_MAKE_UNDEFINED(arena->memory_, arena->bytes_allocated_); + } + } + + if (arena_allocator::kArenaAllocatorPreciseTracking) { + // Do not reuse arenas when tracking. + while (first != nullptr) { + Arena* next = first->next_; + delete first; + first = next; + } + return; + } + + if (first != nullptr) { + Arena* last = first; + while (last->next_ != nullptr) { + last = last->next_; + } + std::lock_guard<std::mutex> lock(lock_); + last->next_ = free_arenas_; + free_arenas_ = first; + } +} + +} // namespace art diff --git a/libartbase/base/malloc_arena_pool.h b/libartbase/base/malloc_arena_pool.h new file mode 100644 index 00000000000..87201893437 --- /dev/null +++ b/libartbase/base/malloc_arena_pool.h @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_LIBARTBASE_BASE_MALLOC_ARENA_POOL_H_ +#define ART_LIBARTBASE_BASE_MALLOC_ARENA_POOL_H_ + +#include <mutex> + +#include "base/arena_allocator.h" + +namespace art { + +class MallocArenaPool FINAL : public ArenaPool { + public: + MallocArenaPool(); + ~MallocArenaPool(); + Arena* AllocArena(size_t size) OVERRIDE; + void FreeArenaChain(Arena* first) OVERRIDE; + size_t GetBytesAllocated() const OVERRIDE; + void ReclaimMemory() OVERRIDE; + void LockReclaimMemory() OVERRIDE; + // Is a nop for malloc pools. + void TrimMaps() OVERRIDE; + + private: + Arena* free_arenas_; + // Use a std::mutex here as Arenas are at the bottom of the lock hierarchy when malloc is used. + mutable std::mutex lock_; + + DISALLOW_COPY_AND_ASSIGN(MallocArenaPool); +}; + +} // namespace art + +#endif // ART_LIBARTBASE_BASE_MALLOC_ARENA_POOL_H_ diff --git a/runtime/mem_map.cc b/libartbase/base/mem_map.cc index b9d51c11252..21634f86b69 100644 --- a/runtime/mem_map.cc +++ b/libartbase/base/mem_map.cc @@ -34,7 +34,6 @@ #include "base/allocator.h" #include "base/bit_utils.h" -#include "base/file_utils.h" #include "base/globals.h" #include "base/logging.h" // For VLOG_IS_ON. #include "base/memory_tool.h" @@ -207,7 +206,8 @@ static bool CheckNonOverlapping(uintptr_t begin, *error_msg = StringPrintf("Requested region 0x%08" PRIxPTR "-0x%08" PRIxPTR " overlaps with " "existing map 0x%08" PRIxPTR "-0x%08" PRIxPTR " (%s)\n%s", begin, end, - static_cast<uintptr_t>(entry->start), static_cast<uintptr_t>(entry->end), + static_cast<uintptr_t>(entry->start), + static_cast<uintptr_t>(entry->end), entry->name.c_str(), map_info.str().c_str()); return false; diff --git a/runtime/mem_map.h b/libartbase/base/mem_map.h index 0ecb4146149..b7beb08dbc3 100644 --- a/runtime/mem_map.h +++ b/libartbase/base/mem_map.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef ART_RUNTIME_MEM_MAP_H_ -#define ART_RUNTIME_MEM_MAP_H_ +#ifndef ART_LIBARTBASE_BASE_MEM_MAP_H_ +#define ART_LIBARTBASE_BASE_MEM_MAP_H_ #include <stddef.h> #include <sys/types.h> @@ -25,6 +25,7 @@ #include <string> #include "android-base/thread_annotations.h" +#include "base/macros.h" namespace art { @@ -297,4 +298,4 @@ void ZeroAndReleasePages(void* address, size_t length); } // namespace art -#endif // ART_RUNTIME_MEM_MAP_H_ +#endif // ART_LIBARTBASE_BASE_MEM_MAP_H_ diff --git a/runtime/mem_map_test.cc b/libartbase/base/mem_map_test.cc index 3adbf18a7a6..3adbf18a7a6 100644 --- a/runtime/mem_map_test.cc +++ b/libartbase/base/mem_map_test.cc diff --git a/runtime/memory_region.cc b/libartbase/base/memory_region.cc index 862ff736396..862ff736396 100644 --- a/runtime/memory_region.cc +++ b/libartbase/base/memory_region.cc diff --git a/runtime/memory_region.h b/libartbase/base/memory_region.h index 23e0aecbda5..7add466cc7a 100644 --- a/runtime/memory_region.h +++ b/libartbase/base/memory_region.h @@ -14,17 +14,17 @@ * limitations under the License. */ -#ifndef ART_RUNTIME_MEMORY_REGION_H_ -#define ART_RUNTIME_MEMORY_REGION_H_ +#ifndef ART_LIBARTBASE_BASE_MEMORY_REGION_H_ +#define ART_LIBARTBASE_BASE_MEMORY_REGION_H_ #include <stdint.h> #include <type_traits> #include <android-base/logging.h> -#include "arch/instruction_set.h" #include "base/bit_utils.h" #include "base/casts.h" +#include "base/enums.h" #include "base/macros.h" #include "base/value_object.h" #include "globals.h" @@ -211,9 +211,8 @@ class MemoryRegion FINAL : public ValueObject { // Is `address` aligned on a machine word? template<typename T> static constexpr bool IsWordAligned(const T* address) { - // Word alignment in bytes. - size_t kWordAlignment = static_cast<size_t>(GetInstructionSetPointerSize(kRuntimeISA)); - return IsAlignedParam(address, kWordAlignment); + // Word alignment in bytes. Determined from pointer size. + return IsAligned<kRuntimePointerSize>(address); } void* pointer_; @@ -222,4 +221,4 @@ class MemoryRegion FINAL : public ValueObject { } // namespace art -#endif // ART_RUNTIME_MEMORY_REGION_H_ +#endif // ART_LIBARTBASE_BASE_MEMORY_REGION_H_ diff --git a/runtime/memory_region_test.cc b/libartbase/base/memory_region_test.cc index e3aead47faa..e3aead47faa 100644 --- a/runtime/memory_region_test.cc +++ b/libartbase/base/memory_region_test.cc diff --git a/libartbase/base/safe_copy_test.cc b/libartbase/base/safe_copy_test.cc index a9ec9528a18..f1d7c55e777 100644 --- a/libartbase/base/safe_copy_test.cc +++ b/libartbase/base/safe_copy_test.cc @@ -16,14 +16,15 @@ #include "safe_copy.h" -#include "common_runtime_test.h" - #include <errno.h> #include <string.h> #include <sys/mman.h> #include <sys/user.h> -#include "globals.h" +#include "android-base/logging.h" +#include "base/globals.h" +#include "gtest/gtest.h" + namespace art { diff --git a/runtime/base/scoped_arena_allocator.cc b/libartbase/base/scoped_arena_allocator.cc index 7240842d552..7240842d552 100644 --- a/runtime/base/scoped_arena_allocator.cc +++ b/libartbase/base/scoped_arena_allocator.cc diff --git a/runtime/base/scoped_arena_allocator.h b/libartbase/base/scoped_arena_allocator.h index a253e2f5359..d5f6df82cc1 100644 --- a/runtime/base/scoped_arena_allocator.h +++ b/libartbase/base/scoped_arena_allocator.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef ART_RUNTIME_BASE_SCOPED_ARENA_ALLOCATOR_H_ -#define ART_RUNTIME_BASE_SCOPED_ARENA_ALLOCATOR_H_ +#ifndef ART_LIBARTBASE_BASE_SCOPED_ARENA_ALLOCATOR_H_ +#define ART_LIBARTBASE_BASE_SCOPED_ARENA_ALLOCATOR_H_ #include <android-base/logging.h> @@ -185,4 +185,4 @@ class ScopedArenaAllocator } // namespace art -#endif // ART_RUNTIME_BASE_SCOPED_ARENA_ALLOCATOR_H_ +#endif // ART_LIBARTBASE_BASE_SCOPED_ARENA_ALLOCATOR_H_ diff --git a/runtime/base/scoped_arena_containers.h b/libartbase/base/scoped_arena_containers.h index f8ee3f33af8..4df02b62057 100644 --- a/runtime/base/scoped_arena_containers.h +++ b/libartbase/base/scoped_arena_containers.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef ART_RUNTIME_BASE_SCOPED_ARENA_CONTAINERS_H_ -#define ART_RUNTIME_BASE_SCOPED_ARENA_CONTAINERS_H_ +#ifndef ART_LIBARTBASE_BASE_SCOPED_ARENA_CONTAINERS_H_ +#define ART_LIBARTBASE_BASE_SCOPED_ARENA_CONTAINERS_H_ #include <deque> #include <queue> @@ -272,4 +272,4 @@ using ArenaUniquePtr = std::unique_ptr<T, ArenaDelete<T>>; } // namespace art -#endif // ART_RUNTIME_BASE_SCOPED_ARENA_CONTAINERS_H_ +#endif // ART_LIBARTBASE_BASE_SCOPED_ARENA_CONTAINERS_H_ diff --git a/runtime/zip_archive.cc b/libartbase/base/zip_archive.cc index 5b3cab42d8c..4185c227c8e 100644 --- a/runtime/zip_archive.cc +++ b/libartbase/base/zip_archive.cc @@ -14,7 +14,7 @@ * limitations under the License. */ -#include "zip_archive.h" +#include "base/zip_archive.h" #include <fcntl.h> #include <stdio.h> @@ -29,7 +29,6 @@ #include "base/bit_utils.h" #include "base/unix_file/fd_file.h" -#include "dex/dex_file.h" namespace art { @@ -55,10 +54,6 @@ bool ZipEntry::IsAlignedTo(size_t alignment) const { return IsAlignedParam(zip_entry_->offset, static_cast<int>(alignment)); } -bool ZipEntry::IsAlignedToDexHeader() const { - return IsAlignedTo(alignof(DexFile::Header)); -} - ZipEntry::~ZipEntry() { delete zip_entry_; } diff --git a/runtime/zip_archive.h b/libartbase/base/zip_archive.h index aa540185744..39c81685198 100644 --- a/runtime/zip_archive.h +++ b/libartbase/base/zip_archive.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef ART_RUNTIME_ZIP_ARCHIVE_H_ -#define ART_RUNTIME_ZIP_ARCHIVE_H_ +#ifndef ART_LIBARTBASE_BASE_ZIP_ARCHIVE_H_ +#define ART_LIBARTBASE_BASE_ZIP_ARCHIVE_H_ #include <stdint.h> #include <memory> @@ -24,10 +24,10 @@ #include <android-base/logging.h> #include "base/os.h" +#include "base/mem_map.h" #include "base/safe_map.h" #include "base/unix_file/random_access_file.h" #include "globals.h" -#include "mem_map.h" // system/core/zip_archive definitions. struct ZipEntry; @@ -64,7 +64,6 @@ class ZipEntry { bool IsUncompressed(); bool IsAlignedTo(size_t alignment) const; - bool IsAlignedToDexHeader() const; private: ZipEntry(ZipArchiveHandle handle, @@ -102,4 +101,4 @@ class ZipArchive { } // namespace art -#endif // ART_RUNTIME_ZIP_ARCHIVE_H_ +#endif // ART_LIBARTBASE_BASE_ZIP_ARCHIVE_H_ diff --git a/runtime/zip_archive_test.cc b/libartbase/base/zip_archive_test.cc index 48ee94ce8c1..03f4cd4d54d 100644 --- a/runtime/zip_archive_test.cc +++ b/libartbase/base/zip_archive_test.cc @@ -14,7 +14,7 @@ * limitations under the License. */ -#include "zip_archive.h" +#include "base/zip_archive.h" #include <fcntl.h> #include <sys/stat.h> diff --git a/libdexfile/Android.bp b/libdexfile/Android.bp index 3fd61ee251a..b2c041c81f5 100644 --- a/libdexfile/Android.bp +++ b/libdexfile/Android.bp @@ -32,6 +32,7 @@ cc_defaults { "dex/modifiers.cc", "dex/primitive.cc", "dex/standard_dex_file.cc", + "dex/type_lookup_table.cc", "dex/utf.cc", ], @@ -123,6 +124,7 @@ art_cc_test { "dex/dex_instruction_test.cc", "dex/primitive_test.cc", "dex/string_reference_test.cc", + "dex/type_lookup_table_test.cc", "dex/utf_test.cc", ], shared_libs: [ diff --git a/libdexfile/dex/dex_file.cc b/libdexfile/dex/dex_file.cc index 1f471106df4..8cfee6655ed 100644 --- a/libdexfile/dex/dex_file.cc +++ b/libdexfile/dex/dex_file.cc @@ -349,25 +349,6 @@ const DexFile::TypeId* DexFile::FindTypeId(const char* string) const { return nullptr; } -const DexFile::StringId* DexFile::FindStringId(const uint16_t* string, size_t length) const { - int32_t lo = 0; - int32_t hi = NumStringIds() - 1; - while (hi >= lo) { - int32_t mid = (hi + lo) / 2; - const DexFile::StringId& str_id = GetStringId(dex::StringIndex(mid)); - const char* str = GetStringData(str_id); - int compare = CompareModifiedUtf8ToUtf16AsCodePointValues(str, string, length); - if (compare > 0) { - lo = mid + 1; - } else if (compare < 0) { - hi = mid - 1; - } else { - return &str_id; - } - } - return nullptr; -} - const DexFile::TypeId* DexFile::FindTypeId(dex::StringIndex string_idx) const { int32_t lo = 0; int32_t hi = NumTypeIds() - 1; diff --git a/libdexfile/dex/dex_file.h b/libdexfile/dex/dex_file.h index 683a8243edc..4098b4250af 100644 --- a/libdexfile/dex/dex_file.h +++ b/libdexfile/dex/dex_file.h @@ -504,9 +504,6 @@ class DexFile { const TypeId* FindTypeId(const char* string) const; - // Looks up a string id for a given utf16 string. - const StringId* FindStringId(const uint16_t* string, size_t length) const; - // Returns the number of type identifiers in the .dex file. uint32_t NumTypeIds() const { DCHECK(header_ != nullptr) << GetLocation(); diff --git a/runtime/type_lookup_table.cc b/libdexfile/dex/type_lookup_table.cc index 7e204fc03a5..ca5ec2f7987 100644 --- a/runtime/type_lookup_table.cc +++ b/libdexfile/dex/type_lookup_table.cc @@ -20,7 +20,6 @@ #include <memory> #include "base/bit_utils.h" -#include "base/utils.h" #include "dex/dex_file-inl.h" #include "dex/utf-inl.h" diff --git a/runtime/type_lookup_table.h b/libdexfile/dex/type_lookup_table.h index 3352d60ed16..0ba2b75dc6b 100644 --- a/runtime/type_lookup_table.h +++ b/libdexfile/dex/type_lookup_table.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef ART_RUNTIME_TYPE_LOOKUP_TABLE_H_ -#define ART_RUNTIME_TYPE_LOOKUP_TABLE_H_ +#ifndef ART_LIBDEXFILE_DEX_TYPE_LOOKUP_TABLE_H_ +#define ART_LIBDEXFILE_DEX_TYPE_LOOKUP_TABLE_H_ #include "base/leb128.h" #include "dex/dex_file_types.h" @@ -172,4 +172,4 @@ class TypeLookupTable { } // namespace art -#endif // ART_RUNTIME_TYPE_LOOKUP_TABLE_H_ +#endif // ART_LIBDEXFILE_DEX_TYPE_LOOKUP_TABLE_H_ diff --git a/runtime/type_lookup_table_test.cc b/libdexfile/dex/type_lookup_table_test.cc index b6ab6da78cc..b6ab6da78cc 100644 --- a/runtime/type_lookup_table_test.cc +++ b/libdexfile/dex/type_lookup_table_test.cc diff --git a/oatdump/oatdump.cc b/oatdump/oatdump.cc index c0d0478bf95..6a68b55fad5 100644 --- a/oatdump/oatdump.cc +++ b/oatdump/oatdump.cc @@ -34,6 +34,7 @@ #include "art_field-inl.h" #include "art_method-inl.h" #include "base/bit_utils_iterator.h" +#include "base/indenter.h" #include "base/os.h" #include "base/safe_map.h" #include "base/stl_util.h" @@ -49,6 +50,7 @@ #include "dex/dex_file-inl.h" #include "dex/dex_instruction-inl.h" #include "dex/string_reference.h" +#include "dex/type_lookup_table.h" #include "disassembler.h" #include "gc/accounting/space_bitmap-inl.h" #include "gc/space/image_space.h" @@ -56,7 +58,6 @@ #include "gc/space/space-inl.h" #include "image-inl.h" #include "imtable-inl.h" -#include "indenter.h" #include "subtype_check.h" #include "index_bss_mapping.h" #include "interpreter/unstarted_runtime.h" @@ -75,7 +76,6 @@ #include "stack.h" #include "stack_map.h" #include "thread_list.h" -#include "type_lookup_table.h" #include "vdex_file.h" #include "verifier/method_verifier.h" #include "verifier/verifier_deps.h" @@ -172,6 +172,7 @@ class OatSymbolizer FINAL { builder_->PrepareDynamicSection(elf_file->GetPath(), rodata_size, text_size, + oat_file_->DataBimgRelRoSize(), oat_file_->BssSize(), oat_file_->BssMethodsOffset(), oat_file_->BssRootsOffset(), @@ -513,6 +514,18 @@ class OatDumper { os << StringPrintf("0x%08x\n\n", resolved_addr2instr_); } + // Dump .data.bimg.rel.ro entries. + DumpDataBimgRelRoEntries(os); + + // Dump .bss summary, individual entries are dumped per dex file. + os << ".bss: "; + if (oat_file_.GetBssMethods().empty() && oat_file_.GetBssGcRoots().empty()) { + os << "empty.\n\n"; + } else { + os << oat_file_.GetBssMethods().size() << " methods, "; + os << oat_file_.GetBssGcRoots().size() << " GC roots.\n\n"; + } + // Dumping the dex file overview is compact enough to do even if header only. DexFileData cumulative; for (size_t i = 0; i < oat_dex_files_.size(); i++) { @@ -1701,7 +1714,10 @@ class OatDumper { CHECK(dex_cache != nullptr); ArtMethod* method = runtime->GetClassLinker()->ResolveMethodWithoutInvokeType( dex_method_idx, dex_cache, *options_.class_loader_); - CHECK(method != nullptr); + if (method == nullptr) { + soa.Self()->ClearException(); + return nullptr; + } return verifier::MethodVerifier::VerifyMethodAndDump( soa.Self(), vios, dex_method_idx, dex_file, dex_cache, *options_.class_loader_, class_def, code_item, method, method_access_flags); @@ -1927,6 +1943,66 @@ class OatDumper { } } + void DumpDataBimgRelRoEntries(std::ostream& os) { + os << ".data.bimg.rel.ro: "; + if (oat_file_.GetBootImageRelocations().empty()) { + os << "empty.\n\n"; + return; + } + + os << oat_file_.GetBootImageRelocations().size() << " entries.\n"; + Runtime* runtime = Runtime::Current(); + if (runtime != nullptr && !runtime->GetHeap()->GetBootImageSpaces().empty()) { + const std::vector<gc::space::ImageSpace*>& boot_image_spaces = + runtime->GetHeap()->GetBootImageSpaces(); + ScopedObjectAccess soa(Thread::Current()); + for (const uint32_t& object_offset : oat_file_.GetBootImageRelocations()) { + uint32_t entry_index = &object_offset - oat_file_.GetBootImageRelocations().data(); + uint32_t entry_offset = entry_index * sizeof(oat_file_.GetBootImageRelocations()[0]); + os << StringPrintf(" 0x%x: 0x%08x", entry_offset, object_offset); + uint8_t* object = boot_image_spaces[0]->Begin() + object_offset; + bool found = false; + for (gc::space::ImageSpace* space : boot_image_spaces) { + uint64_t local_offset = object - space->Begin(); + if (local_offset < space->GetImageHeader().GetImageSize()) { + if (space->GetImageHeader().GetObjectsSection().Contains(local_offset)) { + ObjPtr<mirror::Object> o = reinterpret_cast<mirror::Object*>(object); + if (o->IsString()) { + os << " String: " << o->AsString()->ToModifiedUtf8(); + } else if (o->IsClass()) { + os << " Class: " << o->AsClass()->PrettyDescriptor(); + } else { + os << StringPrintf(" 0x%08x %s", + object_offset, + o->GetClass()->PrettyDescriptor().c_str()); + } + } else if (space->GetImageHeader().GetMethodsSection().Contains(local_offset)) { + ArtMethod* m = reinterpret_cast<ArtMethod*>(object); + os << " ArtMethod: " << m->PrettyMethod(); + } else { + os << StringPrintf(" 0x%08x <unexpected section in %s>", + object_offset, + space->GetImageFilename().c_str()); + } + found = true; + break; + } + } + if (!found) { + os << StringPrintf(" 0x%08x <outside boot image spaces>", object_offset); + } + os << "\n"; + } + } else { + for (const uint32_t& object_offset : oat_file_.GetBootImageRelocations()) { + uint32_t entry_index = &object_offset - oat_file_.GetBootImageRelocations().data(); + uint32_t entry_offset = entry_index * sizeof(oat_file_.GetBootImageRelocations()[0]); + os << StringPrintf(" 0x%x: 0x%08x\n", entry_offset, object_offset); + } + } + os << "\n"; + } + template <typename NameGetter> void DumpBssEntries(std::ostream& os, const char* slot_type, @@ -2201,7 +2277,7 @@ class ImageDumper { // Intern table is 8-byte aligned. uint32_t end_caches = dex_cache_arrays_section.Offset() + dex_cache_arrays_section.Size(); - CHECK_ALIGNED(intern_section.Offset(), sizeof(uint64_t)); + CHECK_EQ(RoundUp(end_caches, 8U), intern_section.Offset()); stats_.alignment_bytes += intern_section.Offset() - end_caches; // Add space between intern table and class table. diff --git a/oatdump/oatdump_app_test.cc b/oatdump/oatdump_app_test.cc index f12522277b8..34b07d2ddf8 100644 --- a/oatdump/oatdump_app_test.cc +++ b/oatdump/oatdump_app_test.cc @@ -20,25 +20,29 @@ namespace art { TEST_F(OatDumpTest, TestAppWithBootImage) { std::string error_msg; - ASSERT_TRUE(GenerateAppOdexFile(kDynamic, {}, &error_msg)) << error_msg; + ASSERT_TRUE(GenerateAppOdexFile(kDynamic, {"--runtime-arg", "-Xmx64M"}, &error_msg)) << error_msg; ASSERT_TRUE(Exec(kDynamic, kModeOatWithBootImage, {}, kListAndCode, &error_msg)) << error_msg; } TEST_F(OatDumpTest, TestAppWithBootImageStatic) { TEST_DISABLED_FOR_NON_STATIC_HOST_BUILDS(); std::string error_msg; - ASSERT_TRUE(GenerateAppOdexFile(kStatic, {}, &error_msg)) << error_msg; + ASSERT_TRUE(GenerateAppOdexFile(kStatic, {"--runtime-arg", "-Xmx64M"}, &error_msg)) << error_msg; ASSERT_TRUE(Exec(kStatic, kModeOatWithBootImage, {}, kListAndCode, &error_msg)) << error_msg; } TEST_F(OatDumpTest, TestPicAppWithBootImage) { std::string error_msg; - ASSERT_TRUE(GenerateAppOdexFile(kDynamic, {"--compile-pic"}, &error_msg)) << error_msg; + ASSERT_TRUE( + GenerateAppOdexFile(kDynamic, {"--runtime-arg", "-Xmx64M", "--compile-pic"}, &error_msg)) + << error_msg; ASSERT_TRUE(Exec(kDynamic, kModeOatWithBootImage, {}, kListAndCode, &error_msg)) << error_msg; } TEST_F(OatDumpTest, TestPicAppWithBootImageStatic) { TEST_DISABLED_FOR_NON_STATIC_HOST_BUILDS(); std::string error_msg; - ASSERT_TRUE(GenerateAppOdexFile(kStatic, {"--compile-pic"}, &error_msg)) << error_msg; + ASSERT_TRUE( + GenerateAppOdexFile(kStatic, {"--runtime-arg", "-Xmx64M", "--compile-pic"}, &error_msg)) + << error_msg; ASSERT_TRUE(Exec(kStatic, kModeOatWithBootImage, {}, kListAndCode, &error_msg)) << error_msg; } diff --git a/openjdkjvmti/events.cc b/openjdkjvmti/events.cc index 4d055d4c270..de678711fc8 100644 --- a/openjdkjvmti/events.cc +++ b/openjdkjvmti/events.cc @@ -1001,6 +1001,27 @@ bool EventHandler::OtherMonitorEventsEnabledAnywhere(ArtJvmtiEvent event) { return false; } +void EventHandler::SetupFramePopTraceListener(bool enable) { + if (enable) { + frame_pop_enabled = true; + SetupTraceListener(method_trace_listener_.get(), ArtJvmtiEvent::kFramePop, enable); + } else { + // remove the listener if we have no outstanding frames. + { + art::ReaderMutexLock mu(art::Thread::Current(), envs_lock_); + for (ArtJvmTiEnv* env : envs) { + art::ReaderMutexLock event_mu(art::Thread::Current(), env->event_info_mutex_); + if (!env->notify_frames.empty()) { + // Leaving FramePop listener since there are unsent FramePop events. + return; + } + } + frame_pop_enabled = false; + } + SetupTraceListener(method_trace_listener_.get(), ArtJvmtiEvent::kFramePop, enable); + } +} + // Handle special work for the given event type, if necessary. void EventHandler::HandleEventType(ArtJvmtiEvent event, bool enable) { switch (event) { @@ -1015,14 +1036,14 @@ void EventHandler::HandleEventType(ArtJvmtiEvent event, bool enable) { case ArtJvmtiEvent::kGarbageCollectionFinish: SetupGcPauseTracking(gc_pause_listener_.get(), event, enable); return; - // FramePop can never be disabled once it's been turned on since we would either need to deal - // with dangling pointers or have missed events. - // TODO We really need to make this not the case anymore. + // FramePop can never be disabled once it's been turned on if it was turned off with outstanding + // pop-events since we would either need to deal with dangling pointers or have missed events. case ArtJvmtiEvent::kFramePop: - if (!enable || (enable && frame_pop_enabled)) { + if (enable && frame_pop_enabled) { + // The frame-pop event was held on by pending events so we don't need to do anything. break; } else { - SetupTraceListener(method_trace_listener_.get(), event, enable); + SetupFramePopTraceListener(enable); break; } case ArtJvmtiEvent::kMethodEntry: diff --git a/openjdkjvmti/events.h b/openjdkjvmti/events.h index 8141eff88c0..bf12cb191e4 100644 --- a/openjdkjvmti/events.h +++ b/openjdkjvmti/events.h @@ -247,6 +247,9 @@ class EventHandler { private: void SetupTraceListener(JvmtiMethodTraceListener* listener, ArtJvmtiEvent event, bool enable); + // Specifically handle the FramePop event which it might not always be possible to turn off. + void SetupFramePopTraceListener(bool enable); + template <ArtJvmtiEvent kEvent, typename ...Args> ALWAYS_INLINE inline std::vector<impl::EventHandlerFunc<kEvent>> CollectEvents(art::Thread* thread, diff --git a/openjdkjvmti/ti_class_definition.h b/openjdkjvmti/ti_class_definition.h index 31c3611e727..f888a7474f6 100644 --- a/openjdkjvmti/ti_class_definition.h +++ b/openjdkjvmti/ti_class_definition.h @@ -39,7 +39,7 @@ #include "art_jvmti.h" #include "base/array_ref.h" -#include "mem_map.h" +#include "base/mem_map.h" namespace openjdkjvmti { diff --git a/openjdkjvmti/ti_class_loader.h b/openjdkjvmti/ti_class_loader.h index 5c9497b0b59..dbe30da4621 100644 --- a/openjdkjvmti/ti_class_loader.h +++ b/openjdkjvmti/ti_class_loader.h @@ -39,6 +39,7 @@ #include "art_jvmti.h" #include "art_method.h" #include "base/array_slice.h" +#include "base/mem_map.h" #include "class_linker.h" #include "dex/dex_file.h" #include "dex/utf.h" @@ -47,7 +48,6 @@ #include "jni_env_ext-inl.h" #include "jvmti.h" #include "linear_alloc.h" -#include "mem_map.h" #include "mirror/array-inl.h" #include "mirror/array.h" #include "mirror/class-inl.h" diff --git a/openjdkjvmti/ti_redefine.h b/openjdkjvmti/ti_redefine.h index fa7d28648de..03238565cc5 100644 --- a/openjdkjvmti/ti_redefine.h +++ b/openjdkjvmti/ti_redefine.h @@ -39,6 +39,7 @@ #include "art_jvmti.h" #include "art_method.h" #include "base/array_ref.h" +#include "base/mem_map.h" #include "class_linker.h" #include "dex/dex_file.h" #include "dex/utf.h" @@ -47,7 +48,6 @@ #include "jni_env_ext-inl.h" #include "jvmti.h" #include "linear_alloc.h" -#include "mem_map.h" #include "mirror/array-inl.h" #include "mirror/array.h" #include "mirror/class-inl.h" diff --git a/openjdkjvmti/transform.cc b/openjdkjvmti/transform.cc index 62094a327de..f31486ba586 100644 --- a/openjdkjvmti/transform.cc +++ b/openjdkjvmti/transform.cc @@ -39,6 +39,7 @@ #include "art_method.h" #include "base/array_ref.h" +#include "base/mem_map.h" #include "class_linker.h" #include "dex/dex_file.h" #include "dex/dex_file_types.h" @@ -51,7 +52,6 @@ #include "jvalue.h" #include "jvmti.h" #include "linear_alloc.h" -#include "mem_map.h" #include "mirror/array.h" #include "mirror/class-inl.h" #include "mirror/class_ext.h" diff --git a/patchoat/patchoat.cc b/patchoat/patchoat.cc index b9a9a69ab59..dc9d990e299 100644 --- a/patchoat/patchoat.cc +++ b/patchoat/patchoat.cc @@ -700,6 +700,7 @@ bool PatchOat::Verify(const std::string& image_location, } bool PatchOat::WriteImage(File* out) { + CHECK(out != nullptr); TimingLogger::ScopedTiming t("Writing image File", timings_); std::string error_msg; @@ -709,7 +710,6 @@ bool PatchOat::WriteImage(File* out) { true /* read_only_mode */, &error_msg); CHECK(image_ != nullptr); - CHECK(out != nullptr); size_t expect = image_->Size(); if (out->WriteFully(reinterpret_cast<char*>(image_->Begin()), expect) && out->SetLength(expect) == 0) { diff --git a/patchoat/patchoat_test.cc b/patchoat/patchoat_test.cc index 974ed3217d1..69728ae0512 100644 --- a/patchoat/patchoat_test.cc +++ b/patchoat/patchoat_test.cc @@ -24,6 +24,7 @@ #include "android-base/stringprintf.h" #include "android-base/strings.h" +#include "base/hex_dump.h" #include "base/leb128.h" #include "dexopt_test.h" #include "runtime.h" @@ -320,6 +321,12 @@ class PatchoatTest : public DexoptTest { if (image1[i] != image2[i]) { *error_msg = StringPrintf("%s and %s differ at offset %zu", filename1.c_str(), filename2.c_str(), i); + size_t hexdump_size = std::min<size_t>(16u, size - i); + HexDump dump1(&image1[i], hexdump_size, /* show_actual_addresses */ false, /* prefix */ ""); + HexDump dump2(&image2[i], hexdump_size, /* show_actual_addresses */ false, /* prefix */ ""); + std::ostringstream oss; + oss << "\n" << dump1 << "\n" << dump2; + *error_msg += oss.str(); return true; } } diff --git a/profman/profman.cc b/profman/profman.cc index 90e342d16aa..f2cec47da3a 100644 --- a/profman/profman.cc +++ b/profman/profman.cc @@ -19,7 +19,6 @@ #include <stdlib.h> #include <sys/file.h> #include <sys/param.h> -#include <sys/stat.h> #include <unistd.h> #include <fstream> @@ -40,6 +39,7 @@ #include "base/time_utils.h" #include "base/unix_file/fd_file.h" #include "base/utils.h" +#include "base/zip_archive.h" #include "boot_image_profile.h" #include "dex/art_dex_file_loader.h" #include "dex/bytecode_utils.h" @@ -51,7 +51,6 @@ #include "jit/profile_compilation_info.h" #include "profile_assistant.h" #include "runtime.h" -#include "zip_archive.h" namespace art { diff --git a/runtime/Android.bp b/runtime/Android.bp index 08025824c37..6b432058d9d 100644 --- a/runtime/Android.bp +++ b/runtime/Android.bp @@ -32,12 +32,10 @@ cc_defaults { "art_field.cc", "art_method.cc", "barrier.cc", - "base/arena_allocator.cc", - "base/arena_bit_vector.cc", "base/file_utils.cc", + "base/mem_map_arena_pool.cc", "base/mutex.cc", "base/quasi_atomic.cc", - "base/scoped_arena_allocator.cc", "base/timing_logger.cc", "cha.cc", "check_jni.cc", @@ -67,7 +65,6 @@ cc_defaults { "gc/collector/garbage_collector.cc", "gc/collector/immune_region.cc", "gc/collector/immune_spaces.cc", - "gc/collector/mark_compact.cc", "gc/collector/mark_sweep.cc", "gc/collector/partial_mark_sweep.cc", "gc/collector/semi_space.cc", @@ -122,8 +119,6 @@ cc_defaults { "jobject_comparator.cc", "linear_alloc.cc", "managed_stack.cc", - "mem_map.cc", - "memory_region.cc", "method_handles.cc", "mirror/array.cc", "mirror/call_site.cc", @@ -203,7 +198,6 @@ cc_defaults { "ti/agent.cc", "trace.cc", "transaction.cc", - "type_lookup_table.cc", "vdex_file.cc", "verifier/instruction_flags.cc", "verifier/method_verifier.cc", @@ -213,7 +207,6 @@ cc_defaults { "verifier/verifier_deps.cc", "verify_object.cc", "well_known_classes.cc", - "zip_archive.cc", "arch/context.cc", "arch/instruction_set.cc", @@ -248,6 +241,9 @@ cc_defaults { "entrypoints/quick/quick_trampoline_entrypoints.cc", ], + // b/77976998, clang lld does not recognize the --keep-unique flag. + use_clang_lld: false, + arch: { arm: { clang_asflags: ["-no-integrated-as"], @@ -453,6 +449,7 @@ gensrcs { "thread.h", "thread_state.h", "ti/agent.h", + "trace.h", "verifier/verifier_enums.h", ], output_extension: "operator_out.cc", @@ -542,7 +539,6 @@ art_cc_test { "arch/x86/instruction_set_features_x86_test.cc", "arch/x86_64/instruction_set_features_x86_64_test.cc", "barrier_test.cc", - "base/arena_allocator_test.cc", "base/file_utils_test.cc", "base/mutex_test.cc", "base/timing_logger_test.cc", @@ -576,7 +572,6 @@ art_cc_test { "handle_scope_test.cc", "hidden_api_test.cc", "imtable_test.cc", - "indenter_test.cc", "indirect_reference_table_test.cc", "instrumentation_test.cc", "intern_table_test.cc", @@ -585,8 +580,6 @@ art_cc_test { "jdwp/jdwp_options_test.cc", "java_vm_ext_test.cc", "jit/profile_compilation_info_test.cc", - "mem_map_test.cc", - "memory_region_test.cc", "method_handles_test.cc", "mirror/dex_cache_test.cc", "mirror/method_type_test.cc", @@ -604,11 +597,9 @@ art_cc_test { "subtype_check_test.cc", "thread_pool_test.cc", "transaction_test.cc", - "type_lookup_table_test.cc", "vdex_file_test.cc", "verifier/method_verifier_test.cc", "verifier/reg_type_test.cc", - "zip_archive_test.cc", ], shared_libs: [ "libbacktrace", diff --git a/runtime/arch/arm/quick_entrypoints_arm.S b/runtime/arch/arm/quick_entrypoints_arm.S index 98214fb6848..0fd239a244a 100644 --- a/runtime/arch/arm/quick_entrypoints_arm.S +++ b/runtime/arch/arm/quick_entrypoints_arm.S @@ -809,6 +809,9 @@ END art_quick_unlock_object_no_inline .extern artInstanceOfFromCode .extern artThrowClassCastExceptionForObject ENTRY art_quick_check_instance_of + // Type check using the bit string passes null as the target class. In that case just throw. + cbz r1, .Lthrow_class_cast_exception_for_bitstring_check + push {r0-r2, lr} @ save arguments, padding (r2) and link register .cfi_adjust_cfa_offset 16 .cfi_rel_offset r0, 0 @@ -827,6 +830,7 @@ ENTRY art_quick_check_instance_of .cfi_restore r2 .cfi_restore lr +.Lthrow_class_cast_exception_for_bitstring_check: SETUP_SAVE_ALL_CALLEE_SAVES_FRAME r2 @ save all registers as basis for long jump context mov r2, r9 @ pass Thread::Current bl artThrowClassCastExceptionForObject @ (Object*, Class*, Thread*) diff --git a/runtime/arch/arm64/quick_entrypoints_arm64.S b/runtime/arch/arm64/quick_entrypoints_arm64.S index e2fce1f7909..9ff5ebede3b 100644 --- a/runtime/arch/arm64/quick_entrypoints_arm64.S +++ b/runtime/arch/arm64/quick_entrypoints_arm64.S @@ -614,56 +614,18 @@ INVOKE_TRAMPOLINE art_quick_invoke_virtual_trampoline_with_access_check, artInvo .macro INVOKE_STUB_CREATE_FRAME +SAVE_SIZE=6*8 // x4, x5, x19, x20, FP, LR saved. + SAVE_TWO_REGS_INCREASE_FRAME x4, x5, SAVE_SIZE + SAVE_TWO_REGS x19, x20, 16 + SAVE_TWO_REGS xFP, xLR, 32 -SAVE_SIZE=15*8 // x4, x5, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, SP, LR, FP saved. -SAVE_SIZE_AND_METHOD=SAVE_SIZE+8 - - - mov x9, sp // Save stack pointer. - .cfi_register sp,x9 - - add x10, x2, # SAVE_SIZE_AND_METHOD // calculate size of frame. - sub x10, sp, x10 // Calculate SP position - saves + ArtMethod* + args - and x10, x10, # ~0xf // Enforce 16 byte stack alignment. - mov sp, x10 // Set new SP. - - sub x10, x9, #SAVE_SIZE // Calculate new FP (later). Done here as we must move SP - .cfi_def_cfa_register x10 // before this. - .cfi_adjust_cfa_offset SAVE_SIZE - - str x28, [x10, #112] - .cfi_rel_offset x28, 112 - - stp x26, x27, [x10, #96] - .cfi_rel_offset x26, 96 - .cfi_rel_offset x27, 104 - - stp x24, x25, [x10, #80] - .cfi_rel_offset x24, 80 - .cfi_rel_offset x25, 88 - - stp x22, x23, [x10, #64] - .cfi_rel_offset x22, 64 - .cfi_rel_offset x23, 72 - - stp x20, x21, [x10, #48] - .cfi_rel_offset x20, 48 - .cfi_rel_offset x21, 56 - - stp x9, x19, [x10, #32] // Save old stack pointer and x19. - .cfi_rel_offset sp, 32 - .cfi_rel_offset x19, 40 - - stp x4, x5, [x10, #16] // Save result and shorty addresses. - .cfi_rel_offset x4, 16 - .cfi_rel_offset x5, 24 + mov xFP, sp // Use xFP for frame pointer, as it's callee-saved. + .cfi_def_cfa_register xFP - stp xFP, xLR, [x10] // Store LR & FP. - .cfi_rel_offset x29, 0 - .cfi_rel_offset x30, 8 + add x10, x2, #(__SIZEOF_POINTER__ + 0xf) // Reserve space for ArtMethod*, arguments and + and x10, x10, # ~0xf // round up for 16-byte stack alignment. + sub sp, sp, x10 // Adjust SP for ArtMethod*, args and alignment padding. - mov xFP, x10 // Use xFP now, as it's callee-saved. - .cfi_def_cfa_register x29 mov xSELF, x3 // Move thread pointer into SELF register. // Copy arguments into stack frame. @@ -678,12 +640,10 @@ SAVE_SIZE_AND_METHOD=SAVE_SIZE+8 // Copy parameters into the stack. Use numeric label as this is a macro and Clang's assembler // does not have unique-id variables. 1: - cmp w2, #0 - beq 2f + cbz w2, 2f sub w2, w2, #4 // Need 65536 bytes of range. ldr w10, [x1, x2] str w10, [x9, x2] - b 1b 2: @@ -700,29 +660,14 @@ SAVE_SIZE_AND_METHOD=SAVE_SIZE+8 // Branch to method. blr x9 - // Restore return value address and shorty address. - ldp x4, x5, [xFP, #16] - .cfi_restore x4 - .cfi_restore x5 - - ldr x28, [xFP, #112] - .cfi_restore x28 - - ldp x26, x27, [xFP, #96] - .cfi_restore x26 - .cfi_restore x27 - - ldp x24, x25, [xFP, #80] - .cfi_restore x24 - .cfi_restore x25 - - ldp x22, x23, [xFP, #64] - .cfi_restore x22 - .cfi_restore x23 + // Pop the ArtMethod* (null), arguments and alignment padding from the stack. + mov sp, xFP + .cfi_def_cfa_register sp - ldp x20, x21, [xFP, #48] - .cfi_restore x20 - .cfi_restore x21 + // Restore saved registers including value address and shorty address. + RESTORE_TWO_REGS x19, x20, 16 + RESTORE_TWO_REGS xFP, xLR, 32 + RESTORE_TWO_REGS_DECREASE_FRAME x4, x5, SAVE_SIZE // Store result (w0/x0/s0/d0) appropriately, depending on resultType. ldrb w10, [x5] @@ -732,33 +677,28 @@ SAVE_SIZE_AND_METHOD=SAVE_SIZE+8 // Don't set anything for a void type. cmp w10, #'V' - beq 3f + beq 1f // Is it a double? cmp w10, #'D' - bne 1f - str d0, [x4] - b 3f + beq 2f -1: // Is it a float? + // Is it a float? cmp w10, #'F' - bne 2f - str s0, [x4] - b 3f + beq 3f -2: // Just store x0. Doesn't matter if it is 64 or 32 bits. + // Just store x0. Doesn't matter if it is 64 or 32 bits. str x0, [x4] -3: // Finish up. - ldp x2, x19, [xFP, #32] // Restore stack pointer and x19. - .cfi_restore x19 - mov sp, x2 - .cfi_restore sp +1: // Finish up. + ret - ldp xFP, xLR, [xFP] // Restore old frame pointer and link register. - .cfi_restore x29 - .cfi_restore x30 +2: // Store double. + str d0, [x4] + ret +3: // Store float. + str s0, [x4] ret .endm @@ -1057,7 +997,7 @@ END art_quick_invoke_static_stub /* extern"C" void art_quick_osr_stub(void** stack, x0 * size_t stack_size_in_bytes, x1 - * const uin8_t* native_pc, x2 + * const uint8_t* native_pc, x2 * JValue *result, x3 * char *shorty, x4 * Thread *self) x5 @@ -1329,6 +1269,9 @@ END art_quick_unlock_object_no_inline .extern artInstanceOfFromCode .extern artThrowClassCastExceptionForObject ENTRY art_quick_check_instance_of + // Type check using the bit string passes null as the target class. In that case just throw. + cbz x1, .Lthrow_class_cast_exception_for_bitstring_check + // Store arguments and link register // Stack needs to be 16B aligned on calls. SAVE_TWO_REGS_INCREASE_FRAME x0, x1, 32 @@ -1354,6 +1297,7 @@ ENTRY art_quick_check_instance_of // Restore RESTORE_TWO_REGS_DECREASE_FRAME x0, x1, 32 +.Lthrow_class_cast_exception_for_bitstring_check: SETUP_SAVE_ALL_CALLEE_SAVES_FRAME // save all registers as basis for long jump context mov x2, xSELF // pass Thread::Current bl artThrowClassCastExceptionForObject // (Object*, Class*, Thread*) diff --git a/runtime/arch/mips/quick_entrypoints_mips.S b/runtime/arch/mips/quick_entrypoints_mips.S index b2f7e10f520..d8fe480719f 100644 --- a/runtime/arch/mips/quick_entrypoints_mips.S +++ b/runtime/arch/mips/quick_entrypoints_mips.S @@ -1423,6 +1423,10 @@ END art_quick_unlock_object_no_inline .extern artInstanceOfFromCode .extern artThrowClassCastExceptionForObject ENTRY art_quick_check_instance_of + // Type check using the bit string passes null as the target class. In that case just throw. + beqz $a1, .Lthrow_class_cast_exception_for_bitstring_check + nop + addiu $sp, $sp, -32 .cfi_adjust_cfa_offset 32 sw $gp, 16($sp) @@ -1441,12 +1445,15 @@ ENTRY art_quick_check_instance_of jalr $zero, $ra addiu $sp, $sp, 32 .cfi_adjust_cfa_offset -32 + .Lthrow_class_cast_exception: lw $t9, 8($sp) lw $a1, 4($sp) lw $a0, 0($sp) addiu $sp, $sp, 32 .cfi_adjust_cfa_offset -32 + +.Lthrow_class_cast_exception_for_bitstring_check: SETUP_SAVE_ALL_CALLEE_SAVES_FRAME la $t9, artThrowClassCastExceptionForObject jalr $zero, $t9 # artThrowClassCastException (Object*, Class*, Thread*) diff --git a/runtime/arch/mips64/quick_entrypoints_mips64.S b/runtime/arch/mips64/quick_entrypoints_mips64.S index 58e0e448131..8d2a7bd6c1b 100644 --- a/runtime/arch/mips64/quick_entrypoints_mips64.S +++ b/runtime/arch/mips64/quick_entrypoints_mips64.S @@ -1364,6 +1364,9 @@ END art_quick_unlock_object_no_inline .extern artInstanceOfFromCode .extern artThrowClassCastExceptionForObject ENTRY art_quick_check_instance_of + // Type check using the bit string passes null as the target class. In that case just throw. + beqzc $a1, .Lthrow_class_cast_exception_for_bitstring_check + daddiu $sp, $sp, -32 .cfi_adjust_cfa_offset 32 sd $ra, 24($sp) @@ -1379,12 +1382,15 @@ ENTRY art_quick_check_instance_of jalr $zero, $ra daddiu $sp, $sp, 32 .cfi_adjust_cfa_offset -32 + .Lthrow_class_cast_exception: ld $t9, 16($sp) ld $a1, 8($sp) ld $a0, 0($sp) daddiu $sp, $sp, 32 .cfi_adjust_cfa_offset -32 + +.Lthrow_class_cast_exception_for_bitstring_check: SETUP_GP SETUP_SAVE_ALL_CALLEE_SAVES_FRAME dla $t9, artThrowClassCastExceptionForObject diff --git a/runtime/arch/x86/quick_entrypoints_x86.S b/runtime/arch/x86/quick_entrypoints_x86.S index 5c4ae4ea12c..df43aef94be 100644 --- a/runtime/arch/x86/quick_entrypoints_x86.S +++ b/runtime/arch/x86/quick_entrypoints_x86.S @@ -1432,6 +1432,10 @@ DEFINE_FUNCTION art_quick_instance_of END_FUNCTION art_quick_instance_of DEFINE_FUNCTION art_quick_check_instance_of + // Type check using the bit string passes null as the target class. In that case just throw. + testl %ecx, %ecx + jz .Lthrow_class_cast_exception_for_bitstring_check + PUSH eax // alignment padding PUSH ecx // pass arg2 - checked class PUSH eax // pass arg1 - obj @@ -1449,6 +1453,7 @@ DEFINE_FUNCTION art_quick_check_instance_of addl LITERAL(4), %esp CFI_ADJUST_CFA_OFFSET(-4) +.Lthrow_class_cast_exception_for_bitstring_check: SETUP_SAVE_ALL_CALLEE_SAVES_FRAME ebx, ebx // save all registers as basis for long jump context // Outgoing argument set up PUSH eax // alignment padding diff --git a/runtime/arch/x86_64/quick_entrypoints_x86_64.S b/runtime/arch/x86_64/quick_entrypoints_x86_64.S index a8132006060..4f941e1c483 100644 --- a/runtime/arch/x86_64/quick_entrypoints_x86_64.S +++ b/runtime/arch/x86_64/quick_entrypoints_x86_64.S @@ -1403,6 +1403,10 @@ DEFINE_FUNCTION art_quick_unlock_object_no_inline END_FUNCTION art_quick_unlock_object_no_inline DEFINE_FUNCTION art_quick_check_instance_of + // Type check using the bit string passes null as the target class. In that case just throw. + testl %esi, %esi + jz .Lthrow_class_cast_exception_for_bitstring_check + // We could check the super classes here but that is usually already checked in the caller. PUSH rdi // Save args for exc PUSH rsi @@ -1426,6 +1430,7 @@ DEFINE_FUNCTION art_quick_check_instance_of POP rsi // Pop arguments POP rdi +.Lthrow_class_cast_exception_for_bitstring_check: SETUP_SAVE_ALL_CALLEE_SAVES_FRAME // save all registers as basis for long jump context mov %gs:THREAD_SELF_OFFSET, %rdx // pass Thread::Current() call SYMBOL(artThrowClassCastExceptionForObject) // (Object* src, Class* dest, Thread*) diff --git a/runtime/barrier_test.cc b/runtime/barrier_test.cc index 04bb6bab1ed..88075ba3688 100644 --- a/runtime/barrier_test.cc +++ b/runtime/barrier_test.cc @@ -69,18 +69,18 @@ TEST_F(BarrierTest, CheckWait) { thread_pool.AddTask(self, new CheckWaitTask(&barrier, &count1, &count2)); } thread_pool.StartWorkers(self); - while (count1.LoadRelaxed() != num_threads) { + while (count1.load(std::memory_order_relaxed) != num_threads) { timeout_barrier.Increment(self, 1, 100); // sleep 100 msecs } // Count 2 should still be zero since no thread should have gone past the barrier. - EXPECT_EQ(0, count2.LoadRelaxed()); + EXPECT_EQ(0, count2.load(std::memory_order_relaxed)); // Perform one additional Wait(), allowing pool threads to proceed. barrier.Wait(self); // Wait for all the threads to finish. thread_pool.Wait(self, true, false); // Both counts should be equal to num_threads now. - EXPECT_EQ(count1.LoadRelaxed(), num_threads); - EXPECT_EQ(count2.LoadRelaxed(), num_threads); + EXPECT_EQ(count1.load(std::memory_order_relaxed), num_threads); + EXPECT_EQ(count2.load(std::memory_order_relaxed), num_threads); timeout_barrier.Init(self, 0); // Reset to zero for destruction. } @@ -124,7 +124,7 @@ TEST_F(BarrierTest, CheckPass) { // Wait for all the tasks to complete using the barrier. barrier.Increment(self, expected_total_tasks); // The total number of completed tasks should be equal to expected_total_tasks. - EXPECT_EQ(count.LoadRelaxed(), expected_total_tasks); + EXPECT_EQ(count.load(std::memory_order_relaxed), expected_total_tasks); } } // namespace art diff --git a/runtime/base/file_utils.cc b/runtime/base/file_utils.cc index 09a9b97cca4..7b0796cf37e 100644 --- a/runtime/base/file_utils.cc +++ b/runtime/base/file_utils.cc @@ -83,59 +83,6 @@ bool ReadFileToString(const std::string& file_name, std::string* result) { } } -bool PrintFileToLog(const std::string& file_name, android::base::LogSeverity level) { - File file(file_name, O_RDONLY, false); - if (!file.IsOpened()) { - return false; - } - - constexpr size_t kBufSize = 256; // Small buffer. Avoid stack overflow and stack size warnings. - char buf[kBufSize + 1]; // +1 for terminator. - size_t filled_to = 0; - while (true) { - DCHECK_LT(filled_to, kBufSize); - int64_t n = TEMP_FAILURE_RETRY(read(file.Fd(), &buf[filled_to], kBufSize - filled_to)); - if (n <= 0) { - // Print the rest of the buffer, if it exists. - if (filled_to > 0) { - buf[filled_to] = 0; - LOG(level) << buf; - } - return n == 0; - } - // Scan for '\n'. - size_t i = filled_to; - bool found_newline = false; - for (; i < filled_to + n; ++i) { - if (buf[i] == '\n') { - // Found a line break, that's something to print now. - buf[i] = 0; - LOG(level) << buf; - // Copy the rest to the front. - if (i + 1 < filled_to + n) { - memmove(&buf[0], &buf[i + 1], filled_to + n - i - 1); - filled_to = filled_to + n - i - 1; - } else { - filled_to = 0; - } - found_newline = true; - break; - } - } - if (found_newline) { - continue; - } else { - filled_to += n; - // Check if we must flush now. - if (filled_to == kBufSize) { - buf[kBufSize] = 0; - LOG(level) << buf; - filled_to = 0; - } - } - } -} - std::string GetAndroidRootSafe(std::string* error_msg) { // Prefer ANDROID_ROOT if it's set. const char* android_dir = getenv("ANDROID_ROOT"); @@ -306,8 +253,8 @@ std::string GetSystemImageFilename(const char* location, const InstructionSet is } std::string ReplaceFileExtension(const std::string& filename, const std::string& new_extension) { - const size_t last_ext = filename.find_last_of('.'); - if (last_ext == std::string::npos) { + const size_t last_ext = filename.find_last_of("./"); + if (last_ext == std::string::npos || filename[last_ext] != '.') { return filename + "." + new_extension; } else { return filename.substr(0, last_ext + 1) + new_extension; diff --git a/runtime/base/file_utils.h b/runtime/base/file_utils.h index 8adb4f7bf8d..d4f6c576c09 100644 --- a/runtime/base/file_utils.h +++ b/runtime/base/file_utils.h @@ -28,7 +28,6 @@ namespace art { bool ReadFileToString(const std::string& file_name, std::string* result); -bool PrintFileToLog(const std::string& file_name, android::base::LogSeverity level); // Find $ANDROID_ROOT, /system, or abort. std::string GetAndroidRoot(); diff --git a/runtime/base/file_utils_test.cc b/runtime/base/file_utils_test.cc index cf6e34d1eab..e74dfe5e645 100644 --- a/runtime/base/file_utils_test.cc +++ b/runtime/base/file_utils_test.cc @@ -94,4 +94,11 @@ TEST_F(FileUtilsTest, GetAndroidRootSafe) { ASSERT_EQ(0, setenv("ANDROID_ROOT", android_root_env.c_str(), 1 /* overwrite */)); } +TEST_F(FileUtilsTest, ReplaceFileExtension) { + EXPECT_EQ("/directory/file.vdex", ReplaceFileExtension("/directory/file.oat", "vdex")); + EXPECT_EQ("/.directory/file.vdex", ReplaceFileExtension("/.directory/file.oat", "vdex")); + EXPECT_EQ("/directory/file.vdex", ReplaceFileExtension("/directory/file", "vdex")); + EXPECT_EQ("/.directory/file.vdex", ReplaceFileExtension("/.directory/file", "vdex")); +} + } // namespace art diff --git a/runtime/base/mem_map_arena_pool.cc b/runtime/base/mem_map_arena_pool.cc new file mode 100644 index 00000000000..9ac7886e5d0 --- /dev/null +++ b/runtime/base/mem_map_arena_pool.cc @@ -0,0 +1,155 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mem_map_arena_pool.h" + +#include <sys/mman.h> + +#include <algorithm> +#include <cstddef> +#include <iomanip> +#include <numeric> + +#include <android-base/logging.h> + +#include "base/arena_allocator-inl.h" +#include "base/mem_map.h" +#include "base/systrace.h" + +namespace art { + +class MemMapArena FINAL : public Arena { + public: + MemMapArena(size_t size, bool low_4gb, const char* name); + virtual ~MemMapArena(); + void Release() OVERRIDE; + + private: + std::unique_ptr<MemMap> map_; +}; + +MemMapArena::MemMapArena(size_t size, bool low_4gb, const char* name) { + // Round up to a full page as that's the smallest unit of allocation for mmap() + // and we want to be able to use all memory that we actually allocate. + size = RoundUp(size, kPageSize); + std::string error_msg; + map_.reset(MemMap::MapAnonymous( + name, nullptr, size, PROT_READ | PROT_WRITE, low_4gb, false, &error_msg)); + CHECK(map_.get() != nullptr) << error_msg; + memory_ = map_->Begin(); + static_assert(ArenaAllocator::kArenaAlignment <= kPageSize, + "Arena should not need stronger alignment than kPageSize."); + DCHECK_ALIGNED(memory_, ArenaAllocator::kArenaAlignment); + size_ = map_->Size(); +} + +MemMapArena::~MemMapArena() { + // Destroys MemMap via std::unique_ptr<>. +} + +void MemMapArena::Release() { + if (bytes_allocated_ > 0) { + map_->MadviseDontNeedAndZero(); + bytes_allocated_ = 0; + } +} + +MemMapArenaPool::MemMapArenaPool(bool low_4gb, const char* name) + : low_4gb_(low_4gb), + name_(name), + free_arenas_(nullptr) { + MemMap::Init(); +} + +MemMapArenaPool::~MemMapArenaPool() { + ReclaimMemory(); +} + +void MemMapArenaPool::ReclaimMemory() { + while (free_arenas_ != nullptr) { + Arena* arena = free_arenas_; + free_arenas_ = free_arenas_->next_; + delete arena; + } +} + +void MemMapArenaPool::LockReclaimMemory() { + std::lock_guard<std::mutex> lock(lock_); + ReclaimMemory(); +} + +Arena* MemMapArenaPool::AllocArena(size_t size) { + Arena* ret = nullptr; + { + std::lock_guard<std::mutex> lock(lock_); + if (free_arenas_ != nullptr && LIKELY(free_arenas_->Size() >= size)) { + ret = free_arenas_; + free_arenas_ = free_arenas_->next_; + } + } + if (ret == nullptr) { + ret = new MemMapArena(size, low_4gb_, name_); + } + ret->Reset(); + return ret; +} + +void MemMapArenaPool::TrimMaps() { + ScopedTrace trace(__PRETTY_FUNCTION__); + std::lock_guard<std::mutex> lock(lock_); + for (Arena* arena = free_arenas_; arena != nullptr; arena = arena->next_) { + arena->Release(); + } +} + +size_t MemMapArenaPool::GetBytesAllocated() const { + size_t total = 0; + std::lock_guard<std::mutex> lock(lock_); + for (Arena* arena = free_arenas_; arena != nullptr; arena = arena->next_) { + total += arena->GetBytesAllocated(); + } + return total; +} + +void MemMapArenaPool::FreeArenaChain(Arena* first) { + if (UNLIKELY(RUNNING_ON_MEMORY_TOOL > 0)) { + for (Arena* arena = first; arena != nullptr; arena = arena->next_) { + MEMORY_TOOL_MAKE_UNDEFINED(arena->memory_, arena->bytes_allocated_); + } + } + + if (arena_allocator::kArenaAllocatorPreciseTracking) { + // Do not reuse arenas when tracking. + while (first != nullptr) { + Arena* next = first->next_; + delete first; + first = next; + } + return; + } + + if (first != nullptr) { + Arena* last = first; + while (last->next_ != nullptr) { + last = last->next_; + } + std::lock_guard<std::mutex> lock(lock_); + last->next_ = free_arenas_; + free_arenas_ = first; + } +} + +} // namespace art diff --git a/runtime/base/mem_map_arena_pool.h b/runtime/base/mem_map_arena_pool.h new file mode 100644 index 00000000000..24e150e1e70 --- /dev/null +++ b/runtime/base/mem_map_arena_pool.h @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_RUNTIME_BASE_MEM_MAP_ARENA_POOL_H_ +#define ART_RUNTIME_BASE_MEM_MAP_ARENA_POOL_H_ + +#include "base/arena_allocator.h" + +namespace art { + +class MemMapArenaPool FINAL : public ArenaPool { + public: + explicit MemMapArenaPool(bool low_4gb = false, const char* name = "LinearAlloc"); + virtual ~MemMapArenaPool(); + Arena* AllocArena(size_t size) OVERRIDE; + void FreeArenaChain(Arena* first) OVERRIDE; + size_t GetBytesAllocated() const OVERRIDE; + void ReclaimMemory() OVERRIDE; + void LockReclaimMemory() OVERRIDE; + // Trim the maps in arenas by madvising, used by JIT to reduce memory usage. + void TrimMaps() OVERRIDE; + + private: + const bool low_4gb_; + const char* name_; + Arena* free_arenas_; + // Use a std::mutex here as Arenas are second-from-the-bottom when using MemMaps, and MemMap + // itself uses std::mutex scoped to within an allocate/free only. + mutable std::mutex lock_; + + DISALLOW_COPY_AND_ASSIGN(MemMapArenaPool); +}; + +} // namespace art + +#endif // ART_RUNTIME_BASE_MEM_MAP_ARENA_POOL_H_ diff --git a/runtime/base/mutator_locked_dumpable.h b/runtime/base/mutator_locked_dumpable.h new file mode 100644 index 00000000000..cf2199c1008 --- /dev/null +++ b/runtime/base/mutator_locked_dumpable.h @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_RUNTIME_BASE_MUTATOR_LOCKED_DUMPABLE_H_ +#define ART_RUNTIME_BASE_MUTATOR_LOCKED_DUMPABLE_H_ + +#include "base/mutex.h" +#include "thread-current-inl.h" + +namespace art { + +template<typename T> +class MutatorLockedDumpable { + public: + explicit MutatorLockedDumpable(T& value) REQUIRES_SHARED(Locks::mutator_lock_) : value_(value) {} + + void Dump(std::ostream& os) const REQUIRES_SHARED(Locks::mutator_lock_) { + value_.Dump(os); + } + + private: + const T& value_; + + DISALLOW_COPY_AND_ASSIGN(MutatorLockedDumpable); +}; + +// template<typename T> +// std::ostream& operator<<(std::ostream& os, const MutatorLockedDumpable<T>& rhs) +// // TODO: should be REQUIRES_SHARED(Locks::mutator_lock_) however annotalysis +// // currently fails for this. +// NO_THREAD_SAFETY_ANALYSIS; + +template<typename T> +inline std::ostream& operator<<(std::ostream& os, const MutatorLockedDumpable<T>& rhs) + NO_THREAD_SAFETY_ANALYSIS { + Locks::mutator_lock_->AssertSharedHeld(Thread::Current()); + rhs.Dump(os); + return os; +} + +} // namespace art + +#endif // ART_RUNTIME_BASE_MUTATOR_LOCKED_DUMPABLE_H_ diff --git a/runtime/base/mutex-inl.h b/runtime/base/mutex-inl.h index d6dbab4606c..dfa14b91f05 100644 --- a/runtime/base/mutex-inl.h +++ b/runtime/base/mutex-inl.h @@ -161,7 +161,7 @@ inline void ReaderWriterMutex::SharedLock(Thread* self) { #if ART_USE_FUTEXES bool done = false; do { - int32_t cur_state = state_.LoadRelaxed(); + int32_t cur_state = state_.load(std::memory_order_relaxed); if (LIKELY(cur_state >= 0)) { // Add as an extra reader. done = state_.CompareAndSetWeakAcquire(cur_state, cur_state + 1); @@ -185,7 +185,7 @@ inline void ReaderWriterMutex::SharedUnlock(Thread* self) { #if ART_USE_FUTEXES bool done = false; do { - int32_t cur_state = state_.LoadRelaxed(); + int32_t cur_state = state_.load(std::memory_order_relaxed); if (LIKELY(cur_state > 0)) { // Reduce state by 1 and impose lock release load/store ordering. // Note, the relaxed loads below musn't reorder before the CompareAndSet. @@ -193,8 +193,8 @@ inline void ReaderWriterMutex::SharedUnlock(Thread* self) { // a status bit into the state on contention. done = state_.CompareAndSetWeakSequentiallyConsistent(cur_state, cur_state - 1); if (done && (cur_state - 1) == 0) { // Weak CAS may fail spuriously. - if (num_pending_writers_.LoadRelaxed() > 0 || - num_pending_readers_.LoadRelaxed() > 0) { + if (num_pending_writers_.load(std::memory_order_relaxed) > 0 || + num_pending_readers_.load(std::memory_order_relaxed) > 0) { // Wake any exclusive waiters as there are now no readers. futex(state_.Address(), FUTEX_WAKE, -1, nullptr, nullptr, 0); } @@ -221,7 +221,7 @@ inline bool Mutex::IsExclusiveHeld(const Thread* self) const { } inline pid_t Mutex::GetExclusiveOwnerTid() const { - return exclusive_owner_.LoadRelaxed(); + return exclusive_owner_.load(std::memory_order_relaxed); } inline void Mutex::AssertExclusiveHeld(const Thread* self) const { @@ -248,16 +248,16 @@ inline bool ReaderWriterMutex::IsExclusiveHeld(const Thread* self) const { inline pid_t ReaderWriterMutex::GetExclusiveOwnerTid() const { #if ART_USE_FUTEXES - int32_t state = state_.LoadRelaxed(); + int32_t state = state_.load(std::memory_order_relaxed); if (state == 0) { return 0; // No owner. } else if (state > 0) { return -1; // Shared. } else { - return exclusive_owner_.LoadRelaxed(); + return exclusive_owner_.load(std::memory_order_relaxed); } #else - return exclusive_owner_.LoadRelaxed(); + return exclusive_owner_.load(std::memory_order_relaxed); #endif } diff --git a/runtime/base/mutex.cc b/runtime/base/mutex.cc index a1f30b67945..73b464119ea 100644 --- a/runtime/base/mutex.cc +++ b/runtime/base/mutex.cc @@ -128,15 +128,15 @@ class ScopedAllMutexesLock FINAL { public: explicit ScopedAllMutexesLock(const BaseMutex* mutex) : mutex_(mutex) { for (uint32_t i = 0; - !gAllMutexData->all_mutexes_guard.CompareAndSetWeakAcquire(0, mutex); + !gAllMutexData->all_mutexes_guard.CompareAndSetWeakAcquire(nullptr, mutex); ++i) { BackOff(i); } } ~ScopedAllMutexesLock() { - DCHECK_EQ(gAllMutexData->all_mutexes_guard.LoadRelaxed(), mutex_); - gAllMutexData->all_mutexes_guard.StoreRelease(0); + DCHECK_EQ(gAllMutexData->all_mutexes_guard.load(std::memory_order_relaxed), mutex_); + gAllMutexData->all_mutexes_guard.store(nullptr, std::memory_order_release); } private: @@ -147,15 +147,17 @@ class Locks::ScopedExpectedMutexesOnWeakRefAccessLock FINAL { public: explicit ScopedExpectedMutexesOnWeakRefAccessLock(const BaseMutex* mutex) : mutex_(mutex) { for (uint32_t i = 0; - !Locks::expected_mutexes_on_weak_ref_access_guard_.CompareAndSetWeakAcquire(0, mutex); + !Locks::expected_mutexes_on_weak_ref_access_guard_.CompareAndSetWeakAcquire(nullptr, + mutex); ++i) { BackOff(i); } } ~ScopedExpectedMutexesOnWeakRefAccessLock() { - DCHECK_EQ(Locks::expected_mutexes_on_weak_ref_access_guard_.LoadRelaxed(), mutex_); - Locks::expected_mutexes_on_weak_ref_access_guard_.StoreRelease(0); + DCHECK_EQ(Locks::expected_mutexes_on_weak_ref_access_guard_.load(std::memory_order_relaxed), + mutex_); + Locks::expected_mutexes_on_weak_ref_access_guard_.store(nullptr, std::memory_order_release); } private: @@ -293,7 +295,7 @@ void BaseMutex::CheckSafeToWait(Thread* self) { void BaseMutex::ContentionLogData::AddToWaitTime(uint64_t value) { if (kLogLockContentions) { // Atomically add value to wait_time. - wait_time.FetchAndAddSequentiallyConsistent(value); + wait_time.fetch_add(value, std::memory_order_seq_cst); } } @@ -306,19 +308,19 @@ void BaseMutex::RecordContention(uint64_t blocked_tid, data->AddToWaitTime(nano_time_blocked); ContentionLogEntry* log = data->contention_log; // This code is intentionally racy as it is only used for diagnostics. - uint32_t slot = data->cur_content_log_entry.LoadRelaxed(); + int32_t slot = data->cur_content_log_entry.load(std::memory_order_relaxed); if (log[slot].blocked_tid == blocked_tid && log[slot].owner_tid == blocked_tid) { ++log[slot].count; } else { uint32_t new_slot; do { - slot = data->cur_content_log_entry.LoadRelaxed(); + slot = data->cur_content_log_entry.load(std::memory_order_relaxed); new_slot = (slot + 1) % kContentionLogSize; } while (!data->cur_content_log_entry.CompareAndSetWeakRelaxed(slot, new_slot)); log[new_slot].blocked_tid = blocked_tid; log[new_slot].owner_tid = owner_tid; - log[new_slot].count.StoreRelaxed(1); + log[new_slot].count.store(1, std::memory_order_relaxed); } } } @@ -327,8 +329,8 @@ void BaseMutex::DumpContention(std::ostream& os) const { if (kLogLockContentions) { const ContentionLogData* data = contention_log_data_; const ContentionLogEntry* log = data->contention_log; - uint64_t wait_time = data->wait_time.LoadRelaxed(); - uint32_t contention_count = data->contention_count.LoadRelaxed(); + uint64_t wait_time = data->wait_time.load(std::memory_order_relaxed); + uint32_t contention_count = data->contention_count.load(std::memory_order_relaxed); if (contention_count == 0) { os << "never contended"; } else { @@ -340,7 +342,7 @@ void BaseMutex::DumpContention(std::ostream& os) const { for (size_t i = 0; i < kContentionLogSize; ++i) { uint64_t blocked_tid = log[i].blocked_tid; uint64_t owner_tid = log[i].owner_tid; - uint32_t count = log[i].count.LoadRelaxed(); + uint32_t count = log[i].count.load(std::memory_order_relaxed); if (count > 0) { auto it = most_common_blocked.find(blocked_tid); if (it != most_common_blocked.end()) { @@ -386,8 +388,8 @@ void BaseMutex::DumpContention(std::ostream& os) const { Mutex::Mutex(const char* name, LockLevel level, bool recursive) : BaseMutex(name, level), exclusive_owner_(0), recursive_(recursive), recursion_count_(0) { #if ART_USE_FUTEXES - DCHECK_EQ(0, state_.LoadRelaxed()); - DCHECK_EQ(0, num_contenders_.LoadRelaxed()); + DCHECK_EQ(0, state_.load(std::memory_order_relaxed)); + DCHECK_EQ(0, num_contenders_.load(std::memory_order_relaxed)); #else CHECK_MUTEX_CALL(pthread_mutex_init, (&mutex_, nullptr)); #endif @@ -402,7 +404,7 @@ static bool IsSafeToCallAbortSafe() { Mutex::~Mutex() { bool safe_to_call_abort = Locks::IsSafeToCallAbortRacy(); #if ART_USE_FUTEXES - if (state_.LoadRelaxed() != 0) { + if (state_.load(std::memory_order_relaxed) != 0) { LOG(safe_to_call_abort ? FATAL : WARNING) << "destroying mutex with owner: " << GetExclusiveOwnerTid(); } else { @@ -410,7 +412,7 @@ Mutex::~Mutex() { LOG(safe_to_call_abort ? FATAL : WARNING) << "unexpectedly found an owner on unlocked mutex " << name_; } - if (num_contenders_.LoadSequentiallyConsistent() != 0) { + if (num_contenders_.load(std::memory_order_seq_cst) != 0) { LOG(safe_to_call_abort ? FATAL : WARNING) << "unexpectedly found a contender on mutex " << name_; } @@ -436,7 +438,7 @@ void Mutex::ExclusiveLock(Thread* self) { #if ART_USE_FUTEXES bool done = false; do { - int32_t cur_state = state_.LoadRelaxed(); + int32_t cur_state = state_.load(std::memory_order_relaxed); if (LIKELY(cur_state == 0)) { // Change state from 0 to 1 and impose load/store ordering appropriate for lock acquisition. done = state_.CompareAndSetWeakAcquire(0 /* cur_state */, 1 /* new state */); @@ -457,12 +459,12 @@ void Mutex::ExclusiveLock(Thread* self) { num_contenders_--; } } while (!done); - DCHECK_EQ(state_.LoadRelaxed(), 1); + DCHECK_EQ(state_.load(std::memory_order_relaxed), 1); #else CHECK_MUTEX_CALL(pthread_mutex_lock, (&mutex_)); #endif DCHECK_EQ(GetExclusiveOwnerTid(), 0); - exclusive_owner_.StoreRelaxed(SafeGetTid(self)); + exclusive_owner_.store(SafeGetTid(self), std::memory_order_relaxed); RegisterAsLocked(self); } recursion_count_++; @@ -482,7 +484,7 @@ bool Mutex::ExclusiveTryLock(Thread* self) { #if ART_USE_FUTEXES bool done = false; do { - int32_t cur_state = state_.LoadRelaxed(); + int32_t cur_state = state_.load(std::memory_order_relaxed); if (cur_state == 0) { // Change state from 0 to 1 and impose load/store ordering appropriate for lock acquisition. done = state_.CompareAndSetWeakAcquire(0 /* cur_state */, 1 /* new state */); @@ -490,7 +492,7 @@ bool Mutex::ExclusiveTryLock(Thread* self) { return false; } } while (!done); - DCHECK_EQ(state_.LoadRelaxed(), 1); + DCHECK_EQ(state_.load(std::memory_order_relaxed), 1); #else int result = pthread_mutex_trylock(&mutex_); if (result == EBUSY) { @@ -502,7 +504,7 @@ bool Mutex::ExclusiveTryLock(Thread* self) { } #endif DCHECK_EQ(GetExclusiveOwnerTid(), 0); - exclusive_owner_.StoreRelaxed(SafeGetTid(self)); + exclusive_owner_.store(SafeGetTid(self), std::memory_order_relaxed); RegisterAsLocked(self); } recursion_count_++; @@ -539,10 +541,10 @@ void Mutex::ExclusiveUnlock(Thread* self) { #if ART_USE_FUTEXES bool done = false; do { - int32_t cur_state = state_.LoadRelaxed(); + int32_t cur_state = state_.load(std::memory_order_relaxed); if (LIKELY(cur_state == 1)) { // We're no longer the owner. - exclusive_owner_.StoreRelaxed(0); + exclusive_owner_.store(0 /* pid */, std::memory_order_relaxed); // Change state to 0 and impose load/store ordering appropriate for lock release. // Note, the relaxed loads below mustn't reorder before the CompareAndSet. // TODO: the ordering here is non-trivial as state is split across 3 fields, fix by placing @@ -550,7 +552,7 @@ void Mutex::ExclusiveUnlock(Thread* self) { done = state_.CompareAndSetWeakSequentiallyConsistent(cur_state, 0 /* new state */); if (LIKELY(done)) { // Spurious fail? // Wake a contender. - if (UNLIKELY(num_contenders_.LoadRelaxed() > 0)) { + if (UNLIKELY(num_contenders_.load(std::memory_order_relaxed) > 0)) { futex(state_.Address(), FUTEX_WAKE, 1, nullptr, nullptr, 0); } } @@ -569,7 +571,7 @@ void Mutex::ExclusiveUnlock(Thread* self) { } } while (!done); #else - exclusive_owner_.StoreRelaxed(0); + exclusive_owner_.store(0 /* pid */, std::memory_order_relaxed); CHECK_MUTEX_CALL(pthread_mutex_unlock, (&mutex_)); #endif } @@ -593,7 +595,7 @@ void Mutex::WakeupToRespondToEmptyCheckpoint() { #if ART_USE_FUTEXES // Wake up all the waiters so they will respond to the emtpy checkpoint. DCHECK(should_respond_to_empty_checkpoint_request_); - if (UNLIKELY(num_contenders_.LoadRelaxed() > 0)) { + if (UNLIKELY(num_contenders_.load(std::memory_order_relaxed) > 0)) { futex(state_.Address(), FUTEX_WAKE, -1, nullptr, nullptr, 0); } #else @@ -610,15 +612,15 @@ ReaderWriterMutex::ReaderWriterMutex(const char* name, LockLevel level) #if !ART_USE_FUTEXES CHECK_MUTEX_CALL(pthread_rwlock_init, (&rwlock_, nullptr)); #endif - exclusive_owner_.StoreRelaxed(0); + exclusive_owner_.store(0 /* pid */, std::memory_order_relaxed); } ReaderWriterMutex::~ReaderWriterMutex() { #if ART_USE_FUTEXES - CHECK_EQ(state_.LoadRelaxed(), 0); + CHECK_EQ(state_.load(std::memory_order_relaxed), 0); CHECK_EQ(GetExclusiveOwnerTid(), 0); - CHECK_EQ(num_pending_readers_.LoadRelaxed(), 0); - CHECK_EQ(num_pending_writers_.LoadRelaxed(), 0); + CHECK_EQ(num_pending_readers_.load(std::memory_order_relaxed), 0); + CHECK_EQ(num_pending_writers_.load(std::memory_order_relaxed), 0); #else // We can't use CHECK_MUTEX_CALL here because on shutdown a suspended daemon thread // may still be using locks. @@ -637,7 +639,7 @@ void ReaderWriterMutex::ExclusiveLock(Thread* self) { #if ART_USE_FUTEXES bool done = false; do { - int32_t cur_state = state_.LoadRelaxed(); + int32_t cur_state = state_.load(std::memory_order_relaxed); if (LIKELY(cur_state == 0)) { // Change state from 0 to -1 and impose load/store ordering appropriate for lock acquisition. done = state_.CompareAndSetWeakAcquire(0 /* cur_state*/, -1 /* new state */); @@ -658,12 +660,12 @@ void ReaderWriterMutex::ExclusiveLock(Thread* self) { --num_pending_writers_; } } while (!done); - DCHECK_EQ(state_.LoadRelaxed(), -1); + DCHECK_EQ(state_.load(std::memory_order_relaxed), -1); #else CHECK_MUTEX_CALL(pthread_rwlock_wrlock, (&rwlock_)); #endif DCHECK_EQ(GetExclusiveOwnerTid(), 0); - exclusive_owner_.StoreRelaxed(SafeGetTid(self)); + exclusive_owner_.store(SafeGetTid(self), std::memory_order_relaxed); RegisterAsLocked(self); AssertExclusiveHeld(self); } @@ -676,10 +678,10 @@ void ReaderWriterMutex::ExclusiveUnlock(Thread* self) { #if ART_USE_FUTEXES bool done = false; do { - int32_t cur_state = state_.LoadRelaxed(); + int32_t cur_state = state_.load(std::memory_order_relaxed); if (LIKELY(cur_state == -1)) { // We're no longer the owner. - exclusive_owner_.StoreRelaxed(0); + exclusive_owner_.store(0 /* pid */, std::memory_order_relaxed); // Change state from -1 to 0 and impose load/store ordering appropriate for lock release. // Note, the relaxed loads below musn't reorder before the CompareAndSet. // TODO: the ordering here is non-trivial as state is split across 3 fields, fix by placing @@ -687,8 +689,8 @@ void ReaderWriterMutex::ExclusiveUnlock(Thread* self) { done = state_.CompareAndSetWeakSequentiallyConsistent(-1 /* cur_state*/, 0 /* new state */); if (LIKELY(done)) { // Weak CAS may fail spuriously. // Wake any waiters. - if (UNLIKELY(num_pending_readers_.LoadRelaxed() > 0 || - num_pending_writers_.LoadRelaxed() > 0)) { + if (UNLIKELY(num_pending_readers_.load(std::memory_order_relaxed) > 0 || + num_pending_writers_.load(std::memory_order_relaxed) > 0)) { futex(state_.Address(), FUTEX_WAKE, -1, nullptr, nullptr, 0); } } @@ -697,7 +699,7 @@ void ReaderWriterMutex::ExclusiveUnlock(Thread* self) { } } while (!done); #else - exclusive_owner_.StoreRelaxed(0); + exclusive_owner_.store(0 /* pid */, std::memory_order_relaxed); CHECK_MUTEX_CALL(pthread_rwlock_unlock, (&rwlock_)); #endif } @@ -710,7 +712,7 @@ bool ReaderWriterMutex::ExclusiveLockWithTimeout(Thread* self, int64_t ms, int32 timespec end_abs_ts; InitTimeSpec(true, CLOCK_MONOTONIC, ms, ns, &end_abs_ts); do { - int32_t cur_state = state_.LoadRelaxed(); + int32_t cur_state = state_.load(std::memory_order_relaxed); if (cur_state == 0) { // Change state from 0 to -1 and impose load/store ordering appropriate for lock acquisition. done = state_.CompareAndSetWeakAcquire(0 /* cur_state */, -1 /* new state */); @@ -753,7 +755,7 @@ bool ReaderWriterMutex::ExclusiveLockWithTimeout(Thread* self, int64_t ms, int32 PLOG(FATAL) << "pthread_rwlock_timedwrlock failed for " << name_; } #endif - exclusive_owner_.StoreRelaxed(SafeGetTid(self)); + exclusive_owner_.store(SafeGetTid(self), std::memory_order_relaxed); RegisterAsLocked(self); AssertSharedHeld(self); return true; @@ -782,7 +784,7 @@ bool ReaderWriterMutex::SharedTryLock(Thread* self) { #if ART_USE_FUTEXES bool done = false; do { - int32_t cur_state = state_.LoadRelaxed(); + int32_t cur_state = state_.load(std::memory_order_relaxed); if (cur_state >= 0) { // Add as an extra reader and impose load/store ordering appropriate for lock acquisition. done = state_.CompareAndSetWeakAcquire(cur_state, cur_state + 1); @@ -822,9 +824,9 @@ void ReaderWriterMutex::Dump(std::ostream& os) const { << " level=" << static_cast<int>(level_) << " owner=" << GetExclusiveOwnerTid() #if ART_USE_FUTEXES - << " state=" << state_.LoadSequentiallyConsistent() - << " num_pending_writers=" << num_pending_writers_.LoadSequentiallyConsistent() - << " num_pending_readers=" << num_pending_readers_.LoadSequentiallyConsistent() + << " state=" << state_.load(std::memory_order_seq_cst) + << " num_pending_writers=" << num_pending_writers_.load(std::memory_order_seq_cst) + << " num_pending_readers=" << num_pending_readers_.load(std::memory_order_seq_cst) #endif << " "; DumpContention(os); @@ -844,8 +846,8 @@ void ReaderWriterMutex::WakeupToRespondToEmptyCheckpoint() { #if ART_USE_FUTEXES // Wake up all the waiters so they will respond to the emtpy checkpoint. DCHECK(should_respond_to_empty_checkpoint_request_); - if (UNLIKELY(num_pending_readers_.LoadRelaxed() > 0 || - num_pending_writers_.LoadRelaxed() > 0)) { + if (UNLIKELY(num_pending_readers_.load(std::memory_order_relaxed) > 0 || + num_pending_writers_.load(std::memory_order_relaxed) > 0)) { futex(state_.Address(), FUTEX_WAKE, -1, nullptr, nullptr, 0); } #else @@ -856,7 +858,7 @@ void ReaderWriterMutex::WakeupToRespondToEmptyCheckpoint() { ConditionVariable::ConditionVariable(const char* name, Mutex& guard) : name_(name), guard_(guard) { #if ART_USE_FUTEXES - DCHECK_EQ(0, sequence_.LoadRelaxed()); + DCHECK_EQ(0, sequence_.load(std::memory_order_relaxed)); num_waiters_ = 0; #else pthread_condattr_t cond_attrs; @@ -899,7 +901,7 @@ void ConditionVariable::Broadcast(Thread* self) { sequence_++; // Indicate the broadcast occurred. bool done = false; do { - int32_t cur_sequence = sequence_.LoadRelaxed(); + int32_t cur_sequence = sequence_.load(std::memory_order_relaxed); // Requeue waiters onto mutex. The waiter holds the contender count on the mutex high ensuring // mutex unlocks will awaken the requeued waiter thread. done = futex(sequence_.Address(), FUTEX_CMP_REQUEUE, 0, @@ -948,7 +950,7 @@ void ConditionVariable::WaitHoldingLocks(Thread* self) { // Ensure the Mutex is contended so that requeued threads are awoken. guard_.num_contenders_++; guard_.recursion_count_ = 1; - int32_t cur_sequence = sequence_.LoadRelaxed(); + int32_t cur_sequence = sequence_.load(std::memory_order_relaxed); guard_.ExclusiveUnlock(self); if (futex(sequence_.Address(), FUTEX_WAIT, cur_sequence, nullptr, nullptr, 0) != 0) { // Futex failed, check it is an expected error. @@ -974,14 +976,14 @@ void ConditionVariable::WaitHoldingLocks(Thread* self) { CHECK_GE(num_waiters_, 0); num_waiters_--; // We awoke and so no longer require awakes from the guard_'s unlock. - CHECK_GE(guard_.num_contenders_.LoadRelaxed(), 0); + CHECK_GE(guard_.num_contenders_.load(std::memory_order_relaxed), 0); guard_.num_contenders_--; #else pid_t old_owner = guard_.GetExclusiveOwnerTid(); - guard_.exclusive_owner_.StoreRelaxed(0); + guard_.exclusive_owner_.store(0 /* pid */, std::memory_order_relaxed); guard_.recursion_count_ = 0; CHECK_MUTEX_CALL(pthread_cond_wait, (&cond_, &guard_.mutex_)); - guard_.exclusive_owner_.StoreRelaxed(old_owner); + guard_.exclusive_owner_.store(old_owner, std::memory_order_relaxed); #endif guard_.recursion_count_ = old_recursion_count; } @@ -999,7 +1001,7 @@ bool ConditionVariable::TimedWait(Thread* self, int64_t ms, int32_t ns) { // Ensure the Mutex is contended so that requeued threads are awoken. guard_.num_contenders_++; guard_.recursion_count_ = 1; - int32_t cur_sequence = sequence_.LoadRelaxed(); + int32_t cur_sequence = sequence_.load(std::memory_order_relaxed); guard_.ExclusiveUnlock(self); if (futex(sequence_.Address(), FUTEX_WAIT, cur_sequence, &rel_ts, nullptr, 0) != 0) { if (errno == ETIMEDOUT) { @@ -1015,7 +1017,7 @@ bool ConditionVariable::TimedWait(Thread* self, int64_t ms, int32_t ns) { CHECK_GE(num_waiters_, 0); num_waiters_--; // We awoke and so no longer require awakes from the guard_'s unlock. - CHECK_GE(guard_.num_contenders_.LoadRelaxed(), 0); + CHECK_GE(guard_.num_contenders_.load(std::memory_order_relaxed), 0); guard_.num_contenders_--; #else #if !defined(__APPLE__) @@ -1024,7 +1026,7 @@ bool ConditionVariable::TimedWait(Thread* self, int64_t ms, int32_t ns) { int clock = CLOCK_REALTIME; #endif pid_t old_owner = guard_.GetExclusiveOwnerTid(); - guard_.exclusive_owner_.StoreRelaxed(0); + guard_.exclusive_owner_.store(0 /* pid */, std::memory_order_relaxed); guard_.recursion_count_ = 0; timespec ts; InitTimeSpec(true, clock, ms, ns, &ts); @@ -1035,7 +1037,7 @@ bool ConditionVariable::TimedWait(Thread* self, int64_t ms, int32_t ns) { errno = rc; PLOG(FATAL) << "TimedWait failed for " << name_; } - guard_.exclusive_owner_.StoreRelaxed(old_owner); + guard_.exclusive_owner_.store(old_owner, std::memory_order_relaxed); #endif guard_.recursion_count_ = old_recursion_count; return timed_out; @@ -1254,12 +1256,13 @@ void Locks::InitConditions() { } void Locks::SetClientCallback(ClientCallback* safe_to_call_abort_cb) { - safe_to_call_abort_callback.StoreRelease(safe_to_call_abort_cb); + safe_to_call_abort_callback.store(safe_to_call_abort_cb, std::memory_order_release); } // Helper to allow checking shutdown while ignoring locking requirements. bool Locks::IsSafeToCallAbortRacy() { - Locks::ClientCallback* safe_to_call_abort_cb = safe_to_call_abort_callback.LoadAcquire(); + Locks::ClientCallback* safe_to_call_abort_cb = + safe_to_call_abort_callback.load(std::memory_order_acquire); return safe_to_call_abort_cb != nullptr && safe_to_call_abort_cb(); } diff --git a/runtime/base/mutex.h b/runtime/base/mutex.h index 437661798f4..1cf4ddded48 100644 --- a/runtime/base/mutex.h +++ b/runtime/base/mutex.h @@ -31,10 +31,10 @@ #include "base/globals.h" #include "base/macros.h" -#if defined(__APPLE__) -#define ART_USE_FUTEXES 0 -#else +#if defined(__linux__) #define ART_USE_FUTEXES 1 +#else +#define ART_USE_FUTEXES 0 #endif // Currently Darwin doesn't support locks with timeouts. @@ -224,7 +224,7 @@ class BaseMutex { public: bool HasEverContended() const { if (kLogLockContentions) { - return contention_log_data_->contention_count.LoadSequentiallyConsistent() > 0; + return contention_log_data_->contention_count.load(std::memory_order_seq_cst) > 0; } return false; } diff --git a/runtime/base/quasi_atomic.h b/runtime/base/quasi_atomic.h index 067d01db01c..0012f6482b8 100644 --- a/runtime/base/quasi_atomic.h +++ b/runtime/base/quasi_atomic.h @@ -152,14 +152,6 @@ class QuasiAtomic { return NeedSwapMutexes(isa); } - static void ThreadFenceAcquire() { - std::atomic_thread_fence(std::memory_order_acquire); - } - - static void ThreadFenceRelease() { - std::atomic_thread_fence(std::memory_order_release); - } - static void ThreadFenceForConstructor() { #if defined(__aarch64__) __asm__ __volatile__("dmb ishst" : : : "memory"); @@ -168,10 +160,6 @@ class QuasiAtomic { #endif } - static void ThreadFenceSequentiallyConsistent() { - std::atomic_thread_fence(std::memory_order_seq_cst); - } - private: static Mutex* GetSwapMutex(const volatile int64_t* addr); static int64_t SwapMutexRead64(volatile const int64_t* addr); diff --git a/runtime/check_jni.cc b/runtime/check_jni.cc index 900ce0eac3d..9a437905750 100644 --- a/runtime/check_jni.cc +++ b/runtime/check_jni.cc @@ -37,6 +37,8 @@ #include "java_vm_ext.h" #include "jni_internal.h" #include "mirror/class-inl.h" +#include "mirror/field.h" +#include "mirror/method.h" #include "mirror/object-inl.h" #include "mirror/object_array-inl.h" #include "mirror/string-inl.h" @@ -273,6 +275,43 @@ class VarArgs { }; }; +// Check whether the current thread is attached. This is usually required +// to be the first check, as ScopedCheck needs a ScopedObjectAccess for +// checking heap values (and that will fail with unattached threads). +bool CheckAttachedThread(const char* function_name) { + Thread* self = Thread::Current(); + if (UNLIKELY(self == nullptr)) { + // Need to attach this thread for a proper abort to work. We prefer this + // to get reasonable stacks and environment, rather than relying on + // tombstoned. + JNIEnv* env; + Runtime::Current()->GetJavaVM()->AttachCurrentThread(&env, /* thread_args */ nullptr); + + std::string tmp = android::base::StringPrintf( + "a thread (tid %" PRId64 " is making JNI calls without being attached", + static_cast<int64_t>(GetTid())); + Runtime::Current()->GetJavaVM()->JniAbort(function_name, tmp.c_str()); + + CHECK_NE(Runtime::Current()->GetJavaVM()->DetachCurrentThread(), JNI_ERR); + return false; + } + return true; +} + +// Macro helpers for the above. +#define CHECK_ATTACHED_THREAD(function_name, fail_val) \ + do { \ + if (!CheckAttachedThread((function_name))) { \ + return fail_val; \ + } \ + } while (false) +#define CHECK_ATTACHED_THREAD_VOID(function_name) \ + do { \ + if (!CheckAttachedThread((function_name))) { \ + return; \ + } \ + } while (false) + class ScopedCheck { public: ScopedCheck(uint16_t flags, const char* functionName, bool has_method = true) @@ -599,9 +638,8 @@ class ScopedCheck { AbortF("expected non-null method"); return false; } - mirror::Class* c = method->GetClass(); - if (soa.Decode<mirror::Class>(WellKnownClasses::java_lang_reflect_Method) != c && - soa.Decode<mirror::Class>(WellKnownClasses::java_lang_reflect_Constructor) != c) { + ObjPtr<mirror::Class> c = method->GetClass(); + if (mirror::Method::StaticClass() != c && mirror::Constructor::StaticClass() != c) { AbortF("expected java.lang.reflect.Method or " "java.lang.reflect.Constructor but got object of type %s: %p", method->PrettyTypeOf().c_str(), jmethod); @@ -630,8 +668,8 @@ class ScopedCheck { AbortF("expected non-null java.lang.reflect.Field"); return false; } - mirror::Class* c = field->GetClass(); - if (soa.Decode<mirror::Class>(WellKnownClasses::java_lang_reflect_Field) != c) { + ObjPtr<mirror::Class> c = field->GetClass(); + if (mirror::Field::StaticClass() != c) { AbortF("expected java.lang.reflect.Field but got object of type %s: %p", field->PrettyTypeOf().c_str(), jfield); return false; @@ -1254,10 +1292,7 @@ class ScopedCheck { bool CheckThread(JNIEnv* env) REQUIRES_SHARED(Locks::mutator_lock_) { Thread* self = Thread::Current(); - if (self == nullptr) { - AbortF("a thread (tid %d) is making JNI calls without being attached", GetTid()); - return false; - } + CHECK(self != nullptr); // Get the current thread's JNIEnv by going through our TLS pointer. JNIEnvExt* threadEnv = self->GetJniEnv(); @@ -1707,6 +1742,7 @@ const char* const GuardedCopy::kCanary = "JNI BUFFER RED ZONE"; class CheckJNI { public: static jint GetVersion(JNIEnv* env) { + CHECK_ATTACHED_THREAD(__FUNCTION__, JNI_ERR); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_Default, __FUNCTION__); JniValueType args[1] = {{.E = env }}; @@ -1721,6 +1757,7 @@ class CheckJNI { } static jint GetJavaVM(JNIEnv *env, JavaVM **vm) { + CHECK_ATTACHED_THREAD(__FUNCTION__, JNI_ERR); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_Default, __FUNCTION__); JniValueType args[2] = {{.E = env }, {.p = vm}}; @@ -1735,6 +1772,7 @@ class CheckJNI { } static jint RegisterNatives(JNIEnv* env, jclass c, const JNINativeMethod* methods, jint nMethods) { + CHECK_ATTACHED_THREAD(__FUNCTION__, JNI_ERR); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_Default, __FUNCTION__); JniValueType args[4] = {{.E = env }, {.c = c}, {.p = methods}, {.I = nMethods}}; @@ -1749,6 +1787,7 @@ class CheckJNI { } static jint UnregisterNatives(JNIEnv* env, jclass c) { + CHECK_ATTACHED_THREAD(__FUNCTION__, JNI_ERR); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_Default, __FUNCTION__); JniValueType args[2] = {{.E = env }, {.c = c}}; @@ -1763,6 +1802,7 @@ class CheckJNI { } static jobjectRefType GetObjectRefType(JNIEnv* env, jobject obj) { + CHECK_ATTACHED_THREAD(__FUNCTION__, JNIInvalidRefType); // Note: we use "EL" here but "Ep" has been used in the past on the basis that we'd like to // know the object is invalid. The spec says that passing invalid objects or even ones that // are deleted isn't supported. @@ -1781,6 +1821,7 @@ class CheckJNI { static jclass DefineClass(JNIEnv* env, const char* name, jobject loader, const jbyte* buf, jsize bufLen) { + CHECK_ATTACHED_THREAD(__FUNCTION__, nullptr); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_Default, __FUNCTION__); JniValueType args[5] = {{.E = env}, {.u = name}, {.L = loader}, {.p = buf}, {.z = bufLen}}; @@ -1795,6 +1836,7 @@ class CheckJNI { } static jclass FindClass(JNIEnv* env, const char* name) { + CHECK_ATTACHED_THREAD(__FUNCTION__, nullptr); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_Default, __FUNCTION__); JniValueType args[2] = {{.E = env}, {.u = name}}; @@ -1809,6 +1851,7 @@ class CheckJNI { } static jclass GetSuperclass(JNIEnv* env, jclass c) { + CHECK_ATTACHED_THREAD(__FUNCTION__, nullptr); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_Default, __FUNCTION__); JniValueType args[2] = {{.E = env}, {.c = c}}; @@ -1823,6 +1866,7 @@ class CheckJNI { } static jboolean IsAssignableFrom(JNIEnv* env, jclass c1, jclass c2) { + CHECK_ATTACHED_THREAD(__FUNCTION__, JNI_FALSE); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_Default, __FUNCTION__); JniValueType args[3] = {{.E = env}, {.c = c1}, {.c = c2}}; @@ -1837,6 +1881,7 @@ class CheckJNI { } static jmethodID FromReflectedMethod(JNIEnv* env, jobject method) { + CHECK_ATTACHED_THREAD(__FUNCTION__, nullptr); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_Default, __FUNCTION__); JniValueType args[2] = {{.E = env}, {.L = method}}; @@ -1851,6 +1896,7 @@ class CheckJNI { } static jfieldID FromReflectedField(JNIEnv* env, jobject field) { + CHECK_ATTACHED_THREAD(__FUNCTION__, nullptr); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_Default, __FUNCTION__); JniValueType args[2] = {{.E = env}, {.L = field}}; @@ -1865,6 +1911,7 @@ class CheckJNI { } static jobject ToReflectedMethod(JNIEnv* env, jclass cls, jmethodID mid, jboolean isStatic) { + CHECK_ATTACHED_THREAD(__FUNCTION__, nullptr); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_Default, __FUNCTION__); JniValueType args[4] = {{.E = env}, {.c = cls}, {.m = mid}, {.I = isStatic}}; @@ -1880,6 +1927,7 @@ class CheckJNI { } static jobject ToReflectedField(JNIEnv* env, jclass cls, jfieldID fid, jboolean isStatic) { + CHECK_ATTACHED_THREAD(__FUNCTION__, nullptr); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_Default, __FUNCTION__); JniValueType args[4] = {{.E = env}, {.c = cls}, {.f = fid}, {.I = isStatic}}; @@ -1895,6 +1943,7 @@ class CheckJNI { } static jint Throw(JNIEnv* env, jthrowable obj) { + CHECK_ATTACHED_THREAD(__FUNCTION__, JNI_ERR); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_Default, __FUNCTION__); JniValueType args[2] = {{.E = env}, {.t = obj}}; @@ -1909,6 +1958,7 @@ class CheckJNI { } static jint ThrowNew(JNIEnv* env, jclass c, const char* message) { + CHECK_ATTACHED_THREAD(__FUNCTION__, JNI_ERR); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_NullableUtf, __FUNCTION__); JniValueType args[3] = {{.E = env}, {.c = c}, {.u = message}}; @@ -1923,6 +1973,7 @@ class CheckJNI { } static jthrowable ExceptionOccurred(JNIEnv* env) { + CHECK_ATTACHED_THREAD(__FUNCTION__, nullptr); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_ExcepOkay, __FUNCTION__); JniValueType args[1] = {{.E = env}}; @@ -1937,6 +1988,7 @@ class CheckJNI { } static void ExceptionDescribe(JNIEnv* env) { + CHECK_ATTACHED_THREAD_VOID(__FUNCTION__); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_ExcepOkay, __FUNCTION__); JniValueType args[1] = {{.E = env}}; @@ -1949,6 +2001,7 @@ class CheckJNI { } static void ExceptionClear(JNIEnv* env) { + CHECK_ATTACHED_THREAD_VOID(__FUNCTION__); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_ExcepOkay, __FUNCTION__); JniValueType args[1] = {{.E = env}}; @@ -1961,6 +2014,7 @@ class CheckJNI { } static jboolean ExceptionCheck(JNIEnv* env) { + CHECK_ATTACHED_THREAD(__FUNCTION__, JNI_FALSE); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_CritOkay | kFlag_ExcepOkay, __FUNCTION__); JniValueType args[1] = {{.E = env}}; @@ -1975,6 +2029,7 @@ class CheckJNI { } static void FatalError(JNIEnv* env, const char* msg) { + CHECK_ATTACHED_THREAD_VOID(__FUNCTION__); // The JNI specification doesn't say it's okay to call FatalError with a pending exception, // but you're about to abort anyway, and it's quite likely that you have a pending exception, // and it's not unimaginable that you don't know that you do. So we allow it. @@ -1991,6 +2046,7 @@ class CheckJNI { } static jint PushLocalFrame(JNIEnv* env, jint capacity) { + CHECK_ATTACHED_THREAD(__FUNCTION__, JNI_ERR); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_ExcepOkay, __FUNCTION__); JniValueType args[2] = {{.E = env}, {.I = capacity}}; @@ -2005,6 +2061,7 @@ class CheckJNI { } static jobject PopLocalFrame(JNIEnv* env, jobject res) { + CHECK_ATTACHED_THREAD(__FUNCTION__, nullptr); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_ExcepOkay, __FUNCTION__); JniValueType args[2] = {{.E = env}, {.L = res}}; @@ -2042,6 +2099,7 @@ class CheckJNI { } static jint EnsureLocalCapacity(JNIEnv *env, jint capacity) { + CHECK_ATTACHED_THREAD(__FUNCTION__, JNI_ERR); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_Default, __FUNCTION__); JniValueType args[2] = {{.E = env}, {.I = capacity}}; @@ -2056,6 +2114,7 @@ class CheckJNI { } static jboolean IsSameObject(JNIEnv* env, jobject ref1, jobject ref2) { + CHECK_ATTACHED_THREAD(__FUNCTION__, JNI_FALSE); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_Default, __FUNCTION__); JniValueType args[3] = {{.E = env}, {.L = ref1}, {.L = ref2}}; @@ -2070,6 +2129,7 @@ class CheckJNI { } static jobject AllocObject(JNIEnv* env, jclass c) { + CHECK_ATTACHED_THREAD(__FUNCTION__, nullptr); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_Default, __FUNCTION__); JniValueType args[2] = {{.E = env}, {.c = c}}; @@ -2084,6 +2144,7 @@ class CheckJNI { } static jobject NewObjectV(JNIEnv* env, jclass c, jmethodID mid, va_list vargs) { + CHECK_ATTACHED_THREAD(__FUNCTION__, nullptr); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_Default, __FUNCTION__); VarArgs rest(mid, vargs); @@ -2100,6 +2161,7 @@ class CheckJNI { } static jobject NewObject(JNIEnv* env, jclass c, jmethodID mid, ...) { + CHECK_ATTACHED_THREAD(__FUNCTION__, nullptr); va_list args; va_start(args, mid); jobject result = NewObjectV(env, c, mid, args); @@ -2108,6 +2170,7 @@ class CheckJNI { } static jobject NewObjectA(JNIEnv* env, jclass c, jmethodID mid, jvalue* vargs) { + CHECK_ATTACHED_THREAD(__FUNCTION__, nullptr); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_Default, __FUNCTION__); VarArgs rest(mid, vargs); @@ -2124,6 +2187,7 @@ class CheckJNI { } static jclass GetObjectClass(JNIEnv* env, jobject obj) { + CHECK_ATTACHED_THREAD(__FUNCTION__, nullptr); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_Default, __FUNCTION__); JniValueType args[2] = {{.E = env}, {.L = obj}}; @@ -2138,6 +2202,7 @@ class CheckJNI { } static jboolean IsInstanceOf(JNIEnv* env, jobject obj, jclass c) { + CHECK_ATTACHED_THREAD(__FUNCTION__, JNI_FALSE); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_Default, __FUNCTION__); JniValueType args[3] = {{.E = env}, {.L = obj}, {.c = c}}; @@ -2313,6 +2378,7 @@ class CheckJNI { #undef CALL static jstring NewString(JNIEnv* env, const jchar* unicode_chars, jsize len) { + CHECK_ATTACHED_THREAD(__FUNCTION__, nullptr); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_Default, __FUNCTION__); JniValueType args[3] = {{.E = env}, {.p = unicode_chars}, {.z = len}}; @@ -2327,6 +2393,7 @@ class CheckJNI { } static jstring NewStringUTF(JNIEnv* env, const char* chars) { + CHECK_ATTACHED_THREAD(__FUNCTION__, nullptr); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_NullableUtf, __FUNCTION__); JniValueType args[2] = {{.E = env}, {.u = chars}}; @@ -2342,6 +2409,7 @@ class CheckJNI { } static jsize GetStringLength(JNIEnv* env, jstring string) { + CHECK_ATTACHED_THREAD(__FUNCTION__, JNI_ERR); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_CritOkay, __FUNCTION__); JniValueType args[2] = {{.E = env}, {.s = string}}; @@ -2356,6 +2424,7 @@ class CheckJNI { } static jsize GetStringUTFLength(JNIEnv* env, jstring string) { + CHECK_ATTACHED_THREAD(__FUNCTION__, JNI_ERR); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_CritOkay, __FUNCTION__); JniValueType args[2] = {{.E = env}, {.s = string}}; @@ -2397,6 +2466,7 @@ class CheckJNI { } static void GetStringRegion(JNIEnv* env, jstring string, jsize start, jsize len, jchar* buf) { + CHECK_ATTACHED_THREAD_VOID(__FUNCTION__); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_CritOkay, __FUNCTION__); JniValueType args[5] = {{.E = env}, {.s = string}, {.z = start}, {.z = len}, {.p = buf}}; @@ -2411,6 +2481,7 @@ class CheckJNI { } static void GetStringUTFRegion(JNIEnv* env, jstring string, jsize start, jsize len, char* buf) { + CHECK_ATTACHED_THREAD_VOID(__FUNCTION__); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_CritOkay, __FUNCTION__); JniValueType args[5] = {{.E = env}, {.s = string}, {.z = start}, {.z = len}, {.p = buf}}; @@ -2425,6 +2496,7 @@ class CheckJNI { } static jsize GetArrayLength(JNIEnv* env, jarray array) { + CHECK_ATTACHED_THREAD(__FUNCTION__, JNI_ERR); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_CritOkay, __FUNCTION__); JniValueType args[2] = {{.E = env}, {.a = array}}; @@ -2440,6 +2512,7 @@ class CheckJNI { static jobjectArray NewObjectArray(JNIEnv* env, jsize length, jclass element_class, jobject initial_element) { + CHECK_ATTACHED_THREAD(__FUNCTION__, nullptr); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_Default, __FUNCTION__); JniValueType args[4] = @@ -2456,6 +2529,7 @@ class CheckJNI { } static jobject GetObjectArrayElement(JNIEnv* env, jobjectArray array, jsize index) { + CHECK_ATTACHED_THREAD(__FUNCTION__, nullptr); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_Default, __FUNCTION__); JniValueType args[3] = {{.E = env}, {.a = array}, {.z = index}}; @@ -2470,6 +2544,7 @@ class CheckJNI { } static void SetObjectArrayElement(JNIEnv* env, jobjectArray array, jsize index, jobject value) { + CHECK_ATTACHED_THREAD_VOID(__FUNCTION__); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_Default, __FUNCTION__); JniValueType args[4] = {{.E = env}, {.a = array}, {.z = index}, {.L = value}}; @@ -2556,6 +2631,7 @@ class CheckJNI { #undef PRIMITIVE_ARRAY_FUNCTIONS static jint MonitorEnter(JNIEnv* env, jobject obj) { + CHECK_ATTACHED_THREAD(__FUNCTION__, JNI_ERR); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_Default, __FUNCTION__); JniValueType args[2] = {{.E = env}, {.L = obj}}; @@ -2573,6 +2649,7 @@ class CheckJNI { } static jint MonitorExit(JNIEnv* env, jobject obj) { + CHECK_ATTACHED_THREAD(__FUNCTION__, JNI_ERR); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_ExcepOkay, __FUNCTION__); JniValueType args[2] = {{.E = env}, {.L = obj}}; @@ -2590,6 +2667,7 @@ class CheckJNI { } static void* GetPrimitiveArrayCritical(JNIEnv* env, jarray array, jboolean* is_copy) { + CHECK_ATTACHED_THREAD(__FUNCTION__, nullptr); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_CritGet, __FUNCTION__); JniValueType args[3] = {{.E = env}, {.a = array}, {.p = is_copy}}; @@ -2608,6 +2686,7 @@ class CheckJNI { } static void ReleasePrimitiveArrayCritical(JNIEnv* env, jarray array, void* carray, jint mode) { + CHECK_ATTACHED_THREAD_VOID(__FUNCTION__); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_CritRelease | kFlag_ExcepOkay, __FUNCTION__); sc.CheckNonNull(carray); @@ -2624,6 +2703,7 @@ class CheckJNI { } static jobject NewDirectByteBuffer(JNIEnv* env, void* address, jlong capacity) { + CHECK_ATTACHED_THREAD(__FUNCTION__, nullptr); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_Default, __FUNCTION__); JniValueType args[3] = {{.E = env}, {.p = address}, {.J = capacity}}; @@ -2639,6 +2719,7 @@ class CheckJNI { } static void* GetDirectBufferAddress(JNIEnv* env, jobject buf) { + CHECK_ATTACHED_THREAD(__FUNCTION__, nullptr); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_Default, __FUNCTION__); JniValueType args[2] = {{.E = env}, {.L = buf}}; @@ -2655,6 +2736,7 @@ class CheckJNI { } static jlong GetDirectBufferCapacity(JNIEnv* env, jobject buf) { + CHECK_ATTACHED_THREAD(__FUNCTION__, JNI_ERR); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_Default, __FUNCTION__); JniValueType args[2] = {{.E = env}, {.L = buf}}; @@ -2680,6 +2762,7 @@ class CheckJNI { } static jobject NewRef(const char* function_name, JNIEnv* env, jobject obj, IndirectRefKind kind) { + CHECK_ATTACHED_THREAD(function_name, nullptr); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_Default, function_name); JniValueType args[2] = {{.E = env}, {.L = obj}}; @@ -2708,6 +2791,7 @@ class CheckJNI { } static void DeleteRef(const char* function_name, JNIEnv* env, jobject obj, IndirectRefKind kind) { + CHECK_ATTACHED_THREAD_VOID(function_name); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_ExcepOkay, function_name); JniValueType args[2] = {{.E = env}, {.L = obj}}; @@ -2734,6 +2818,7 @@ class CheckJNI { static jmethodID GetMethodIDInternal(const char* function_name, JNIEnv* env, jclass c, const char* name, const char* sig, bool is_static) { + CHECK_ATTACHED_THREAD(function_name, nullptr); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_Default, function_name); JniValueType args[4] = {{.E = env}, {.c = c}, {.u = name}, {.u = sig}}; @@ -2753,6 +2838,7 @@ class CheckJNI { static jfieldID GetFieldIDInternal(const char* function_name, JNIEnv* env, jclass c, const char* name, const char* sig, bool is_static) { + CHECK_ATTACHED_THREAD(function_name, nullptr); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_Default, function_name); JniValueType args[4] = {{.E = env}, {.c = c}, {.u = name}, {.u = sig}}; @@ -2772,6 +2858,7 @@ class CheckJNI { static JniValueType GetField(const char* function_name, JNIEnv* env, jobject obj, jfieldID fid, bool is_static, Primitive::Type type) { + CHECK_ATTACHED_THREAD(function_name, JniValueType()); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_Default, function_name); JniValueType args[3] = {{.E = env}, {.L = obj}, {.f = fid}}; @@ -2866,6 +2953,7 @@ class CheckJNI { static void SetField(const char* function_name, JNIEnv* env, jobject obj, jfieldID fid, bool is_static, Primitive::Type type, JniValueType value) { + CHECK_ATTACHED_THREAD_VOID(function_name); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_Default, function_name); JniValueType args[4] = {{.E = env}, {.L = obj}, {.f = fid}, value}; @@ -2980,6 +3068,7 @@ class CheckJNI { static JniValueType CallMethodA(const char* function_name, JNIEnv* env, jobject obj, jclass c, jmethodID mid, jvalue* vargs, Primitive::Type type, InvokeType invoke) { + CHECK_ATTACHED_THREAD(function_name, JniValueType()); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_Default, function_name); JniValueType result; @@ -3164,6 +3253,7 @@ class CheckJNI { static JniValueType CallMethodV(const char* function_name, JNIEnv* env, jobject obj, jclass c, jmethodID mid, va_list vargs, Primitive::Type type, InvokeType invoke) { + CHECK_ATTACHED_THREAD(function_name, JniValueType()); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_Default, function_name); JniValueType result; @@ -3347,6 +3437,7 @@ class CheckJNI { static const void* GetStringCharsInternal(const char* function_name, JNIEnv* env, jstring string, jboolean* is_copy, bool utf, bool critical) { + CHECK_ATTACHED_THREAD(function_name, nullptr); ScopedObjectAccess soa(env); int flags = critical ? kFlag_CritGet : kFlag_CritOkay; ScopedCheck sc(flags, function_name); @@ -3387,6 +3478,7 @@ class CheckJNI { static void ReleaseStringCharsInternal(const char* function_name, JNIEnv* env, jstring string, const void* chars, bool utf, bool critical) { + CHECK_ATTACHED_THREAD_VOID(function_name); ScopedObjectAccess soa(env); int flags = kFlag_ExcepOkay | kFlag_Release; if (critical) { @@ -3419,6 +3511,7 @@ class CheckJNI { static jarray NewPrimitiveArray(const char* function_name, JNIEnv* env, jsize length, Primitive::Type type) { + CHECK_ATTACHED_THREAD(function_name, nullptr); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_Default, function_name); JniValueType args[2] = {{.E = env}, {.z = length}}; @@ -3461,6 +3554,7 @@ class CheckJNI { static void* GetPrimitiveArrayElements(const char* function_name, Primitive::Type type, JNIEnv* env, jarray array, jboolean* is_copy) { + CHECK_ATTACHED_THREAD(function_name, nullptr); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_Default, function_name); JniValueType args[3] = {{.E = env}, {.a = array}, {.p = is_copy}}; @@ -3512,6 +3606,7 @@ class CheckJNI { static void ReleasePrimitiveArrayElements(const char* function_name, Primitive::Type type, JNIEnv* env, jarray array, void* elems, jint mode) { + CHECK_ATTACHED_THREAD_VOID(function_name); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_ExcepOkay, function_name); if (sc.CheckNonNull(elems) && sc.CheckPrimitiveArrayType(soa, array, type)) { @@ -3567,6 +3662,7 @@ class CheckJNI { static void GetPrimitiveArrayRegion(const char* function_name, Primitive::Type type, JNIEnv* env, jarray array, jsize start, jsize len, void* buf) { + CHECK_ATTACHED_THREAD_VOID(function_name); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_Default, function_name); JniValueType args[5] = {{.E = env}, {.a = array}, {.z = start}, {.z = len}, {.p = buf}}; @@ -3617,6 +3713,7 @@ class CheckJNI { static void SetPrimitiveArrayRegion(const char* function_name, Primitive::Type type, JNIEnv* env, jarray array, jsize start, jsize len, const void* buf) { + CHECK_ATTACHED_THREAD_VOID(function_name); ScopedObjectAccess soa(env); ScopedCheck sc(kFlag_Default, function_name); JniValueType args[5] = {{.E = env}, {.a = array}, {.z = start}, {.z = len}, {.p = buf}}; diff --git a/runtime/class_linker.cc b/runtime/class_linker.cc index d98e0b2c54c..45d1381297e 100644 --- a/runtime/class_linker.cc +++ b/runtime/class_linker.cc @@ -270,9 +270,13 @@ static void WrapExceptionInInitializer(Handle<mirror::Class> klass) // cannot in general be guaranteed, but in all likelihood leads to breakage down the line. if (klass->GetClassLoader() == nullptr && !Runtime::Current()->IsAotCompiler()) { std::string tmp; - LOG(kIsDebugBuild ? FATAL : WARNING) << klass->GetDescriptor(&tmp) - << " failed initialization: " - << self->GetException()->Dump(); + // We want to LOG(FATAL) on debug builds since this really shouldn't be happening but we need to + // make sure to only do it if we don't have AsyncExceptions being thrown around since those + // could have caused the error. + bool known_impossible = kIsDebugBuild && !Runtime::Current()->AreAsyncExceptionsThrown(); + LOG(known_impossible ? FATAL : WARNING) << klass->GetDescriptor(&tmp) + << " failed initialization: " + << self->GetException()->Dump(); } env->ExceptionClear(); @@ -1828,21 +1832,6 @@ bool ClassLinker::AddImageSpace( header.VisitPackedArtMethods(&visitor, space->Begin(), image_pointer_size_); } - if (!app_image) { - // Make the string intern table and class table immutable for boot image. - // PIC app oat files may mmap a read-only copy into their own .bss section, - // so enforce that the data in the boot image tables remains unchanged. - // - // We cannot do that for app image even after the fixup as some interned - // String references may actually end up pointing to moveable Strings. - uint8_t* const_section_begin = space->Begin() + header.GetBootImageConstantTablesOffset(); - CheckedCall(mprotect, - "protect constant tables", - const_section_begin, - header.GetBootImageConstantTablesSize(), - PROT_READ); - } - ClassTable* class_table = nullptr; { WriterMutexLock mu(self, *Locks::classlinker_classes_lock_); @@ -3390,9 +3379,10 @@ void ClassLinker::RegisterDexFileLocked(const DexFile& dex_file, CHECK_EQ(dex_cache_location, dex_file_suffix); const OatFile* oat_file = (dex_file.GetOatDexFile() != nullptr) ? dex_file.GetOatDexFile()->GetOatFile() : nullptr; - // Clean up pass to remove null dex caches. Also check if we need to initialize OatFile .bss. - // Null dex caches can occur due to class unloading and we are lazily removing null entries. - bool initialize_oat_file_bss = (oat_file != nullptr); + // Clean up pass to remove null dex caches; null dex caches can occur due to class unloading + // and we are lazily removing null entries. Also check if we need to initialize OatFile data + // (.data.bimg.rel.ro and .bss sections) needed for code execution. + bool initialize_oat_file_data = (oat_file != nullptr) && oat_file->IsExecutable(); JavaVMExt* const vm = self->GetJniEnv()->GetVm(); for (auto it = dex_caches_.begin(); it != dex_caches_.end(); ) { DexCacheData data = *it; @@ -3400,15 +3390,36 @@ void ClassLinker::RegisterDexFileLocked(const DexFile& dex_file, vm->DeleteWeakGlobalRef(self, data.weak_root); it = dex_caches_.erase(it); } else { - if (initialize_oat_file_bss && + if (initialize_oat_file_data && it->dex_file->GetOatDexFile() != nullptr && it->dex_file->GetOatDexFile()->GetOatFile() == oat_file) { - initialize_oat_file_bss = false; // Already initialized. + initialize_oat_file_data = false; // Already initialized. } ++it; } } - if (initialize_oat_file_bss) { + if (initialize_oat_file_data) { + // Initialize the .data.bimg.rel.ro section. + if (!oat_file->GetBootImageRelocations().empty()) { + uint8_t* reloc_begin = const_cast<uint8_t*>(oat_file->DataBimgRelRoBegin()); + CheckedCall(mprotect, + "un-protect boot image relocations", + reloc_begin, + oat_file->DataBimgRelRoSize(), + PROT_READ | PROT_WRITE); + uint32_t boot_image_begin = dchecked_integral_cast<uint32_t>(reinterpret_cast<uintptr_t>( + Runtime::Current()->GetHeap()->GetBootImageSpaces().front()->Begin())); + for (const uint32_t& relocation : oat_file->GetBootImageRelocations()) { + const_cast<uint32_t&>(relocation) += boot_image_begin; + } + CheckedCall(mprotect, + "protect boot image relocations", + reloc_begin, + oat_file->DataBimgRelRoSize(), + PROT_READ); + } + + // Initialize the .bss section. // TODO: Pre-initialize from boot/app image? ArtMethod* resolution_method = Runtime::Current()->GetResolutionMethod(); for (ArtMethod*& entry : oat_file->GetBssMethods()) { @@ -6169,7 +6180,7 @@ ArtMethod* ClassLinker::AddMethodToConflictTable(ObjPtr<mirror::Class> klass, // Note that there is a race in the presence of multiple threads and we may leak // memory from the LinearAlloc, but that's a tradeoff compared to using // atomic operations. - QuasiAtomic::ThreadFenceRelease(); + std::atomic_thread_fence(std::memory_order_release); new_conflict_method->SetImtConflictTable(new_table, image_pointer_size_); return new_conflict_method; } @@ -8225,29 +8236,34 @@ mirror::MethodHandle* ClassLinker::ResolveMethodHandleForField( DexFile::MethodHandleType handle_type = static_cast<DexFile::MethodHandleType>(method_handle.method_handle_type_); mirror::MethodHandle::Kind kind; + bool is_put; bool is_static; int32_t num_params; switch (handle_type) { case DexFile::MethodHandleType::kStaticPut: { kind = mirror::MethodHandle::Kind::kStaticPut; + is_put = true; is_static = true; num_params = 1; break; } case DexFile::MethodHandleType::kStaticGet: { kind = mirror::MethodHandle::Kind::kStaticGet; + is_put = false; is_static = true; num_params = 0; break; } case DexFile::MethodHandleType::kInstancePut: { kind = mirror::MethodHandle::Kind::kInstancePut; + is_put = true; is_static = false; num_params = 2; break; } case DexFile::MethodHandleType::kInstanceGet: { kind = mirror::MethodHandle::Kind::kInstanceGet; + is_put = false; is_static = false; num_params = 1; break; @@ -8269,6 +8285,10 @@ mirror::MethodHandle* ClassLinker::ResolveMethodHandleForField( ThrowIllegalAccessErrorField(referring_class, target_field); return nullptr; } + if (UNLIKELY(is_put && target_field->IsFinal())) { + ThrowIllegalAccessErrorField(referring_class, target_field); + return nullptr; + } } else { DCHECK(Thread::Current()->IsExceptionPending()); return nullptr; @@ -8849,6 +8869,15 @@ void ClassLinker::VisitClassLoaders(ClassLoaderVisitor* visitor) const { } } +void ClassLinker::VisitAllocators(AllocatorVisitor* visitor) const { + for (const ClassLoaderData& data : class_loaders_) { + LinearAlloc* alloc = data.allocator; + if (alloc != nullptr && !visitor->Visit(alloc)) { + break; + } + } +} + void ClassLinker::InsertDexFileInToClassLoader(ObjPtr<mirror::Object> dex_file, ObjPtr<mirror::ClassLoader> class_loader) { DCHECK(dex_file != nullptr); diff --git a/runtime/class_linker.h b/runtime/class_linker.h index c46e8271e5c..e935d1dfb8b 100644 --- a/runtime/class_linker.h +++ b/runtime/class_linker.h @@ -97,6 +97,14 @@ class ClassLoaderVisitor { REQUIRES_SHARED(Locks::classlinker_classes_lock_, Locks::mutator_lock_) = 0; }; +class AllocatorVisitor { + public: + virtual ~AllocatorVisitor() {} + // Return true to continue visiting. + virtual bool Visit(LinearAlloc* alloc) + REQUIRES_SHARED(Locks::classlinker_classes_lock_, Locks::mutator_lock_) = 0; +}; + class ClassLinker { public: // Well known mirror::Class roots accessed via GetClassRoot. @@ -673,6 +681,11 @@ class ClassLinker { REQUIRES(!Locks::classlinker_classes_lock_) REQUIRES_SHARED(Locks::mutator_lock_); + // Visit all of the allocators that belong to classloaders except boot classloader. + // This is used by 616-cha-unloading test to confirm memory reuse. + void VisitAllocators(AllocatorVisitor* visitor) const + REQUIRES_SHARED(Locks::classlinker_classes_lock_, Locks::mutator_lock_); + // Throw the class initialization failure recorded when first trying to initialize the given // class. void ThrowEarlierClassFailure(ObjPtr<mirror::Class> c, bool wrap_in_no_class_def = false) @@ -1327,7 +1340,6 @@ class ClassLinker { friend class ImageDumper; // for DexLock friend struct linker::CompilationHelper; // For Compile in ImageTest. friend class linker::ImageWriter; // for GetClassRoots - friend class linker::OatWriter; // for boot image string/class table slot address lookup. friend class JniCompilerTest; // for GetRuntimeQuickGenericJniStub friend class JniInternalTest; // for GetRuntimeQuickGenericJniStub friend class VMClassLoader; // for LookupClass and FindClassInBaseDexClassLoader. diff --git a/runtime/class_table-inl.h b/runtime/class_table-inl.h index c59e2e881dd..5da5470c1a9 100644 --- a/runtime/class_table-inl.h +++ b/runtime/class_table-inl.h @@ -88,7 +88,7 @@ bool ClassTable::Visit(const Visitor& visitor) { template<ReadBarrierOption kReadBarrierOption> inline mirror::Class* ClassTable::TableSlot::Read() const { - const uint32_t before = data_.LoadRelaxed(); + const uint32_t before = data_.load(std::memory_order_relaxed); ObjPtr<mirror::Class> const before_ptr(ExtractPtr(before)); ObjPtr<mirror::Class> const after_ptr( GcRoot<mirror::Class>(before_ptr).Read<kReadBarrierOption>()); @@ -102,7 +102,7 @@ inline mirror::Class* ClassTable::TableSlot::Read() const { template<typename Visitor> inline void ClassTable::TableSlot::VisitRoot(const Visitor& visitor) const { - const uint32_t before = data_.LoadRelaxed(); + const uint32_t before = data_.load(std::memory_order_relaxed); ObjPtr<mirror::Class> before_ptr(ExtractPtr(before)); GcRoot<mirror::Class> root(before_ptr); visitor.VisitRoot(root.AddressWithoutBarrier()); diff --git a/runtime/class_table.h b/runtime/class_table.h index 19c29b5768f..0b08041dbd3 100644 --- a/runtime/class_table.h +++ b/runtime/class_table.h @@ -53,14 +53,14 @@ class ClassTable { public: TableSlot() : data_(0u) {} - TableSlot(const TableSlot& copy) : data_(copy.data_.LoadRelaxed()) {} + TableSlot(const TableSlot& copy) : data_(copy.data_.load(std::memory_order_relaxed)) {} explicit TableSlot(ObjPtr<mirror::Class> klass); TableSlot(ObjPtr<mirror::Class> klass, uint32_t descriptor_hash); TableSlot& operator=(const TableSlot& copy) { - data_.StoreRelaxed(copy.data_.LoadRelaxed()); + data_.store(copy.data_.load(std::memory_order_relaxed), std::memory_order_relaxed); return *this; } @@ -69,7 +69,7 @@ class ClassTable { } uint32_t Hash() const { - return MaskHash(data_.LoadRelaxed()); + return MaskHash(data_.load(std::memory_order_relaxed)); } static uint32_t MaskHash(uint32_t hash) { @@ -295,7 +295,6 @@ class ClassTable { std::vector<const OatFile*> oat_files_ GUARDED_BY(lock_); friend class linker::ImageWriter; // for InsertWithoutLocks. - friend class linker::OatWriter; // for boot class TableSlot address lookup. }; } // namespace art diff --git a/runtime/common_runtime_test.cc b/runtime/common_runtime_test.cc index 05159e253d8..f6a5efc5478 100644 --- a/runtime/common_runtime_test.cc +++ b/runtime/common_runtime_test.cc @@ -30,6 +30,7 @@ #include "base/file_utils.h" #include "base/logging.h" #include "base/macros.h" +#include "base/mem_map.h" #include "base/mutex.h" #include "base/os.h" #include "base/runtime_debug.h" @@ -49,7 +50,6 @@ #include "interpreter/unstarted_runtime.h" #include "java_vm_ext.h" #include "jni_internal.h" -#include "mem_map.h" #include "mirror/class-inl.h" #include "mirror/class_loader.h" #include "native/dalvik_system_DexFile.h" diff --git a/runtime/common_runtime_test.h b/runtime/common_runtime_test.h index 7fc70e294fd..83a1f9a58af 100644 --- a/runtime/common_runtime_test.h +++ b/runtime/common_runtime_test.h @@ -332,12 +332,6 @@ class CheckJniAbortCatcher { return; \ } -#define TEST_DISABLED_FOR_COMPACT_DEX() \ - if (kDefaultCompactDexLevel != CompactDexLevel::kCompactDexLevelNone) { \ - printf("WARNING: TEST DISABLED FOR COMPACT DEX\n"); \ - return; \ - } - #define TEST_DISABLED_FOR_HEAP_POISONING() \ if (kPoisonHeapReferences) { \ printf("WARNING: TEST DISABLED FOR HEAP POISONING\n"); \ diff --git a/runtime/common_throws.cc b/runtime/common_throws.cc index b72dc3ff4c3..8fd95ed890a 100644 --- a/runtime/common_throws.cc +++ b/runtime/common_throws.cc @@ -878,8 +878,8 @@ void ThrowVerifyError(ObjPtr<mirror::Class> referrer, const char* fmt, ...) { // WrongMethodTypeException -void ThrowWrongMethodTypeException(mirror::MethodType* expected_type, - mirror::MethodType* actual_type) { +void ThrowWrongMethodTypeException(ObjPtr<mirror::MethodType> expected_type, + ObjPtr<mirror::MethodType> actual_type) { ThrowException("Ljava/lang/invoke/WrongMethodTypeException;", nullptr, StringPrintf("Expected %s but was %s", diff --git a/runtime/common_throws.h b/runtime/common_throws.h index e9baa4fef0b..29a056e9ea6 100644 --- a/runtime/common_throws.h +++ b/runtime/common_throws.h @@ -270,8 +270,8 @@ void ThrowVerifyError(ObjPtr<mirror::Class> referrer, const char* fmt, ...) // WrongMethodTypeException -void ThrowWrongMethodTypeException(mirror::MethodType* callee_type, - mirror::MethodType* callsite_type) +void ThrowWrongMethodTypeException(ObjPtr<mirror::MethodType> callee_type, + ObjPtr<mirror::MethodType> callsite_type) REQUIRES_SHARED(Locks::mutator_lock_) COLD_ATTR; } // namespace art diff --git a/runtime/debugger.cc b/runtime/debugger.cc index 0481c03d52d..28659cb11d7 100644 --- a/runtime/debugger.cc +++ b/runtime/debugger.cc @@ -4359,9 +4359,11 @@ bool Dbg::DdmHandleChunk(JNIEnv* env, WellKnownClasses::org_apache_harmony_dalvik_ddmc_DdmServer_dispatch, type, dataArray.get(), 0, data.size())); if (env->ExceptionCheck()) { - LOG(INFO) << StringPrintf("Exception thrown by dispatcher for 0x%08x", type); - env->ExceptionDescribe(); - env->ExceptionClear(); + Thread* self = Thread::Current(); + ScopedObjectAccess soa(self); + LOG(INFO) << StringPrintf("Exception thrown by dispatcher for 0x%08x", type) << std::endl + << self->GetException()->Dump(); + self->ClearException(); return false; } @@ -4405,10 +4407,11 @@ bool Dbg::DdmHandleChunk(JNIEnv* env, reinterpret_cast<jbyte*>(out_data->data())); if (env->ExceptionCheck()) { + Thread* self = Thread::Current(); + ScopedObjectAccess soa(self); LOG(INFO) << StringPrintf("Exception thrown when reading response data from dispatcher 0x%08x", - type); - env->ExceptionDescribe(); - env->ExceptionClear(); + type) << std::endl << self->GetException()->Dump(); + self->ClearException(); return false; } diff --git a/runtime/dex/art_dex_file_loader.cc b/runtime/dex/art_dex_file_loader.cc index f3e6a692795..415e451098e 100644 --- a/runtime/dex/art_dex_file_loader.cc +++ b/runtime/dex/art_dex_file_loader.cc @@ -26,11 +26,11 @@ #include "base/stl_util.h" #include "base/systrace.h" #include "base/unix_file/fd_file.h" +#include "base/zip_archive.h" #include "dex/compact_dex_file.h" #include "dex/dex_file.h" #include "dex/dex_file_verifier.h" #include "dex/standard_dex_file.h" -#include "zip_archive.h" namespace art { @@ -128,7 +128,7 @@ bool ArtDexFileLoader::GetMultiDexChecksums(const char* filename, do { if (zip_file_only_contains_uncompressed_dex != nullptr) { - if (!(zip_entry->IsUncompressed() && zip_entry->IsAlignedToDexHeader())) { + if (!(zip_entry->IsUncompressed() && zip_entry->IsAlignedTo(alignof(DexFile::Header)))) { *zip_file_only_contains_uncompressed_dex = false; } } diff --git a/runtime/dex/art_dex_file_loader_test.cc b/runtime/dex/art_dex_file_loader_test.cc index 3e0d6662c70..afc2599ea02 100644 --- a/runtime/dex/art_dex_file_loader_test.cc +++ b/runtime/dex/art_dex_file_loader_test.cc @@ -21,6 +21,7 @@ #include "art_dex_file_loader.h" #include "base/file_utils.h" +#include "base/mem_map.h" #include "base/os.h" #include "base/stl_util.h" #include "base/unix_file/fd_file.h" @@ -31,7 +32,6 @@ #include "dex/dex_file.h" #include "dex/dex_file-inl.h" #include "dex/dex_file_loader.h" -#include "mem_map.h" #include "scoped_thread_state_change-inl.h" #include "thread-current-inl.h" diff --git a/runtime/dexopt_test.cc b/runtime/dexopt_test.cc index c19fa82877c..8f0f9c61dc9 100644 --- a/runtime/dexopt_test.cc +++ b/runtime/dexopt_test.cc @@ -21,12 +21,12 @@ #include <gtest/gtest.h> #include "base/file_utils.h" +#include "base/mem_map.h" #include "common_runtime_test.h" #include "compiler_callbacks.h" #include "dex2oat_environment_test.h" #include "dexopt_test.h" #include "gc/space/image_space.h" -#include "mem_map.h" namespace art { void DexoptTest::SetUp() { diff --git a/runtime/elf_file_impl.h b/runtime/elf_file_impl.h index 3143df537f0..a5808e27baa 100644 --- a/runtime/elf_file_impl.h +++ b/runtime/elf_file_impl.h @@ -24,7 +24,7 @@ // Explicitly include our own elf.h to avoid Linux and other dependencies. #include "./elf.h" -#include "mem_map.h" +#include "base/mem_map.h" namespace art { diff --git a/runtime/entrypoints/quick/quick_throw_entrypoints.cc b/runtime/entrypoints/quick/quick_throw_entrypoints.cc index 9b0756b529b..ba7fb6b9db5 100644 --- a/runtime/entrypoints/quick/quick_throw_entrypoints.cc +++ b/runtime/entrypoints/quick/quick_throw_entrypoints.cc @@ -16,8 +16,11 @@ #include "art_method-inl.h" #include "callee_save_frame.h" +#include "dex/code_item_accessors-inl.h" +#include "dex/dex_instruction-inl.h" #include "common_throws.h" #include "mirror/object-inl.h" +#include "nth_caller_visitor.h" #include "thread.h" #include "well_known_classes.h" @@ -112,6 +115,26 @@ extern "C" NO_RETURN void artThrowClassCastException(mirror::Class* dest_type, Thread* self) REQUIRES_SHARED(Locks::mutator_lock_) { ScopedQuickEntrypointChecks sqec(self); + if (dest_type == nullptr) { + // Find the target class for check cast using the bitstring check (dest_type == null). + NthCallerVisitor visitor(self, 0u); + visitor.WalkStack(); + DCHECK(visitor.caller != nullptr); + uint32_t dex_pc = visitor.GetDexPc(); + CodeItemDataAccessor accessor(*visitor.caller->GetDexFile(), visitor.caller->GetCodeItem()); + const Instruction& check_cast = accessor.InstructionAt(dex_pc); + DCHECK_EQ(check_cast.Opcode(), Instruction::CHECK_CAST); + dex::TypeIndex type_index(check_cast.VRegB_21c()); + ClassLinker* linker = Runtime::Current()->GetClassLinker(); + dest_type = linker->LookupResolvedType(type_index, visitor.caller).Ptr(); + CHECK(dest_type != nullptr) << "Target class should have been previously resolved: " + << visitor.caller->GetDexFile()->PrettyType(type_index); + CHECK(!dest_type->IsAssignableFrom(src_type)) + << " " << std::hex << dest_type->PrettyDescriptor() << ";" << dest_type->Depth() + << "/" << dest_type->GetField32(mirror::Class::StatusOffset()) + << " <: " << src_type->PrettyDescriptor() << ";" << src_type->Depth() + << "/" << src_type->GetField32(mirror::Class::StatusOffset()); + } DCHECK(!dest_type->IsAssignableFrom(src_type)); ThrowClassCastException(dest_type, src_type); self->QuickDeliverException(); diff --git a/runtime/entrypoints/quick/quick_trampoline_entrypoints.cc b/runtime/entrypoints/quick/quick_trampoline_entrypoints.cc index 66cec7113e7..39429c5b414 100644 --- a/runtime/entrypoints/quick/quick_trampoline_entrypoints.cc +++ b/runtime/entrypoints/quick/quick_trampoline_entrypoints.cc @@ -1170,7 +1170,7 @@ extern "C" TwoWordReturn artInstrumentationMethodExitFromCode(Thread* self, instrumentation::Instrumentation* instrumentation = Runtime::Current()->GetInstrumentation(); TwoWordReturn return_or_deoptimize_pc = instrumentation->PopInstrumentationStackFrame( self, return_pc, gpr_result, fpr_result); - if (self->IsExceptionPending()) { + if (self->IsExceptionPending() || self->ObserveAsyncException()) { return GetTwoWordFailureValue(); } return return_or_deoptimize_pc; @@ -2356,10 +2356,6 @@ extern "C" TwoWordReturn artQuickGenericJniTrampoline(Thread* self, ArtMethod** ArtMethod* called = *sp; DCHECK(called->IsNative()) << called->PrettyMethod(true); Runtime* runtime = Runtime::Current(); - jit::Jit* jit = runtime->GetJit(); - if (jit != nullptr) { - jit->AddSamples(self, called, 1u, /*with_backedges*/ false); - } uint32_t shorty_len = 0; const char* shorty = called->GetShorty(&shorty_len); bool critical_native = called->IsCriticalNative(); @@ -2385,6 +2381,12 @@ extern "C" TwoWordReturn artQuickGenericJniTrampoline(Thread* self, ArtMethod** self->VerifyStack(); + // We can now walk the stack if needed by JIT GC from MethodEntered() for JIT-on-first-use. + jit::Jit* jit = runtime->GetJit(); + if (jit != nullptr) { + jit->MethodEntered(self, called); + } + uint32_t cookie; uint32_t* sp32; // Skip calling JniMethodStart for @CriticalNative. diff --git a/runtime/gc/accounting/atomic_stack.h b/runtime/gc/accounting/atomic_stack.h index 6b103bfe1b7..e30fef4fc25 100644 --- a/runtime/gc/accounting/atomic_stack.h +++ b/runtime/gc/accounting/atomic_stack.h @@ -27,7 +27,7 @@ #include "base/atomic.h" #include "base/macros.h" -#include "mem_map.h" +#include "base/mem_map.h" #include "stack_reference.h" // This implements a double-ended queue (deque) with various flavors of PushBack operations, @@ -74,8 +74,8 @@ class AtomicStack { void Reset() { DCHECK(mem_map_.get() != nullptr); DCHECK(begin_ != nullptr); - front_index_.StoreRelaxed(0); - back_index_.StoreRelaxed(0); + front_index_.store(0, std::memory_order_relaxed); + back_index_.store(0, std::memory_order_relaxed); debug_is_sorted_ = true; mem_map_->MadviseDontNeedAndZero(); } @@ -103,7 +103,7 @@ class AtomicStack { int32_t index; int32_t new_index; do { - index = back_index_.LoadRelaxed(); + index = back_index_.load(std::memory_order_relaxed); new_index = index + num_slots; if (UNLIKELY(static_cast<size_t>(new_index) >= growth_limit_)) { // Stack overflow. @@ -134,31 +134,32 @@ class AtomicStack { if (kIsDebugBuild) { debug_is_sorted_ = false; } - const int32_t index = back_index_.LoadRelaxed(); + const int32_t index = back_index_.load(std::memory_order_relaxed); DCHECK_LT(static_cast<size_t>(index), growth_limit_); - back_index_.StoreRelaxed(index + 1); + back_index_.store(index + 1, std::memory_order_relaxed); begin_[index].Assign(value); } T* PopBack() REQUIRES_SHARED(Locks::mutator_lock_) { - DCHECK_GT(back_index_.LoadRelaxed(), front_index_.LoadRelaxed()); + DCHECK_GT(back_index_.load(std::memory_order_relaxed), + front_index_.load(std::memory_order_relaxed)); // Decrement the back index non atomically. - back_index_.StoreRelaxed(back_index_.LoadRelaxed() - 1); - return begin_[back_index_.LoadRelaxed()].AsMirrorPtr(); + back_index_.store(back_index_.load(std::memory_order_relaxed) - 1, std::memory_order_relaxed); + return begin_[back_index_.load(std::memory_order_relaxed)].AsMirrorPtr(); } // Take an item from the front of the stack. T PopFront() { - int32_t index = front_index_.LoadRelaxed(); - DCHECK_LT(index, back_index_.LoadRelaxed()); - front_index_.StoreRelaxed(index + 1); + int32_t index = front_index_.load(std::memory_order_relaxed); + DCHECK_LT(index, back_index_.load(std::memory_order_relaxed)); + front_index_.store(index + 1, std::memory_order_relaxed); return begin_[index]; } // Pop a number of elements. void PopBackCount(int32_t n) { DCHECK_GE(Size(), static_cast<size_t>(n)); - back_index_.StoreRelaxed(back_index_.LoadRelaxed() - n); + back_index_.store(back_index_.load(std::memory_order_relaxed) - n, std::memory_order_relaxed); } bool IsEmpty() const { @@ -170,15 +171,17 @@ class AtomicStack { } size_t Size() const { - DCHECK_LE(front_index_.LoadRelaxed(), back_index_.LoadRelaxed()); - return back_index_.LoadRelaxed() - front_index_.LoadRelaxed(); + DCHECK_LE(front_index_.load(std::memory_order_relaxed), + back_index_.load(std::memory_order_relaxed)); + return + back_index_.load(std::memory_order_relaxed) - front_index_.load(std::memory_order_relaxed); } StackReference<T>* Begin() const { - return begin_ + front_index_.LoadRelaxed(); + return begin_ + front_index_.load(std::memory_order_relaxed); } StackReference<T>* End() const { - return begin_ + back_index_.LoadRelaxed(); + return begin_ + back_index_.load(std::memory_order_relaxed); } size_t Capacity() const { @@ -193,11 +196,11 @@ class AtomicStack { } void Sort() { - int32_t start_back_index = back_index_.LoadRelaxed(); - int32_t start_front_index = front_index_.LoadRelaxed(); + int32_t start_back_index = back_index_.load(std::memory_order_relaxed); + int32_t start_front_index = front_index_.load(std::memory_order_relaxed); std::sort(Begin(), End(), ObjectComparator()); - CHECK_EQ(start_back_index, back_index_.LoadRelaxed()); - CHECK_EQ(start_front_index, front_index_.LoadRelaxed()); + CHECK_EQ(start_back_index, back_index_.load(std::memory_order_relaxed)); + CHECK_EQ(start_front_index, front_index_.load(std::memory_order_relaxed)); if (kIsDebugBuild) { debug_is_sorted_ = true; } @@ -236,7 +239,7 @@ class AtomicStack { } int32_t index; do { - index = back_index_.LoadRelaxed(); + index = back_index_.load(std::memory_order_relaxed); if (UNLIKELY(static_cast<size_t>(index) >= limit)) { // Stack overflow. return false; diff --git a/runtime/gc/accounting/bitmap-inl.h b/runtime/gc/accounting/bitmap-inl.h index a71b212af31..a4273e5ff6a 100644 --- a/runtime/gc/accounting/bitmap-inl.h +++ b/runtime/gc/accounting/bitmap-inl.h @@ -37,7 +37,7 @@ inline bool Bitmap::AtomicTestAndSetBit(uintptr_t bit_index) { auto* atomic_entry = reinterpret_cast<Atomic<uintptr_t>*>(&bitmap_begin_[word_index]); uintptr_t old_word; do { - old_word = atomic_entry->LoadRelaxed(); + old_word = atomic_entry->load(std::memory_order_relaxed); // Fast path: The bit is already set. if ((old_word & word_mask) != 0) { DCHECK(TestBit(bit_index)); diff --git a/runtime/gc/accounting/bitmap.cc b/runtime/gc/accounting/bitmap.cc index e5353807a9c..d45a0cc018e 100644 --- a/runtime/gc/accounting/bitmap.cc +++ b/runtime/gc/accounting/bitmap.cc @@ -19,9 +19,9 @@ #include <sys/mman.h> // For the PROT_* and MAP_* constants. #include "base/bit_utils.h" +#include "base/mem_map.h" #include "card_table.h" #include "jit/jit_code_cache.h" -#include "mem_map.h" namespace art { namespace gc { diff --git a/runtime/gc/accounting/card_table-inl.h b/runtime/gc/accounting/card_table-inl.h index 14f5d0e1c60..357a4985b68 100644 --- a/runtime/gc/accounting/card_table-inl.h +++ b/runtime/gc/accounting/card_table-inl.h @@ -23,7 +23,7 @@ #include "base/atomic.h" #include "base/bit_utils.h" -#include "mem_map.h" +#include "base/mem_map.h" #include "space_bitmap.h" namespace art { @@ -43,7 +43,7 @@ static inline bool byte_cas(uint8_t old_value, uint8_t new_value, uint8_t* addre Atomic<uintptr_t>* word_atomic = reinterpret_cast<Atomic<uintptr_t>*>(address); // Word with the byte we are trying to cas cleared. - const uintptr_t cur_word = word_atomic->LoadRelaxed() & + const uintptr_t cur_word = word_atomic->load(std::memory_order_relaxed) & ~(static_cast<uintptr_t>(0xFF) << shift_in_bits); const uintptr_t old_word = cur_word | (static_cast<uintptr_t>(old_value) << shift_in_bits); const uintptr_t new_word = cur_word | (static_cast<uintptr_t>(new_value) << shift_in_bits); diff --git a/runtime/gc/accounting/card_table.cc b/runtime/gc/accounting/card_table.cc index 4eea607c393..c7f936fb116 100644 --- a/runtime/gc/accounting/card_table.cc +++ b/runtime/gc/accounting/card_table.cc @@ -18,13 +18,13 @@ #include <sys/mman.h> +#include "base/mem_map.h" #include "base/systrace.h" #include "base/utils.h" #include "card_table-inl.h" #include "gc/heap.h" #include "gc/space/space.h" #include "heap_bitmap.h" -#include "mem_map.h" #include "runtime.h" namespace art { diff --git a/runtime/gc/accounting/read_barrier_table.h b/runtime/gc/accounting/read_barrier_table.h index 775746f68dd..57e4db9b2b1 100644 --- a/runtime/gc/accounting/read_barrier_table.h +++ b/runtime/gc/accounting/read_barrier_table.h @@ -20,10 +20,10 @@ #include <sys/mman.h> // For the PROT_* and MAP_* constants. #include "base/bit_utils.h" +#include "base/mem_map.h" #include "base/mutex.h" #include "gc/space/space.h" #include "globals.h" -#include "mem_map.h" namespace art { namespace gc { diff --git a/runtime/gc/accounting/space_bitmap-inl.h b/runtime/gc/accounting/space_bitmap-inl.h index 384e3c2f4c2..d460e000752 100644 --- a/runtime/gc/accounting/space_bitmap-inl.h +++ b/runtime/gc/accounting/space_bitmap-inl.h @@ -41,7 +41,7 @@ inline bool SpaceBitmap<kAlignment>::AtomicTestAndSet(const mirror::Object* obj) DCHECK_LT(index, bitmap_size_ / sizeof(intptr_t)) << " bitmap_size_ = " << bitmap_size_; uintptr_t old_word; do { - old_word = atomic_entry->LoadRelaxed(); + old_word = atomic_entry->load(std::memory_order_relaxed); // Fast path: The bit is already set. if ((old_word & mask) != 0) { DCHECK(Test(obj)); @@ -59,7 +59,8 @@ inline bool SpaceBitmap<kAlignment>::Test(const mirror::Object* obj) const { DCHECK(bitmap_begin_ != nullptr); DCHECK_GE(addr, heap_begin_); const uintptr_t offset = addr - heap_begin_; - return (bitmap_begin_[OffsetToIndex(offset)].LoadRelaxed() & OffsetToMask(offset)) != 0; + size_t index = OffsetToIndex(offset); + return (bitmap_begin_[index].load(std::memory_order_relaxed) & OffsetToMask(offset)) != 0; } template<size_t kAlignment> @@ -119,7 +120,7 @@ inline void SpaceBitmap<kAlignment>::VisitMarkedRange(uintptr_t visit_begin, // Traverse the middle, full part. for (size_t i = index_start + 1; i < index_end; ++i) { - uintptr_t w = bitmap_begin_[i].LoadRelaxed(); + uintptr_t w = bitmap_begin_[i].load(std::memory_order_relaxed); if (w != 0) { const uintptr_t ptr_base = IndexToOffset(i) + heap_begin_; // Iterate on the bits set in word `w`, from the least to the most significant bit. @@ -168,7 +169,7 @@ void SpaceBitmap<kAlignment>::Walk(Visitor&& visitor) { uintptr_t end = OffsetToIndex(HeapLimit() - heap_begin_ - 1); Atomic<uintptr_t>* bitmap_begin = bitmap_begin_; for (uintptr_t i = 0; i <= end; ++i) { - uintptr_t w = bitmap_begin[i].LoadRelaxed(); + uintptr_t w = bitmap_begin[i].load(std::memory_order_relaxed); if (w != 0) { uintptr_t ptr_base = IndexToOffset(i) + heap_begin_; do { @@ -192,7 +193,7 @@ inline bool SpaceBitmap<kAlignment>::Modify(const mirror::Object* obj) { const uintptr_t mask = OffsetToMask(offset); DCHECK_LT(index, bitmap_size_ / sizeof(intptr_t)) << " bitmap_size_ = " << bitmap_size_; Atomic<uintptr_t>* atomic_entry = &bitmap_begin_[index]; - uintptr_t old_word = atomic_entry->LoadRelaxed(); + uintptr_t old_word = atomic_entry->load(std::memory_order_relaxed); if (kSetBit) { // Check the bit before setting the word incase we are trying to mark a read only bitmap // like an image space bitmap. This bitmap is mapped as read only and will fault if we @@ -200,10 +201,10 @@ inline bool SpaceBitmap<kAlignment>::Modify(const mirror::Object* obj) { // occur if we check before setting the bit. This also prevents dirty pages that would // occur if the bitmap was read write and we did not check the bit. if ((old_word & mask) == 0) { - atomic_entry->StoreRelaxed(old_word | mask); + atomic_entry->store(old_word | mask, std::memory_order_relaxed); } } else { - atomic_entry->StoreRelaxed(old_word & ~mask); + atomic_entry->store(old_word & ~mask, std::memory_order_relaxed); } DCHECK_EQ(Test(obj), kSetBit); return (old_word & mask) != 0; diff --git a/runtime/gc/accounting/space_bitmap.cc b/runtime/gc/accounting/space_bitmap.cc index 0247564a8cd..ced62cd2498 100644 --- a/runtime/gc/accounting/space_bitmap.cc +++ b/runtime/gc/accounting/space_bitmap.cc @@ -19,8 +19,8 @@ #include "android-base/stringprintf.h" #include "art_field-inl.h" +#include "base/mem_map.h" #include "dex/dex_file-inl.h" -#include "mem_map.h" #include "mirror/class-inl.h" #include "mirror/object-inl.h" #include "mirror/object_array.h" @@ -145,7 +145,7 @@ void SpaceBitmap<kAlignment>::CopyFrom(SpaceBitmap* source_bitmap) { Atomic<uintptr_t>* const src = source_bitmap->Begin(); Atomic<uintptr_t>* const dest = Begin(); for (size_t i = 0; i < count; ++i) { - dest[i].StoreRelaxed(src[i].LoadRelaxed()); + dest[i].store(src[i].load(std::memory_order_relaxed), std::memory_order_relaxed); } } @@ -184,7 +184,8 @@ void SpaceBitmap<kAlignment>::SweepWalk(const SpaceBitmap<kAlignment>& live_bitm Atomic<uintptr_t>* live = live_bitmap.bitmap_begin_; Atomic<uintptr_t>* mark = mark_bitmap.bitmap_begin_; for (size_t i = start; i <= end; i++) { - uintptr_t garbage = live[i].LoadRelaxed() & ~mark[i].LoadRelaxed(); + uintptr_t garbage = + live[i].load(std::memory_order_relaxed) & ~mark[i].load(std::memory_order_relaxed); if (UNLIKELY(garbage != 0)) { uintptr_t ptr_base = IndexToOffset(i) + live_bitmap.heap_begin_; do { diff --git a/runtime/gc/allocator/rosalloc.cc b/runtime/gc/allocator/rosalloc.cc index 928abe873eb..a4095d815fd 100644 --- a/runtime/gc/allocator/rosalloc.cc +++ b/runtime/gc/allocator/rosalloc.cc @@ -25,9 +25,9 @@ #include "base/logging.h" // For VLOG #include "base/memory_tool.h" +#include "base/mem_map.h" #include "base/mutex-inl.h" #include "gc/space/memory_tool_settings.h" -#include "mem_map.h" #include "mirror/class-inl.h" #include "mirror/object-inl.h" #include "mirror/object.h" diff --git a/runtime/gc/collector/concurrent_copying-inl.h b/runtime/gc/collector/concurrent_copying-inl.h index 56983be8fa6..6e345fb2f24 100644 --- a/runtime/gc/collector/concurrent_copying-inl.h +++ b/runtime/gc/collector/concurrent_copying-inl.h @@ -78,13 +78,13 @@ inline mirror::Object* ConcurrentCopying::MarkImmuneSpace(mirror::Object* ref) { if (kIsDebugBuild) { if (Thread::Current() == thread_running_gc_) { DCHECK(!kGrayImmuneObject || - updated_all_immune_objects_.LoadRelaxed() || + updated_all_immune_objects_.load(std::memory_order_relaxed) || gc_grays_immune_objects_); } else { DCHECK(kGrayImmuneObject); } } - if (!kGrayImmuneObject || updated_all_immune_objects_.LoadRelaxed()) { + if (!kGrayImmuneObject || updated_all_immune_objects_.load(std::memory_order_relaxed)) { return ref; } // This may or may not succeed, which is ok because the object may already be gray. diff --git a/runtime/gc/collector/concurrent_copying.cc b/runtime/gc/collector/concurrent_copying.cc index 81e86d4b8df..0747c3c77bf 100644 --- a/runtime/gc/collector/concurrent_copying.cc +++ b/runtime/gc/collector/concurrent_copying.cc @@ -291,14 +291,14 @@ void ConcurrentCopying::InitializePhase() { rb_mark_bit_stack_full_ = false; mark_from_read_barrier_measurements_ = measure_read_barrier_slow_path_; if (measure_read_barrier_slow_path_) { - rb_slow_path_ns_.StoreRelaxed(0); - rb_slow_path_count_.StoreRelaxed(0); - rb_slow_path_count_gc_.StoreRelaxed(0); + rb_slow_path_ns_.store(0, std::memory_order_relaxed); + rb_slow_path_count_.store(0, std::memory_order_relaxed); + rb_slow_path_count_gc_.store(0, std::memory_order_relaxed); } immune_spaces_.Reset(); - bytes_moved_.StoreRelaxed(0); - objects_moved_.StoreRelaxed(0); + bytes_moved_.store(0, std::memory_order_relaxed); + objects_moved_.store(0, std::memory_order_relaxed); GcCause gc_cause = GetCurrentIteration()->GetGcCause(); if (gc_cause == kGcCauseExplicit || gc_cause == kGcCauseCollectorTransition || @@ -308,7 +308,7 @@ void ConcurrentCopying::InitializePhase() { force_evacuate_all_ = false; } if (kUseBakerReadBarrier) { - updated_all_immune_objects_.StoreRelaxed(false); + updated_all_immune_objects_.store(false, std::memory_order_relaxed); // GC may gray immune objects in the thread flip. gc_grays_immune_objects_ = true; if (kIsDebugBuild) { @@ -350,7 +350,7 @@ class ConcurrentCopying::ThreadFlipVisitor : public Closure, public RootVisitor concurrent_copying_->region_space_->RevokeThreadLocalBuffers(thread); reinterpret_cast<Atomic<size_t>*>( &concurrent_copying_->from_space_num_objects_at_first_pause_)-> - FetchAndAddSequentiallyConsistent(thread_local_objects); + fetch_add(thread_local_objects, std::memory_order_seq_cst); } else { concurrent_copying_->region_space_->RevokeThreadLocalBuffers(thread); } @@ -430,7 +430,8 @@ class ConcurrentCopying::FlipCallback : public Closure { cc->from_space_num_bytes_at_first_pause_ = cc->region_space_->GetBytesAllocated(); } cc->is_marking_ = true; - cc->mark_stack_mode_.StoreRelaxed(ConcurrentCopying::kMarkStackModeThreadLocal); + cc->mark_stack_mode_.store(ConcurrentCopying::kMarkStackModeThreadLocal, + std::memory_order_relaxed); if (kIsDebugBuild) { cc->region_space_->AssertAllRegionLiveBytesZeroOrCleared(); } @@ -728,7 +729,7 @@ void ConcurrentCopying::GrayAllNewlyDirtyImmuneObjects() { } // Since all of the objects that may point to other spaces are gray, we can avoid all the read // barriers in the immune spaces. - updated_all_immune_objects_.StoreRelaxed(true); + updated_all_immune_objects_.store(true, std::memory_order_relaxed); } void ConcurrentCopying::SwapStacks() { @@ -816,7 +817,7 @@ void ConcurrentCopying::MarkingPhase() { if (kUseBakerReadBarrier) { // This release fence makes the field updates in the above loop visible before allowing mutator // getting access to immune objects without graying it first. - updated_all_immune_objects_.StoreRelease(true); + updated_all_immune_objects_.store(true, std::memory_order_release); // Now whiten immune objects concurrently accessed and grayed by mutators. We can't do this in // the above loop because we would incorrectly disable the read barrier by whitening an object // which may point to an unscanned, white object, breaking the to-space invariant. @@ -1018,8 +1019,8 @@ void ConcurrentCopying::DisableMarking() { heap_->rb_table_->ClearAll(); DCHECK(heap_->rb_table_->IsAllCleared()); } - is_mark_stack_push_disallowed_.StoreSequentiallyConsistent(1); - mark_stack_mode_.StoreSequentiallyConsistent(kMarkStackModeOff); + is_mark_stack_push_disallowed_.store(1, std::memory_order_seq_cst); + mark_stack_mode_.store(kMarkStackModeOff, std::memory_order_seq_cst); } void ConcurrentCopying::PushOntoFalseGrayStack(mirror::Object* ref) { @@ -1069,11 +1070,11 @@ void ConcurrentCopying::ExpandGcMarkStack() { } void ConcurrentCopying::PushOntoMarkStack(mirror::Object* to_ref) { - CHECK_EQ(is_mark_stack_push_disallowed_.LoadRelaxed(), 0) + CHECK_EQ(is_mark_stack_push_disallowed_.load(std::memory_order_relaxed), 0) << " " << to_ref << " " << mirror::Object::PrettyTypeOf(to_ref); Thread* self = Thread::Current(); // TODO: pass self as an argument from call sites? CHECK(thread_running_gc_ != nullptr); - MarkStackMode mark_stack_mode = mark_stack_mode_.LoadRelaxed(); + MarkStackMode mark_stack_mode = mark_stack_mode_.load(std::memory_order_relaxed); if (LIKELY(mark_stack_mode == kMarkStackModeThreadLocal)) { if (LIKELY(self == thread_running_gc_)) { // If GC-running thread, use the GC mark stack instead of a thread-local mark stack. @@ -1412,7 +1413,7 @@ bool ConcurrentCopying::ProcessMarkStackOnce() { CHECK(self == thread_running_gc_); CHECK(self->GetThreadLocalMarkStack() == nullptr); size_t count = 0; - MarkStackMode mark_stack_mode = mark_stack_mode_.LoadRelaxed(); + MarkStackMode mark_stack_mode = mark_stack_mode_.load(std::memory_order_relaxed); if (mark_stack_mode == kMarkStackModeThreadLocal) { // Process the thread-local mark stacks and the GC mark stack. count += ProcessThreadLocalMarkStacks(/* disable_weak_ref_access */ false, @@ -1597,10 +1598,10 @@ void ConcurrentCopying::SwitchToSharedMarkStackMode() { CHECK(thread_running_gc_ != nullptr); CHECK_EQ(self, thread_running_gc_); CHECK(self->GetThreadLocalMarkStack() == nullptr); - MarkStackMode before_mark_stack_mode = mark_stack_mode_.LoadRelaxed(); + MarkStackMode before_mark_stack_mode = mark_stack_mode_.load(std::memory_order_relaxed); CHECK_EQ(static_cast<uint32_t>(before_mark_stack_mode), static_cast<uint32_t>(kMarkStackModeThreadLocal)); - mark_stack_mode_.StoreRelaxed(kMarkStackModeShared); + mark_stack_mode_.store(kMarkStackModeShared, std::memory_order_relaxed); DisableWeakRefAccessCallback dwrac(this); // Process the thread local mark stacks one last time after switching to the shared mark stack // mode and disable weak ref accesses. @@ -1615,10 +1616,10 @@ void ConcurrentCopying::SwitchToGcExclusiveMarkStackMode() { CHECK(thread_running_gc_ != nullptr); CHECK_EQ(self, thread_running_gc_); CHECK(self->GetThreadLocalMarkStack() == nullptr); - MarkStackMode before_mark_stack_mode = mark_stack_mode_.LoadRelaxed(); + MarkStackMode before_mark_stack_mode = mark_stack_mode_.load(std::memory_order_relaxed); CHECK_EQ(static_cast<uint32_t>(before_mark_stack_mode), static_cast<uint32_t>(kMarkStackModeShared)); - mark_stack_mode_.StoreRelaxed(kMarkStackModeGcExclusive); + mark_stack_mode_.store(kMarkStackModeGcExclusive, std::memory_order_relaxed); QuasiAtomic::ThreadFenceForConstructor(); if (kVerboseMode) { LOG(INFO) << "Switched to GC exclusive mark stack mode"; @@ -1630,7 +1631,7 @@ void ConcurrentCopying::CheckEmptyMarkStack() { CHECK(thread_running_gc_ != nullptr); CHECK_EQ(self, thread_running_gc_); CHECK(self->GetThreadLocalMarkStack() == nullptr); - MarkStackMode mark_stack_mode = mark_stack_mode_.LoadRelaxed(); + MarkStackMode mark_stack_mode = mark_stack_mode_.load(std::memory_order_relaxed); if (mark_stack_mode == kMarkStackModeThreadLocal) { // Thread-local mark stack mode. RevokeThreadLocalMarkStacks(false, nullptr); @@ -1738,9 +1739,9 @@ void ConcurrentCopying::ReclaimPhase() { } IssueEmptyCheckpoint(); // Disable the check. - is_mark_stack_push_disallowed_.StoreSequentiallyConsistent(0); + is_mark_stack_push_disallowed_.store(0, std::memory_order_seq_cst); if (kUseBakerReadBarrier) { - updated_all_immune_objects_.StoreSequentiallyConsistent(false); + updated_all_immune_objects_.store(false, std::memory_order_seq_cst); } CheckEmptyMarkStack(); } @@ -1753,10 +1754,10 @@ void ConcurrentCopying::ReclaimPhase() { const uint64_t from_objects = region_space_->GetObjectsAllocatedInFromSpace(); const uint64_t unevac_from_bytes = region_space_->GetBytesAllocatedInUnevacFromSpace(); const uint64_t unevac_from_objects = region_space_->GetObjectsAllocatedInUnevacFromSpace(); - uint64_t to_bytes = bytes_moved_.LoadSequentiallyConsistent(); - cumulative_bytes_moved_.FetchAndAddRelaxed(to_bytes); - uint64_t to_objects = objects_moved_.LoadSequentiallyConsistent(); - cumulative_objects_moved_.FetchAndAddRelaxed(to_objects); + uint64_t to_bytes = bytes_moved_.load(std::memory_order_seq_cst); + cumulative_bytes_moved_.fetch_add(to_bytes, std::memory_order_relaxed); + uint64_t to_objects = objects_moved_.load(std::memory_order_seq_cst); + cumulative_objects_moved_.fetch_add(to_objects, std::memory_order_relaxed); if (kEnableFromSpaceAccountingCheck) { CHECK_EQ(from_space_num_objects_at_first_pause_, from_objects + unevac_from_objects); CHECK_EQ(from_space_num_bytes_at_first_pause_, from_bytes + unevac_from_bytes); @@ -1787,12 +1788,12 @@ void ConcurrentCopying::ReclaimPhase() { << " unevac_from_space size=" << region_space_->UnevacFromSpaceSize() << " to_space size=" << region_space_->ToSpaceSize(); LOG(INFO) << "(before) num_bytes_allocated=" - << heap_->num_bytes_allocated_.LoadSequentiallyConsistent(); + << heap_->num_bytes_allocated_.load(std::memory_order_seq_cst); } RecordFree(ObjectBytePair(freed_objects, freed_bytes)); if (kVerboseMode) { LOG(INFO) << "(after) num_bytes_allocated=" - << heap_->num_bytes_allocated_.LoadSequentiallyConsistent(); + << heap_->num_bytes_allocated_.load(std::memory_order_seq_cst); } } @@ -2042,7 +2043,7 @@ void ConcurrentCopying::AssertToSpaceInvariantInNonMovingSpace(mirror::Object* o if (Thread::Current() == thread_running_gc_ && !gc_grays_immune_objects_) { return; } - bool updated_all_immune_objects = updated_all_immune_objects_.LoadSequentiallyConsistent(); + bool updated_all_immune_objects = updated_all_immune_objects_.load(std::memory_order_seq_cst); CHECK(updated_all_immune_objects || ref->GetReadBarrierState() == ReadBarrier::GrayState()) << "Unmarked immune space ref. obj=" << obj << " rb_state=" << (obj != nullptr ? obj->GetReadBarrierState() : 0U) @@ -2165,7 +2166,7 @@ inline void ConcurrentCopying::VisitRoots( mirror::Object* expected_ref = ref; mirror::Object* new_ref = to_ref; do { - if (expected_ref != addr->LoadRelaxed()) { + if (expected_ref != addr->load(std::memory_order_relaxed)) { // It was updated by the mutator. break; } @@ -2184,7 +2185,7 @@ inline void ConcurrentCopying::MarkRoot(mirror::CompressedReference<mirror::Obje auto new_ref = mirror::CompressedReference<mirror::Object>::FromMirrorPtr(to_ref); // If the cas fails, then it was updated by the mutator. do { - if (ref != addr->LoadRelaxed().AsMirrorPtr()) { + if (ref != addr->load(std::memory_order_relaxed).AsMirrorPtr()) { // It was updated by the mutator. break; } @@ -2380,8 +2381,9 @@ mirror::Object* ConcurrentCopying::Copy(mirror::Object* from_ref, fall_back_to_non_moving = true; if (kVerboseMode) { LOG(INFO) << "Out of memory in the to-space. Fall back to non-moving. skipped_bytes=" - << to_space_bytes_skipped_.LoadSequentiallyConsistent() - << " skipped_objects=" << to_space_objects_skipped_.LoadSequentiallyConsistent(); + << to_space_bytes_skipped_.load(std::memory_order_seq_cst) + << " skipped_objects=" + << to_space_objects_skipped_.load(std::memory_order_seq_cst); } to_ref = heap_->non_moving_space_->Alloc(Thread::Current(), obj_size, &non_moving_space_bytes_allocated, nullptr, &dummy); @@ -2432,9 +2434,9 @@ mirror::Object* ConcurrentCopying::Copy(mirror::Object* from_ref, region_space_->FreeLarge</*kForEvac*/ true>(to_ref, bytes_allocated); } else { // Record the lost copy for later reuse. - heap_->num_bytes_allocated_.FetchAndAddSequentiallyConsistent(bytes_allocated); - to_space_bytes_skipped_.FetchAndAddSequentiallyConsistent(bytes_allocated); - to_space_objects_skipped_.FetchAndAddSequentiallyConsistent(1); + heap_->num_bytes_allocated_.fetch_add(bytes_allocated, std::memory_order_seq_cst); + to_space_bytes_skipped_.fetch_add(bytes_allocated, std::memory_order_seq_cst); + to_space_objects_skipped_.fetch_add(1, std::memory_order_seq_cst); MutexLock mu(Thread::Current(), skipped_blocks_lock_); skipped_blocks_map_.insert(std::make_pair(bytes_allocated, reinterpret_cast<uint8_t*>(to_ref))); @@ -2470,7 +2472,7 @@ mirror::Object* ConcurrentCopying::Copy(mirror::Object* from_ref, // Do a fence to prevent the field CAS in ConcurrentCopying::Process from possibly reordering // before the object copy. - QuasiAtomic::ThreadFenceRelease(); + std::atomic_thread_fence(std::memory_order_release); LockWord new_lock_word = LockWord::FromForwardingAddress(reinterpret_cast<size_t>(to_ref)); @@ -2478,8 +2480,8 @@ mirror::Object* ConcurrentCopying::Copy(mirror::Object* from_ref, bool success = from_ref->CasLockWordWeakRelaxed(old_lock_word, new_lock_word); if (LIKELY(success)) { // The CAS succeeded. - objects_moved_.FetchAndAddRelaxed(1); - bytes_moved_.FetchAndAddRelaxed(region_space_alloc_size); + objects_moved_.fetch_add(1, std::memory_order_relaxed); + bytes_moved_.fetch_add(region_space_alloc_size, std::memory_order_relaxed); if (LIKELY(!fall_back_to_non_moving)) { DCHECK(region_space_->IsInToSpace(to_ref)); } else { @@ -2565,7 +2567,7 @@ mirror::Object* ConcurrentCopying::IsMarked(mirror::Object* from_ref) { bool ConcurrentCopying::IsOnAllocStack(mirror::Object* ref) { // TODO: Explain why this is here. What release operation does it pair with? - QuasiAtomic::ThreadFenceAcquire(); + std::atomic_thread_fence(std::memory_order_acquire); accounting::ObjectStack* alloc_stack = GetAllocationStack(); return alloc_stack->Contains(ref); } @@ -2705,9 +2707,10 @@ void ConcurrentCopying::FinishPhase() { } if (measure_read_barrier_slow_path_) { MutexLock mu(self, rb_slow_path_histogram_lock_); - rb_slow_path_time_histogram_.AdjustAndAddValue(rb_slow_path_ns_.LoadRelaxed()); - rb_slow_path_count_total_ += rb_slow_path_count_.LoadRelaxed(); - rb_slow_path_count_gc_total_ += rb_slow_path_count_gc_.LoadRelaxed(); + rb_slow_path_time_histogram_.AdjustAndAddValue( + rb_slow_path_ns_.load(std::memory_order_relaxed)); + rb_slow_path_count_total_ += rb_slow_path_count_.load(std::memory_order_relaxed); + rb_slow_path_count_gc_total_ += rb_slow_path_count_gc_.load(std::memory_order_relaxed); } } @@ -2761,15 +2764,15 @@ void ConcurrentCopying::RevokeAllThreadLocalBuffers() { mirror::Object* ConcurrentCopying::MarkFromReadBarrierWithMeasurements(mirror::Object* from_ref) { if (Thread::Current() != thread_running_gc_) { - rb_slow_path_count_.FetchAndAddRelaxed(1u); + rb_slow_path_count_.fetch_add(1u, std::memory_order_relaxed); } else { - rb_slow_path_count_gc_.FetchAndAddRelaxed(1u); + rb_slow_path_count_gc_.fetch_add(1u, std::memory_order_relaxed); } ScopedTrace tr(__FUNCTION__); const uint64_t start_time = measure_read_barrier_slow_path_ ? NanoTime() : 0u; mirror::Object* ret = Mark(from_ref); if (measure_read_barrier_slow_path_) { - rb_slow_path_ns_.FetchAndAddRelaxed(NanoTime() - start_time); + rb_slow_path_ns_.fetch_add(NanoTime() - start_time, std::memory_order_relaxed); } return ret; } @@ -2788,8 +2791,10 @@ void ConcurrentCopying::DumpPerformanceInfo(std::ostream& os) { if (rb_slow_path_count_gc_total_ > 0) { os << "GC slow path count " << rb_slow_path_count_gc_total_ << "\n"; } - os << "Cumulative bytes moved " << cumulative_bytes_moved_.LoadRelaxed() << "\n"; - os << "Cumulative objects moved " << cumulative_objects_moved_.LoadRelaxed() << "\n"; + os << "Cumulative bytes moved " + << cumulative_bytes_moved_.load(std::memory_order_relaxed) << "\n"; + os << "Cumulative objects moved " + << cumulative_objects_moved_.load(std::memory_order_relaxed) << "\n"; os << "Peak regions allocated " << region_space_->GetMaxPeakNumNonFreeRegions() << " (" diff --git a/runtime/gc/collector/mark_compact.cc b/runtime/gc/collector/mark_compact.cc deleted file mode 100644 index 34cc129ce8c..00000000000 --- a/runtime/gc/collector/mark_compact.cc +++ /dev/null @@ -1,642 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "mark_compact.h" - -#include <android-base/logging.h> - -#include "base/macros.h" -#include "base/mutex-inl.h" -#include "base/timing_logger.h" -#include "gc/accounting/heap_bitmap-inl.h" -#include "gc/accounting/mod_union_table.h" -#include "gc/accounting/space_bitmap-inl.h" -#include "gc/heap.h" -#include "gc/reference_processor.h" -#include "gc/space/bump_pointer_space-inl.h" -#include "gc/space/large_object_space.h" -#include "gc/space/space-inl.h" -#include "mirror/class-inl.h" -#include "mirror/object-inl.h" -#include "mirror/object-refvisitor-inl.h" -#include "runtime.h" -#include "stack.h" -#include "thread-current-inl.h" -#include "thread_list.h" - -namespace art { -namespace gc { -namespace collector { - -void MarkCompact::BindBitmaps() { - TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings()); - WriterMutexLock mu(Thread::Current(), *Locks::heap_bitmap_lock_); - // Mark all of the spaces we never collect as immune. - for (const auto& space : GetHeap()->GetContinuousSpaces()) { - if (space->GetGcRetentionPolicy() == space::kGcRetentionPolicyNeverCollect || - space->GetGcRetentionPolicy() == space::kGcRetentionPolicyFullCollect) { - immune_spaces_.AddSpace(space); - } - } -} - -MarkCompact::MarkCompact(Heap* heap, const std::string& name_prefix) - : GarbageCollector(heap, name_prefix + (name_prefix.empty() ? "" : " ") + "mark compact"), - mark_stack_(nullptr), - space_(nullptr), - mark_bitmap_(nullptr), - collector_name_(name_), - bump_pointer_(nullptr), - live_objects_in_space_(0), - updating_references_(false) {} - -void MarkCompact::RunPhases() { - Thread* self = Thread::Current(); - InitializePhase(); - CHECK(!Locks::mutator_lock_->IsExclusiveHeld(self)); - { - ScopedPause pause(this); - GetHeap()->PreGcVerificationPaused(this); - GetHeap()->PrePauseRosAllocVerification(this); - MarkingPhase(); - ReclaimPhase(); - } - GetHeap()->PostGcVerification(this); - FinishPhase(); -} - -void MarkCompact::ForwardObject(mirror::Object* obj) { - const size_t alloc_size = RoundUp(obj->SizeOf(), space::BumpPointerSpace::kAlignment); - LockWord lock_word = obj->GetLockWord(false); - // If we have a non empty lock word, store it and restore it later. - if (!LockWord::IsDefault(lock_word)) { - // Set the bit in the bitmap so that we know to restore it later. - objects_with_lockword_->Set(obj); - lock_words_to_restore_.push_back(lock_word); - } - obj->SetLockWord(LockWord::FromForwardingAddress(reinterpret_cast<size_t>(bump_pointer_)), - false); - bump_pointer_ += alloc_size; - ++live_objects_in_space_; -} - - -void MarkCompact::CalculateObjectForwardingAddresses() { - TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings()); - // The bump pointer in the space where the next forwarding address will be. - bump_pointer_ = reinterpret_cast<uint8_t*>(space_->Begin()); - // Visit all the marked objects in the bitmap. - objects_before_forwarding_->VisitMarkedRange(reinterpret_cast<uintptr_t>(space_->Begin()), - reinterpret_cast<uintptr_t>(space_->End()), - [this](mirror::Object* obj) - REQUIRES(Locks::mutator_lock_, Locks::heap_bitmap_lock_) { - DCHECK_ALIGNED(obj, space::BumpPointerSpace::kAlignment); - DCHECK(IsMarked(obj) != nullptr); - ForwardObject(obj); - }); -} - -void MarkCompact::InitializePhase() { - TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings()); - mark_stack_ = heap_->GetMarkStack(); - DCHECK(mark_stack_ != nullptr); - immune_spaces_.Reset(); - CHECK(space_->CanMoveObjects()) << "Attempting compact non-movable space from " << *space_; - // TODO: I don't think we should need heap bitmap lock to Get the mark bitmap. - ReaderMutexLock mu(Thread::Current(), *Locks::heap_bitmap_lock_); - mark_bitmap_ = heap_->GetMarkBitmap(); - live_objects_in_space_ = 0; -} - -void MarkCompact::ProcessReferences(Thread* self) { - WriterMutexLock mu(self, *Locks::heap_bitmap_lock_); - heap_->GetReferenceProcessor()->ProcessReferences( - false, GetTimings(), GetCurrentIteration()->GetClearSoftReferences(), this); -} - -inline mirror::Object* MarkCompact::MarkObject(mirror::Object* obj) { - if (obj == nullptr) { - return nullptr; - } - if (kUseBakerReadBarrier) { - // Verify all the objects have the correct forward state installed. - obj->AssertReadBarrierState(); - } - if (!immune_spaces_.IsInImmuneRegion(obj)) { - if (objects_before_forwarding_->HasAddress(obj)) { - if (!objects_before_forwarding_->Set(obj)) { - MarkStackPush(obj); // This object was not previously marked. - } - } else { - DCHECK(!space_->HasAddress(obj)); - auto slow_path = [](const mirror::Object* ref) - REQUIRES_SHARED(Locks::mutator_lock_) { - // Marking a large object, make sure its aligned as a sanity check. - if (!IsAligned<kPageSize>(ref)) { - Runtime::Current()->GetHeap()->DumpSpaces(LOG_STREAM(ERROR)); - LOG(FATAL) << ref; - } - }; - if (!mark_bitmap_->Set(obj, slow_path)) { - // This object was not previously marked. - MarkStackPush(obj); - } - } - } - return obj; -} - -void MarkCompact::MarkingPhase() { - TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings()); - Thread* self = Thread::Current(); - // Bitmap which describes which objects we have to move. - objects_before_forwarding_.reset(accounting::ContinuousSpaceBitmap::Create( - "objects before forwarding", space_->Begin(), space_->Size())); - // Bitmap which describes which lock words we need to restore. - objects_with_lockword_.reset(accounting::ContinuousSpaceBitmap::Create( - "objects with lock words", space_->Begin(), space_->Size())); - CHECK(Locks::mutator_lock_->IsExclusiveHeld(self)); - // Assume the cleared space is already empty. - BindBitmaps(); - t.NewTiming("ProcessCards"); - // Process dirty cards and add dirty cards to mod-union tables. - heap_->ProcessCards(GetTimings(), false, false, true); - // Clear the whole card table since we cannot get any additional dirty cards during the - // paused GC. This saves memory but only works for pause the world collectors. - t.NewTiming("ClearCardTable"); - heap_->GetCardTable()->ClearCardTable(); - // Need to do this before the checkpoint since we don't want any threads to add references to - // the live stack during the recursive mark. - if (kUseThreadLocalAllocationStack) { - t.NewTiming("RevokeAllThreadLocalAllocationStacks"); - heap_->RevokeAllThreadLocalAllocationStacks(self); - } - t.NewTiming("SwapStacks"); - heap_->SwapStacks(); - { - WriterMutexLock mu(self, *Locks::heap_bitmap_lock_); - MarkRoots(); - // Mark roots of immune spaces. - UpdateAndMarkModUnion(); - // Recursively mark remaining objects. - MarkReachableObjects(); - } - ProcessReferences(self); - { - ReaderMutexLock mu(self, *Locks::heap_bitmap_lock_); - SweepSystemWeaks(); - } - Runtime::Current()->GetClassLinker()->CleanupClassLoaders(); - // Revoke buffers before measuring how many objects were moved since the TLABs need to be revoked - // before they are properly counted. - RevokeAllThreadLocalBuffers(); - // Disabled due to an issue where we have objects in the bump pointer space which reference dead - // objects. - // heap_->PreSweepingGcVerification(this); -} - -void MarkCompact::UpdateAndMarkModUnion() { - TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings()); - for (auto& space : heap_->GetContinuousSpaces()) { - // If the space is immune then we need to mark the references to other spaces. - if (immune_spaces_.ContainsSpace(space)) { - accounting::ModUnionTable* table = heap_->FindModUnionTableFromSpace(space); - if (table != nullptr) { - // TODO: Improve naming. - TimingLogger::ScopedTiming t2( - space->IsZygoteSpace() ? "UpdateAndMarkZygoteModUnionTable" : - "UpdateAndMarkImageModUnionTable", GetTimings()); - table->UpdateAndMarkReferences(this); - } - } - } -} - -void MarkCompact::MarkReachableObjects() { - TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings()); - accounting::ObjectStack* live_stack = heap_->GetLiveStack(); - { - TimingLogger::ScopedTiming t2("MarkAllocStackAsLive", GetTimings()); - heap_->MarkAllocStackAsLive(live_stack); - } - live_stack->Reset(); - // Recursively process the mark stack. - ProcessMarkStack(); -} - -void MarkCompact::ReclaimPhase() { - TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings()); - WriterMutexLock mu(Thread::Current(), *Locks::heap_bitmap_lock_); - // Reclaim unmarked objects. - Sweep(false); - // Swap the live and mark bitmaps for each space which we modified space. This is an - // optimization that enables us to not clear live bits inside of the sweep. Only swaps unbound - // bitmaps. - SwapBitmaps(); - GetHeap()->UnBindBitmaps(); // Unbind the live and mark bitmaps. - Compact(); -} - -void MarkCompact::ResizeMarkStack(size_t new_size) { - std::vector<StackReference<mirror::Object>> temp(mark_stack_->Begin(), mark_stack_->End()); - CHECK_LE(mark_stack_->Size(), new_size); - mark_stack_->Resize(new_size); - for (auto& obj : temp) { - mark_stack_->PushBack(obj.AsMirrorPtr()); - } -} - -inline void MarkCompact::MarkStackPush(mirror::Object* obj) { - if (UNLIKELY(mark_stack_->Size() >= mark_stack_->Capacity())) { - ResizeMarkStack(mark_stack_->Capacity() * 2); - } - // The object must be pushed on to the mark stack. - mark_stack_->PushBack(obj); -} - -void MarkCompact::MarkHeapReference(mirror::HeapReference<mirror::Object>* obj_ptr, - bool do_atomic_update ATTRIBUTE_UNUSED) { - if (updating_references_) { - UpdateHeapReference(obj_ptr); - } else { - MarkObject(obj_ptr->AsMirrorPtr()); - } -} - -void MarkCompact::VisitRoots( - mirror::Object*** roots, size_t count, const RootInfo& info ATTRIBUTE_UNUSED) { - for (size_t i = 0; i < count; ++i) { - MarkObject(*roots[i]); - } -} - -void MarkCompact::VisitRoots( - mirror::CompressedReference<mirror::Object>** roots, size_t count, - const RootInfo& info ATTRIBUTE_UNUSED) { - for (size_t i = 0; i < count; ++i) { - MarkObject(roots[i]->AsMirrorPtr()); - } -} - -class MarkCompact::UpdateRootVisitor : public RootVisitor { - public: - explicit UpdateRootVisitor(MarkCompact* collector) : collector_(collector) {} - - void VisitRoots(mirror::Object*** roots, size_t count, const RootInfo& info ATTRIBUTE_UNUSED) - OVERRIDE REQUIRES(Locks::mutator_lock_) - REQUIRES_SHARED(Locks::heap_bitmap_lock_) { - for (size_t i = 0; i < count; ++i) { - mirror::Object* obj = *roots[i]; - mirror::Object* new_obj = collector_->GetMarkedForwardAddress(obj); - if (obj != new_obj) { - *roots[i] = new_obj; - DCHECK(new_obj != nullptr); - } - } - } - - void VisitRoots(mirror::CompressedReference<mirror::Object>** roots, size_t count, - const RootInfo& info ATTRIBUTE_UNUSED) - OVERRIDE REQUIRES(Locks::mutator_lock_) - REQUIRES_SHARED(Locks::heap_bitmap_lock_) { - for (size_t i = 0; i < count; ++i) { - mirror::Object* obj = roots[i]->AsMirrorPtr(); - mirror::Object* new_obj = collector_->GetMarkedForwardAddress(obj); - if (obj != new_obj) { - roots[i]->Assign(new_obj); - DCHECK(new_obj != nullptr); - } - } - } - - private: - MarkCompact* const collector_; -}; - -class MarkCompact::UpdateObjectReferencesVisitor { - public: - explicit UpdateObjectReferencesVisitor(MarkCompact* collector) : collector_(collector) {} - - void operator()(mirror::Object* obj) const REQUIRES_SHARED(Locks::heap_bitmap_lock_) - REQUIRES(Locks::mutator_lock_) ALWAYS_INLINE { - collector_->UpdateObjectReferences(obj); - } - - private: - MarkCompact* const collector_; -}; - -void MarkCompact::UpdateReferences() { - TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings()); - updating_references_ = true; - Runtime* runtime = Runtime::Current(); - // Update roots. - UpdateRootVisitor update_root_visitor(this); - runtime->VisitRoots(&update_root_visitor); - // Update object references in mod union tables and spaces. - for (const auto& space : heap_->GetContinuousSpaces()) { - // If the space is immune then we need to mark the references to other spaces. - accounting::ModUnionTable* table = heap_->FindModUnionTableFromSpace(space); - if (table != nullptr) { - // TODO: Improve naming. - TimingLogger::ScopedTiming t2( - space->IsZygoteSpace() ? "UpdateZygoteModUnionTableReferences" : - "UpdateImageModUnionTableReferences", - GetTimings()); - table->UpdateAndMarkReferences(this); - } else { - // No mod union table, so we need to scan the space using bitmap visit. - // Scan the space using bitmap visit. - accounting::ContinuousSpaceBitmap* bitmap = space->GetLiveBitmap(); - if (bitmap != nullptr) { - UpdateObjectReferencesVisitor visitor(this); - bitmap->VisitMarkedRange(reinterpret_cast<uintptr_t>(space->Begin()), - reinterpret_cast<uintptr_t>(space->End()), - visitor); - } - } - } - CHECK(!kMovingClasses) - << "Didn't update large object classes since they are assumed to not move."; - // Update the system weaks, these should already have been swept. - runtime->SweepSystemWeaks(this); - // Update the objects in the bump pointer space last, these objects don't have a bitmap. - UpdateObjectReferencesVisitor visitor(this); - objects_before_forwarding_->VisitMarkedRange(reinterpret_cast<uintptr_t>(space_->Begin()), - reinterpret_cast<uintptr_t>(space_->End()), - visitor); - // Update the reference processor cleared list. - heap_->GetReferenceProcessor()->UpdateRoots(this); - updating_references_ = false; -} - -void MarkCompact::Compact() { - TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings()); - CalculateObjectForwardingAddresses(); - UpdateReferences(); - MoveObjects(); - // Space - int64_t objects_freed = space_->GetObjectsAllocated() - live_objects_in_space_; - int64_t bytes_freed = reinterpret_cast<int64_t>(space_->End()) - - reinterpret_cast<int64_t>(bump_pointer_); - t.NewTiming("RecordFree"); - space_->RecordFree(objects_freed, bytes_freed); - RecordFree(ObjectBytePair(objects_freed, bytes_freed)); - space_->SetEnd(bump_pointer_); - // Need to zero out the memory we freed. TODO: Use madvise for pages. - memset(bump_pointer_, 0, bytes_freed); -} - -// Marks all objects in the root set. -void MarkCompact::MarkRoots() { - TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings()); - Runtime::Current()->VisitRoots(this); -} - -inline void MarkCompact::UpdateHeapReference(mirror::HeapReference<mirror::Object>* reference) { - mirror::Object* obj = reference->AsMirrorPtr(); - if (obj != nullptr) { - mirror::Object* new_obj = GetMarkedForwardAddress(obj); - if (obj != new_obj) { - DCHECK(new_obj != nullptr); - reference->Assign(new_obj); - } - } -} - -class MarkCompact::UpdateReferenceVisitor { - public: - explicit UpdateReferenceVisitor(MarkCompact* collector) : collector_(collector) {} - - void operator()(mirror::Object* obj, MemberOffset offset, bool /*is_static*/) const - ALWAYS_INLINE REQUIRES(Locks::mutator_lock_, Locks::heap_bitmap_lock_) { - collector_->UpdateHeapReference(obj->GetFieldObjectReferenceAddr<kVerifyNone>(offset)); - } - - void operator()(ObjPtr<mirror::Class> /*klass*/, mirror::Reference* ref) const - REQUIRES(Locks::mutator_lock_, Locks::heap_bitmap_lock_) { - collector_->UpdateHeapReference( - ref->GetFieldObjectReferenceAddr<kVerifyNone>(mirror::Reference::ReferentOffset())); - } - - // TODO: Remove NO_THREAD_SAFETY_ANALYSIS when clang better understands visitors. - void VisitRootIfNonNull(mirror::CompressedReference<mirror::Object>* root) const - NO_THREAD_SAFETY_ANALYSIS { - if (!root->IsNull()) { - VisitRoot(root); - } - } - - void VisitRoot(mirror::CompressedReference<mirror::Object>* root) const - NO_THREAD_SAFETY_ANALYSIS { - root->Assign(collector_->GetMarkedForwardAddress(root->AsMirrorPtr())); - } - - private: - MarkCompact* const collector_; -}; - -void MarkCompact::UpdateObjectReferences(mirror::Object* obj) { - UpdateReferenceVisitor visitor(this); - obj->VisitReferences(visitor, visitor); -} - -inline mirror::Object* MarkCompact::GetMarkedForwardAddress(mirror::Object* obj) { - DCHECK(obj != nullptr); - if (objects_before_forwarding_->HasAddress(obj)) { - DCHECK(objects_before_forwarding_->Test(obj)); - mirror::Object* ret = - reinterpret_cast<mirror::Object*>(obj->GetLockWord(false).ForwardingAddress()); - DCHECK(ret != nullptr); - return ret; - } - DCHECK(!space_->HasAddress(obj)); - return obj; -} - -mirror::Object* MarkCompact::IsMarked(mirror::Object* object) { - if (immune_spaces_.IsInImmuneRegion(object)) { - return object; - } - if (updating_references_) { - return GetMarkedForwardAddress(object); - } - if (objects_before_forwarding_->HasAddress(object)) { - return objects_before_forwarding_->Test(object) ? object : nullptr; - } - return mark_bitmap_->Test(object) ? object : nullptr; -} - -bool MarkCompact::IsNullOrMarkedHeapReference(mirror::HeapReference<mirror::Object>* ref_ptr, - // MarkCompact does the GC in a pause. No CAS needed. - bool do_atomic_update ATTRIBUTE_UNUSED) { - // Side effect free since we call this before ever moving objects. - mirror::Object* obj = ref_ptr->AsMirrorPtr(); - if (obj == nullptr) { - return true; - } - return IsMarked(obj) != nullptr; -} - -void MarkCompact::SweepSystemWeaks() { - TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings()); - Runtime::Current()->SweepSystemWeaks(this); -} - -bool MarkCompact::ShouldSweepSpace(space::ContinuousSpace* space) const { - return space != space_ && !immune_spaces_.ContainsSpace(space); -} - -void MarkCompact::MoveObject(mirror::Object* obj, size_t len) { - // Look at the forwarding address stored in the lock word to know where to copy. - DCHECK(space_->HasAddress(obj)) << obj; - uintptr_t dest_addr = obj->GetLockWord(false).ForwardingAddress(); - mirror::Object* dest_obj = reinterpret_cast<mirror::Object*>(dest_addr); - DCHECK(space_->HasAddress(dest_obj)) << dest_obj; - // Use memmove since there may be overlap. - memmove(reinterpret_cast<void*>(dest_addr), reinterpret_cast<const void*>(obj), len); - // Restore the saved lock word if needed. - LockWord lock_word = LockWord::Default(); - if (UNLIKELY(objects_with_lockword_->Test(obj))) { - lock_word = lock_words_to_restore_.front(); - lock_words_to_restore_.pop_front(); - } - dest_obj->SetLockWord(lock_word, false); -} - -void MarkCompact::MoveObjects() { - TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings()); - // Move the objects in the before forwarding bitmap. - objects_before_forwarding_->VisitMarkedRange(reinterpret_cast<uintptr_t>(space_->Begin()), - reinterpret_cast<uintptr_t>(space_->End()), - [this](mirror::Object* obj) - REQUIRES_SHARED(Locks::heap_bitmap_lock_) - REQUIRES(Locks::mutator_lock_) ALWAYS_INLINE { - MoveObject(obj, obj->SizeOf()); - }); - CHECK(lock_words_to_restore_.empty()); -} - -void MarkCompact::Sweep(bool swap_bitmaps) { - TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings()); - DCHECK(mark_stack_->IsEmpty()); - for (const auto& space : GetHeap()->GetContinuousSpaces()) { - if (space->IsContinuousMemMapAllocSpace()) { - space::ContinuousMemMapAllocSpace* alloc_space = space->AsContinuousMemMapAllocSpace(); - if (!ShouldSweepSpace(alloc_space)) { - continue; - } - TimingLogger::ScopedTiming t2( - alloc_space->IsZygoteSpace() ? "SweepZygoteSpace" : "SweepAllocSpace", GetTimings()); - RecordFree(alloc_space->Sweep(swap_bitmaps)); - } - } - SweepLargeObjects(swap_bitmaps); -} - -void MarkCompact::SweepLargeObjects(bool swap_bitmaps) { - space::LargeObjectSpace* los = heap_->GetLargeObjectsSpace(); - if (los != nullptr) { - TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings());\ - RecordFreeLOS(los->Sweep(swap_bitmaps)); - } -} - -// Process the "referent" field in a java.lang.ref.Reference. If the referent has not yet been -// marked, put it on the appropriate list in the heap for later processing. -void MarkCompact::DelayReferenceReferent(ObjPtr<mirror::Class> klass, - ObjPtr<mirror::Reference> reference) { - heap_->GetReferenceProcessor()->DelayReferenceReferent(klass, reference, this); -} - -class MarkCompact::MarkObjectVisitor { - public: - explicit MarkObjectVisitor(MarkCompact* collector) : collector_(collector) {} - - void operator()(ObjPtr<mirror::Object> obj, - MemberOffset offset, - bool /*is_static*/) const ALWAYS_INLINE - REQUIRES(Locks::mutator_lock_, Locks::heap_bitmap_lock_) { - // Object was already verified when we scanned it. - collector_->MarkObject(obj->GetFieldObject<mirror::Object, kVerifyNone>(offset)); - } - - void operator()(ObjPtr<mirror::Class> klass, - ObjPtr<mirror::Reference> ref) const - REQUIRES_SHARED(Locks::mutator_lock_) - REQUIRES(Locks::heap_bitmap_lock_) { - collector_->DelayReferenceReferent(klass, ref); - } - - // TODO: Remove NO_THREAD_SAFETY_ANALYSIS when clang better understands visitors. - void VisitRootIfNonNull(mirror::CompressedReference<mirror::Object>* root) const - NO_THREAD_SAFETY_ANALYSIS { - if (!root->IsNull()) { - VisitRoot(root); - } - } - - void VisitRoot(mirror::CompressedReference<mirror::Object>* root) const - NO_THREAD_SAFETY_ANALYSIS { - collector_->MarkObject(root->AsMirrorPtr()); - } - - private: - MarkCompact* const collector_; -}; - -// Visit all of the references of an object and update. -void MarkCompact::ScanObject(mirror::Object* obj) { - MarkObjectVisitor visitor(this); - obj->VisitReferences(visitor, visitor); -} - -// Scan anything that's on the mark stack. -void MarkCompact::ProcessMarkStack() { - TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings()); - while (!mark_stack_->IsEmpty()) { - mirror::Object* obj = mark_stack_->PopBack(); - DCHECK(obj != nullptr); - ScanObject(obj); - } -} - -void MarkCompact::SetSpace(space::BumpPointerSpace* space) { - DCHECK(space != nullptr); - space_ = space; -} - -void MarkCompact::FinishPhase() { - TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings()); - space_ = nullptr; - CHECK(mark_stack_->IsEmpty()); - mark_stack_->Reset(); - // Clear all of the spaces' mark bitmaps. - WriterMutexLock mu(Thread::Current(), *Locks::heap_bitmap_lock_); - heap_->ClearMarkedObjects(); - // Release our bitmaps. - objects_before_forwarding_.reset(nullptr); - objects_with_lockword_.reset(nullptr); -} - -void MarkCompact::RevokeAllThreadLocalBuffers() { - TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings()); - GetHeap()->RevokeAllThreadLocalBuffers(); -} - -} // namespace collector -} // namespace gc -} // namespace art diff --git a/runtime/gc/collector/mark_compact.h b/runtime/gc/collector/mark_compact.h deleted file mode 100644 index e7749597cd4..00000000000 --- a/runtime/gc/collector/mark_compact.h +++ /dev/null @@ -1,238 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef ART_RUNTIME_GC_COLLECTOR_MARK_COMPACT_H_ -#define ART_RUNTIME_GC_COLLECTOR_MARK_COMPACT_H_ - -#include <deque> -#include <memory> // For unique_ptr. - -#include "base/atomic.h" -#include "base/macros.h" -#include "base/mutex.h" -#include "garbage_collector.h" -#include "gc/accounting/heap_bitmap.h" -#include "gc_root.h" -#include "immune_spaces.h" -#include "lock_word.h" -#include "offsets.h" - -namespace art { - -class Thread; - -namespace mirror { -class Class; -class Object; -} // namespace mirror - -namespace gc { - -class Heap; - -namespace accounting { -template <typename T> class AtomicStack; -typedef AtomicStack<mirror::Object> ObjectStack; -} // namespace accounting - -namespace space { -class BumpPointerSpace; -class ContinuousMemMapAllocSpace; -class ContinuousSpace; -} // namespace space - -namespace collector { - -class MarkCompact : public GarbageCollector { - public: - explicit MarkCompact(Heap* heap, const std::string& name_prefix = ""); - ~MarkCompact() {} - - virtual void RunPhases() OVERRIDE NO_THREAD_SAFETY_ANALYSIS; - void InitializePhase(); - void MarkingPhase() REQUIRES(Locks::mutator_lock_) - REQUIRES(!Locks::heap_bitmap_lock_); - void ReclaimPhase() REQUIRES(Locks::mutator_lock_) - REQUIRES(!Locks::heap_bitmap_lock_); - void FinishPhase() REQUIRES(Locks::mutator_lock_); - void MarkReachableObjects() - REQUIRES(Locks::mutator_lock_, Locks::heap_bitmap_lock_); - virtual GcType GetGcType() const OVERRIDE { - return kGcTypePartial; - } - virtual CollectorType GetCollectorType() const OVERRIDE { - return kCollectorTypeMC; - } - - // Sets which space we will be copying objects in. - void SetSpace(space::BumpPointerSpace* space); - - // Initializes internal structures. - void Init(); - - // Find the default mark bitmap. - void FindDefaultMarkBitmap(); - - void ScanObject(mirror::Object* obj) - REQUIRES(Locks::heap_bitmap_lock_, Locks::mutator_lock_); - - // Marks the root set at the start of a garbage collection. - void MarkRoots() - REQUIRES(Locks::heap_bitmap_lock_, Locks::mutator_lock_); - - // Bind the live bits to the mark bits of bitmaps for spaces that are never collected, ie - // the image. Mark that portion of the heap as immune. - void BindBitmaps() REQUIRES_SHARED(Locks::mutator_lock_) - REQUIRES(!Locks::heap_bitmap_lock_); - - void UnBindBitmaps() - REQUIRES(Locks::heap_bitmap_lock_); - - void ProcessReferences(Thread* self) REQUIRES(Locks::mutator_lock_) - REQUIRES(Locks::mutator_lock_); - - // Sweeps unmarked objects to complete the garbage collection. - void Sweep(bool swap_bitmaps) REQUIRES(Locks::heap_bitmap_lock_, Locks::mutator_lock_); - - // Sweeps unmarked objects to complete the garbage collection. - void SweepLargeObjects(bool swap_bitmaps) REQUIRES(Locks::heap_bitmap_lock_); - - void SweepSystemWeaks() - REQUIRES_SHARED(Locks::heap_bitmap_lock_, Locks::mutator_lock_); - - virtual void VisitRoots(mirror::Object*** roots, size_t count, const RootInfo& info) - OVERRIDE REQUIRES(Locks::mutator_lock_, Locks::heap_bitmap_lock_); - - virtual void VisitRoots(mirror::CompressedReference<mirror::Object>** roots, size_t count, - const RootInfo& info) - OVERRIDE REQUIRES(Locks::mutator_lock_, Locks::heap_bitmap_lock_); - - // Schedules an unmarked object for reference processing. - void DelayReferenceReferent(ObjPtr<mirror::Class> klass, ObjPtr<mirror::Reference> reference) - REQUIRES_SHARED(Locks::heap_bitmap_lock_, Locks::mutator_lock_); - - protected: - // Returns null if the object is not marked, otherwise returns the forwarding address (same as - // object for non movable things). - mirror::Object* GetMarkedForwardAddress(mirror::Object* object) - REQUIRES(Locks::mutator_lock_) - REQUIRES_SHARED(Locks::heap_bitmap_lock_); - - // Marks or unmarks a large object based on whether or not set is true. If set is true, then we - // mark, otherwise we unmark. - bool MarkLargeObject(const mirror::Object* obj) - REQUIRES(Locks::heap_bitmap_lock_) - REQUIRES_SHARED(Locks::mutator_lock_); - - // Expand mark stack to 2x its current size. - void ResizeMarkStack(size_t new_size) REQUIRES_SHARED(Locks::mutator_lock_); - - // Returns true if we should sweep the space. - bool ShouldSweepSpace(space::ContinuousSpace* space) const; - - // Push an object onto the mark stack. - void MarkStackPush(mirror::Object* obj) REQUIRES_SHARED(Locks::mutator_lock_); - - void UpdateAndMarkModUnion() - REQUIRES(Locks::heap_bitmap_lock_) - REQUIRES_SHARED(Locks::mutator_lock_); - - // Recursively blackens objects on the mark stack. - void ProcessMarkStack() - REQUIRES(Locks::mutator_lock_, Locks::heap_bitmap_lock_); - - // 3 pass mark compact approach. - void Compact() REQUIRES(Locks::mutator_lock_, Locks::heap_bitmap_lock_); - // Calculate the forwarding address of objects marked as "live" in the objects_before_forwarding - // bitmap. - void CalculateObjectForwardingAddresses() - REQUIRES(Locks::mutator_lock_, Locks::heap_bitmap_lock_); - // Update the references of objects by using the forwarding addresses. - void UpdateReferences() REQUIRES(Locks::mutator_lock_, Locks::heap_bitmap_lock_); - // Move objects and restore lock words. - void MoveObjects() REQUIRES(Locks::mutator_lock_); - // Move a single object to its forward address. - void MoveObject(mirror::Object* obj, size_t len) REQUIRES(Locks::mutator_lock_); - // Mark a single object. - virtual mirror::Object* MarkObject(mirror::Object* obj) OVERRIDE - REQUIRES(Locks::heap_bitmap_lock_, Locks::mutator_lock_); - virtual void MarkHeapReference(mirror::HeapReference<mirror::Object>* obj_ptr, - bool do_atomic_update) OVERRIDE - REQUIRES(Locks::heap_bitmap_lock_, Locks::mutator_lock_); - virtual mirror::Object* IsMarked(mirror::Object* obj) OVERRIDE - REQUIRES_SHARED(Locks::heap_bitmap_lock_) - REQUIRES(Locks::mutator_lock_); - virtual bool IsNullOrMarkedHeapReference(mirror::HeapReference<mirror::Object>* obj, - bool do_atomic_update) OVERRIDE - REQUIRES_SHARED(Locks::heap_bitmap_lock_) - REQUIRES(Locks::mutator_lock_); - void ForwardObject(mirror::Object* obj) REQUIRES(Locks::heap_bitmap_lock_, - Locks::mutator_lock_); - // Update a single heap reference. - void UpdateHeapReference(mirror::HeapReference<mirror::Object>* reference) - REQUIRES_SHARED(Locks::heap_bitmap_lock_) - REQUIRES(Locks::mutator_lock_); - // Update all of the references of a single object. - void UpdateObjectReferences(mirror::Object* obj) - REQUIRES_SHARED(Locks::heap_bitmap_lock_) - REQUIRES(Locks::mutator_lock_); - - // Revoke all the thread-local buffers. - void RevokeAllThreadLocalBuffers(); - - accounting::ObjectStack* mark_stack_; - - // Every object inside the immune spaces is assumed to be marked. - ImmuneSpaces immune_spaces_; - - // Bump pointer space which we are collecting. - space::BumpPointerSpace* space_; - // Cached mark bitmap as an optimization. - accounting::HeapBitmap* mark_bitmap_; - - // The name of the collector. - std::string collector_name_; - - // The bump pointer in the space where the next forwarding address will be. - uint8_t* bump_pointer_; - // How many live objects we have in the space. - size_t live_objects_in_space_; - - // Bitmap which describes which objects we have to move, need to do / 2 so that we can handle - // objects which are only 8 bytes. - std::unique_ptr<accounting::ContinuousSpaceBitmap> objects_before_forwarding_; - // Bitmap which describes which lock words we need to restore. - std::unique_ptr<accounting::ContinuousSpaceBitmap> objects_with_lockword_; - // Which lock words we need to restore as we are moving objects. - std::deque<LockWord> lock_words_to_restore_; - - // State whether or not we are updating references. - bool updating_references_; - - private: - class MarkObjectVisitor; - class UpdateObjectReferencesVisitor; - class UpdateReferenceVisitor; - class UpdateRootVisitor; - - DISALLOW_IMPLICIT_CONSTRUCTORS(MarkCompact); -}; - -} // namespace collector -} // namespace gc -} // namespace art - -#endif // ART_RUNTIME_GC_COLLECTOR_MARK_COMPACT_H_ diff --git a/runtime/gc/collector/mark_sweep.cc b/runtime/gc/collector/mark_sweep.cc index 9ab965ec78c..23359640fe3 100644 --- a/runtime/gc/collector/mark_sweep.cc +++ b/runtime/gc/collector/mark_sweep.cc @@ -116,21 +116,21 @@ void MarkSweep::InitializePhase() { mark_stack_ = heap_->GetMarkStack(); DCHECK(mark_stack_ != nullptr); immune_spaces_.Reset(); - no_reference_class_count_.StoreRelaxed(0); - normal_count_.StoreRelaxed(0); - class_count_.StoreRelaxed(0); - object_array_count_.StoreRelaxed(0); - other_count_.StoreRelaxed(0); - reference_count_.StoreRelaxed(0); - large_object_test_.StoreRelaxed(0); - large_object_mark_.StoreRelaxed(0); - overhead_time_ .StoreRelaxed(0); - work_chunks_created_.StoreRelaxed(0); - work_chunks_deleted_.StoreRelaxed(0); - mark_null_count_.StoreRelaxed(0); - mark_immune_count_.StoreRelaxed(0); - mark_fastpath_count_.StoreRelaxed(0); - mark_slowpath_count_.StoreRelaxed(0); + no_reference_class_count_.store(0, std::memory_order_relaxed); + normal_count_.store(0, std::memory_order_relaxed); + class_count_.store(0, std::memory_order_relaxed); + object_array_count_.store(0, std::memory_order_relaxed); + other_count_.store(0, std::memory_order_relaxed); + reference_count_.store(0, std::memory_order_relaxed); + large_object_test_.store(0, std::memory_order_relaxed); + large_object_mark_.store(0, std::memory_order_relaxed); + overhead_time_ .store(0, std::memory_order_relaxed); + work_chunks_created_.store(0, std::memory_order_relaxed); + work_chunks_deleted_.store(0, std::memory_order_relaxed); + mark_null_count_.store(0, std::memory_order_relaxed); + mark_immune_count_.store(0, std::memory_order_relaxed); + mark_fastpath_count_.store(0, std::memory_order_relaxed); + mark_slowpath_count_.store(0, std::memory_order_relaxed); { // TODO: I don't think we should need heap bitmap lock to Get the mark bitmap. ReaderMutexLock mu(Thread::Current(), *Locks::heap_bitmap_lock_); @@ -724,7 +724,7 @@ class MarkSweep::MarkStackTask : public Task { if (kUseFinger) { std::atomic_thread_fence(std::memory_order_seq_cst); if (reinterpret_cast<uintptr_t>(ref) >= - static_cast<uintptr_t>(mark_sweep_->atomic_finger_.LoadRelaxed())) { + static_cast<uintptr_t>(mark_sweep_->atomic_finger_.load(std::memory_order_relaxed))) { return; } } @@ -1046,7 +1046,7 @@ void MarkSweep::RecursiveMark() { // This function does not handle heap end increasing, so we must use the space end. uintptr_t begin = reinterpret_cast<uintptr_t>(space->Begin()); uintptr_t end = reinterpret_cast<uintptr_t>(space->End()); - atomic_finger_.StoreRelaxed(AtomicInteger::MaxValue()); + atomic_finger_.store(AtomicInteger::MaxValue(), std::memory_order_relaxed); // Create a few worker tasks. const size_t n = thread_count * 2; @@ -1405,8 +1405,8 @@ void MarkSweep::ProcessMarkStackParallel(size_t thread_count) { thread_pool->Wait(self, true, true); thread_pool->StopWorkers(self); mark_stack_->Reset(); - CHECK_EQ(work_chunks_created_.LoadSequentiallyConsistent(), - work_chunks_deleted_.LoadSequentiallyConsistent()) + CHECK_EQ(work_chunks_created_.load(std::memory_order_seq_cst), + work_chunks_deleted_.load(std::memory_order_seq_cst)) << " some of the work chunks were leaked"; } @@ -1462,28 +1462,32 @@ void MarkSweep::FinishPhase() { if (kCountScannedTypes) { VLOG(gc) << "MarkSweep scanned" - << " no reference objects=" << no_reference_class_count_.LoadRelaxed() - << " normal objects=" << normal_count_.LoadRelaxed() - << " classes=" << class_count_.LoadRelaxed() - << " object arrays=" << object_array_count_.LoadRelaxed() - << " references=" << reference_count_.LoadRelaxed() - << " other=" << other_count_.LoadRelaxed(); + << " no reference objects=" << no_reference_class_count_.load(std::memory_order_relaxed) + << " normal objects=" << normal_count_.load(std::memory_order_relaxed) + << " classes=" << class_count_.load(std::memory_order_relaxed) + << " object arrays=" << object_array_count_.load(std::memory_order_relaxed) + << " references=" << reference_count_.load(std::memory_order_relaxed) + << " other=" << other_count_.load(std::memory_order_relaxed); } if (kCountTasks) { - VLOG(gc) << "Total number of work chunks allocated: " << work_chunks_created_.LoadRelaxed(); + VLOG(gc) + << "Total number of work chunks allocated: " + << work_chunks_created_.load(std::memory_order_relaxed); } if (kMeasureOverhead) { - VLOG(gc) << "Overhead time " << PrettyDuration(overhead_time_.LoadRelaxed()); + VLOG(gc) << "Overhead time " << PrettyDuration(overhead_time_.load(std::memory_order_relaxed)); } if (kProfileLargeObjects) { - VLOG(gc) << "Large objects tested " << large_object_test_.LoadRelaxed() - << " marked " << large_object_mark_.LoadRelaxed(); + VLOG(gc) + << "Large objects tested " << large_object_test_.load(std::memory_order_relaxed) + << " marked " << large_object_mark_.load(std::memory_order_relaxed); } if (kCountMarkedObjects) { - VLOG(gc) << "Marked: null=" << mark_null_count_.LoadRelaxed() - << " immune=" << mark_immune_count_.LoadRelaxed() - << " fastpath=" << mark_fastpath_count_.LoadRelaxed() - << " slowpath=" << mark_slowpath_count_.LoadRelaxed(); + VLOG(gc) + << "Marked: null=" << mark_null_count_.load(std::memory_order_relaxed) + << " immune=" << mark_immune_count_.load(std::memory_order_relaxed) + << " fastpath=" << mark_fastpath_count_.load(std::memory_order_relaxed) + << " slowpath=" << mark_slowpath_count_.load(std::memory_order_relaxed); } CHECK(mark_stack_->IsEmpty()); // Ensure that the mark stack is empty. mark_stack_->Reset(); diff --git a/runtime/gc/collector_type.h b/runtime/gc/collector_type.h index 8979e742c87..4759fca46c5 100644 --- a/runtime/gc/collector_type.h +++ b/runtime/gc/collector_type.h @@ -34,8 +34,6 @@ enum CollectorType { kCollectorTypeSS, // A generational variant of kCollectorTypeSS. kCollectorTypeGSS, - // Mark compact collector. - kCollectorTypeMC, // Heap trimming collector, doesn't do any actual collecting. kCollectorTypeHeapTrim, // A (mostly) concurrent copying collector. diff --git a/runtime/gc/heap-inl.h b/runtime/gc/heap-inl.h index 41ee18350db..948d23303cd 100644 --- a/runtime/gc/heap-inl.h +++ b/runtime/gc/heap-inl.h @@ -156,7 +156,7 @@ inline mirror::Object* Heap::AllocObjectWithAllocator(Thread* self, pre_fence_visitor(obj, usable_size); QuasiAtomic::ThreadFenceForConstructor(); size_t num_bytes_allocated_before = - num_bytes_allocated_.FetchAndAddRelaxed(bytes_tl_bulk_allocated); + num_bytes_allocated_.fetch_add(bytes_tl_bulk_allocated, std::memory_order_relaxed); new_num_bytes_allocated = num_bytes_allocated_before + bytes_tl_bulk_allocated; if (bytes_tl_bulk_allocated > 0) { // Only trace when we get an increase in the number of bytes allocated. This happens when @@ -187,7 +187,7 @@ inline mirror::Object* Heap::AllocObjectWithAllocator(Thread* self, DCHECK(allocation_records_ != nullptr); allocation_records_->RecordAllocation(self, &obj, bytes_allocated); } - AllocationListener* l = alloc_listener_.LoadSequentiallyConsistent(); + AllocationListener* l = alloc_listener_.load(std::memory_order_seq_cst); if (l != nullptr) { // Same as above. We assume that a listener that was once stored will never be deleted. // Otherwise we'd have to perform this under a lock. @@ -393,7 +393,7 @@ inline bool Heap::ShouldAllocLargeObject(ObjPtr<mirror::Class> c, size_t byte_co inline bool Heap::IsOutOfMemoryOnAllocation(AllocatorType allocator_type, size_t alloc_size, bool grow) { - size_t new_footprint = num_bytes_allocated_.LoadSequentiallyConsistent() + alloc_size; + size_t new_footprint = num_bytes_allocated_.load(std::memory_order_seq_cst) + alloc_size; if (UNLIKELY(new_footprint > max_allowed_footprint_)) { if (UNLIKELY(new_footprint > growth_limit_)) { return true; diff --git a/runtime/gc/heap.cc b/runtime/gc/heap.cc index 3011c37f3af..e85824de702 100644 --- a/runtime/gc/heap.cc +++ b/runtime/gc/heap.cc @@ -48,7 +48,6 @@ #include "gc/accounting/remembered_set.h" #include "gc/accounting/space_bitmap-inl.h" #include "gc/collector/concurrent_copying.h" -#include "gc/collector/mark_compact.h" #include "gc/collector/mark_sweep.h" #include "gc/collector/partial_mark_sweep.h" #include "gc/collector/semi_space.h" @@ -143,6 +142,10 @@ static constexpr bool kLogAllGCs = false; static constexpr size_t kPartialTlabSize = 16 * KB; static constexpr bool kUsePartialTlabs = true; +// Use Max heap for 2 seconds, this is smaller than the usual 5s window since we don't want to leave +// allocate with relaxed ergonomics for that long. +static constexpr size_t kPostForkMaxHeapDurationMS = 2000; + #if defined(__LP64__) || !defined(ADDRESS_SANITIZER) // 300 MB (0x12c00000) - (default non-moving space capacity). uint8_t* const Heap::kPreferredAllocSpaceBegin = @@ -258,7 +261,6 @@ Heap::Heap(size_t initial_size, verify_object_mode_(kVerifyObjectModeDisabled), disable_moving_gc_count_(0), semi_space_collector_(nullptr), - mark_compact_collector_(nullptr), concurrent_copying_collector_(nullptr), is_running_on_memory_tool_(Runtime::Current()->IsRunningOnMemoryTool()), use_tlab_(use_tlab), @@ -545,7 +547,7 @@ Heap::Heap(size_t initial_size, AddRememberedSet(non_moving_space_rem_set); } // TODO: Count objects in the image space here? - num_bytes_allocated_.StoreRelaxed(0); + num_bytes_allocated_.store(0, std::memory_order_relaxed); mark_stack_.reset(accounting::ObjectStack::Create("mark stack", kDefaultMarkStackSize, kDefaultMarkStackSize)); const size_t alloc_stack_capacity = max_allocation_stack_size_ + kAllocationStackReserveSize; @@ -599,10 +601,6 @@ Heap::Heap(size_t initial_size, concurrent_copying_collector_->SetRegionSpace(region_space_); garbage_collectors_.push_back(concurrent_copying_collector_); } - if (MayUseCollector(kCollectorTypeMC)) { - mark_compact_collector_ = new collector::MarkCompact(this); - garbage_collectors_.push_back(mark_compact_collector_); - } } if (!GetBootImageSpaces().empty() && non_moving_space_ != nullptr && (is_zygote || separate_non_moving_space || foreground_collector_type_ == kCollectorTypeGSS)) { @@ -1049,7 +1047,8 @@ void Heap::DumpGcPerformanceInfo(std::ostream& os) { } os << "Registered native bytes allocated: " - << old_native_bytes_allocated_.LoadRelaxed() + new_native_bytes_allocated_.LoadRelaxed() + << (old_native_bytes_allocated_.load(std::memory_order_relaxed) + + new_native_bytes_allocated_.load(std::memory_order_relaxed)) << "\n"; BaseMutex::DumpAll(os); @@ -1116,11 +1115,7 @@ void Heap::DumpBlockingGcCountRateHistogram(std::ostream& os) const { ALWAYS_INLINE static inline AllocationListener* GetAndOverwriteAllocationListener( Atomic<AllocationListener*>* storage, AllocationListener* new_value) { - AllocationListener* old; - do { - old = storage->LoadSequentiallyConsistent(); - } while (!storage->CompareAndSetStrongSequentiallyConsistent(old, new_value)); - return old; + return storage->exchange(new_value); } Heap::~Heap() { @@ -1138,12 +1133,11 @@ Heap::~Heap() { delete thread_flip_lock_; delete pending_task_lock_; delete backtrace_lock_; - if (unique_backtrace_count_.LoadRelaxed() != 0 || seen_backtrace_count_.LoadRelaxed() != 0) { - LOG(INFO) << "gc stress unique=" << unique_backtrace_count_.LoadRelaxed() - << " total=" << seen_backtrace_count_.LoadRelaxed() + - unique_backtrace_count_.LoadRelaxed(); + uint64_t unique_count = unique_backtrace_count_.load(std::memory_order_relaxed); + uint64_t seen_count = seen_backtrace_count_.load(std::memory_order_relaxed); + if (unique_count != 0 || seen_count != 0) { + LOG(INFO) << "gc stress unique=" << unique_count << " total=" << (unique_count + seen_count); } - VLOG(heap) << "Finished ~Heap()"; } @@ -1209,7 +1203,8 @@ void Heap::ThrowOutOfMemoryError(Thread* self, size_t byte_count, AllocatorType // If we're in a stack overflow, do not create a new exception. It would require running the // constructor, which will of course still be in a stack overflow. if (self->IsHandlingStackOverflow()) { - self->SetException(Runtime::Current()->GetPreAllocatedOutOfMemoryError()); + self->SetException( + Runtime::Current()->GetPreAllocatedOutOfMemoryErrorWhenHandlingStackOverflow()); return; } @@ -1489,7 +1484,7 @@ void Heap::VerifyObjectBody(ObjPtr<mirror::Object> obj) { } // Ignore early dawn of the universe verifications. - if (UNLIKELY(static_cast<size_t>(num_bytes_allocated_.LoadRelaxed()) < 10 * KB)) { + if (UNLIKELY(num_bytes_allocated_.load(std::memory_order_relaxed) < 10 * KB)) { return; } CHECK_ALIGNED(obj.Ptr(), kObjectAlignment) << "Object isn't aligned"; @@ -1521,9 +1516,10 @@ void Heap::RecordFree(uint64_t freed_objects, int64_t freed_bytes) { // Use signed comparison since freed bytes can be negative when background compaction foreground // transitions occurs. This is caused by the moving objects from a bump pointer space to a // free list backed space typically increasing memory footprint due to padding and binning. - DCHECK_LE(freed_bytes, static_cast<int64_t>(num_bytes_allocated_.LoadRelaxed())); + DCHECK_LE(freed_bytes, + static_cast<int64_t>(num_bytes_allocated_.load(std::memory_order_relaxed))); // Note: This relies on 2s complement for handling negative freed_bytes. - num_bytes_allocated_.FetchAndSubSequentiallyConsistent(static_cast<ssize_t>(freed_bytes)); + num_bytes_allocated_.fetch_sub(static_cast<ssize_t>(freed_bytes)); if (Runtime::Current()->HasStatsEnabled()) { RuntimeStats* thread_stats = Thread::Current()->GetStats(); thread_stats->freed_objects += freed_objects; @@ -1540,10 +1536,10 @@ void Heap::RecordFreeRevoke() { // ahead-of-time, bulk counting of bytes allocated in rosalloc thread-local buffers. // If there's a concurrent revoke, ok to not necessarily reset num_bytes_freed_revoke_ // all the way to zero exactly as the remainder will be subtracted at the next GC. - size_t bytes_freed = num_bytes_freed_revoke_.LoadSequentiallyConsistent(); - CHECK_GE(num_bytes_freed_revoke_.FetchAndSubSequentiallyConsistent(bytes_freed), + size_t bytes_freed = num_bytes_freed_revoke_.load(); + CHECK_GE(num_bytes_freed_revoke_.fetch_sub(bytes_freed), bytes_freed) << "num_bytes_freed_revoke_ underflow"; - CHECK_GE(num_bytes_allocated_.FetchAndSubSequentiallyConsistent(bytes_freed), + CHECK_GE(num_bytes_allocated_.fetch_sub(bytes_freed), bytes_freed) << "num_bytes_allocated_ underflow"; GetCurrentGcIteration()->SetFreedRevoke(bytes_freed); } @@ -1699,13 +1695,13 @@ mirror::Object* Heap::AllocateInternalWithGc(Thread* self, // Always print that we ran homogeneous space compation since this can cause jank. VLOG(heap) << "Ran heap homogeneous space compaction, " << " requested defragmentation " - << count_requested_homogeneous_space_compaction_.LoadSequentiallyConsistent() + << count_requested_homogeneous_space_compaction_.load() << " performed defragmentation " - << count_performed_homogeneous_space_compaction_.LoadSequentiallyConsistent() + << count_performed_homogeneous_space_compaction_.load() << " ignored homogeneous space compaction " - << count_ignored_homogeneous_space_compaction_.LoadSequentiallyConsistent() + << count_ignored_homogeneous_space_compaction_.load() << " delayed count = " - << count_delayed_oom_.LoadSequentiallyConsistent(); + << count_delayed_oom_.load(); } break; } @@ -1968,7 +1964,7 @@ void Heap::TransitionCollector(CollectorType collector_type) { VLOG(heap) << "TransitionCollector: " << static_cast<int>(collector_type_) << " -> " << static_cast<int>(collector_type); uint64_t start_time = NanoTime(); - uint32_t before_allocated = num_bytes_allocated_.LoadSequentiallyConsistent(); + uint32_t before_allocated = num_bytes_allocated_.load(); Runtime* const runtime = Runtime::Current(); Thread* const self = Thread::Current(); ScopedThreadStateChange tsc(self, kWaitingPerformingGc); @@ -2106,7 +2102,7 @@ void Heap::TransitionCollector(CollectorType collector_type) { ScopedObjectAccess soa(self); soa.Vm()->UnloadNativeLibraries(); } - int32_t after_allocated = num_bytes_allocated_.LoadSequentiallyConsistent(); + int32_t after_allocated = num_bytes_allocated_.load(std::memory_order_seq_cst); int32_t delta_allocated = before_allocated - after_allocated; std::string saved_str; if (delta_allocated >= 0) { @@ -2121,10 +2117,6 @@ void Heap::TransitionCollector(CollectorType collector_type) { void Heap::ChangeCollector(CollectorType collector_type) { // TODO: Only do this with all mutators suspended to avoid races. if (collector_type != collector_type_) { - if (collector_type == kCollectorTypeMC) { - // Don't allow mark compact unless support is compiled in. - CHECK(kMarkCompactSupport); - } collector_type_ = collector_type; gc_plan_.clear(); switch (collector_type_) { @@ -2137,7 +2129,6 @@ void Heap::ChangeCollector(CollectorType collector_type) { } break; } - case kCollectorTypeMC: // Fall-through. case kCollectorTypeSS: // Fall-through. case kCollectorTypeGSS: { gc_plan_.push_back(collector::kGcTypeFull); @@ -2486,13 +2477,9 @@ collector::GarbageCollector* Heap::Compact(space::ContinuousMemMapAllocSpace* ta semi_space_collector_->SetToSpace(target_space); semi_space_collector_->Run(gc_cause, false); return semi_space_collector_; - } else { - CHECK(target_space->IsBumpPointerSpace()) - << "In-place compaction is only supported for bump pointer spaces"; - mark_compact_collector_->SetSpace(target_space->AsBumpPointerSpace()); - mark_compact_collector_->Run(kGcCauseCollectorTransition, false); - return mark_compact_collector_; } + LOG(FATAL) << "Unsupported"; + UNREACHABLE(); } void Heap::TraceHeapSize(size_t heap_size) { @@ -2555,7 +2542,9 @@ collector::GcType Heap::CollectGarbageInternal(collector::GcType gc_type, // Move all bytes from new_native_bytes_allocated_ to // old_native_bytes_allocated_ now that GC has been triggered, resetting // new_native_bytes_allocated_ to zero in the process. - old_native_bytes_allocated_.FetchAndAddRelaxed(new_native_bytes_allocated_.ExchangeRelaxed(0)); + old_native_bytes_allocated_.fetch_add( + new_native_bytes_allocated_.exchange(0, std::memory_order_relaxed), + std::memory_order_relaxed); } DCHECK_LT(gc_type, collector::kGcTypeMax); @@ -2580,14 +2569,10 @@ collector::GcType Heap::CollectGarbageInternal(collector::GcType gc_type, case kCollectorTypeCC: collector = concurrent_copying_collector_; break; - case kCollectorTypeMC: - mark_compact_collector_->SetSpace(bump_pointer_space_); - collector = mark_compact_collector_; - break; default: LOG(FATAL) << "Invalid collector type " << static_cast<size_t>(collector_type_); } - if (collector != mark_compact_collector_ && collector != concurrent_copying_collector_) { + if (collector != concurrent_copying_collector_) { temp_space_->GetMemMap()->Protect(PROT_READ | PROT_WRITE); if (kIsDebugBuild) { // Try to read each page of the memory map in case mprotect didn't work properly b/19894268. @@ -2750,12 +2735,10 @@ class ScanVisitor { // Verify a reference from an object. class VerifyReferenceVisitor : public SingleRootVisitor { public: - VerifyReferenceVisitor(Heap* heap, Atomic<size_t>* fail_count, bool verify_referent) + VerifyReferenceVisitor(Thread* self, Heap* heap, size_t* fail_count, bool verify_referent) REQUIRES_SHARED(Locks::mutator_lock_) - : heap_(heap), fail_count_(fail_count), verify_referent_(verify_referent) {} - - size_t GetFailureCount() const { - return fail_count_->LoadSequentiallyConsistent(); + : self_(self), heap_(heap), fail_count_(fail_count), verify_referent_(verify_referent) { + CHECK_EQ(self_, Thread::Current()); } void operator()(ObjPtr<mirror::Class> klass ATTRIBUTE_UNUSED, ObjPtr<mirror::Reference> ref) const @@ -2807,8 +2790,10 @@ class VerifyReferenceVisitor : public SingleRootVisitor { // Verify that the reference is live. return true; } - if (fail_count_->FetchAndAddSequentiallyConsistent(1) == 0) { - // Print message on only on first failure to prevent spam. + CHECK_EQ(self_, Thread::Current()); // fail_count_ is private to the calling thread. + *fail_count_ += 1; + if (*fail_count_ == 1) { + // Only print message for the first failure to prevent spam. LOG(ERROR) << "!!!!!!!!!!!!!!Heap corruption detected!!!!!!!!!!!!!!!!!!!"; } if (obj != nullptr) { @@ -2894,38 +2879,41 @@ class VerifyReferenceVisitor : public SingleRootVisitor { return false; } + Thread* const self_; Heap* const heap_; - Atomic<size_t>* const fail_count_; + size_t* const fail_count_; const bool verify_referent_; }; // Verify all references within an object, for use with HeapBitmap::Visit. class VerifyObjectVisitor { public: - VerifyObjectVisitor(Heap* heap, Atomic<size_t>* fail_count, bool verify_referent) - : heap_(heap), fail_count_(fail_count), verify_referent_(verify_referent) {} + VerifyObjectVisitor(Thread* self, Heap* heap, size_t* fail_count, bool verify_referent) + : self_(self), heap_(heap), fail_count_(fail_count), verify_referent_(verify_referent) {} void operator()(mirror::Object* obj) REQUIRES_SHARED(Locks::mutator_lock_) { // Note: we are verifying the references in obj but not obj itself, this is because obj must // be live or else how did we find it in the live bitmap? - VerifyReferenceVisitor visitor(heap_, fail_count_, verify_referent_); + VerifyReferenceVisitor visitor(self_, heap_, fail_count_, verify_referent_); // The class doesn't count as a reference but we should verify it anyways. obj->VisitReferences(visitor, visitor); } void VerifyRoots() REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(!Locks::heap_bitmap_lock_) { ReaderMutexLock mu(Thread::Current(), *Locks::heap_bitmap_lock_); - VerifyReferenceVisitor visitor(heap_, fail_count_, verify_referent_); + VerifyReferenceVisitor visitor(self_, heap_, fail_count_, verify_referent_); Runtime::Current()->VisitRoots(&visitor); } - size_t GetFailureCount() const { - return fail_count_->LoadSequentiallyConsistent(); + uint32_t GetFailureCount() const REQUIRES(Locks::mutator_lock_) { + CHECK_EQ(self_, Thread::Current()); + return *fail_count_; } private: + Thread* const self_; Heap* const heap_; - Atomic<size_t>* const fail_count_; + size_t* const fail_count_; const bool verify_referent_; }; @@ -2977,8 +2965,8 @@ size_t Heap::VerifyHeapReferences(bool verify_referents) { // Since we sorted the allocation stack content, need to revoke all // thread-local allocation stacks. RevokeAllThreadLocalAllocationStacks(self); - Atomic<size_t> fail_count_(0); - VerifyObjectVisitor visitor(this, &fail_count_, verify_referents); + size_t fail_count = 0; + VerifyObjectVisitor visitor(self, this, &fail_count, verify_referents); // Verify objects in the allocation stack since these will be objects which were: // 1. Allocated prior to the GC (pre GC verification). // 2. Allocated during the GC (pre sweep GC verification). @@ -3539,6 +3527,12 @@ void Heap::ClampGrowthLimit() { } void Heap::ClearGrowthLimit() { + if (max_allowed_footprint_ == growth_limit_ && growth_limit_ < capacity_) { + max_allowed_footprint_ = capacity_; + concurrent_start_bytes_ = + std::max(max_allowed_footprint_, kMinConcurrentRemainingBytes) - + kMinConcurrentRemainingBytes; + } growth_limit_ = capacity_; ScopedObjectAccess soa(Thread::Current()); for (const auto& space : continuous_spaces_) { @@ -3595,7 +3589,7 @@ static bool CanAddHeapTask(Thread* self) REQUIRES(!Locks::runtime_shutdown_lock_ } void Heap::ClearConcurrentGCRequest() { - concurrent_gc_pending_.StoreRelaxed(false); + concurrent_gc_pending_.store(false, std::memory_order_relaxed); } void Heap::RequestConcurrentGC(Thread* self, GcCause cause, bool force_full) { @@ -3718,12 +3712,21 @@ void Heap::RequestTrim(Thread* self) { task_processor_->AddTask(self, added_task); } +void Heap::IncrementNumberOfBytesFreedRevoke(size_t freed_bytes_revoke) { + size_t previous_num_bytes_freed_revoke = + num_bytes_freed_revoke_.fetch_add(freed_bytes_revoke, std::memory_order_seq_cst); + // Check the updated value is less than the number of bytes allocated. There is a risk of + // execution being suspended between the increment above and the CHECK below, leading to + // the use of previous_num_bytes_freed_revoke in the comparison. + CHECK_GE(num_bytes_allocated_.load(std::memory_order_relaxed), + previous_num_bytes_freed_revoke + freed_bytes_revoke); +} + void Heap::RevokeThreadLocalBuffers(Thread* thread) { if (rosalloc_space_ != nullptr) { size_t freed_bytes_revoke = rosalloc_space_->RevokeThreadLocalBuffers(thread); if (freed_bytes_revoke > 0U) { - num_bytes_freed_revoke_.FetchAndAddSequentiallyConsistent(freed_bytes_revoke); - CHECK_GE(num_bytes_allocated_.LoadRelaxed(), num_bytes_freed_revoke_.LoadRelaxed()); + IncrementNumberOfBytesFreedRevoke(freed_bytes_revoke); } } if (bump_pointer_space_ != nullptr) { @@ -3738,8 +3741,7 @@ void Heap::RevokeRosAllocThreadLocalBuffers(Thread* thread) { if (rosalloc_space_ != nullptr) { size_t freed_bytes_revoke = rosalloc_space_->RevokeThreadLocalBuffers(thread); if (freed_bytes_revoke > 0U) { - num_bytes_freed_revoke_.FetchAndAddSequentiallyConsistent(freed_bytes_revoke); - CHECK_GE(num_bytes_allocated_.LoadRelaxed(), num_bytes_freed_revoke_.LoadRelaxed()); + IncrementNumberOfBytesFreedRevoke(freed_bytes_revoke); } } } @@ -3748,8 +3750,7 @@ void Heap::RevokeAllThreadLocalBuffers() { if (rosalloc_space_ != nullptr) { size_t freed_bytes_revoke = rosalloc_space_->RevokeAllThreadLocalBuffers(); if (freed_bytes_revoke > 0U) { - num_bytes_freed_revoke_.FetchAndAddSequentiallyConsistent(freed_bytes_revoke); - CHECK_GE(num_bytes_allocated_.LoadRelaxed(), num_bytes_freed_revoke_.LoadRelaxed()); + IncrementNumberOfBytesFreedRevoke(freed_bytes_revoke); } } if (bump_pointer_space_ != nullptr) { @@ -3761,7 +3762,7 @@ void Heap::RevokeAllThreadLocalBuffers() { } bool Heap::IsGCRequestPending() const { - return concurrent_gc_pending_.LoadRelaxed(); + return concurrent_gc_pending_.load(std::memory_order_relaxed); } void Heap::RunFinalization(JNIEnv* env, uint64_t timeout) { @@ -3771,7 +3772,7 @@ void Heap::RunFinalization(JNIEnv* env, uint64_t timeout) { } void Heap::RegisterNativeAllocation(JNIEnv* env, size_t bytes) { - size_t old_value = new_native_bytes_allocated_.FetchAndAddRelaxed(bytes); + size_t old_value = new_native_bytes_allocated_.fetch_add(bytes, std::memory_order_relaxed); if (old_value > NativeAllocationGcWatermark() * HeapGrowthMultiplier() && !IsGCRequestPending()) { @@ -3793,12 +3794,12 @@ void Heap::RegisterNativeFree(JNIEnv*, size_t bytes) { size_t allocated; size_t new_freed_bytes; do { - allocated = new_native_bytes_allocated_.LoadRelaxed(); + allocated = new_native_bytes_allocated_.load(std::memory_order_relaxed); new_freed_bytes = std::min(allocated, bytes); } while (!new_native_bytes_allocated_.CompareAndSetWeakRelaxed(allocated, allocated - new_freed_bytes)); if (new_freed_bytes < bytes) { - old_native_bytes_allocated_.FetchAndSubRelaxed(bytes - new_freed_bytes); + old_native_bytes_allocated_.fetch_sub(bytes - new_freed_bytes, std::memory_order_relaxed); } } @@ -3932,9 +3933,9 @@ void Heap::CheckGcStressMode(Thread* self, ObjPtr<mirror::Object>* obj) { StackHandleScope<1> hs(self); auto h = hs.NewHandleWrapper(obj); CollectGarbage(/* clear_soft_references */ false); - unique_backtrace_count_.FetchAndAddSequentiallyConsistent(1); + unique_backtrace_count_.fetch_add(1, std::memory_order_seq_cst); } else { - seen_backtrace_count_.FetchAndAddSequentiallyConsistent(1); + seen_backtrace_count_.fetch_add(1, std::memory_order_seq_cst); } } } @@ -4010,11 +4011,11 @@ void Heap::RemoveAllocationListener() { } void Heap::SetGcPauseListener(GcPauseListener* l) { - gc_pause_listener_.StoreRelaxed(l); + gc_pause_listener_.store(l, std::memory_order_relaxed); } void Heap::RemoveGcPauseListener() { - gc_pause_listener_.StoreRelaxed(nullptr); + gc_pause_listener_.store(nullptr, std::memory_order_relaxed); } mirror::Object* Heap::AllocWithNewTLAB(Thread* self, @@ -4110,5 +4111,32 @@ void Heap::VlogHeapGrowth(size_t max_allowed_footprint, size_t new_footprint, si << PrettySize(new_footprint) << " for a " << PrettySize(alloc_size) << " allocation"; } +class Heap::TriggerPostForkCCGcTask : public HeapTask { + public: + explicit TriggerPostForkCCGcTask(uint64_t target_time) : HeapTask(target_time) {} + void Run(Thread* self) OVERRIDE { + gc::Heap* heap = Runtime::Current()->GetHeap(); + // Trigger a GC, if not already done. The first GC after fork, whenever + // takes place, will adjust the thresholds to normal levels. + if (heap->max_allowed_footprint_ == heap->growth_limit_) { + heap->RequestConcurrentGC(self, kGcCauseBackground, false); + } + } +}; + +void Heap::PostForkChildAction(Thread* self) { + // Temporarily increase max_allowed_footprint_ and concurrent_start_bytes_ to + // max values to avoid GC during app launch. + if (collector_type_ == kCollectorTypeCC && !IsLowMemoryMode()) { + // Set max_allowed_footprint_ to the largest allowed value. + SetIdealFootprint(growth_limit_); + // Set concurrent_start_bytes_ to half of the heap size. + concurrent_start_bytes_ = std::max(max_allowed_footprint_ / 2, GetBytesAllocated()); + + GetTaskProcessor()->AddTask( + self, new TriggerPostForkCCGcTask(NanoTime() + MsToNs(kPostForkMaxHeapDurationMS))); + } +} + } // namespace gc } // namespace art diff --git a/runtime/gc/heap.h b/runtime/gc/heap.h index 5ce01bc9d2f..99ebab9357f 100644 --- a/runtime/gc/heap.h +++ b/runtime/gc/heap.h @@ -84,7 +84,6 @@ class RememberedSet; namespace collector { class ConcurrentCopying; class GarbageCollector; -class MarkCompact; class MarkSweep; class SemiSpace; } // namespace collector @@ -496,7 +495,7 @@ class Heap { // Returns the number of bytes currently allocated. size_t GetBytesAllocated() const { - return num_bytes_allocated_.LoadSequentiallyConsistent(); + return num_bytes_allocated_.load(std::memory_order_seq_cst); } // Returns the number of objects currently allocated. @@ -546,7 +545,7 @@ class Heap { // Returns how much free memory we have until we need to grow the heap to perform an allocation. // Similar to GetFreeMemoryUntilGC. Implements java.lang.Runtime.freeMemory. size_t GetFreeMemory() const { - size_t byte_allocated = num_bytes_allocated_.LoadSequentiallyConsistent(); + size_t byte_allocated = num_bytes_allocated_.load(std::memory_order_seq_cst); size_t total_memory = GetTotalMemory(); // Make sure we don't get a negative number. return total_memory - std::min(total_memory, byte_allocated); @@ -775,11 +774,11 @@ class Heap { // Allocation tracking support // Callers to this function use double-checked locking to ensure safety on allocation_records_ bool IsAllocTrackingEnabled() const { - return alloc_tracking_enabled_.LoadRelaxed(); + return alloc_tracking_enabled_.load(std::memory_order_relaxed); } void SetAllocTrackingEnabled(bool enabled) REQUIRES(Locks::alloc_tracker_lock_) { - alloc_tracking_enabled_.StoreRelaxed(enabled); + alloc_tracking_enabled_.store(enabled, std::memory_order_relaxed); } AllocRecordObjectMap* GetAllocationRecords() const @@ -825,7 +824,7 @@ class Heap { void SetGcPauseListener(GcPauseListener* l); // Get the currently installed gc pause listener, or null. GcPauseListener* GetGcPauseListener() { - return gc_pause_listener_.LoadAcquire(); + return gc_pause_listener_.load(std::memory_order_acquire); } // Remove a gc pause listener. Note: the listener must not be deleted, as for performance // reasons, we assume it stays valid when we read it (so that we don't require a lock). @@ -833,10 +832,13 @@ class Heap { const Verification* GetVerification() const; + void PostForkChildAction(Thread* self); + private: class ConcurrentGCTask; class CollectorTransitionTask; class HeapTrimTask; + class TriggerPostForkCCGcTask; // Compact source space to target space. Returns the collector used. collector::GarbageCollector* Compact(space::ContinuousMemMapAllocSpace* target_space, @@ -880,7 +882,6 @@ class Heap { collector_type == kCollectorTypeGSS || collector_type == kCollectorTypeCC || collector_type == kCollectorTypeCCBackground || - collector_type == kCollectorTypeMC || collector_type == kCollectorTypeHomogeneousSpaceCompact; } bool ShouldAllocLargeObject(ObjPtr<mirror::Class> c, size_t byte_count) const @@ -1088,6 +1089,8 @@ class Heap { return max_free_; } + ALWAYS_INLINE void IncrementNumberOfBytesFreedRevoke(size_t freed_bytes_revoke); + void TraceHeapSize(size_t heap_size); // Remove a vlog code from heap-inl.h which is transitively included in half the world. @@ -1349,7 +1352,6 @@ class Heap { std::vector<collector::GarbageCollector*> garbage_collectors_; collector::SemiSpace* semi_space_collector_; - collector::MarkCompact* mark_compact_collector_; collector::ConcurrentCopying* concurrent_copying_collector_; const bool is_running_on_memory_tool_; @@ -1437,7 +1439,6 @@ class Heap { friend class CollectorTransitionTask; friend class collector::GarbageCollector; - friend class collector::MarkCompact; friend class collector::ConcurrentCopying; friend class collector::MarkSweep; friend class collector::SemiSpace; diff --git a/runtime/gc/space/bump_pointer_space-inl.h b/runtime/gc/space/bump_pointer_space-inl.h index 9ebb131ad10..4c585497ec7 100644 --- a/runtime/gc/space/bump_pointer_space-inl.h +++ b/runtime/gc/space/bump_pointer_space-inl.h @@ -46,16 +46,18 @@ inline mirror::Object* BumpPointerSpace::AllocThreadUnsafe(Thread* self, size_t size_t* bytes_tl_bulk_allocated) { Locks::mutator_lock_->AssertExclusiveHeld(self); num_bytes = RoundUp(num_bytes, kAlignment); - uint8_t* end = end_.LoadRelaxed(); + uint8_t* end = end_.load(std::memory_order_relaxed); if (end + num_bytes > growth_end_) { return nullptr; } mirror::Object* obj = reinterpret_cast<mirror::Object*>(end); - end_.StoreRelaxed(end + num_bytes); + end_.store(end + num_bytes, std::memory_order_relaxed); *bytes_allocated = num_bytes; // Use the CAS free versions as an optimization. - objects_allocated_.StoreRelaxed(objects_allocated_.LoadRelaxed() + 1); - bytes_allocated_.StoreRelaxed(bytes_allocated_.LoadRelaxed() + num_bytes); + objects_allocated_.store(objects_allocated_.load(std::memory_order_relaxed) + 1, + std::memory_order_relaxed); + bytes_allocated_.store(bytes_allocated_.load(std::memory_order_relaxed) + num_bytes, + std::memory_order_relaxed); if (UNLIKELY(usable_size != nullptr)) { *usable_size = num_bytes; } @@ -68,7 +70,7 @@ inline mirror::Object* BumpPointerSpace::AllocNonvirtualWithoutAccounting(size_t uint8_t* old_end; uint8_t* new_end; do { - old_end = end_.LoadRelaxed(); + old_end = end_.load(std::memory_order_relaxed); new_end = old_end + num_bytes; // If there is no more room in the region, we are out of memory. if (UNLIKELY(new_end > growth_end_)) { @@ -81,8 +83,8 @@ inline mirror::Object* BumpPointerSpace::AllocNonvirtualWithoutAccounting(size_t inline mirror::Object* BumpPointerSpace::AllocNonvirtual(size_t num_bytes) { mirror::Object* ret = AllocNonvirtualWithoutAccounting(num_bytes); if (ret != nullptr) { - objects_allocated_.FetchAndAddSequentiallyConsistent(1); - bytes_allocated_.FetchAndAddSequentiallyConsistent(num_bytes); + objects_allocated_.fetch_add(1, std::memory_order_seq_cst); + bytes_allocated_.fetch_add(num_bytes, std::memory_order_seq_cst); } return ret; } diff --git a/runtime/gc/space/bump_pointer_space.cc b/runtime/gc/space/bump_pointer_space.cc index ce0e0f36308..e95da01d8c9 100644 --- a/runtime/gc/space/bump_pointer_space.cc +++ b/runtime/gc/space/bump_pointer_space.cc @@ -72,8 +72,8 @@ void BumpPointerSpace::Clear() { // Reset the end of the space back to the beginning, we move the end forward as we allocate // objects. SetEnd(Begin()); - objects_allocated_.StoreRelaxed(0); - bytes_allocated_.StoreRelaxed(0); + objects_allocated_.store(0, std::memory_order_relaxed); + bytes_allocated_.store(0, std::memory_order_relaxed); growth_end_ = Limit(); { MutexLock mu(Thread::Current(), block_lock_); @@ -160,7 +160,7 @@ accounting::ContinuousSpaceBitmap::SweepCallback* BumpPointerSpace::GetSweepCall uint64_t BumpPointerSpace::GetBytesAllocated() { // Start out pre-determined amount (blocks which are not being allocated into). - uint64_t total = static_cast<uint64_t>(bytes_allocated_.LoadRelaxed()); + uint64_t total = static_cast<uint64_t>(bytes_allocated_.load(std::memory_order_relaxed)); Thread* self = Thread::Current(); MutexLock mu(self, *Locks::runtime_shutdown_lock_); MutexLock mu2(self, *Locks::thread_list_lock_); @@ -178,7 +178,7 @@ uint64_t BumpPointerSpace::GetBytesAllocated() { uint64_t BumpPointerSpace::GetObjectsAllocated() { // Start out pre-determined amount (blocks which are not being allocated into). - uint64_t total = static_cast<uint64_t>(objects_allocated_.LoadRelaxed()); + uint64_t total = static_cast<uint64_t>(objects_allocated_.load(std::memory_order_relaxed)); Thread* self = Thread::Current(); MutexLock mu(self, *Locks::runtime_shutdown_lock_); MutexLock mu2(self, *Locks::thread_list_lock_); @@ -195,8 +195,8 @@ uint64_t BumpPointerSpace::GetObjectsAllocated() { } void BumpPointerSpace::RevokeThreadLocalBuffersLocked(Thread* thread) { - objects_allocated_.FetchAndAddSequentiallyConsistent(thread->GetThreadLocalObjectsAllocated()); - bytes_allocated_.FetchAndAddSequentiallyConsistent(thread->GetThreadLocalBytesAllocated()); + objects_allocated_.fetch_add(thread->GetThreadLocalObjectsAllocated(), std::memory_order_seq_cst); + bytes_allocated_.fetch_add(thread->GetThreadLocalBytesAllocated(), std::memory_order_seq_cst); thread->SetTlab(nullptr, nullptr, nullptr); } diff --git a/runtime/gc/space/bump_pointer_space.h b/runtime/gc/space/bump_pointer_space.h index 7b43362c2d3..5ba13ca3ff1 100644 --- a/runtime/gc/space/bump_pointer_space.h +++ b/runtime/gc/space/bump_pointer_space.h @@ -155,8 +155,8 @@ class BumpPointerSpace FINAL : public ContinuousMemMapAllocSpace { // Record objects / bytes freed. void RecordFree(int32_t objects, int32_t bytes) { - objects_allocated_.FetchAndSubSequentiallyConsistent(objects); - bytes_allocated_.FetchAndSubSequentiallyConsistent(bytes); + objects_allocated_.fetch_sub(objects, std::memory_order_seq_cst); + bytes_allocated_.fetch_sub(bytes, std::memory_order_seq_cst); } void LogFragmentationAllocFailure(std::ostream& os, size_t failed_alloc_bytes) OVERRIDE diff --git a/runtime/gc/space/image_space.cc b/runtime/gc/space/image_space.cc index c100bc0c755..e2154b8e4da 100644 --- a/runtime/gc/space/image_space.cc +++ b/runtime/gc/space/image_space.cc @@ -672,7 +672,7 @@ class ImageSpaceLoader { // Loaded the map, use the image header from the file now in case we patch it with // RelocateInPlace. image_header = reinterpret_cast<ImageHeader*>(map->Begin()); - const uint32_t bitmap_index = ImageSpace::bitmap_index_.FetchAndAddSequentiallyConsistent(1); + const uint32_t bitmap_index = ImageSpace::bitmap_index_.fetch_add(1, std::memory_order_seq_cst); std::string bitmap_name(StringPrintf("imagespace %s live-bitmap %u", image_filename, bitmap_index)); diff --git a/runtime/gc/space/region_space-inl.h b/runtime/gc/space/region_space-inl.h index 410931cbe53..c6ec174a138 100644 --- a/runtime/gc/space/region_space-inl.h +++ b/runtime/gc/space/region_space-inl.h @@ -100,13 +100,13 @@ inline mirror::Object* RegionSpace::Region::Alloc(size_t num_bytes, uint8_t* old_top; uint8_t* new_top; do { - old_top = top_.LoadRelaxed(); + old_top = top_.load(std::memory_order_relaxed); new_top = old_top + num_bytes; if (UNLIKELY(new_top > end_)) { return nullptr; } } while (!top_.CompareAndSetWeakRelaxed(old_top, new_top)); - objects_allocated_.FetchAndAddRelaxed(1); + objects_allocated_.fetch_add(1, std::memory_order_relaxed); DCHECK_LE(Top(), end_); DCHECK_LT(old_top, end_); DCHECK_LE(new_top, end_); @@ -258,16 +258,79 @@ inline mirror::Object* RegionSpace::AllocLarge(size_t num_bytes, return nullptr; } } + // Find a large enough set of contiguous free regions. - size_t left = 0; - while (left + num_regs - 1 < num_regions_) { + if (kCyclicRegionAllocation) { + // Try to find a range of free regions within [cyclic_alloc_region_index_, num_regions_). + size_t next_region1 = -1; + mirror::Object* region1 = AllocLargeInRange<kForEvac>(cyclic_alloc_region_index_, + num_regions_, + num_regs, + bytes_allocated, + usable_size, + bytes_tl_bulk_allocated, + &next_region1); + if (region1 != nullptr) { + DCHECK_LT(0u, next_region1); + DCHECK_LE(next_region1, num_regions_); + // Move the cyclic allocation region marker to the region + // following the large region that was just allocated. + cyclic_alloc_region_index_ = next_region1 % num_regions_; + return region1; + } + + // If the previous attempt failed, try to find a range of free regions within + // [0, cyclic_alloc_region_index_ + num_regions_ - 1). + size_t next_region2 = -1; + mirror::Object* region2 = + AllocLargeInRange<kForEvac>(0, + cyclic_alloc_region_index_ + num_regions_ - 1, + num_regs, + bytes_allocated, + usable_size, + bytes_tl_bulk_allocated, + &next_region2); + if (region2 != nullptr) { + DCHECK_LT(0u, next_region2); + DCHECK_LE(next_region2, num_regions_); + // Move the cyclic allocation region marker to the region + // following the large region that was just allocated. + cyclic_alloc_region_index_ = next_region2 % num_regions_; + return region2; + } + } else { + // Try to find a range of free regions within [0, num_regions_). + mirror::Object* region = AllocLargeInRange<kForEvac>(0, + num_regions_, + num_regs, + bytes_allocated, + usable_size, + bytes_tl_bulk_allocated); + if (region != nullptr) { + return region; + } + } + return nullptr; +} + +template<bool kForEvac> +inline mirror::Object* RegionSpace::AllocLargeInRange(size_t begin, + size_t end, + size_t num_regs, + /* out */ size_t* bytes_allocated, + /* out */ size_t* usable_size, + /* out */ size_t* bytes_tl_bulk_allocated, + /* out */ size_t* next_region) { + size_t left = begin; + while (left + num_regs - 1 < end) { bool found = true; size_t right = left; - DCHECK_LT(right, left + num_regs) - << "The inner loop Should iterate at least once"; + DCHECK_LT(right, left + num_regs) << "The inner loop should iterate at least once"; while (right < left + num_regs) { if (regions_[right].IsFree()) { ++right; + // Ensure `right` is not going beyond the past-the-end index of the region space. + DCHECK_LE(right, num_regions_); } else { found = false; break; @@ -303,9 +366,15 @@ inline mirror::Object* RegionSpace::AllocLarge(size_t num_bytes, *usable_size = allocated; } *bytes_tl_bulk_allocated = allocated; - return reinterpret_cast<mirror::Object*>(first_reg->Begin()); + mirror::Object* large_region = reinterpret_cast<mirror::Object*>(first_reg->Begin()); + DCHECK(large_region != nullptr); + if (next_region != nullptr) { + // Return the index to the region next to the allocated large region via `next_region`. + *next_region = right; + } + return large_region; } else { - // right points to the non-free region. Start with the one after it. + // `right` points to the non-free region. Start with the one after it. left = right + 1; } } @@ -365,11 +434,11 @@ inline size_t RegionSpace::Region::BytesAllocated() const { inline size_t RegionSpace::Region::ObjectsAllocated() const { if (IsLarge()) { DCHECK_LT(begin_ + kRegionSize, Top()); - DCHECK_EQ(objects_allocated_.LoadRelaxed(), 0U); + DCHECK_EQ(objects_allocated_.load(std::memory_order_relaxed), 0U); return 1; } else if (IsLargeTail()) { DCHECK_EQ(begin_, Top()); - DCHECK_EQ(objects_allocated_.LoadRelaxed(), 0U); + DCHECK_EQ(objects_allocated_.load(std::memory_order_relaxed), 0U); return 0; } else { DCHECK(IsAllocated()) << "state=" << state_; diff --git a/runtime/gc/space/region_space.cc b/runtime/gc/space/region_space.cc index 8d94c867019..74abe1c2ab7 100644 --- a/runtime/gc/space/region_space.cc +++ b/runtime/gc/space/region_space.cc @@ -29,10 +29,19 @@ namespace space { // value of the region size, evaculate the region. static constexpr uint kEvacuateLivePercentThreshold = 75U; -// If we protect the cleared regions. +// Whether we protect the cleared regions. // Only protect for target builds to prevent flaky test failures (b/63131961). static constexpr bool kProtectClearedRegions = kIsTargetBuild; +// Wether we poison memory areas occupied by dead objects in unevacuated regions. +static constexpr bool kPoisonDeadObjectsInUnevacuatedRegions = kIsDebugBuild; + +// Special 32-bit value used to poison memory areas occupied by dead +// objects in unevacuated regions. Dereferencing this value is expected +// to trigger a memory protection fault, as it is unlikely that it +// points to a valid, non-protected memory area. +static constexpr uint32_t kPoisonDeadObject = 0xBADDB01D; // "BADDROID" + MemMap* RegionSpace::CreateMemMap(const std::string& name, size_t capacity, uint8_t* requested_begin) { CHECK_ALIGNED(capacity, kRegionSize); @@ -92,7 +101,8 @@ RegionSpace::RegionSpace(const std::string& name, MemMap* mem_map) max_peak_num_non_free_regions_(0U), non_free_region_index_limit_(0U), current_region_(&full_region_), - evac_region_(nullptr) { + evac_region_(nullptr), + cyclic_alloc_region_index_(0U) { CHECK_ALIGNED(mem_map->Size(), kRegionSize); CHECK_ALIGNED(mem_map->Begin(), kRegionSize); DCHECK_GT(num_regions_, 0U); @@ -369,6 +379,13 @@ void RegionSpace::ClearFromSpace(/* out */ uint64_t* cleared_bytes, // as they are unevac regions that are live. // Subtract one for the for-loop. i += regions_to_clear_bitmap - 1; + } else { + // Only some allocated bytes are live in this unevac region. + // This should only happen for an allocated non-large region. + DCHECK(r->IsAllocated()) << r->State(); + if (kPoisonDeadObjectsInUnevacuatedRegions) { + PoisonDeadObjectsInUnevacuatedRegion(r); + } } } // Note r != last_checked_region if r->IsInUnevacFromSpace() was true above. @@ -387,6 +404,55 @@ void RegionSpace::ClearFromSpace(/* out */ uint64_t* cleared_bytes, num_evac_regions_ = 0; } +// Poison the memory area in range [`begin`, `end`) with value `kPoisonDeadObject`. +static void PoisonUnevacuatedRange(uint8_t* begin, uint8_t* end) { + static constexpr size_t kPoisonDeadObjectSize = sizeof(kPoisonDeadObject); + static_assert(IsPowerOfTwo(kPoisonDeadObjectSize) && + IsPowerOfTwo(RegionSpace::kAlignment) && + (kPoisonDeadObjectSize < RegionSpace::kAlignment), + "RegionSpace::kAlignment should be a multiple of kPoisonDeadObjectSize" + " and both should be powers of 2"); + DCHECK_ALIGNED(begin, kPoisonDeadObjectSize); + DCHECK_ALIGNED(end, kPoisonDeadObjectSize); + uint32_t* begin_addr = reinterpret_cast<uint32_t*>(begin); + uint32_t* end_addr = reinterpret_cast<uint32_t*>(end); + std::fill(begin_addr, end_addr, kPoisonDeadObject); +} + +void RegionSpace::PoisonDeadObjectsInUnevacuatedRegion(Region* r) { + // The live byte count of `r` should be different from -1, as this + // region should neither be a newly allocated region nor an + // evacuated region. + DCHECK_NE(r->LiveBytes(), static_cast<size_t>(-1)); + + // Past-the-end address of the previously visited (live) object (or + // the beginning of the region, if `maybe_poison` has not run yet). + uint8_t* prev_obj_end = reinterpret_cast<uint8_t*>(r->Begin()); + + // Functor poisoning the space between `obj` and the previously + // visited (live) object (or the beginng of the region), if any. + auto maybe_poison = [this, &prev_obj_end](mirror::Object* obj) REQUIRES(Locks::mutator_lock_) { + DCHECK_ALIGNED(obj, kAlignment); + uint8_t* cur_obj_begin = reinterpret_cast<uint8_t*>(obj); + if (cur_obj_begin != prev_obj_end) { + // There is a gap (dead object(s)) between the previously + // visited (live) object (or the beginning of the region) and + // `obj`; poison that space. + PoisonUnevacuatedRange(prev_obj_end, cur_obj_begin); + } + prev_obj_end = reinterpret_cast<uint8_t*>(GetNextObject(obj)); + }; + + // Visit live objects in `r` and poison gaps (dead objects) between them. + GetLiveBitmap()->VisitMarkedRange(reinterpret_cast<uintptr_t>(r->Begin()), + reinterpret_cast<uintptr_t>(r->Top()), + maybe_poison); + // Poison memory between the last live object and the end of the region, if any. + if (prev_obj_end < r->Top()) { + PoisonUnevacuatedRange(prev_obj_end, r->Top()); + } +} + void RegionSpace::LogFragmentationAllocFailure(std::ostream& os, size_t /* failed_alloc_bytes */) { size_t max_contiguous_allocation = 0; @@ -437,6 +503,7 @@ void RegionSpace::Clear() { r->Clear(/*zero_and_release_pages*/true); } SetNonFreeRegionLimit(0); + DCHECK_EQ(num_non_free_regions_, 0u); current_region_ = &full_region_; evac_region_ = &full_region_; } @@ -450,6 +517,9 @@ void RegionSpace::ClampGrowthLimit(size_t new_capacity) { return; } num_regions_ = new_num_regions; + if (kCyclicRegionAllocation && cyclic_alloc_region_index_ >= num_regions_) { + cyclic_alloc_region_index_ = 0u; + } SetLimit(Begin() + new_capacity); if (Size() > new_capacity) { SetEnd(Limit()); @@ -489,7 +559,7 @@ void RegionSpace::DumpNonFreeRegions(std::ostream& os) { void RegionSpace::RecordAlloc(mirror::Object* ref) { CHECK(ref != nullptr); Region* r = RefToRegion(ref); - r->objects_allocated_.FetchAndAddSequentiallyConsistent(1); + r->objects_allocated_.fetch_add(1, std::memory_order_seq_cst); } bool RegionSpace::AllocNewTlab(Thread* self, size_t min_bytes) { @@ -589,10 +659,10 @@ size_t RegionSpace::AllocationSizeNonvirtual(mirror::Object* obj, size_t* usable } void RegionSpace::Region::Clear(bool zero_and_release_pages) { - top_.StoreRelaxed(begin_); + top_.store(begin_, std::memory_order_relaxed); state_ = RegionState::kRegionStateFree; type_ = RegionType::kRegionTypeNone; - objects_allocated_.StoreRelaxed(0); + objects_allocated_.store(0, std::memory_order_relaxed); alloc_time_ = 0; live_bytes_ = static_cast<size_t>(-1); if (zero_and_release_pages) { @@ -608,7 +678,14 @@ RegionSpace::Region* RegionSpace::AllocateRegion(bool for_evac) { return nullptr; } for (size_t i = 0; i < num_regions_; ++i) { - Region* r = ®ions_[i]; + // When using the cyclic region allocation strategy, try to + // allocate a region starting from the last cyclic allocated + // region marker. Otherwise, try to allocate a region starting + // from the beginning of the region space. + size_t region_index = kCyclicRegionAllocation + ? ((cyclic_alloc_region_index_ + i) % num_regions_) + : i; + Region* r = ®ions_[region_index]; if (r->IsFree()) { r->Unfree(this, time_); if (for_evac) { @@ -618,6 +695,11 @@ RegionSpace::Region* RegionSpace::AllocateRegion(bool for_evac) { r->SetNewlyAllocated(); ++num_non_free_regions_; } + if (kCyclicRegionAllocation) { + // Move the cyclic allocation region marker to the region + // following the one that was just allocated. + cyclic_alloc_region_index_ = (region_index + 1) % num_regions_; + } return r; } } diff --git a/runtime/gc/space/region_space.h b/runtime/gc/space/region_space.h index d63257d928f..ab18b1bcb9c 100644 --- a/runtime/gc/space/region_space.h +++ b/runtime/gc/space/region_space.h @@ -31,6 +31,13 @@ class ReadBarrierTable; namespace space { +// Cyclic region allocation strategy. If `true`, region allocation +// will not try to allocate a new region from the beginning of the +// region space, but from the last allocated region. This allocation +// strategy reduces region reuse and should help catch some GC bugs +// earlier. +static constexpr bool kCyclicRegionAllocation = kIsDebugBuild; + // A space that consists of equal-sized regions. class RegionSpace FINAL : public ContinuousMemMapAllocSpace { public: @@ -300,11 +307,11 @@ class RegionSpace FINAL : public ContinuousMemMapAllocSpace { void Init(size_t idx, uint8_t* begin, uint8_t* end) { idx_ = idx; begin_ = begin; - top_.StoreRelaxed(begin); + top_.store(begin, std::memory_order_relaxed); end_ = end; state_ = RegionState::kRegionStateFree; type_ = RegionType::kRegionTypeNone; - objects_allocated_.StoreRelaxed(0); + objects_allocated_.store(0, std::memory_order_relaxed); alloc_time_ = 0; live_bytes_ = static_cast<size_t>(-1); is_newly_allocated_ = false; @@ -334,7 +341,7 @@ class RegionSpace FINAL : public ContinuousMemMapAllocSpace { if (is_free) { DCHECK(IsInNoSpace()); DCHECK_EQ(begin_, Top()); - DCHECK_EQ(objects_allocated_.LoadRelaxed(), 0U); + DCHECK_EQ(objects_allocated_.load(std::memory_order_relaxed), 0U); } return is_free; } @@ -461,11 +468,11 @@ class RegionSpace FINAL : public ContinuousMemMapAllocSpace { } ALWAYS_INLINE uint8_t* Top() const { - return top_.LoadRelaxed(); + return top_.load(std::memory_order_relaxed); } void SetTop(uint8_t* new_top) { - top_.StoreRelaxed(new_top); + top_.store(new_top, std::memory_order_relaxed); } uint8_t* End() const { @@ -480,10 +487,10 @@ class RegionSpace FINAL : public ContinuousMemMapAllocSpace { void RecordThreadLocalAllocations(size_t num_objects, size_t num_bytes) { DCHECK(IsAllocated()); - DCHECK_EQ(objects_allocated_.LoadRelaxed(), 0U); + DCHECK_EQ(objects_allocated_.load(std::memory_order_relaxed), 0U); DCHECK_EQ(Top(), end_); - objects_allocated_.StoreRelaxed(num_objects); - top_.StoreRelaxed(begin_ + num_bytes); + objects_allocated_.store(num_objects, std::memory_order_relaxed); + top_.store(begin_ + num_bytes, std::memory_order_relaxed); DCHECK_LE(Top(), end_); } @@ -571,6 +578,28 @@ class RegionSpace FINAL : public ContinuousMemMapAllocSpace { Region* AllocateRegion(bool for_evac) REQUIRES(region_lock_); + // Scan region range [`begin`, `end`) in increasing order to try to + // allocate a large region having a size of `num_regs` regions. If + // there is no space in the region space to allocate this large + // region, return null. + // + // If argument `next_region` is not null, use `*next_region` to + // return the index to the region next to the allocated large region + // returned by this method. + template<bool kForEvac> + mirror::Object* AllocLargeInRange(size_t num_regs, + size_t begin, + size_t end, + /* out */ size_t* bytes_allocated, + /* out */ size_t* usable_size, + /* out */ size_t* bytes_tl_bulk_allocated, + /* out */ size_t* next_region = nullptr) REQUIRES(region_lock_); + + // Poison memory areas used by dead objects within unevacuated + // region `r`. This is meant to detect dangling references to dead + // objects earlier in debug mode. + void PoisonDeadObjectsInUnevacuatedRegion(Region* r); + Mutex region_lock_ DEFAULT_MUTEX_ACQUIRED_AFTER; uint32_t time_; // The time as the number of collections since the startup. @@ -600,6 +629,11 @@ class RegionSpace FINAL : public ContinuousMemMapAllocSpace { Region* evac_region_; // The region currently used for evacuation. Region full_region_; // The dummy/sentinel region that looks full. + // Index into the region array pointing to the starting region when + // trying to allocate a new region. Only used when + // `kCyclicRegionAllocation` is true. + size_t cyclic_alloc_region_index_ GUARDED_BY(region_lock_); + // Mark bitmap used by the GC. std::unique_ptr<accounting::ContinuousSpaceBitmap> mark_bitmap_; diff --git a/runtime/gc/space/space.h b/runtime/gc/space/space.h index 7af19fae61a..d888935ff2e 100644 --- a/runtime/gc/space/space.h +++ b/runtime/gc/space/space.h @@ -22,11 +22,11 @@ #include "base/atomic.h" #include "base/macros.h" +#include "base/mem_map.h" #include "base/mutex.h" #include "gc/accounting/space_bitmap.h" #include "gc/collector/object_byte_pair.h" #include "globals.h" -#include "mem_map.h" namespace art { namespace mirror { @@ -272,7 +272,7 @@ class ContinuousSpace : public Space { // Current address at which the space ends, which may vary as the space is filled. uint8_t* End() const { - return end_.LoadRelaxed(); + return end_.load(std::memory_order_relaxed); } // The end of the address range covered by the space. @@ -283,7 +283,7 @@ class ContinuousSpace : public Space { // Change the end of the space. Be careful with use since changing the end of a space to an // invalid value may break the GC. void SetEnd(uint8_t* end) { - end_.StoreRelaxed(end); + end_.store(end, std::memory_order_relaxed); } void SetLimit(uint8_t* limit) { diff --git a/runtime/gc/space/zygote_space.cc b/runtime/gc/space/zygote_space.cc index cde155fb221..8c73ef91161 100644 --- a/runtime/gc/space/zygote_space.cc +++ b/runtime/gc/space/zygote_space.cc @@ -122,7 +122,7 @@ void ZygoteSpace::SweepCallback(size_t num_ptrs, mirror::Object** ptrs, void* ar // Need to mark the card since this will update the mod-union table next GC cycle. card_table->MarkCard(ptrs[i]); } - zygote_space->objects_allocated_.FetchAndSubSequentiallyConsistent(num_ptrs); + zygote_space->objects_allocated_.fetch_sub(num_ptrs, std::memory_order_seq_cst); } } // namespace space diff --git a/runtime/gc/space/zygote_space.h b/runtime/gc/space/zygote_space.h index 08231017e7f..6fe21d99a83 100644 --- a/runtime/gc/space/zygote_space.h +++ b/runtime/gc/space/zygote_space.h @@ -17,9 +17,9 @@ #ifndef ART_RUNTIME_GC_SPACE_ZYGOTE_SPACE_H_ #define ART_RUNTIME_GC_SPACE_ZYGOTE_SPACE_H_ +#include "base/mem_map.h" #include "gc/accounting/space_bitmap.h" #include "malloc_space.h" -#include "mem_map.h" namespace art { namespace gc { @@ -67,7 +67,7 @@ class ZygoteSpace FINAL : public ContinuousMemMapAllocSpace { } uint64_t GetObjectsAllocated() { - return objects_allocated_.LoadSequentiallyConsistent(); + return objects_allocated_.load(std::memory_order_seq_cst); } void Clear() OVERRIDE; diff --git a/runtime/gc/task_processor_test.cc b/runtime/gc/task_processor_test.cc index 77b40e45939..38581ce8075 100644 --- a/runtime/gc/task_processor_test.cc +++ b/runtime/gc/task_processor_test.cc @@ -37,7 +37,7 @@ class RecursiveTask : public HeapTask { if (max_recursion_ > 0) { task_processor_->AddTask(self, new RecursiveTask(task_processor_, counter_, max_recursion_ - 1)); - counter_->FetchAndAddSequentiallyConsistent(1U); + counter_->fetch_add(1U, std::memory_order_seq_cst); } } @@ -54,7 +54,7 @@ class WorkUntilDoneTask : public SelfDeletingTask { } virtual void Run(Thread* self) OVERRIDE { task_processor_->RunAllTasks(self); - done_running_->StoreSequentiallyConsistent(true); + done_running_->store(true, std::memory_order_seq_cst); } private: @@ -76,7 +76,7 @@ TEST_F(TaskProcessorTest, Interrupt) { thread_pool.StartWorkers(self); ASSERT_FALSE(done_running); // Wait until all the tasks are done, but since we didn't interrupt, done_running should be 0. - while (counter.LoadSequentiallyConsistent() != kRecursion) { + while (counter.load(std::memory_order_seq_cst) != kRecursion) { usleep(10); } ASSERT_FALSE(done_running); @@ -84,11 +84,11 @@ TEST_F(TaskProcessorTest, Interrupt) { thread_pool.Wait(self, true, false); // After the interrupt and wait, the WorkUntilInterruptedTasktask should have terminated and // set done_running_ to true. - ASSERT_TRUE(done_running.LoadSequentiallyConsistent()); + ASSERT_TRUE(done_running.load(std::memory_order_seq_cst)); // Test that we finish remaining tasks before returning from RunTasksUntilInterrupted. - counter.StoreSequentiallyConsistent(0); - done_running.StoreSequentiallyConsistent(false); + counter.store(0, std::memory_order_seq_cst); + done_running.store(false, std::memory_order_seq_cst); // Self interrupt before any of the other tasks run, but since we added them we should keep on // working until all the tasks are completed. task_processor.Stop(self); @@ -96,8 +96,8 @@ TEST_F(TaskProcessorTest, Interrupt) { thread_pool.AddTask(self, new WorkUntilDoneTask(&task_processor, &done_running)); thread_pool.StartWorkers(self); thread_pool.Wait(self, true, false); - ASSERT_TRUE(done_running.LoadSequentiallyConsistent()); - ASSERT_EQ(counter.LoadSequentiallyConsistent(), kRecursion); + ASSERT_TRUE(done_running.load(std::memory_order_seq_cst)); + ASSERT_EQ(counter.load(std::memory_order_seq_cst), kRecursion); } class TestOrderTask : public HeapTask { @@ -137,10 +137,10 @@ TEST_F(TaskProcessorTest, Ordering) { Atomic<bool> done_running(false); // Add a task which will wait until interrupted to the thread pool. thread_pool.AddTask(self, new WorkUntilDoneTask(&task_processor, &done_running)); - ASSERT_FALSE(done_running.LoadSequentiallyConsistent()); + ASSERT_FALSE(done_running.load(std::memory_order_seq_cst)); thread_pool.StartWorkers(self); thread_pool.Wait(self, true, false); - ASSERT_TRUE(done_running.LoadSequentiallyConsistent()); + ASSERT_TRUE(done_running.load(std::memory_order_seq_cst)); ASSERT_EQ(counter, kNumTasks); } diff --git a/runtime/image.cc b/runtime/image.cc index 3b13d05d808..316f7a5c63c 100644 --- a/runtime/image.cc +++ b/runtime/image.cc @@ -26,7 +26,7 @@ namespace art { const uint8_t ImageHeader::kImageMagic[] = { 'a', 'r', 't', '\n' }; -const uint8_t ImageHeader::kImageVersion[] = { '0', '5', '6', '\0' }; // ReachabilityFence. +const uint8_t ImageHeader::kImageVersion[] = { '0', '5', '9', '\0' }; // ReachabilityFence. ImageHeader::ImageHeader(uint32_t image_begin, uint32_t image_size, diff --git a/runtime/image.h b/runtime/image.h index 159a308fb30..fe544cc44b6 100644 --- a/runtime/image.h +++ b/runtime/image.h @@ -19,7 +19,6 @@ #include <string.h> -#include "base/bit_utils.h" #include "base/enums.h" #include "globals.h" #include "mirror/object.h" @@ -327,22 +326,6 @@ class PACKED(4) ImageHeader { return boot_image_size_ != 0u; } - uint32_t GetBootImageConstantTablesOffset() const { - // Interned strings table and class table for boot image are mmapped read only. - DCHECK(!IsAppImage()); - const ImageSection& interned_strings = GetInternedStringsSection(); - DCHECK_ALIGNED(interned_strings.Offset(), kPageSize); - return interned_strings.Offset(); - } - - uint32_t GetBootImageConstantTablesSize() const { - uint32_t start_offset = GetBootImageConstantTablesOffset(); - const ImageSection& class_table = GetClassTableSection(); - DCHECK_LE(start_offset, class_table.Offset()); - size_t tables_size = class_table.Offset() + class_table.Size() - start_offset; - return RoundUp(tables_size, kPageSize); - } - // Visit mirror::Objects in the section starting at base. // TODO: Delete base parameter if it is always equal to GetImageBegin. void VisitObjects(ObjectVisitor* visitor, diff --git a/runtime/indirect_reference_table.cc b/runtime/indirect_reference_table.cc index 3b9cc0f9469..6143ba6fd4a 100644 --- a/runtime/indirect_reference_table.cc +++ b/runtime/indirect_reference_table.cc @@ -16,7 +16,7 @@ #include "indirect_reference_table-inl.h" -#include "base/dumpable-inl.h" +#include "base/mutator_locked_dumpable.h" #include "base/systrace.h" #include "base/utils.h" #include "java_vm_ext.h" diff --git a/runtime/intern_table.h b/runtime/intern_table.h index cb976917e6e..c9127d69877 100644 --- a/runtime/intern_table.h +++ b/runtime/intern_table.h @@ -227,7 +227,6 @@ class InternTable { // modifying the zygote intern table. The back of table is modified when strings are interned. std::vector<UnorderedSet> tables_; - friend class linker::OatWriter; // for boot image string table slot address lookup. ART_FRIEND_TEST(InternTableTest, CrossHash); }; @@ -287,7 +286,6 @@ class InternTable { // Weak root state, used for concurrent system weak processing and more. gc::WeakRootState weak_root_state_ GUARDED_BY(Locks::intern_table_lock_); - friend class linker::OatWriter; // for boot image string table slot address lookup. friend class Transaction; ART_FRIEND_TEST(InternTableTest, CrossHash); DISALLOW_COPY_AND_ASSIGN(InternTable); diff --git a/runtime/interpreter/interpreter.cc b/runtime/interpreter/interpreter.cc index 735c0e815ab..f23304c391f 100644 --- a/runtime/interpreter/interpreter.cc +++ b/runtime/interpreter/interpreter.cc @@ -243,11 +243,13 @@ static inline JValue Execute( const CodeItemDataAccessor& accessor, ShadowFrame& shadow_frame, JValue result_register, - bool stay_in_interpreter = false) REQUIRES_SHARED(Locks::mutator_lock_) { + bool stay_in_interpreter = false, + bool from_deoptimize = false) REQUIRES_SHARED(Locks::mutator_lock_) { DCHECK(!shadow_frame.GetMethod()->IsAbstract()); DCHECK(!shadow_frame.GetMethod()->IsNative()); - if (LIKELY(shadow_frame.GetDexPC() == 0)) { // Entering the method, but not via deoptimization. + if (LIKELY(!from_deoptimize)) { // Entering the method, but not via deoptimization. if (kIsDebugBuild) { + CHECK_EQ(shadow_frame.GetDexPC(), 0u); self->AssertNoPendingException(); } instrumentation::Instrumentation* instrumentation = Runtime::Current()->GetInstrumentation(); @@ -568,7 +570,12 @@ void EnterInterpreterFromDeoptimize(Thread* self, } if (new_dex_pc != dex::kDexNoIndex) { shadow_frame->SetDexPC(new_dex_pc); - value = Execute(self, accessor, *shadow_frame, value); + value = Execute(self, + accessor, + *shadow_frame, + value, + /* stay_in_interpreter */ true, + /* from_deoptimize */ true); } ShadowFrame* old_frame = shadow_frame; shadow_frame = shadow_frame->GetLink(); diff --git a/runtime/interpreter/interpreter_common.cc b/runtime/interpreter/interpreter_common.cc index 380a981d094..8a85ee41f82 100644 --- a/runtime/interpreter/interpreter_common.cc +++ b/runtime/interpreter/interpreter_common.cc @@ -900,171 +900,489 @@ bool DoInvokePolymorphic(Thread* self, } } +static JValue ConvertScalarBootstrapArgument(jvalue value) { + // value either contains a primitive scalar value if it corresponds + // to a primitive type, or it contains an integer value if it + // corresponds to an object instance reference id (e.g. a string id). + return JValue::FromPrimitive(value.j); +} + +static ObjPtr<mirror::Class> GetClassForBootstrapArgument(EncodedArrayValueIterator::ValueType type) + REQUIRES_SHARED(Locks::mutator_lock_) { + ClassLinker* class_linker = Runtime::Current()->GetClassLinker(); + switch (type) { + case EncodedArrayValueIterator::ValueType::kBoolean: + case EncodedArrayValueIterator::ValueType::kByte: + case EncodedArrayValueIterator::ValueType::kChar: + case EncodedArrayValueIterator::ValueType::kShort: + // These types are disallowed by JVMS. Treat as integers. This + // will result in CCE's being raised if the BSM has one of these + // types. + case EncodedArrayValueIterator::ValueType::kInt: + return class_linker->FindPrimitiveClass('I'); + case EncodedArrayValueIterator::ValueType::kLong: + return class_linker->FindPrimitiveClass('J'); + case EncodedArrayValueIterator::ValueType::kFloat: + return class_linker->FindPrimitiveClass('F'); + case EncodedArrayValueIterator::ValueType::kDouble: + return class_linker->FindPrimitiveClass('D'); + case EncodedArrayValueIterator::ValueType::kMethodType: + return mirror::MethodType::StaticClass(); + case EncodedArrayValueIterator::ValueType::kMethodHandle: + return mirror::MethodHandle::StaticClass(); + case EncodedArrayValueIterator::ValueType::kString: + return mirror::String::GetJavaLangString(); + case EncodedArrayValueIterator::ValueType::kType: + return mirror::Class::GetJavaLangClass(); + case EncodedArrayValueIterator::ValueType::kField: + case EncodedArrayValueIterator::ValueType::kMethod: + case EncodedArrayValueIterator::ValueType::kEnum: + case EncodedArrayValueIterator::ValueType::kArray: + case EncodedArrayValueIterator::ValueType::kAnnotation: + case EncodedArrayValueIterator::ValueType::kNull: + return nullptr; + } +} + +static bool GetArgumentForBootstrapMethod(Thread* self, + ArtMethod* referrer, + EncodedArrayValueIterator::ValueType type, + const JValue* encoded_value, + JValue* decoded_value) + REQUIRES_SHARED(Locks::mutator_lock_) { + // The encoded_value contains either a scalar value (IJDF) or a + // scalar DEX file index to a reference type to be materialized. + switch (type) { + case EncodedArrayValueIterator::ValueType::kInt: + case EncodedArrayValueIterator::ValueType::kFloat: + decoded_value->SetI(encoded_value->GetI()); + return true; + case EncodedArrayValueIterator::ValueType::kLong: + case EncodedArrayValueIterator::ValueType::kDouble: + decoded_value->SetJ(encoded_value->GetJ()); + return true; + case EncodedArrayValueIterator::ValueType::kMethodType: { + StackHandleScope<2> hs(self); + Handle<mirror::ClassLoader> class_loader(hs.NewHandle(referrer->GetClassLoader())); + Handle<mirror::DexCache> dex_cache(hs.NewHandle(referrer->GetDexCache())); + uint32_t index = static_cast<uint32_t>(encoded_value->GetI()); + ClassLinker* cl = Runtime::Current()->GetClassLinker(); + ObjPtr<mirror::MethodType> o = cl->ResolveMethodType(self, index, dex_cache, class_loader); + if (UNLIKELY(o.IsNull())) { + DCHECK(self->IsExceptionPending()); + return false; + } + decoded_value->SetL(o); + return true; + } + case EncodedArrayValueIterator::ValueType::kMethodHandle: { + uint32_t index = static_cast<uint32_t>(encoded_value->GetI()); + ClassLinker* cl = Runtime::Current()->GetClassLinker(); + ObjPtr<mirror::MethodHandle> o = cl->ResolveMethodHandle(self, index, referrer); + if (UNLIKELY(o.IsNull())) { + DCHECK(self->IsExceptionPending()); + return false; + } + decoded_value->SetL(o); + return true; + } + case EncodedArrayValueIterator::ValueType::kString: { + StackHandleScope<1> hs(self); + Handle<mirror::DexCache> dex_cache(hs.NewHandle(referrer->GetDexCache())); + dex::StringIndex index(static_cast<uint32_t>(encoded_value->GetI())); + ClassLinker* cl = Runtime::Current()->GetClassLinker(); + ObjPtr<mirror::String> o = cl->ResolveString(index, dex_cache); + if (UNLIKELY(o.IsNull())) { + DCHECK(self->IsExceptionPending()); + return false; + } + decoded_value->SetL(o); + return true; + } + case EncodedArrayValueIterator::ValueType::kType: { + StackHandleScope<2> hs(self); + Handle<mirror::ClassLoader> class_loader(hs.NewHandle(referrer->GetClassLoader())); + Handle<mirror::DexCache> dex_cache(hs.NewHandle(referrer->GetDexCache())); + dex::TypeIndex index(static_cast<uint32_t>(encoded_value->GetI())); + ClassLinker* cl = Runtime::Current()->GetClassLinker(); + ObjPtr<mirror::Class> o = cl->ResolveType(index, dex_cache, class_loader); + if (UNLIKELY(o.IsNull())) { + DCHECK(self->IsExceptionPending()); + return false; + } + decoded_value->SetL(o); + return true; + } + case EncodedArrayValueIterator::ValueType::kBoolean: + case EncodedArrayValueIterator::ValueType::kByte: + case EncodedArrayValueIterator::ValueType::kChar: + case EncodedArrayValueIterator::ValueType::kShort: + case EncodedArrayValueIterator::ValueType::kField: + case EncodedArrayValueIterator::ValueType::kMethod: + case EncodedArrayValueIterator::ValueType::kEnum: + case EncodedArrayValueIterator::ValueType::kArray: + case EncodedArrayValueIterator::ValueType::kAnnotation: + case EncodedArrayValueIterator::ValueType::kNull: + // Unreachable - unsupported types that have been checked when + // determining the effect call site type based on the bootstrap + // argument types. + UNREACHABLE(); + } +} + +static bool PackArgumentForBootstrapMethod(Thread* self, + ArtMethod* referrer, + CallSiteArrayValueIterator* it, + ShadowFrameSetter* setter) + REQUIRES_SHARED(Locks::mutator_lock_) { + auto type = it->GetValueType(); + const JValue encoded_value = ConvertScalarBootstrapArgument(it->GetJavaValue()); + JValue decoded_value; + if (!GetArgumentForBootstrapMethod(self, referrer, type, &encoded_value, &decoded_value)) { + return false; + } + switch (it->GetValueType()) { + case EncodedArrayValueIterator::ValueType::kInt: + case EncodedArrayValueIterator::ValueType::kFloat: + setter->Set(static_cast<uint32_t>(decoded_value.GetI())); + return true; + case EncodedArrayValueIterator::ValueType::kLong: + case EncodedArrayValueIterator::ValueType::kDouble: + setter->SetLong(decoded_value.GetJ()); + return true; + case EncodedArrayValueIterator::ValueType::kMethodType: + case EncodedArrayValueIterator::ValueType::kMethodHandle: + case EncodedArrayValueIterator::ValueType::kString: + case EncodedArrayValueIterator::ValueType::kType: + setter->SetReference(decoded_value.GetL()); + return true; + case EncodedArrayValueIterator::ValueType::kBoolean: + case EncodedArrayValueIterator::ValueType::kByte: + case EncodedArrayValueIterator::ValueType::kChar: + case EncodedArrayValueIterator::ValueType::kShort: + case EncodedArrayValueIterator::ValueType::kField: + case EncodedArrayValueIterator::ValueType::kMethod: + case EncodedArrayValueIterator::ValueType::kEnum: + case EncodedArrayValueIterator::ValueType::kArray: + case EncodedArrayValueIterator::ValueType::kAnnotation: + case EncodedArrayValueIterator::ValueType::kNull: + // Unreachable - unsupported types that have been checked when + // determining the effect call site type based on the bootstrap + // argument types. + UNREACHABLE(); + } +} + +static bool PackCollectorArrayForBootstrapMethod(Thread* self, + ArtMethod* referrer, + ObjPtr<mirror::Class> array_type, + int32_t array_length, + CallSiteArrayValueIterator* it, + ShadowFrameSetter* setter) + REQUIRES_SHARED(Locks::mutator_lock_) { + StackHandleScope<1> hs(self); + ClassLinker* class_linker = Runtime::Current()->GetClassLinker(); + JValue decoded_value; + +#define COLLECT_PRIMITIVE_ARRAY(Descriptor, Type) \ + Handle<mirror::Type ## Array> array = \ + hs.NewHandle(mirror::Type ## Array::Alloc(self, array_length)); \ + if (array.IsNull()) { \ + return false; \ + } \ + for (int32_t i = 0; it->HasNext(); it->Next(), ++i) { \ + auto type = it->GetValueType(); \ + DCHECK_EQ(type, EncodedArrayValueIterator::ValueType::k ## Type); \ + const JValue encoded_value = \ + ConvertScalarBootstrapArgument(it->GetJavaValue()); \ + GetArgumentForBootstrapMethod(self, \ + referrer, \ + type, \ + &encoded_value, \ + &decoded_value); \ + array->Set(i, decoded_value.Get ## Descriptor()); \ + } \ + setter->SetReference(array.Get()); \ + return true; + +#define COLLECT_REFERENCE_ARRAY(T, Type) \ + Handle<mirror::ObjectArray<T>> array = \ + hs.NewHandle(mirror::ObjectArray<T>::Alloc(self, \ + array_type, \ + array_length)); \ + if (array.IsNull()) { \ + return false; \ + } \ + for (int32_t i = 0; it->HasNext(); it->Next(), ++i) { \ + auto type = it->GetValueType(); \ + DCHECK_EQ(type, EncodedArrayValueIterator::ValueType::k ## Type); \ + const JValue encoded_value = \ + ConvertScalarBootstrapArgument(it->GetJavaValue()); \ + if (!GetArgumentForBootstrapMethod(self, \ + referrer, \ + type, \ + &encoded_value, \ + &decoded_value)) { \ + return false; \ + } \ + ObjPtr<mirror::Object> o = decoded_value.GetL(); \ + if (Runtime::Current()->IsActiveTransaction()) { \ + array->Set<true>(i, ObjPtr<T>::DownCast(o)); \ + } else { \ + array->Set<false>(i, ObjPtr<T>::DownCast(o)); \ + } \ + } \ + setter->SetReference(array.Get()); \ + return true; + + if (array_type->GetComponentType() == class_linker->FindPrimitiveClass('I')) { + COLLECT_PRIMITIVE_ARRAY(I, Int); + } else if (array_type->GetComponentType() == class_linker->FindPrimitiveClass('J')) { + COLLECT_PRIMITIVE_ARRAY(J, Long); + } else if (array_type->GetComponentType() == class_linker->FindPrimitiveClass('F')) { + COLLECT_PRIMITIVE_ARRAY(F, Float); + } else if (array_type->GetComponentType() == class_linker->FindPrimitiveClass('D')) { + COLLECT_PRIMITIVE_ARRAY(D, Double); + } else if (array_type->GetComponentType() == mirror::MethodType::StaticClass()) { + COLLECT_REFERENCE_ARRAY(mirror::MethodType, MethodType); + } else if (array_type->GetComponentType() == mirror::MethodHandle::StaticClass()) { + COLLECT_REFERENCE_ARRAY(mirror::MethodHandle, MethodHandle); + } else if (array_type->GetComponentType() == mirror::String::GetJavaLangString()) { + COLLECT_REFERENCE_ARRAY(mirror::String, String); + } else if (array_type->GetComponentType() == mirror::Class::GetJavaLangClass()) { + COLLECT_REFERENCE_ARRAY(mirror::Class, Type); + } else { + UNREACHABLE(); + } + #undef COLLECT_PRIMITIVE_ARRAY + #undef COLLECT_REFERENCE_ARRAY +} + +static ObjPtr<mirror::MethodType> BuildCallSiteForBootstrapMethod(Thread* self, + const DexFile* dex_file, + uint32_t call_site_idx) + REQUIRES_SHARED(Locks::mutator_lock_) { + const DexFile::CallSiteIdItem& csi = dex_file->GetCallSiteId(call_site_idx); + CallSiteArrayValueIterator it(*dex_file, csi); + DCHECK_GE(it.Size(), 1u); + + StackHandleScope<2> hs(self); + // Create array for parameter types. + ObjPtr<mirror::Class> class_type = mirror::Class::GetJavaLangClass(); + mirror::Class* class_array_type = + Runtime::Current()->GetClassLinker()->FindArrayClass(self, &class_type); + Handle<mirror::ObjectArray<mirror::Class>> ptypes = hs.NewHandle( + mirror::ObjectArray<mirror::Class>::Alloc(self, + class_array_type, + static_cast<int>(it.Size()))); + if (ptypes.IsNull()) { + DCHECK(self->IsExceptionPending()); + return nullptr; + } + + // Populate the first argument with an instance of j.l.i.MethodHandles.Lookup + // that the runtime will construct. + ptypes->Set(0, mirror::MethodHandlesLookup::StaticClass()); + it.Next(); + + // The remaining parameter types are derived from the types of + // arguments present in the DEX file. + int index = 1; + while (it.HasNext()) { + ObjPtr<mirror::Class> ptype = GetClassForBootstrapArgument(it.GetValueType()); + if (ptype.IsNull()) { + ThrowClassCastException("Unsupported bootstrap argument type"); + return nullptr; + } + ptypes->Set(index, ptype); + index++; + it.Next(); + } + DCHECK_EQ(static_cast<size_t>(index), it.Size()); + + // By definition, the return type is always a j.l.i.CallSite. + Handle<mirror::Class> rtype = hs.NewHandle(mirror::CallSite::StaticClass()); + return mirror::MethodType::Create(self, rtype, ptypes); +} + static ObjPtr<mirror::CallSite> InvokeBootstrapMethod(Thread* self, ShadowFrame& shadow_frame, uint32_t call_site_idx) REQUIRES_SHARED(Locks::mutator_lock_) { + StackHandleScope<7> hs(self); + // There are three mandatory arguments expected from the call site + // value array in the DEX file: the bootstrap method handle, the + // method name to pass to the bootstrap method, and the method type + // to pass to the bootstrap method. + static constexpr size_t kMandatoryArgumentsCount = 3; ArtMethod* referrer = shadow_frame.GetMethod(); const DexFile* dex_file = referrer->GetDexFile(); const DexFile::CallSiteIdItem& csi = dex_file->GetCallSiteId(call_site_idx); + CallSiteArrayValueIterator it(*dex_file, csi); + if (it.Size() < kMandatoryArgumentsCount) { + ThrowBootstrapMethodError("Truncated bootstrap arguments (%zu < %zu)", + it.Size(), kMandatoryArgumentsCount); + return nullptr; + } - StackHandleScope<10> hs(self); - Handle<mirror::ClassLoader> class_loader(hs.NewHandle(referrer->GetClassLoader())); - Handle<mirror::DexCache> dex_cache(hs.NewHandle(referrer->GetDexCache())); + if (it.GetValueType() != EncodedArrayValueIterator::ValueType::kMethodHandle) { + ThrowBootstrapMethodError("First bootstrap argument is not a method handle"); + return nullptr; + } + + uint32_t bsm_index = static_cast<uint32_t>(it.GetJavaValue().i); + it.Next(); - CallSiteArrayValueIterator it(*dex_file, csi); - uint32_t method_handle_idx = static_cast<uint32_t>(it.GetJavaValue().i); ClassLinker* class_linker = Runtime::Current()->GetClassLinker(); - Handle<mirror::MethodHandle> - bootstrap(hs.NewHandle(class_linker->ResolveMethodHandle(self, method_handle_idx, referrer))); - if (bootstrap.IsNull()) { + Handle<mirror::MethodHandle> bsm = + hs.NewHandle(class_linker->ResolveMethodHandle(self, bsm_index, referrer)); + if (bsm.IsNull()) { DCHECK(self->IsExceptionPending()); return nullptr; } - Handle<mirror::MethodType> bootstrap_method_type = hs.NewHandle(bootstrap->GetMethodType()); - it.Next(); - DCHECK_EQ(static_cast<size_t>(bootstrap->GetMethodType()->GetPTypes()->GetLength()), it.Size()); - const size_t num_bootstrap_vregs = bootstrap->GetMethodType()->NumberOfVRegs(); + if (bsm->GetHandleKind() != mirror::MethodHandle::Kind::kInvokeStatic) { + // JLS suggests also accepting constructors. This is currently + // hard as constructor invocations happen via transformers in ART + // today. The constructor would need to be a class derived from java.lang.invoke.CallSite. + ThrowBootstrapMethodError("Unsupported bootstrap method invocation kind"); + return nullptr; + } - // Set-up a shadow frame for invoking the bootstrap method handle. - ShadowFrameAllocaUniquePtr bootstrap_frame = - CREATE_SHADOW_FRAME(num_bootstrap_vregs, nullptr, referrer, shadow_frame.GetDexPC()); - ScopedStackedShadowFramePusher pusher( - self, bootstrap_frame.get(), StackedShadowFrameType::kShadowFrameUnderConstruction); - size_t vreg = 0; + // Construct the local call site type information based on the 3 + // mandatory arguments provided by the runtime and the static arguments + // in the DEX file. We will use these arguments to build a shadow frame. + MutableHandle<mirror::MethodType> call_site_type = + hs.NewHandle(BuildCallSiteForBootstrapMethod(self, dex_file, call_site_idx)); + if (call_site_type.IsNull()) { + DCHECK(self->IsExceptionPending()); + return nullptr; + } - // The first parameter is a MethodHandles lookup instance. - { - Handle<mirror::Class> lookup_class = - hs.NewHandle(shadow_frame.GetMethod()->GetDeclaringClass()); - ObjPtr<mirror::MethodHandlesLookup> lookup = - mirror::MethodHandlesLookup::Create(self, lookup_class); - if (lookup.IsNull()) { + // Check if this BSM is targeting a variable arity method. If so, + // we'll need to collect the trailing arguments into an array. + Handle<mirror::Array> collector_arguments; + int32_t collector_arguments_length; + if (bsm->GetTargetMethod()->IsVarargs()) { + int number_of_bsm_parameters = bsm->GetMethodType()->GetNumberOfPTypes(); + if (number_of_bsm_parameters == 0) { + ThrowBootstrapMethodError("Variable arity BSM does not have any arguments"); + return nullptr; + } + Handle<mirror::Class> collector_array_class = + hs.NewHandle(bsm->GetMethodType()->GetPTypes()->Get(number_of_bsm_parameters - 1)); + if (!collector_array_class->IsArrayClass()) { + ThrowBootstrapMethodError("Variable arity BSM does not have array as final argument"); + return nullptr; + } + // The call site may include no arguments to be collected. In this + // case the number of arguments must be at least the number of BSM + // parameters less the collector array. + if (call_site_type->GetNumberOfPTypes() < number_of_bsm_parameters - 1) { + ThrowWrongMethodTypeException(bsm->GetMethodType(), call_site_type.Get()); + return nullptr; + } + // Check all the arguments to be collected match the collector array component type. + for (int i = number_of_bsm_parameters - 1; i < call_site_type->GetNumberOfPTypes(); ++i) { + if (call_site_type->GetPTypes()->Get(i) != collector_array_class->GetComponentType()) { + ThrowClassCastException(collector_array_class->GetComponentType(), + call_site_type->GetPTypes()->Get(i)); + return nullptr; + } + } + // Update the call site method type so it now includes the collector array. + int32_t collector_arguments_start = number_of_bsm_parameters - 1; + collector_arguments_length = call_site_type->GetNumberOfPTypes() - number_of_bsm_parameters + 1; + call_site_type.Assign( + mirror::MethodType::CollectTrailingArguments(self, + call_site_type.Get(), + collector_array_class.Get(), + collector_arguments_start)); + if (call_site_type.IsNull()) { DCHECK(self->IsExceptionPending()); return nullptr; } - bootstrap_frame->SetVRegReference(vreg++, lookup.Ptr()); + } else { + collector_arguments_length = 0; } - // The second parameter is the name to lookup. - { - dex::StringIndex name_idx(static_cast<uint32_t>(it.GetJavaValue().i)); - ObjPtr<mirror::String> name = class_linker->ResolveString(name_idx, dex_cache); - if (name.IsNull()) { - DCHECK(self->IsExceptionPending()); + if (call_site_type->GetNumberOfPTypes() != bsm->GetMethodType()->GetNumberOfPTypes()) { + ThrowWrongMethodTypeException(bsm->GetMethodType(), call_site_type.Get()); + return nullptr; + } + + // BSM invocation has a different set of exceptions that + // j.l.i.MethodHandle.invoke(). Scan arguments looking for CCE + // "opportunities". Unfortunately we cannot just leave this to the + // method handle invocation as this might generate a WMTE. + for (int32_t i = 0; i < call_site_type->GetNumberOfPTypes(); ++i) { + ObjPtr<mirror::Class> from = call_site_type->GetPTypes()->Get(i); + ObjPtr<mirror::Class> to = bsm->GetMethodType()->GetPTypes()->Get(i); + if (!IsParameterTypeConvertible(from, to)) { + ThrowClassCastException(from, to); return nullptr; } - bootstrap_frame->SetVRegReference(vreg++, name.Ptr()); } - it.Next(); + if (!IsReturnTypeConvertible(call_site_type->GetRType(), bsm->GetMethodType()->GetRType())) { + ThrowClassCastException(bsm->GetMethodType()->GetRType(), call_site_type->GetRType()); + return nullptr; + } + + // Set-up a shadow frame for invoking the bootstrap method handle. + ShadowFrameAllocaUniquePtr bootstrap_frame = + CREATE_SHADOW_FRAME(call_site_type->NumberOfVRegs(), + nullptr, + referrer, + shadow_frame.GetDexPC()); + ScopedStackedShadowFramePusher pusher( + self, bootstrap_frame.get(), StackedShadowFrameType::kShadowFrameUnderConstruction); + ShadowFrameSetter setter(bootstrap_frame.get(), 0u); - // The third parameter is the method type associated with the name. - uint32_t method_type_idx = static_cast<uint32_t>(it.GetJavaValue().i); - Handle<mirror::MethodType> method_type(hs.NewHandle( - class_linker->ResolveMethodType(self, method_type_idx, dex_cache, class_loader))); - if (method_type.IsNull()) { + // The first parameter is a MethodHandles lookup instance. + Handle<mirror::Class> lookup_class = + hs.NewHandle(shadow_frame.GetMethod()->GetDeclaringClass()); + ObjPtr<mirror::MethodHandlesLookup> lookup = + mirror::MethodHandlesLookup::Create(self, lookup_class); + if (lookup.IsNull()) { DCHECK(self->IsExceptionPending()); return nullptr; } - bootstrap_frame->SetVRegReference(vreg++, method_type.Get()); - it.Next(); - - // Append remaining arguments (if any). - while (it.HasNext()) { - const jvalue& jvalue = it.GetJavaValue(); - switch (it.GetValueType()) { - case EncodedArrayValueIterator::ValueType::kBoolean: - case EncodedArrayValueIterator::ValueType::kByte: - case EncodedArrayValueIterator::ValueType::kChar: - case EncodedArrayValueIterator::ValueType::kShort: - case EncodedArrayValueIterator::ValueType::kInt: - bootstrap_frame->SetVReg(vreg, jvalue.i); - vreg += 1; - break; - case EncodedArrayValueIterator::ValueType::kLong: - bootstrap_frame->SetVRegLong(vreg, jvalue.j); - vreg += 2; - break; - case EncodedArrayValueIterator::ValueType::kFloat: - bootstrap_frame->SetVRegFloat(vreg, jvalue.f); - vreg += 1; - break; - case EncodedArrayValueIterator::ValueType::kDouble: - bootstrap_frame->SetVRegDouble(vreg, jvalue.d); - vreg += 2; - break; - case EncodedArrayValueIterator::ValueType::kMethodType: { - uint32_t idx = static_cast<uint32_t>(jvalue.i); - ObjPtr<mirror::MethodType> ref = - class_linker->ResolveMethodType(self, idx, dex_cache, class_loader); - if (ref.IsNull()) { - DCHECK(self->IsExceptionPending()); - return nullptr; - } - bootstrap_frame->SetVRegReference(vreg, ref.Ptr()); - vreg += 1; - break; - } - case EncodedArrayValueIterator::ValueType::kMethodHandle: { - uint32_t idx = static_cast<uint32_t>(jvalue.i); - ObjPtr<mirror::MethodHandle> ref = - class_linker->ResolveMethodHandle(self, idx, referrer); - if (ref.IsNull()) { - DCHECK(self->IsExceptionPending()); - return nullptr; - } - bootstrap_frame->SetVRegReference(vreg, ref.Ptr()); - vreg += 1; - break; - } - case EncodedArrayValueIterator::ValueType::kString: { - dex::StringIndex idx(static_cast<uint32_t>(jvalue.i)); - ObjPtr<mirror::String> ref = class_linker->ResolveString(idx, dex_cache); - if (ref.IsNull()) { - DCHECK(self->IsExceptionPending()); - return nullptr; - } - bootstrap_frame->SetVRegReference(vreg, ref.Ptr()); - vreg += 1; - break; - } - case EncodedArrayValueIterator::ValueType::kType: { - dex::TypeIndex idx(static_cast<uint32_t>(jvalue.i)); - ObjPtr<mirror::Class> ref = class_linker->ResolveType(idx, dex_cache, class_loader); - if (ref.IsNull()) { - DCHECK(self->IsExceptionPending()); - return nullptr; - } - bootstrap_frame->SetVRegReference(vreg, ref.Ptr()); - vreg += 1; - break; + setter.SetReference(lookup); + + // Pack the remaining arguments into the frame. + int number_of_arguments = call_site_type->GetNumberOfPTypes(); + int argument_index; + for (argument_index = 1; argument_index < number_of_arguments; ++argument_index) { + if (argument_index == number_of_arguments - 1 && + call_site_type->GetPTypes()->Get(argument_index)->IsArrayClass()) { + ObjPtr<mirror::Class> array_type = call_site_type->GetPTypes()->Get(argument_index); + if (!PackCollectorArrayForBootstrapMethod(self, + referrer, + array_type, + collector_arguments_length, + &it, + &setter)) { + DCHECK(self->IsExceptionPending()); + return nullptr; } - case EncodedArrayValueIterator::ValueType::kNull: - bootstrap_frame->SetVRegReference(vreg, nullptr); - vreg += 1; - break; - case EncodedArrayValueIterator::ValueType::kField: - case EncodedArrayValueIterator::ValueType::kMethod: - case EncodedArrayValueIterator::ValueType::kEnum: - case EncodedArrayValueIterator::ValueType::kArray: - case EncodedArrayValueIterator::ValueType::kAnnotation: - // Unreachable based on current EncodedArrayValueIterator::Next(). - UNREACHABLE(); + } else if (!PackArgumentForBootstrapMethod(self, referrer, &it, &setter)) { + DCHECK(self->IsExceptionPending()); + return nullptr; } - it.Next(); } + DCHECK(!it.HasNext()); + DCHECK(setter.Done()); // Invoke the bootstrap method handle. JValue result; - RangeInstructionOperands operands(0, vreg); - bool invoke_success = MethodHandleInvokeExact(self, - *bootstrap_frame, - bootstrap, - bootstrap_method_type, - &operands, - &result); + RangeInstructionOperands operands(0, bootstrap_frame->NumberOfVRegs()); + bool invoke_success = MethodHandleInvoke(self, + *bootstrap_frame, + bsm, + call_site_type, + &operands, + &result); if (!invoke_success) { DCHECK(self->IsExceptionPending()); return nullptr; @@ -1077,31 +1395,20 @@ static ObjPtr<mirror::CallSite> InvokeBootstrapMethod(Thread* self, return nullptr; } - // Check the result type is a subclass of CallSite. + // Check the result type is a subclass of j.l.i.CallSite. if (UNLIKELY(!object->InstanceOf(mirror::CallSite::StaticClass()))) { ThrowClassCastException(object->GetClass(), mirror::CallSite::StaticClass()); return nullptr; } + // Check the call site target is not null as we're going to invoke it. Handle<mirror::CallSite> call_site = hs.NewHandle(ObjPtr<mirror::CallSite>::DownCast(ObjPtr<mirror::Object>(result.GetL()))); - // Check the call site target is not null as we're going to invoke it. Handle<mirror::MethodHandle> target = hs.NewHandle(call_site->GetTarget()); if (UNLIKELY(target.IsNull())) { - ThrowClassCastException("Bootstrap method did not return a callsite"); - return nullptr; - } - - // Check the target method type matches the method type requested modulo the receiver - // needs to be compatible rather than exact. - Handle<mirror::MethodType> target_method_type = hs.NewHandle(target->GetMethodType()); - if (UNLIKELY(!target_method_type->IsExactMatch(method_type.Get()) && - !IsParameterTypeConvertible(target_method_type->GetPTypes()->GetWithoutChecks(0), - method_type->GetPTypes()->GetWithoutChecks(0)))) { - ThrowWrongMethodTypeException(target_method_type.Get(), method_type.Get()); + ThrowClassCastException("Bootstrap method returned a CallSite with a null target"); return nullptr; } - return call_site.Get(); } @@ -1129,8 +1436,11 @@ bool DoInvokeCustom(Thread* self, call_site.Assign(InvokeBootstrapMethod(self, shadow_frame, call_site_idx)); if (UNLIKELY(call_site.IsNull())) { CHECK(self->IsExceptionPending()); - ThrowWrappedBootstrapMethodError("Exception from call site #%u bootstrap method", - call_site_idx); + if (!self->GetException()->IsError()) { + // Use a BootstrapMethodError if the exception is not an instance of java.lang.Error. + ThrowWrappedBootstrapMethodError("Exception from call site #%u bootstrap method", + call_site_idx); + } result->SetJ(0); return false; } @@ -1139,9 +1449,6 @@ bool DoInvokeCustom(Thread* self, call_site.Assign(winning_call_site); } - // CallSite.java checks the re-assignment of the call site target - // when mutating call site targets. We only check the target is - // non-null and has the right type during bootstrap method execution. Handle<mirror::MethodHandle> target = hs.NewHandle(call_site->GetTarget()); Handle<mirror::MethodType> target_method_type = hs.NewHandle(target->GetMethodType()); DCHECK_EQ(static_cast<size_t>(inst->VRegA()), target_method_type->NumberOfVRegs()); diff --git a/runtime/interpreter/interpreter_intrinsics.cc b/runtime/interpreter/interpreter_intrinsics.cc index f0cf2af7c75..69dae31b37c 100644 --- a/runtime/interpreter/interpreter_intrinsics.cc +++ b/runtime/interpreter/interpreter_intrinsics.cc @@ -91,7 +91,7 @@ BINARY_II_INTRINSIC(MterpIntegerRotateLeft, (Rot<int32_t, true>), SetI); // java.lang.Integer.signum(I)I UNARY_INTRINSIC(MterpIntegerSignum, Signum, GetVReg, SetI); -// java.lang.Long.reverse(I)I +// java.lang.Long.reverse(J)J UNARY_INTRINSIC(MterpLongReverse, ReverseBits64, GetVRegLong, SetJ); // java.lang.Long.reverseBytes(J)J diff --git a/runtime/interpreter/unstarted_runtime.cc b/runtime/interpreter/unstarted_runtime.cc index 6e920363e8e..791ebf09b7d 100644 --- a/runtime/interpreter/unstarted_runtime.cc +++ b/runtime/interpreter/unstarted_runtime.cc @@ -34,6 +34,7 @@ #include "base/enums.h" #include "base/macros.h" #include "base/quasi_atomic.h" +#include "base/zip_archive.h" #include "class_linker.h" #include "common_throws.h" #include "dex/descriptors_names.h" @@ -56,7 +57,6 @@ #include "thread-inl.h" #include "transaction.h" #include "well_known_classes.h" -#include "zip_archive.h" namespace art { namespace interpreter { @@ -1540,7 +1540,7 @@ void UnstartedRuntime::UnstartedUnsafePutOrderedObject( } int64_t offset = shadow_frame->GetVRegLong(arg_offset + 2); mirror::Object* newValue = shadow_frame->GetVRegReference(arg_offset + 4); - QuasiAtomic::ThreadFenceRelease(); + std::atomic_thread_fence(std::memory_order_release); if (Runtime::Current()->IsActiveTransaction()) { obj->SetFieldObject<true>(MemberOffset(offset), newValue); } else { diff --git a/runtime/java_vm_ext.cc b/runtime/java_vm_ext.cc index da4c4b2fa48..8fe68bd318e 100644 --- a/runtime/java_vm_ext.cc +++ b/runtime/java_vm_ext.cc @@ -736,14 +736,14 @@ void JavaVMExt::DisallowNewWeakGlobals() { // mutator lock exclusively held so that we don't have any threads in the middle of // DecodeWeakGlobal. Locks::mutator_lock_->AssertExclusiveHeld(self); - allow_accessing_weak_globals_.StoreSequentiallyConsistent(false); + allow_accessing_weak_globals_.store(false, std::memory_order_seq_cst); } void JavaVMExt::AllowNewWeakGlobals() { CHECK(!kUseReadBarrier); Thread* self = Thread::Current(); MutexLock mu(self, *Locks::jni_weak_globals_lock_); - allow_accessing_weak_globals_.StoreSequentiallyConsistent(true); + allow_accessing_weak_globals_.store(true, std::memory_order_seq_cst); weak_globals_add_condition_.Broadcast(self); } @@ -770,7 +770,7 @@ inline bool JavaVMExt::MayAccessWeakGlobalsUnlocked(Thread* self) const { DCHECK(self != nullptr); return kUseReadBarrier ? self->GetWeakRefAccessEnabled() : - allow_accessing_weak_globals_.LoadSequentiallyConsistent(); + allow_accessing_weak_globals_.load(std::memory_order_seq_cst); } ObjPtr<mirror::Object> JavaVMExt::DecodeWeakGlobal(Thread* self, IndirectRef ref) { @@ -809,7 +809,7 @@ ObjPtr<mirror::Object> JavaVMExt::DecodeWeakGlobalDuringShutdown(Thread* self, I } // self can be null during a runtime shutdown. ~Runtime()->~ClassLinker()->DecodeWeakGlobal(). if (!kUseReadBarrier) { - DCHECK(allow_accessing_weak_globals_.LoadSequentiallyConsistent()); + DCHECK(allow_accessing_weak_globals_.load(std::memory_order_seq_cst)); } return weak_globals_.SynchronizedGet(ref); } diff --git a/runtime/jdwp/jdwp_handler.cc b/runtime/jdwp/jdwp_handler.cc index 291a983e75c..1e61ba0f2d5 100644 --- a/runtime/jdwp/jdwp_handler.cc +++ b/runtime/jdwp/jdwp_handler.cc @@ -1625,7 +1625,7 @@ size_t JdwpState::ProcessRequest(Request* request, ExpandBuf* pReply, bool* skip * so waitForDebugger() doesn't return if we stall for a bit here. */ Dbg::GoActive(); - last_activity_time_ms_.StoreSequentiallyConsistent(0); + last_activity_time_ms_.store(0, std::memory_order_seq_cst); } /* @@ -1703,7 +1703,7 @@ size_t JdwpState::ProcessRequest(Request* request, ExpandBuf* pReply, bool* skip * the initial setup. Only update if this is a non-DDMS packet. */ if (request->GetCommandSet() != kJDWPDdmCmdSet) { - last_activity_time_ms_.StoreSequentiallyConsistent(MilliTime()); + last_activity_time_ms_.store(MilliTime(), std::memory_order_seq_cst); } return replyLength; diff --git a/runtime/jdwp/jdwp_main.cc b/runtime/jdwp/jdwp_main.cc index 557b032154b..447e3bf45ba 100644 --- a/runtime/jdwp/jdwp_main.cc +++ b/runtime/jdwp/jdwp_main.cc @@ -729,7 +729,7 @@ int64_t JdwpState::LastDebuggerActivity() { return -1; } - int64_t last = last_activity_time_ms_.LoadSequentiallyConsistent(); + int64_t last = last_activity_time_ms_.load(std::memory_order_seq_cst); /* initializing or in the middle of something? */ if (last == 0) { diff --git a/runtime/jit/jit.cc b/runtime/jit/jit.cc index a757960e3e6..813430f0bb7 100644 --- a/runtime/jit/jit.cc +++ b/runtime/jit/jit.cc @@ -717,12 +717,15 @@ void Jit::AddSamples(Thread* self, ArtMethod* method, uint16_t count, bool with_ void Jit::MethodEntered(Thread* thread, ArtMethod* method) { Runtime* runtime = Runtime::Current(); if (UNLIKELY(runtime->UseJitCompilation() && runtime->GetJit()->JitAtFirstUse())) { - // The compiler requires a ProfilingInfo object. - ProfilingInfo::Create(thread, - method->GetInterfaceMethodIfProxy(kRuntimePointerSize), - /* retry_allocation */ true); - JitCompileTask compile_task(method, JitCompileTask::kCompile); - compile_task.Run(thread); + ArtMethod* np_method = method->GetInterfaceMethodIfProxy(kRuntimePointerSize); + if (np_method->IsCompilable()) { + if (!np_method->IsNative()) { + // The compiler requires a ProfilingInfo object for non-native methods. + ProfilingInfo::Create(thread, np_method, /* retry_allocation */ true); + } + JitCompileTask compile_task(method, JitCompileTask::kCompile); + compile_task.Run(thread); + } return; } diff --git a/runtime/jit/jit_code_cache.cc b/runtime/jit/jit_code_cache.cc index f0f0df4f5e7..6dcc87179bb 100644 --- a/runtime/jit/jit_code_cache.cc +++ b/runtime/jit/jit_code_cache.cc @@ -22,6 +22,7 @@ #include "art_method-inl.h" #include "base/enums.h" #include "base/logging.h" // For VLOG. +#include "base/mem_map.h" #include "base/quasi_atomic.h" #include "base/stl_util.h" #include "base/systrace.h" @@ -37,7 +38,6 @@ #include "jit/jit.h" #include "jit/profiling_info.h" #include "linear_alloc.h" -#include "mem_map.h" #include "oat_file-inl.h" #include "oat_quick_method_header.h" #include "object_callbacks.h" @@ -248,7 +248,6 @@ JitCodeCache::JitCodeCache(MemMap* code_map, code_end_(initial_code_capacity), data_end_(initial_data_capacity), last_collection_increased_code_cache_(false), - last_update_time_ns_(0), garbage_collect_code_(garbage_collect_code), used_memory_for_data_(0), used_memory_for_code_(0), @@ -328,33 +327,20 @@ const void* JitCodeCache::GetJniStubCode(ArtMethod* method) { class ScopedCodeCacheWrite : ScopedTrace { public: - explicit ScopedCodeCacheWrite(MemMap* code_map, bool only_for_tlb_shootdown = false) + explicit ScopedCodeCacheWrite(MemMap* code_map) : ScopedTrace("ScopedCodeCacheWrite"), - code_map_(code_map), - only_for_tlb_shootdown_(only_for_tlb_shootdown) { + code_map_(code_map) { ScopedTrace trace("mprotect all"); - CheckedCall(mprotect, - "make code writable", - code_map_->Begin(), - only_for_tlb_shootdown_ ? kPageSize : code_map_->Size(), - kProtAll); + CheckedCall(mprotect, "make code writable", code_map_->Begin(), code_map_->Size(), kProtAll); } ~ScopedCodeCacheWrite() { ScopedTrace trace("mprotect code"); - CheckedCall(mprotect, - "make code protected", - code_map_->Begin(), - only_for_tlb_shootdown_ ? kPageSize : code_map_->Size(), - kProtCode); + CheckedCall(mprotect, "make code protected", code_map_->Begin(), code_map_->Size(), kProtCode); } private: MemMap* const code_map_; - // If we're using ScopedCacheWrite only for TLB shootdown, we limit the scope of mprotect to - // one page. - const bool only_for_tlb_shootdown_; - DISALLOW_COPY_AND_ASSIGN(ScopedCodeCacheWrite); }; @@ -636,7 +622,7 @@ void JitCodeCache::RemoveMethodsIn(Thread* self, const LinearAlloc& alloc) { bool JitCodeCache::IsWeakAccessEnabled(Thread* self) const { return kUseReadBarrier ? self->GetWeakRefAccessEnabled() - : is_weak_access_enabled_.LoadSequentiallyConsistent(); + : is_weak_access_enabled_.load(std::memory_order_seq_cst); } void JitCodeCache::WaitUntilInlineCacheAccessible(Thread* self) { @@ -658,13 +644,13 @@ void JitCodeCache::BroadcastForInlineCacheAccess() { void JitCodeCache::AllowInlineCacheAccess() { DCHECK(!kUseReadBarrier); - is_weak_access_enabled_.StoreSequentiallyConsistent(true); + is_weak_access_enabled_.store(true, std::memory_order_seq_cst); BroadcastForInlineCacheAccess(); } void JitCodeCache::DisallowInlineCacheAccess() { DCHECK(!kUseReadBarrier); - is_weak_access_enabled_.StoreSequentiallyConsistent(false); + is_weak_access_enabled_.store(false, std::memory_order_seq_cst); } void JitCodeCache::CopyInlineCacheInto(const InlineCache& ic, @@ -816,8 +802,6 @@ uint8_t* JitCodeCache::CommitCodeInternal(Thread* self, FillRootTable(roots_data, roots); { // Flush data cache, as compiled code references literals in it. - // We also need a TLB shootdown to act as memory barrier across cores. - ScopedCodeCacheWrite ccw(code_map_.get(), /* only_for_tlb_shootdown */ true); FlushDataCache(reinterpret_cast<char*>(roots_data), reinterpret_cast<char*>(roots_data + data_size)); } @@ -835,7 +819,6 @@ uint8_t* JitCodeCache::CommitCodeInternal(Thread* self, // code. GetLiveBitmap()->AtomicTestAndSet(FromCodeToAllocation(code_ptr)); } - last_update_time_ns_.StoreRelease(NanoTime()); VLOG(jit) << "JIT added (osr=" << std::boolalpha << osr << std::noboolalpha << ") " << ArtMethod::PrettyMethod(method) << "@" << method @@ -1556,7 +1539,7 @@ ProfilingInfo* JitCodeCache::AddProfilingInfoInternal(Thread* self ATTRIBUTE_UNU // Make sure other threads see the data in the profiling info object before the // store in the ArtMethod's ProfilingInfo pointer. - QuasiAtomic::ThreadFenceRelease(); + std::atomic_thread_fence(std::memory_order_release); method->SetProfilingInfo(info); profiling_infos_.push_back(info); @@ -1661,10 +1644,6 @@ void JitCodeCache::GetProfiledMethods(const std::set<std::string>& dex_base_loca } } -uint64_t JitCodeCache::GetLastUpdateTimeNs() const { - return last_update_time_ns_.LoadAcquire(); -} - bool JitCodeCache::IsOsrCompiled(ArtMethod* method) { MutexLock mu(Thread::Current(), lock_); return osr_code_map_.find(method) != osr_code_map_.end(); diff --git a/runtime/jit/jit_code_cache.h b/runtime/jit/jit_code_cache.h index dfa7ac0970d..f1c99fb85af 100644 --- a/runtime/jit/jit_code_cache.h +++ b/runtime/jit/jit_code_cache.h @@ -407,10 +407,6 @@ class JitCodeCache { // Whether the last collection round increased the code cache. bool last_collection_increased_code_cache_ GUARDED_BY(lock_); - // Last time the the code_cache was updated. - // It is atomic to avoid locking when reading it. - Atomic<uint64_t> last_update_time_ns_; - // Whether we can do garbage collection. Not 'const' as tests may override this. bool garbage_collect_code_; diff --git a/runtime/jit/profile_compilation_info.cc b/runtime/jit/profile_compilation_info.cc index 1bbce4f414f..f5c2715cda6 100644 --- a/runtime/jit/profile_compilation_info.cc +++ b/runtime/jit/profile_compilation_info.cc @@ -34,9 +34,8 @@ #include "base/arena_allocator.h" #include "base/dumpable.h" -#include "base/file_utils.h" #include "base/logging.h" // For VLOG. -#include "base/mutex.h" +#include "base/malloc_arena_pool.h" #include "base/os.h" #include "base/safe_map.h" #include "base/scoped_flock.h" @@ -45,9 +44,9 @@ #include "base/time_utils.h" #include "base/unix_file/fd_file.h" #include "base/utils.h" +#include "base/zip_archive.h" #include "dex/dex_file_loader.h" #include "jit/profiling_info.h" -#include "zip_archive.h" namespace art { @@ -90,7 +89,7 @@ ProfileCompilationInfo::ProfileCompilationInfo(ArenaPool* custom_arena_pool) } ProfileCompilationInfo::ProfileCompilationInfo() - : default_arena_pool_(/*use_malloc*/true, /*low_4gb*/false, "ProfileCompilationInfo"), + : default_arena_pool_(), allocator_(&default_arena_pool_), info_(allocator_.Adapter(kArenaAllocProfile)), profile_key_map_(std::less<const std::string>(), allocator_.Adapter(kArenaAllocProfile)) { diff --git a/runtime/jit/profile_compilation_info.h b/runtime/jit/profile_compilation_info.h index 5c4b9e7202b..f4c8c723b3b 100644 --- a/runtime/jit/profile_compilation_info.h +++ b/runtime/jit/profile_compilation_info.h @@ -23,14 +23,15 @@ #include "base/arena_containers.h" #include "base/arena_object.h" #include "base/atomic.h" +#include "base/bit_memory_region.h" +#include "base/malloc_arena_pool.h" +#include "base/mem_map.h" #include "base/safe_map.h" -#include "bit_memory_region.h" #include "dex/dex_cache_resolved_classes.h" #include "dex/dex_file.h" #include "dex/dex_file_types.h" #include "dex/method_reference.h" #include "dex/type_reference.h" -#include "mem_map.h" namespace art { @@ -792,7 +793,7 @@ class ProfileCompilationInfo { friend class ProfileAssistantTest; friend class Dex2oatLayoutTest; - ArenaPool default_arena_pool_; + MallocArenaPool default_arena_pool_; ArenaAllocator allocator_; // Vector containing the actual profile info. diff --git a/runtime/jni_internal_test.cc b/runtime/jni_internal_test.cc index 293e18a5b55..5d74181cef4 100644 --- a/runtime/jni_internal_test.cc +++ b/runtime/jni_internal_test.cc @@ -2517,4 +2517,32 @@ TEST_F(JniInternalTest, JNIEnvExtTableOverride) { env_->DeleteGlobalRef(global2); } +TEST_F(JniInternalTest, NonAttachedThread) { + // This tests leads to warnings and errors in the log. + ScopedLogSeverity sls(LogSeverity::FATAL); + CheckJniAbortCatcher check_jni_abort_catcher; + + auto callee = [](void* env_ptr) -> void* { + JNIEnv* env = reinterpret_cast<JNIEnv*>(env_ptr); + env->NewStringUTF("test"); + return nullptr; + }; + + bool old_check_jni = vm_->SetCheckJniEnabled(false); + vm_->SetCheckJniEnabled(true); + { + pthread_t pthread; + int pthread_create_result = pthread_create(&pthread, + /* pthread_attr */ nullptr, + callee, + reinterpret_cast<void*>(env_)); + CHECK_EQ(pthread_create_result, 0); + int pthread_join_result = pthread_join(pthread, /* thread_return */ nullptr); + CHECK_EQ(pthread_join_result, 0); + } + vm_->SetCheckJniEnabled(old_check_jni); + + check_jni_abort_catcher.Check("is making JNI calls without being attached"); +} + } // namespace art diff --git a/runtime/linear_alloc.h b/runtime/linear_alloc.h index 384b2e32434..87086f29d44 100644 --- a/runtime/linear_alloc.h +++ b/runtime/linear_alloc.h @@ -18,6 +18,7 @@ #define ART_RUNTIME_LINEAR_ALLOC_H_ #include "base/arena_allocator.h" +#include "base/mutex.h" namespace art { diff --git a/runtime/lock_word.h b/runtime/lock_word.h index e89beb6d41b..09d856f89b3 100644 --- a/runtime/lock_word.h +++ b/runtime/lock_word.h @@ -32,9 +32,9 @@ class Object; class Monitor; -/* The lock value itself as stored in mirror::Object::monitor_. The two most significant bits of - * the state. The four possible states are fat locked, thin/unlocked, hash code, and forwarding - * address. +/* The lock value itself as stored in mirror::Object::monitor_. The two most significant bits + * encode the state. The four possible states are fat locked, thin/unlocked, hash code, and + * forwarding address. * * When the lock word is in the "thin" state and its bits are formatted as follows: * diff --git a/runtime/method_handles.h b/runtime/method_handles.h index 7e60a5c170d..fce3d066393 100644 --- a/runtime/method_handles.h +++ b/runtime/method_handles.h @@ -174,19 +174,26 @@ class ShadowFrameSetter { : shadow_frame_(shadow_frame), arg_index_(first_dst_reg) {} ALWAYS_INLINE void Set(uint32_t value) REQUIRES_SHARED(Locks::mutator_lock_) { + DCHECK_LT(arg_index_, shadow_frame_->NumberOfVRegs()); shadow_frame_->SetVReg(arg_index_++, value); } ALWAYS_INLINE void SetReference(ObjPtr<mirror::Object> value) REQUIRES_SHARED(Locks::mutator_lock_) { + DCHECK_LT(arg_index_, shadow_frame_->NumberOfVRegs()); shadow_frame_->SetVRegReference(arg_index_++, value.Ptr()); } ALWAYS_INLINE void SetLong(int64_t value) REQUIRES_SHARED(Locks::mutator_lock_) { + DCHECK_LT(arg_index_, shadow_frame_->NumberOfVRegs()); shadow_frame_->SetVRegLong(arg_index_, value); arg_index_ += 2; } + ALWAYS_INLINE bool Done() const { + return arg_index_ == shadow_frame_->NumberOfVRegs(); + } + private: ShadowFrame* shadow_frame_; size_t arg_index_; diff --git a/runtime/method_info.h b/runtime/method_info.h index fe062564f14..b00ddc660fc 100644 --- a/runtime/method_info.h +++ b/runtime/method_info.h @@ -21,7 +21,7 @@ #include "base/leb128.h" #include "base/macros.h" -#include "memory_region.h" +#include "base/memory_region.h" namespace art { diff --git a/runtime/mirror/dex_cache-inl.h b/runtime/mirror/dex_cache-inl.h index 3ffedcac4bb..7a4876c4128 100644 --- a/runtime/mirror/dex_cache-inl.h +++ b/runtime/mirror/dex_cache-inl.h @@ -154,7 +154,7 @@ inline CallSite* DexCache::GetResolvedCallSite(uint32_t call_site_idx) { GcRoot<mirror::CallSite>& target = GetResolvedCallSites()[call_site_idx]; Atomic<GcRoot<mirror::CallSite>>& ref = reinterpret_cast<Atomic<GcRoot<mirror::CallSite>>&>(target); - return ref.LoadSequentiallyConsistent().Read(); + return ref.load(std::memory_order_seq_cst).Read(); } inline CallSite* DexCache::SetResolvedCallSite(uint32_t call_site_idx, CallSite* call_site) { diff --git a/runtime/mirror/method_type.cc b/runtime/mirror/method_type.cc index 6ac5012ca32..45f7a87951f 100644 --- a/runtime/mirror/method_type.cc +++ b/runtime/mirror/method_type.cc @@ -23,6 +23,18 @@ namespace art { namespace mirror { +namespace { + +ObjPtr<ObjectArray<Class>> AllocatePTypesArray(Thread* self, int count) + REQUIRES_SHARED(Locks::mutator_lock_) { + ObjPtr<Class> class_type = Class::GetJavaLangClass(); + ObjPtr<Class> class_array_type = + Runtime::Current()->GetClassLinker()->FindArrayClass(self, &class_type); + return ObjectArray<Class>::Alloc(self, class_array_type, count); +} + +} // namespace + GcRoot<Class> MethodType::static_class_; MethodType* MethodType::Create(Thread* const self, @@ -47,18 +59,41 @@ MethodType* MethodType::Create(Thread* const self, MethodType* MethodType::CloneWithoutLeadingParameter(Thread* const self, ObjPtr<MethodType> method_type) { StackHandleScope<3> hs(self); - Handle<Class> rtype = hs.NewHandle(method_type->GetRType()); Handle<ObjectArray<Class>> src_ptypes = hs.NewHandle(method_type->GetPTypes()); - ObjPtr<Class> class_type = Class::GetJavaLangClass(); - ObjPtr<Class> class_array_type = - Runtime::Current()->GetClassLinker()->FindArrayClass(self, &class_type); - const int32_t dst_ptypes_count = src_ptypes->GetLength() - 1; - Handle<ObjectArray<Class>> dst_ptypes = hs.NewHandle( - ObjectArray<Class>::Alloc(self, class_array_type, dst_ptypes_count)); + Handle<Class> dst_rtype = hs.NewHandle(method_type->GetRType()); + const int32_t dst_ptypes_count = method_type->GetNumberOfPTypes() - 1; + Handle<ObjectArray<Class>> dst_ptypes = hs.NewHandle(AllocatePTypesArray(self, dst_ptypes_count)); + if (dst_ptypes.IsNull()) { + return nullptr; + } for (int32_t i = 0; i < dst_ptypes_count; ++i) { dst_ptypes->Set(i, src_ptypes->Get(i + 1)); } - return Create(self, rtype, dst_ptypes); + return Create(self, dst_rtype, dst_ptypes); +} + +MethodType* MethodType::CollectTrailingArguments(Thread* self, + ObjPtr<MethodType> method_type, + ObjPtr<Class> collector_array_class, + int32_t start_index) { + int32_t ptypes_length = method_type->GetNumberOfPTypes(); + if (start_index > ptypes_length) { + return method_type.Ptr(); + } + + StackHandleScope<4> hs(self); + Handle<Class> collector_class = hs.NewHandle(collector_array_class); + Handle<Class> dst_rtype = hs.NewHandle(method_type->GetRType()); + Handle<ObjectArray<Class>> src_ptypes = hs.NewHandle(method_type->GetPTypes()); + Handle<ObjectArray<Class>> dst_ptypes = hs.NewHandle(AllocatePTypesArray(self, start_index + 1)); + if (dst_ptypes.IsNull()) { + return nullptr; + } + for (int32_t i = 0; i < start_index; ++i) { + dst_ptypes->Set(i, src_ptypes->Get(i)); + } + dst_ptypes->Set(start_index, collector_class.Get()); + return Create(self, dst_rtype, dst_ptypes); } size_t MethodType::NumberOfVRegs() REQUIRES_SHARED(Locks::mutator_lock_) { diff --git a/runtime/mirror/method_type.h b/runtime/mirror/method_type.h index 3627214d90f..771162a2de5 100644 --- a/runtime/mirror/method_type.h +++ b/runtime/mirror/method_type.h @@ -40,6 +40,14 @@ class MANAGED MethodType : public Object { ObjPtr<MethodType> method_type) REQUIRES_SHARED(Locks::mutator_lock_); + // Collects trailing parameter types into an array. Assumes caller + // has checked trailing arguments are all of the same type. + static MethodType* CollectTrailingArguments(Thread* const self, + ObjPtr<MethodType> method_type, + ObjPtr<Class> collector_array_class, + int32_t start_index) + REQUIRES_SHARED(Locks::mutator_lock_); + static Class* StaticClass() REQUIRES_SHARED(Locks::mutator_lock_) { return static_class_.Read(); } diff --git a/runtime/mirror/object-inl.h b/runtime/mirror/object-inl.h index 55dd51427ca..c7561f42784 100644 --- a/runtime/mirror/object-inl.h +++ b/runtime/mirror/object-inl.h @@ -673,7 +673,7 @@ template<typename kSize> inline kSize Object::GetFieldAcquire(MemberOffset field_offset) { const uint8_t* raw_addr = reinterpret_cast<const uint8_t*>(this) + field_offset.Int32Value(); const kSize* addr = reinterpret_cast<const kSize*>(raw_addr); - return reinterpret_cast<const Atomic<kSize>*>(addr)->LoadAcquire(); + return reinterpret_cast<const Atomic<kSize>*>(addr)->load(std::memory_order_acquire); } template<bool kTransactionActive, bool kCheckTransaction, VerifyObjectFlags kVerifyFlags> @@ -956,7 +956,7 @@ inline ObjPtr<Object> Object::CompareAndExchangeFieldObject(MemberOffset field_o uint32_t new_ref(PtrCompression<kPoisonHeapReferences, Object>::Compress(new_value)); uint8_t* raw_addr = reinterpret_cast<uint8_t*>(this) + field_offset.Int32Value(); Atomic<uint32_t>* atomic_addr = reinterpret_cast<Atomic<uint32_t>*>(raw_addr); - bool success = atomic_addr->CompareAndExchangeStrongSequentiallyConsistent(&old_ref, new_ref); + bool success = atomic_addr->compare_exchange_strong(old_ref, new_ref, std::memory_order_seq_cst); ObjPtr<Object> witness_value(PtrCompression<kPoisonHeapReferences, Object>::Decompress(old_ref)); if (kIsDebugBuild) { // Ensure caller has done read barrier on the reference field so it's in the to-space. @@ -986,7 +986,7 @@ inline ObjPtr<Object> Object::ExchangeFieldObject(MemberOffset field_offset, uint32_t new_ref(PtrCompression<kPoisonHeapReferences, Object>::Compress(new_value)); uint8_t* raw_addr = reinterpret_cast<uint8_t*>(this) + field_offset.Int32Value(); Atomic<uint32_t>* atomic_addr = reinterpret_cast<Atomic<uint32_t>*>(raw_addr); - uint32_t old_ref = atomic_addr->ExchangeSequentiallyConsistent(new_ref); + uint32_t old_ref = atomic_addr->exchange(new_ref, std::memory_order_seq_cst); ObjPtr<Object> old_value(PtrCompression<kPoisonHeapReferences, Object>::Decompress(old_ref)); if (kIsDebugBuild) { // Ensure caller has done read barrier on the reference field so it's in the to-space. diff --git a/runtime/mirror/object.cc b/runtime/mirror/object.cc index f274cfc2fa6..0e03e3741ca 100644 --- a/runtime/mirror/object.cc +++ b/runtime/mirror/object.cc @@ -87,16 +87,18 @@ Object* Object::CopyObject(ObjPtr<mirror::Object> dest, DCHECK_ALIGNED(dst_bytes, sizeof(uintptr_t)); // Use word sized copies to begin. while (num_bytes >= sizeof(uintptr_t)) { - reinterpret_cast<Atomic<uintptr_t>*>(dst_bytes)->StoreRelaxed( - reinterpret_cast<Atomic<uintptr_t>*>(src_bytes)->LoadRelaxed()); + reinterpret_cast<Atomic<uintptr_t>*>(dst_bytes)->store( + reinterpret_cast<Atomic<uintptr_t>*>(src_bytes)->load(std::memory_order_relaxed), + std::memory_order_relaxed); src_bytes += sizeof(uintptr_t); dst_bytes += sizeof(uintptr_t); num_bytes -= sizeof(uintptr_t); } // Copy possible 32 bit word. if (sizeof(uintptr_t) != sizeof(uint32_t) && num_bytes >= sizeof(uint32_t)) { - reinterpret_cast<Atomic<uint32_t>*>(dst_bytes)->StoreRelaxed( - reinterpret_cast<Atomic<uint32_t>*>(src_bytes)->LoadRelaxed()); + reinterpret_cast<Atomic<uint32_t>*>(dst_bytes)->store( + reinterpret_cast<Atomic<uint32_t>*>(src_bytes)->load(std::memory_order_relaxed), + std::memory_order_relaxed); src_bytes += sizeof(uint32_t); dst_bytes += sizeof(uint32_t); num_bytes -= sizeof(uint32_t); @@ -104,8 +106,9 @@ Object* Object::CopyObject(ObjPtr<mirror::Object> dest, // Copy remaining bytes, avoid going past the end of num_bytes since there may be a redzone // there. while (num_bytes > 0) { - reinterpret_cast<Atomic<uint8_t>*>(dst_bytes)->StoreRelaxed( - reinterpret_cast<Atomic<uint8_t>*>(src_bytes)->LoadRelaxed()); + reinterpret_cast<Atomic<uint8_t>*>(dst_bytes)->store( + reinterpret_cast<Atomic<uint8_t>*>(src_bytes)->load(std::memory_order_relaxed), + std::memory_order_relaxed); src_bytes += sizeof(uint8_t); dst_bytes += sizeof(uint8_t); num_bytes -= sizeof(uint8_t); @@ -173,7 +176,7 @@ Object* Object::Clone(Thread* self) { uint32_t Object::GenerateIdentityHashCode() { uint32_t expected_value, new_value; do { - expected_value = hash_code_seed.LoadRelaxed(); + expected_value = hash_code_seed.load(std::memory_order_relaxed); new_value = expected_value * 1103515245 + 12345; } while (!hash_code_seed.CompareAndSetWeakRelaxed(expected_value, new_value) || (expected_value & LockWord::kHashMask) == 0); @@ -181,7 +184,7 @@ uint32_t Object::GenerateIdentityHashCode() { } void Object::SetHashCodeSeed(uint32_t new_seed) { - hash_code_seed.StoreRelaxed(new_seed); + hash_code_seed.store(new_seed, std::memory_order_relaxed); } int32_t Object::IdentityHashCode() { diff --git a/runtime/mirror/object.h b/runtime/mirror/object.h index 95f82cb1475..d00c90bcc0b 100644 --- a/runtime/mirror/object.h +++ b/runtime/mirror/object.h @@ -730,7 +730,7 @@ class MANAGED LOCKABLE Object { uint8_t* raw_addr = reinterpret_cast<uint8_t*>(this) + field_offset.Int32Value(); kSize* addr = reinterpret_cast<kSize*>(raw_addr); if (kIsVolatile) { - reinterpret_cast<Atomic<kSize>*>(addr)->StoreSequentiallyConsistent(new_value); + reinterpret_cast<Atomic<kSize>*>(addr)->store(new_value, std::memory_order_seq_cst); } else { reinterpret_cast<Atomic<kSize>*>(addr)->StoreJavaData(new_value); } @@ -742,7 +742,7 @@ class MANAGED LOCKABLE Object { const uint8_t* raw_addr = reinterpret_cast<const uint8_t*>(this) + field_offset.Int32Value(); const kSize* addr = reinterpret_cast<const kSize*>(raw_addr); if (kIsVolatile) { - return reinterpret_cast<const Atomic<kSize>*>(addr)->LoadSequentiallyConsistent(); + return reinterpret_cast<const Atomic<kSize>*>(addr)->load(std::memory_order_seq_cst); } else { return reinterpret_cast<const Atomic<kSize>*>(addr)->LoadJavaData(); } diff --git a/runtime/mirror/object_reference.h b/runtime/mirror/object_reference.h index cf1f85d2369..356fef0d266 100644 --- a/runtime/mirror/object_reference.h +++ b/runtime/mirror/object_reference.h @@ -110,13 +110,13 @@ class MANAGED HeapReference { template <bool kIsVolatile = false> MirrorType* AsMirrorPtr() const REQUIRES_SHARED(Locks::mutator_lock_) { return Compression::Decompress( - kIsVolatile ? reference_.LoadSequentiallyConsistent() : reference_.LoadJavaData()); + kIsVolatile ? reference_.load(std::memory_order_seq_cst) : reference_.LoadJavaData()); } template <bool kIsVolatile = false> void Assign(MirrorType* other) REQUIRES_SHARED(Locks::mutator_lock_) { if (kIsVolatile) { - reference_.StoreSequentiallyConsistent(Compression::Compress(other)); + reference_.store(Compression::Compress(other), std::memory_order_seq_cst); } else { reference_.StoreJavaData(Compression::Compress(other)); } diff --git a/runtime/mirror/throwable.cc b/runtime/mirror/throwable.cc index b6173d422bd..70069733d17 100644 --- a/runtime/mirror/throwable.cc +++ b/runtime/mirror/throwable.cc @@ -75,6 +75,10 @@ bool Throwable::IsCheckedException() { return !InstanceOf(WellKnownClasses::ToClass(WellKnownClasses::java_lang_RuntimeException)); } +bool Throwable::IsError() { + return InstanceOf(WellKnownClasses::ToClass(WellKnownClasses::java_lang_Error)); +} + int32_t Throwable::GetStackDepth() { ObjPtr<Object> stack_state = GetStackState(); if (stack_state == nullptr || !stack_state->IsObjectArray()) { diff --git a/runtime/mirror/throwable.h b/runtime/mirror/throwable.h index fb45228f49c..b901ca2d006 100644 --- a/runtime/mirror/throwable.h +++ b/runtime/mirror/throwable.h @@ -44,6 +44,7 @@ class MANAGED Throwable : public Object { void SetCause(ObjPtr<Throwable> cause) REQUIRES_SHARED(Locks::mutator_lock_); void SetStackState(ObjPtr<Object> state) REQUIRES_SHARED(Locks::mutator_lock_); bool IsCheckedException() REQUIRES_SHARED(Locks::mutator_lock_); + bool IsError() REQUIRES_SHARED(Locks::mutator_lock_); static Class* GetJavaLangThrowable() REQUIRES_SHARED(Locks::mutator_lock_) { DCHECK(!java_lang_Throwable_.IsNull()); diff --git a/runtime/monitor.cc b/runtime/monitor.cc index 2a938da15bd..2c38de5dae0 100644 --- a/runtime/monitor.cc +++ b/runtime/monitor.cc @@ -59,8 +59,8 @@ static constexpr uint64_t kLongWaitMs = 100 * kDebugThresholdFudgeFactor; * though, because we have a full 32 bits to work with. * * The two states of an Object's lock are referred to as "thin" and "fat". A lock may transition - * from the "thin" state to the "fat" state and this transition is referred to as inflation. Once - * a lock has been inflated it remains in the "fat" state indefinitely. + * from the "thin" state to the "fat" state and this transition is referred to as inflation. We + * deflate locks from time to time as part of heap trimming. * * The lock value itself is stored in mirror::Object::monitor_ and the representation is described * in the LockWord value type. @@ -140,7 +140,7 @@ int32_t Monitor::GetHashCode() { } } DCHECK(HasHashCode()); - return hash_code_.LoadRelaxed(); + return hash_code_.load(std::memory_order_relaxed); } bool Monitor::Install(Thread* self) { @@ -155,7 +155,7 @@ bool Monitor::Install(Thread* self) { break; } case LockWord::kHashCode: { - CHECK_EQ(hash_code_.LoadRelaxed(), static_cast<int32_t>(lw.GetHashCode())); + CHECK_EQ(hash_code_.load(std::memory_order_relaxed), static_cast<int32_t>(lw.GetHashCode())); break; } case LockWord::kFatLocked: { @@ -1101,7 +1101,7 @@ mirror::Object* Monitor::MonitorEnter(Thread* self, mirror::Object* obj, bool tr case LockWord::kFatLocked: { // We should have done an acquire read of the lockword initially, to ensure // visibility of the monitor data structure. Use an explicit fence instead. - QuasiAtomic::ThreadFenceAcquire(); + std::atomic_thread_fence(std::memory_order_acquire); Monitor* mon = lock_word.FatLockMonitor(); if (trylock) { return mon->TryLock(self) ? h_obj.Get() : nullptr; diff --git a/runtime/monitor.h b/runtime/monitor.h index 384ebbedaa7..6b7604ec8a8 100644 --- a/runtime/monitor.h +++ b/runtime/monitor.h @@ -130,7 +130,7 @@ class Monitor { bool IsLocked() REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(!monitor_lock_); bool HasHashCode() const { - return hash_code_.LoadRelaxed() != 0; + return hash_code_.load(std::memory_order_relaxed) != 0; } MonitorId GetMonitorId() const { diff --git a/runtime/native/dalvik_system_DexFile.cc b/runtime/native/dalvik_system_DexFile.cc index 13319c4c579..8320d9c7bac 100644 --- a/runtime/native/dalvik_system_DexFile.cc +++ b/runtime/native/dalvik_system_DexFile.cc @@ -25,6 +25,7 @@ #include "base/os.h" #include "base/stl_util.h" #include "base/utils.h" +#include "base/zip_archive.h" #include "class_linker.h" #include <class_loader_context.h> #include "common_throws.h" @@ -48,7 +49,6 @@ #include "runtime.h" #include "scoped_thread_state_change-inl.h" #include "well_known_classes.h" -#include "zip_archive.h" namespace art { diff --git a/runtime/native/dalvik_system_VMDebug.cc b/runtime/native/dalvik_system_VMDebug.cc index fc9426650ea..3692a308d8f 100644 --- a/runtime/native/dalvik_system_VMDebug.cc +++ b/runtime/native/dalvik_system_VMDebug.cc @@ -89,17 +89,27 @@ static void VMDebug_resetAllocCount(JNIEnv*, jclass, jint kinds) { static void VMDebug_startMethodTracingDdmsImpl(JNIEnv*, jclass, jint bufferSize, jint flags, jboolean samplingEnabled, jint intervalUs) { - Trace::Start("[DDMS]", -1, bufferSize, flags, Trace::TraceOutputMode::kDDMS, - samplingEnabled ? Trace::TraceMode::kSampling : Trace::TraceMode::kMethodTracing, - intervalUs); -} - -static void VMDebug_startMethodTracingFd(JNIEnv* env, jclass, jstring javaTraceFilename, - jint javaFd, jint bufferSize, jint flags, - jboolean samplingEnabled, jint intervalUs, + Trace::StartDDMS(bufferSize, + flags, + samplingEnabled ? Trace::TraceMode::kSampling : Trace::TraceMode::kMethodTracing, + intervalUs); +} + +static void VMDebug_startMethodTracingFd(JNIEnv* env, + jclass, + jstring javaTraceFilename ATTRIBUTE_UNUSED, + jint javaFd, + jint bufferSize, + jint flags, + jboolean samplingEnabled, + jint intervalUs, jboolean streamingOutput) { int originalFd = javaFd; if (originalFd < 0) { + ScopedObjectAccess soa(env); + soa.Self()->ThrowNewExceptionF("Ljava/lang/RuntimeException;", + "Trace fd is invalid: %d", + originalFd); return; } @@ -107,18 +117,20 @@ static void VMDebug_startMethodTracingFd(JNIEnv* env, jclass, jstring javaTraceF if (fd < 0) { ScopedObjectAccess soa(env); soa.Self()->ThrowNewExceptionF("Ljava/lang/RuntimeException;", - "dup(%d) failed: %s", originalFd, strerror(errno)); + "dup(%d) failed: %s", + originalFd, + strerror(errno)); return; } - ScopedUtfChars traceFilename(env, javaTraceFilename); - if (traceFilename.c_str() == nullptr) { - return; - } + // Ignore the traceFilename. Trace::TraceOutputMode outputMode = streamingOutput ? Trace::TraceOutputMode::kStreaming : Trace::TraceOutputMode::kFile; - Trace::Start(traceFilename.c_str(), fd, bufferSize, flags, outputMode, + Trace::Start(fd, + bufferSize, + flags, + outputMode, samplingEnabled ? Trace::TraceMode::kSampling : Trace::TraceMode::kMethodTracing, intervalUs); } @@ -130,7 +142,10 @@ static void VMDebug_startMethodTracingFilename(JNIEnv* env, jclass, jstring java if (traceFilename.c_str() == nullptr) { return; } - Trace::Start(traceFilename.c_str(), -1, bufferSize, flags, Trace::TraceOutputMode::kFile, + Trace::Start(traceFilename.c_str(), + bufferSize, + flags, + Trace::TraceOutputMode::kFile, samplingEnabled ? Trace::TraceMode::kSampling : Trace::TraceMode::kMethodTracing, intervalUs); } diff --git a/runtime/native/dalvik_system_ZygoteHooks.cc b/runtime/native/dalvik_system_ZygoteHooks.cc index 3dd3e43779d..84c7926a118 100644 --- a/runtime/native/dalvik_system_ZygoteHooks.cc +++ b/runtime/native/dalvik_system_ZygoteHooks.cc @@ -312,6 +312,8 @@ static void ZygoteHooks_nativePostForkChild(JNIEnv* env, LOG(ERROR) << StringPrintf("Unknown bits set in runtime_flags: %#x", runtime_flags); } + Runtime::Current()->GetHeap()->PostForkChildAction(thread); + // Update tracing. if (Trace::GetMethodTracingMode() != TracingMode::kTracingInactive) { Trace::TraceOutputMode output_mode = Trace::GetOutputMode(); @@ -342,7 +344,6 @@ static void ZygoteHooks_nativePostForkChild(JNIEnv* env, std::string trace_file = StringPrintf("/data/misc/trace/%s.trace.bin", proc_name.c_str()); Trace::Start(trace_file.c_str(), - -1, buffer_size, 0, // TODO: Expose flags. output_mode, diff --git a/runtime/native/java_lang_Class.cc b/runtime/native/java_lang_Class.cc index e161ed49f70..2c1c963ed6f 100644 --- a/runtime/native/java_lang_Class.cc +++ b/runtime/native/java_lang_Class.cc @@ -701,8 +701,7 @@ static jobject Class_getEnclosingConstructorNative(JNIEnv* env, jobject javaThis } ObjPtr<mirror::Object> method = annotations::GetEnclosingMethod(klass); if (method != nullptr) { - if (soa.Decode<mirror::Class>(WellKnownClasses::java_lang_reflect_Constructor) == - method->GetClass()) { + if (mirror::Constructor::StaticClass() == method->GetClass()) { return soa.AddLocalReference<jobject>(method); } } @@ -718,8 +717,7 @@ static jobject Class_getEnclosingMethodNative(JNIEnv* env, jobject javaThis) { } ObjPtr<mirror::Object> method = annotations::GetEnclosingMethod(klass); if (method != nullptr) { - if (soa.Decode<mirror::Class>(WellKnownClasses::java_lang_reflect_Method) == - method->GetClass()) { + if (mirror::Method::StaticClass() == method->GetClass()) { return soa.AddLocalReference<jobject>(method); } } diff --git a/runtime/native/java_lang_VMClassLoader.cc b/runtime/native/java_lang_VMClassLoader.cc index 3a0d76032e3..44585fc4538 100644 --- a/runtime/native/java_lang_VMClassLoader.cc +++ b/runtime/native/java_lang_VMClassLoader.cc @@ -16,6 +16,7 @@ #include "java_lang_VMClassLoader.h" +#include "base/zip_archive.h" #include "class_linker.h" #include "dex/descriptors_names.h" #include "dex/dex_file_loader.h" @@ -29,7 +30,6 @@ #include "obj_ptr.h" #include "scoped_fast_native_object_access-inl.h" #include "well_known_classes.h" -#include "zip_archive.h" namespace art { diff --git a/runtime/native/sun_misc_Unsafe.cc b/runtime/native/sun_misc_Unsafe.cc index 25f984f6be4..fb00ae39672 100644 --- a/runtime/native/sun_misc_Unsafe.cc +++ b/runtime/native/sun_misc_Unsafe.cc @@ -116,7 +116,7 @@ static void Unsafe_putOrderedInt(JNIEnv* env, jobject, jobject javaObj, jlong of ScopedFastNativeObjectAccess soa(env); ObjPtr<mirror::Object> obj = soa.Decode<mirror::Object>(javaObj); // TODO: A release store is likely to be faster on future processors. - QuasiAtomic::ThreadFenceRelease(); + std::atomic_thread_fence(std::memory_order_release); // JNI must use non transactional mode. obj->SetField32<false>(MemberOffset(offset), newValue); } @@ -152,7 +152,7 @@ static void Unsafe_putOrderedLong(JNIEnv* env, jobject, jobject javaObj, jlong o jlong newValue) { ScopedFastNativeObjectAccess soa(env); ObjPtr<mirror::Object> obj = soa.Decode<mirror::Object>(javaObj); - QuasiAtomic::ThreadFenceRelease(); + std::atomic_thread_fence(std::memory_order_release); // JNI must use non transactional mode. obj->SetField64<false>(MemberOffset(offset), newValue); } @@ -194,7 +194,7 @@ static void Unsafe_putOrderedObject(JNIEnv* env, jobject, jobject javaObj, jlong ScopedFastNativeObjectAccess soa(env); ObjPtr<mirror::Object> obj = soa.Decode<mirror::Object>(javaObj); ObjPtr<mirror::Object> newValue = soa.Decode<mirror::Object>(javaNewValue); - QuasiAtomic::ThreadFenceRelease(); + std::atomic_thread_fence(std::memory_order_release); // JNI must use non transactional mode. obj->SetFieldObject<false>(MemberOffset(offset), newValue); } diff --git a/runtime/oat.h b/runtime/oat.h index 292c9d6f413..01d391401d1 100644 --- a/runtime/oat.h +++ b/runtime/oat.h @@ -32,8 +32,8 @@ class InstructionSetFeatures; class PACKED(4) OatHeader { public: static constexpr uint8_t kOatMagic[] = { 'o', 'a', 't', '\n' }; - // Last oat version changed reason: Math.pow() intrinsic. - static constexpr uint8_t kOatVersion[] = { '1', '3', '8', '\0' }; + // Last oat version changed reason: Retrieve Class* and String* from .data.bimg.rel.ro . + static constexpr uint8_t kOatVersion[] = { '1', '4', '0', '\0' }; static constexpr const char* kImageLocationKey = "image-location"; static constexpr const char* kDex2OatCmdLineKey = "dex2oat-cmdline"; diff --git a/runtime/oat_file.cc b/runtime/oat_file.cc index b0e1de2b81a..c7a558cc94f 100644 --- a/runtime/oat_file.cc +++ b/runtime/oat_file.cc @@ -40,6 +40,7 @@ #include "base/enums.h" #include "base/file_utils.h" #include "base/logging.h" // For VLOG_IS_ON. +#include "base/mem_map.h" #include "base/os.h" #include "base/stl_util.h" #include "base/systrace.h" @@ -49,19 +50,18 @@ #include "dex/dex_file_loader.h" #include "dex/dex_file_types.h" #include "dex/standard_dex_file.h" +#include "dex/type_lookup_table.h" #include "dex/utf-inl.h" #include "elf_file.h" #include "elf_utils.h" #include "gc_root.h" #include "gc/space/image_space.h" -#include "mem_map.h" #include "mirror/class.h" #include "mirror/object-inl.h" #include "oat.h" #include "oat_file-inl.h" #include "oat_file_manager.h" #include "runtime.h" -#include "type_lookup_table.h" #include "vdex_file.h" namespace art { @@ -343,6 +343,19 @@ bool OatFileBase::ComputeFields(uint8_t* requested_base, // Readjust to be non-inclusive upper bound. end_ += sizeof(uint32_t); + data_bimg_rel_ro_begin_ = FindDynamicSymbolAddress("oatdatabimgrelro", &symbol_error_msg); + if (data_bimg_rel_ro_begin_ != nullptr) { + data_bimg_rel_ro_end_ = + FindDynamicSymbolAddress("oatdatabimgrelrolastword", &symbol_error_msg); + if (data_bimg_rel_ro_end_ == nullptr) { + *error_msg = + StringPrintf("Failed to find oatdatabimgrelrolastword symbol in '%s'", file_path.c_str()); + return false; + } + // Readjust to be non-inclusive upper bound. + data_bimg_rel_ro_end_ += sizeof(uint32_t); + } + bss_begin_ = const_cast<uint8_t*>(FindDynamicSymbolAddress("oatbss", &symbol_error_msg)); if (bss_begin_ == nullptr) { // No .bss section. @@ -399,37 +412,6 @@ inline static bool ReadOatDexFileData(const OatFile& oat_file, return true; } -static inline bool MapConstantTables(const gc::space::ImageSpace* space, - uint8_t* address) { - // If MREMAP_DUP is ever merged to Linux kernel, use it to avoid the unnecessary open()/close(). - // Note: The current approach relies on the filename still referencing the same inode. - - File file(space->GetImageFilename(), O_RDONLY, /* checkUsage */ false); - if (!file.IsOpened()) { - LOG(ERROR) << "Failed to open boot image file " << space->GetImageFilename(); - return false; - } - - uint32_t offset = space->GetImageHeader().GetBootImageConstantTablesOffset(); - uint32_t size = space->GetImageHeader().GetBootImageConstantTablesSize(); - std::string error_msg; - std::unique_ptr<MemMap> mem_map(MemMap::MapFileAtAddress(address, - size, - PROT_READ, - MAP_PRIVATE, - file.Fd(), - offset, - /* low_4gb */ false, - /* reuse */ true, - file.GetPath().c_str(), - &error_msg)); - if (mem_map == nullptr) { - LOG(ERROR) << "Failed to mmap boot image tables from file " << space->GetImageFilename(); - return false; - } - return true; -} - static bool ReadIndexBssMapping(OatFile* oat_file, /*inout*/const uint8_t** oat, size_t dex_file_index, @@ -536,6 +518,17 @@ bool OatFileBase::Setup(const char* abs_dex_location, std::string* error_msg) { } const uint8_t* oat = Begin() + oat_dex_files_offset; // Jump to the OatDexFile records. + if (!IsAligned<sizeof(uint32_t)>(data_bimg_rel_ro_begin_) || + !IsAligned<sizeof(uint32_t)>(data_bimg_rel_ro_end_) || + data_bimg_rel_ro_begin_ > data_bimg_rel_ro_end_) { + *error_msg = StringPrintf("In oat file '%s' found unaligned or unordered databimgrelro " + "symbol(s): begin = %p, end = %p", + GetLocation().c_str(), + data_bimg_rel_ro_begin_, + data_bimg_rel_ro_end_); + return false; + } + DCHECK_GE(static_cast<size_t>(pointer_size), alignof(GcRoot<mirror::Object>)); if (!IsAligned<kPageSize>(bss_begin_) || !IsAlignedParam(bss_methods_, static_cast<size_t>(pointer_size)) || @@ -564,12 +557,15 @@ bool OatFileBase::Setup(const char* abs_dex_location, std::string* error_msg) { return false; } - uint8_t* after_tables = - (bss_methods_ != nullptr) ? bss_methods_ : bss_roots_; // May be null. - uint8_t* boot_image_tables = (bss_begin_ == after_tables) ? nullptr : bss_begin_; - uint8_t* boot_image_tables_end = - (bss_begin_ == after_tables) ? nullptr : (after_tables != nullptr) ? after_tables : bss_end_; - DCHECK_EQ(boot_image_tables != nullptr, boot_image_tables_end != nullptr); + if (bss_methods_ != nullptr && bss_methods_ != bss_begin_) { + *error_msg = StringPrintf("In oat file '%s' found unexpected .bss gap before 'oatbssmethods': " + "begin = %p, methods = %p", + GetLocation().c_str(), + bss_begin_, + bss_methods_); + return false; + } + uint32_t dex_file_count = GetOatHeader().GetDexFileCount(); oat_dex_files_storage_.reserve(dex_file_count); for (size_t i = 0; i < dex_file_count; i++) { @@ -849,39 +845,28 @@ bool OatFileBase::Setup(const char* abs_dex_location, std::string* error_msg) { } } - if (boot_image_tables != nullptr) { - Runtime* runtime = Runtime::Current(); + Runtime* runtime = Runtime::Current(); + + if (DataBimgRelRoBegin() != nullptr) { + // Make .data.bimg.rel.ro read only. ClassLinker shall make it writable for relocation. + uint8_t* reloc_begin = const_cast<uint8_t*>(DataBimgRelRoBegin()); + CheckedCall(mprotect, "protect relocations", reloc_begin, DataBimgRelRoSize(), PROT_READ); if (UNLIKELY(runtime == nullptr)) { - // This must be oatdump without boot image. Make sure the .bss is inaccessible. - CheckedCall(mprotect, "protect bss", const_cast<uint8_t*>(BssBegin()), BssSize(), PROT_NONE); + // This must be oatdump without boot image. } else if (!IsExecutable()) { - // Do not try to mmap boot image tables into .bss if the oat file is not executable. + // Do not check whether we have a boot image if the oat file is not executable. + } else if (UNLIKELY(runtime->GetHeap()->GetBootImageSpaces().empty())) { + *error_msg = StringPrintf("Cannot load oat file '%s' with .data.bimg.rel.ro as executable " + "without boot image.", + GetLocation().c_str()); + return false; } else { - // Map boot image tables into the .bss. The reserved size must match size of the tables. - size_t reserved_size = static_cast<size_t>(boot_image_tables_end - boot_image_tables); - size_t tables_size = 0u; - for (gc::space::ImageSpace* space : runtime->GetHeap()->GetBootImageSpaces()) { - tables_size += space->GetImageHeader().GetBootImageConstantTablesSize(); - DCHECK_ALIGNED(tables_size, kPageSize); - } - if (tables_size != reserved_size) { - *error_msg = StringPrintf("In oat file '%s' found unexpected boot image table sizes, " - " %zu bytes, should be %zu.", - GetLocation().c_str(), - reserved_size, - tables_size); - return false; - } - for (gc::space::ImageSpace* space : Runtime::Current()->GetHeap()->GetBootImageSpaces()) { - uint32_t current_tables_size = space->GetImageHeader().GetBootImageConstantTablesSize(); - if (current_tables_size != 0u && !MapConstantTables(space, boot_image_tables)) { - return false; - } - boot_image_tables += current_tables_size; - } - DCHECK(boot_image_tables == boot_image_tables_end); + // ClassLinker shall perform the relocation when we register a dex file from + // this oat file. We do not do the relocation here to avoid dirtying the pages + // if the code is never actually ready to be executed. } } + return true; } @@ -1513,6 +1498,8 @@ OatFile::OatFile(const std::string& location, bool is_executable) vdex_(nullptr), begin_(nullptr), end_(nullptr), + data_bimg_rel_ro_begin_(nullptr), + data_bimg_rel_ro_end_(nullptr), bss_begin_(nullptr), bss_end_(nullptr), bss_methods_(nullptr), @@ -1542,22 +1529,6 @@ const uint8_t* OatFile::End() const { return end_; } -const uint8_t* OatFile::BssBegin() const { - return bss_begin_; -} - -const uint8_t* OatFile::BssEnd() const { - return bss_end_; -} - -const uint8_t* OatFile::VdexBegin() const { - return vdex_begin_; -} - -const uint8_t* OatFile::VdexEnd() const { - return vdex_end_; -} - const uint8_t* OatFile::DexBegin() const { return vdex_->Begin(); } @@ -1566,6 +1537,16 @@ const uint8_t* OatFile::DexEnd() const { return vdex_->End(); } +ArrayRef<const uint32_t> OatFile::GetBootImageRelocations() const { + if (data_bimg_rel_ro_begin_ != nullptr) { + const uint32_t* relocations = reinterpret_cast<const uint32_t*>(data_bimg_rel_ro_begin_); + const uint32_t* relocations_end = reinterpret_cast<const uint32_t*>(data_bimg_rel_ro_end_); + return ArrayRef<const uint32_t>(relocations, relocations_end - relocations); + } else { + return ArrayRef<const uint32_t>(); + } +} + ArrayRef<ArtMethod*> OatFile::GetBssMethods() const { if (bss_methods_ != nullptr) { ArtMethod** methods = reinterpret_cast<ArtMethod**>(bss_methods_); diff --git a/runtime/oat_file.h b/runtime/oat_file.h index 3c2cd00c8d6..6494b4c58ef 100644 --- a/runtime/oat_file.h +++ b/runtime/oat_file.h @@ -32,11 +32,11 @@ #include "compiler_filter.h" #include "dex/dex_file.h" #include "dex/dex_file_layout.h" +#include "dex/type_lookup_table.h" #include "dex/utf.h" #include "index_bss_mapping.h" #include "mirror/object.h" #include "oat.h" -#include "type_lookup_table.h" namespace art { @@ -275,6 +275,10 @@ class OatFile { return p >= Begin() && p < End(); } + size_t DataBimgRelRoSize() const { + return DataBimgRelRoEnd() - DataBimgRelRoBegin(); + } + size_t BssSize() const { return BssEnd() - BssBegin(); } @@ -300,15 +304,19 @@ class OatFile { const uint8_t* Begin() const; const uint8_t* End() const; - const uint8_t* BssBegin() const; - const uint8_t* BssEnd() const; + const uint8_t* DataBimgRelRoBegin() const { return data_bimg_rel_ro_begin_; } + const uint8_t* DataBimgRelRoEnd() const { return data_bimg_rel_ro_end_; } + + const uint8_t* BssBegin() const { return bss_begin_; } + const uint8_t* BssEnd() const { return bss_end_; } - const uint8_t* VdexBegin() const; - const uint8_t* VdexEnd() const; + const uint8_t* VdexBegin() const { return vdex_begin_; } + const uint8_t* VdexEnd() const { return vdex_end_; } const uint8_t* DexBegin() const; const uint8_t* DexEnd() const; + ArrayRef<const uint32_t> GetBootImageRelocations() const; ArrayRef<ArtMethod*> GetBssMethods() const; ArrayRef<GcRoot<mirror::Object>> GetBssGcRoots() const; @@ -355,6 +363,12 @@ class OatFile { // Pointer to end of oat region for bounds checking. const uint8_t* end_; + // Pointer to the .data.bimg.rel.ro section, if present, otherwise null. + const uint8_t* data_bimg_rel_ro_begin_; + + // Pointer to the end of the .data.bimg.rel.ro section, if present, otherwise null. + const uint8_t* data_bimg_rel_ro_end_; + // Pointer to the .bss section, if present, otherwise null. uint8_t* bss_begin_; diff --git a/runtime/oat_file_assistant_test.cc b/runtime/oat_file_assistant_test.cc index 676071b2472..99bc0b2c6e1 100644 --- a/runtime/oat_file_assistant_test.cc +++ b/runtime/oat_file_assistant_test.cc @@ -1487,7 +1487,7 @@ TEST_F(OatFileAssistantTest, SystemOdex) { true, false); int status = oat_file_assistant.MakeUpToDate(false, kSpecialSharedLibraryContext, &error_msg); - EXPECT_EQ(OatFileAssistant::kUpdateSucceeded, status) << error_msg; + ASSERT_EQ(OatFileAssistant::kUpdateSucceeded, status) << error_msg; EXPECT_TRUE(oat_file_assistant.GetBestOatFile()->IsExecutable()); } @@ -1497,7 +1497,7 @@ TEST_F(OatFileAssistantTest, SystemOdex) { true, true); int status = oat_file_assistant.MakeUpToDate(false, kSpecialSharedLibraryContext, &error_msg); - EXPECT_EQ(OatFileAssistant::kUpdateSucceeded, status) << error_msg; + ASSERT_EQ(OatFileAssistant::kUpdateSucceeded, status) << error_msg; EXPECT_FALSE(oat_file_assistant.GetBestOatFile()->IsExecutable()); } @@ -1510,7 +1510,7 @@ TEST_F(OatFileAssistantTest, SystemOdex) { true, false); int status = oat_file_assistant.MakeUpToDate(false, kSpecialSharedLibraryContext, &error_msg); - EXPECT_EQ(OatFileAssistant::kUpdateSucceeded, status) << error_msg; + ASSERT_EQ(OatFileAssistant::kUpdateSucceeded, status) << error_msg; EXPECT_TRUE(oat_file_assistant.GetBestOatFile()->IsExecutable()); } @@ -1520,7 +1520,7 @@ TEST_F(OatFileAssistantTest, SystemOdex) { true, true); int status = oat_file_assistant.MakeUpToDate(false, kSpecialSharedLibraryContext, &error_msg); - EXPECT_EQ(OatFileAssistant::kUpdateSucceeded, status) << error_msg; + ASSERT_EQ(OatFileAssistant::kUpdateSucceeded, status) << error_msg; EXPECT_TRUE(oat_file_assistant.GetBestOatFile()->IsExecutable()); } } diff --git a/runtime/parsed_options_test.cc b/runtime/parsed_options_test.cc index 8948c710db3..9e5d9abe3b5 100644 --- a/runtime/parsed_options_test.cc +++ b/runtime/parsed_options_test.cc @@ -112,7 +112,7 @@ TEST_F(ParsedOptionsTest, ParsedOptions) { TEST_F(ParsedOptionsTest, ParsedOptionsGc) { RuntimeOptions options; - options.push_back(std::make_pair("-Xgc:MC", nullptr)); + options.push_back(std::make_pair("-Xgc:SS", nullptr)); RuntimeArgumentMap map; bool parsed = ParsedOptions::Parse(options, false, &map); @@ -124,7 +124,7 @@ TEST_F(ParsedOptionsTest, ParsedOptionsGc) { EXPECT_TRUE(map.Exists(Opt::GcOption)); XGcOption xgc = map.GetOrDefault(Opt::GcOption); - EXPECT_EQ(gc::kCollectorTypeMC, xgc.collector_type_); + EXPECT_EQ(gc::kCollectorTypeSS, xgc.collector_type_); } TEST_F(ParsedOptionsTest, ParsedOptionsInstructionSet) { diff --git a/runtime/quick_exception_handler.cc b/runtime/quick_exception_handler.cc index 006405f0957..077aa33925a 100644 --- a/runtime/quick_exception_handler.cc +++ b/runtime/quick_exception_handler.cc @@ -143,14 +143,14 @@ class CatchBlockStackVisitor FINAL : public StackVisitor { void QuickExceptionHandler::FindCatch(ObjPtr<mirror::Throwable> exception) { DCHECK(!is_deoptimization_); + StackHandleScope<1> hs(self_); + Handle<mirror::Throwable> exception_ref(hs.NewHandle(exception)); if (kDebugExceptionDelivery) { - mirror::String* msg = exception->GetDetailMessage(); + ObjPtr<mirror::String> msg = exception_ref->GetDetailMessage(); std::string str_msg(msg != nullptr ? msg->ToModifiedUtf8() : ""); - self_->DumpStack(LOG_STREAM(INFO) << "Delivering exception: " << exception->PrettyTypeOf() + self_->DumpStack(LOG_STREAM(INFO) << "Delivering exception: " << exception_ref->PrettyTypeOf() << ": " << str_msg << "\n"); } - StackHandleScope<1> hs(self_); - Handle<mirror::Throwable> exception_ref(hs.NewHandle(exception)); // Walk the stack to find catch handler. CatchBlockStackVisitor visitor(self_, context_, &exception_ref, this); diff --git a/runtime/read_barrier-inl.h b/runtime/read_barrier-inl.h index 58f6c04c3e8..5035ba077c2 100644 --- a/runtime/read_barrier-inl.h +++ b/runtime/read_barrier-inl.h @@ -130,7 +130,7 @@ inline MirrorType* ReadBarrier::BarrierForRoot(MirrorType** root, ref = reinterpret_cast<MirrorType*>(Mark(old_ref)); // Update the field atomically. This may fail if mutator updates before us, but it's ok. if (ref != old_ref) { - Atomic<mirror::Object*>* atomic_root = reinterpret_cast<Atomic<mirror::Object*>*>(root); + Atomic<MirrorType*>* atomic_root = reinterpret_cast<Atomic<MirrorType*>*>(root); atomic_root->CompareAndSetStrongRelaxed(old_ref, ref); } } diff --git a/runtime/runtime.cc b/runtime/runtime.cc index 9f7f5cd7639..c8e7e67beb4 100644 --- a/runtime/runtime.cc +++ b/runtime/runtime.cc @@ -62,6 +62,8 @@ #include "base/dumpable.h" #include "base/enums.h" #include "base/file_utils.h" +#include "base/malloc_arena_pool.h" +#include "base/mem_map_arena_pool.h" #include "base/memory_tool.h" #include "base/mutex.h" #include "base/os.h" @@ -574,19 +576,7 @@ void Runtime::Abort(const char* msg) { LOG(FATAL_WITHOUT_ABORT) << "Unexpectedly returned from abort hook!"; } -#if defined(__GLIBC__) - // TODO: we ought to be able to use pthread_kill(3) here (or abort(3), - // which POSIX defines in terms of raise(3), which POSIX defines in terms - // of pthread_kill(3)). On Linux, though, libcorkscrew can't unwind through - // libpthread, which means the stacks we dump would be useless. Calling - // tgkill(2) directly avoids that. - syscall(__NR_tgkill, getpid(), GetTid(), SIGABRT); - // TODO: LLVM installs it's own SIGABRT handler so exit to be safe... Can we disable that in LLVM? - // If not, we could use sigaction(3) before calling tgkill(2) and lose this call to exit(3). - exit(1); -#else abort(); -#endif // notreached } @@ -841,7 +831,6 @@ bool Runtime::Start() { if (trace_config_.get() != nullptr && trace_config_->trace_file != "") { ScopedThreadStateChange tsc(self, kWaitingForMethodTracingStart); Trace::Start(trace_config_->trace_file.c_str(), - -1, static_cast<int>(trace_config_->trace_file_size), 0, trace_config_->trace_output_mode, @@ -1334,13 +1323,17 @@ bool Runtime::Init(RuntimeArgumentMap&& runtime_options_in) { // Use MemMap arena pool for jit, malloc otherwise. Malloc arenas are faster to allocate but // can't be trimmed as easily. const bool use_malloc = IsAotCompiler(); - arena_pool_.reset(new ArenaPool(use_malloc, /* low_4gb */ false)); - jit_arena_pool_.reset( - new ArenaPool(/* use_malloc */ false, /* low_4gb */ false, "CompilerMetadata")); + if (use_malloc) { + arena_pool_.reset(new MallocArenaPool()); + jit_arena_pool_.reset(new MallocArenaPool()); + } else { + arena_pool_.reset(new MemMapArenaPool(/* low_4gb */ false)); + jit_arena_pool_.reset(new MemMapArenaPool(/* low_4gb */ false, "CompilerMetadata")); + } if (IsAotCompiler() && Is64BitInstructionSet(kRuntimeISA)) { // 4gb, no malloc. Explanation in header. - low_4gb_arena_pool_.reset(new ArenaPool(/* use_malloc */ false, /* low_4gb */ true)); + low_4gb_arena_pool_.reset(new MemMapArenaPool(/* low_4gb */ true)); } linear_alloc_.reset(CreateLinearAlloc()); @@ -1511,19 +1504,34 @@ bool Runtime::Init(RuntimeArgumentMap&& runtime_options_in) { // TODO: move this to just be an Trace::Start argument Trace::SetDefaultClockSource(runtime_options.GetOrDefault(Opt::ProfileClock)); + // Pre-allocate an OutOfMemoryError for the case when we fail to + // allocate the exception to be thrown. + InitPreAllocatedException(self, + &Runtime::pre_allocated_OutOfMemoryError_when_throwing_exception_, + "Ljava/lang/OutOfMemoryError;", + "OutOfMemoryError thrown while trying to throw an exception; " + "no stack trace available"); // Pre-allocate an OutOfMemoryError for the double-OOME case. - self->ThrowNewException("Ljava/lang/OutOfMemoryError;", - "OutOfMemoryError thrown while trying to throw OutOfMemoryError; " - "no stack trace available"); - pre_allocated_OutOfMemoryError_ = GcRoot<mirror::Throwable>(self->GetException()); - self->ClearException(); + InitPreAllocatedException(self, + &Runtime::pre_allocated_OutOfMemoryError_when_throwing_oome_, + "Ljava/lang/OutOfMemoryError;", + "OutOfMemoryError thrown while trying to throw OutOfMemoryError; " + "no stack trace available"); + // Pre-allocate an OutOfMemoryError for the case when we fail to + // allocate while handling a stack overflow. + InitPreAllocatedException(self, + &Runtime::pre_allocated_OutOfMemoryError_when_handling_stack_overflow_, + "Ljava/lang/OutOfMemoryError;", + "OutOfMemoryError thrown while trying to handle a stack overflow; " + "no stack trace available"); // Pre-allocate a NoClassDefFoundError for the common case of failing to find a system class // ahead of checking the application's class loader. - self->ThrowNewException("Ljava/lang/NoClassDefFoundError;", - "Class not found using the boot class loader; no stack trace available"); - pre_allocated_NoClassDefFoundError_ = GcRoot<mirror::Throwable>(self->GetException()); - self->ClearException(); + InitPreAllocatedException(self, + &Runtime::pre_allocated_NoClassDefFoundError_, + "Ljava/lang/NoClassDefFoundError;", + "Class not found using the boot class loader; " + "no stack trace available"); // Runtime initialization is largely done now. // We load plugins first since that can modify the runtime state slightly. @@ -1674,6 +1682,16 @@ void Runtime::AttachAgent(JNIEnv* env, } } +void Runtime::InitPreAllocatedException(Thread* self, + GcRoot<mirror::Throwable> Runtime::* exception, + const char* exception_class_descriptor, + const char* msg) { + DCHECK_EQ(self, Thread::Current()); + self->ThrowNewException(exception_class_descriptor, msg); + this->*exception = GcRoot<mirror::Throwable>(self->GetException()); + self->ClearException(); +} + void Runtime::InitNativeMethods() { VLOG(startup) << "Runtime::InitNativeMethods entering"; Thread* self = Thread::Current(); @@ -1925,10 +1943,26 @@ void Runtime::DetachCurrentThread() { thread_list_->Unregister(self); } -mirror::Throwable* Runtime::GetPreAllocatedOutOfMemoryError() { - mirror::Throwable* oome = pre_allocated_OutOfMemoryError_.Read(); +mirror::Throwable* Runtime::GetPreAllocatedOutOfMemoryErrorWhenThrowingException() { + mirror::Throwable* oome = pre_allocated_OutOfMemoryError_when_throwing_exception_.Read(); + if (oome == nullptr) { + LOG(ERROR) << "Failed to return pre-allocated OOME-when-throwing-exception"; + } + return oome; +} + +mirror::Throwable* Runtime::GetPreAllocatedOutOfMemoryErrorWhenThrowingOOME() { + mirror::Throwable* oome = pre_allocated_OutOfMemoryError_when_throwing_oome_.Read(); + if (oome == nullptr) { + LOG(ERROR) << "Failed to return pre-allocated OOME-when-throwing-OOME"; + } + return oome; +} + +mirror::Throwable* Runtime::GetPreAllocatedOutOfMemoryErrorWhenHandlingStackOverflow() { + mirror::Throwable* oome = pre_allocated_OutOfMemoryError_when_handling_stack_overflow_.Read(); if (oome == nullptr) { - LOG(ERROR) << "Failed to return pre-allocated OOME"; + LOG(ERROR) << "Failed to return pre-allocated OOME-when-handling-stack-overflow"; } return oome; } @@ -2013,7 +2047,12 @@ void Runtime::VisitTransactionRoots(RootVisitor* visitor) { void Runtime::VisitNonThreadRoots(RootVisitor* visitor) { java_vm_->VisitRoots(visitor); sentinel_.VisitRootIfNonNull(visitor, RootInfo(kRootVMInternal)); - pre_allocated_OutOfMemoryError_.VisitRootIfNonNull(visitor, RootInfo(kRootVMInternal)); + pre_allocated_OutOfMemoryError_when_throwing_exception_ + .VisitRootIfNonNull(visitor, RootInfo(kRootVMInternal)); + pre_allocated_OutOfMemoryError_when_throwing_oome_ + .VisitRootIfNonNull(visitor, RootInfo(kRootVMInternal)); + pre_allocated_OutOfMemoryError_when_handling_stack_overflow_ + .VisitRootIfNonNull(visitor, RootInfo(kRootVMInternal)); pre_allocated_NoClassDefFoundError_.VisitRootIfNonNull(visitor, RootInfo(kRootVMInternal)); verifier::MethodVerifier::VisitStaticRoots(visitor); VisitTransactionRoots(visitor); diff --git a/runtime/runtime.h b/runtime/runtime.h index 3bf0edabfae..3c0c6a08a51 100644 --- a/runtime/runtime.h +++ b/runtime/runtime.h @@ -291,7 +291,12 @@ class Runtime { // Get the special object used to mark a cleared JNI weak global. mirror::Object* GetClearedJniWeakGlobal() REQUIRES_SHARED(Locks::mutator_lock_); - mirror::Throwable* GetPreAllocatedOutOfMemoryError() REQUIRES_SHARED(Locks::mutator_lock_); + mirror::Throwable* GetPreAllocatedOutOfMemoryErrorWhenThrowingException() + REQUIRES_SHARED(Locks::mutator_lock_); + mirror::Throwable* GetPreAllocatedOutOfMemoryErrorWhenThrowingOOME() + REQUIRES_SHARED(Locks::mutator_lock_); + mirror::Throwable* GetPreAllocatedOutOfMemoryErrorWhenHandlingStackOverflow() + REQUIRES_SHARED(Locks::mutator_lock_); mirror::Throwable* GetPreAllocatedNoClassDefFoundError() REQUIRES_SHARED(Locks::mutator_lock_); @@ -772,6 +777,11 @@ class Runtime { bool Init(RuntimeArgumentMap&& runtime_options) SHARED_TRYLOCK_FUNCTION(true, Locks::mutator_lock_); + void InitPreAllocatedException(Thread* self, + GcRoot<mirror::Throwable> Runtime::* exception, + const char* exception_class_descriptor, + const char* msg) + REQUIRES_SHARED(Locks::mutator_lock_); void InitNativeMethods() REQUIRES(!Locks::mutator_lock_); void RegisterRuntimeNativeMethods(JNIEnv* env); @@ -804,7 +814,10 @@ class Runtime { // 64 bit so that we can share the same asm offsets for both 32 and 64 bits. uint64_t callee_save_methods_[kCalleeSaveSize]; - GcRoot<mirror::Throwable> pre_allocated_OutOfMemoryError_; + // Pre-allocated exceptions (see Runtime::Init). + GcRoot<mirror::Throwable> pre_allocated_OutOfMemoryError_when_throwing_exception_; + GcRoot<mirror::Throwable> pre_allocated_OutOfMemoryError_when_throwing_oome_; + GcRoot<mirror::Throwable> pre_allocated_OutOfMemoryError_when_handling_stack_overflow_; GcRoot<mirror::Throwable> pre_allocated_NoClassDefFoundError_; ArtMethod* resolution_method_; ArtMethod* imt_conflict_method_; diff --git a/runtime/runtime_callbacks_test.cc b/runtime/runtime_callbacks_test.cc index 56035261772..72d9919971d 100644 --- a/runtime/runtime_callbacks_test.cc +++ b/runtime/runtime_callbacks_test.cc @@ -28,13 +28,13 @@ #include "jni.h" #include "art_method-inl.h" +#include "base/mem_map.h" #include "base/mutex.h" #include "class_linker.h" #include "common_runtime_test.h" #include "dex/class_reference.h" #include "handle.h" #include "handle_scope-inl.h" -#include "mem_map.h" #include "mirror/class-inl.h" #include "mirror/class_loader.h" #include "monitor.h" diff --git a/runtime/runtime_common.cc b/runtime/runtime_common.cc index 59af9187f9b..eae2505ce92 100644 --- a/runtime/runtime_common.cc +++ b/runtime/runtime_common.cc @@ -371,30 +371,11 @@ static bool IsTimeoutSignal(int signal_number) { #pragma GCC diagnostic ignored "-Wframe-larger-than=" #endif -void HandleUnexpectedSignalCommon(int signal_number, - siginfo_t* info, - void* raw_context, - bool handle_timeout_signal, - bool dump_on_stderr) { - static bool handling_unexpected_signal = false; - if (handling_unexpected_signal) { - LogHelper::LogLineLowStack(__FILE__, - __LINE__, - ::android::base::FATAL_WITHOUT_ABORT, - "HandleUnexpectedSignal reentered\n"); - if (handle_timeout_signal) { - if (IsTimeoutSignal(signal_number)) { - // Ignore a recursive timeout. - return; - } - } - _exit(1); - } - handling_unexpected_signal = true; - - gAborting++; // set before taking any locks - MutexLock mu(Thread::Current(), *Locks::unexpected_signal_lock_); - +static void HandleUnexpectedSignalCommonDump(int signal_number, + siginfo_t* info, + void* raw_context, + bool handle_timeout_signal, + bool dump_on_stderr) { auto logger = [&](auto& stream) { bool has_address = (signal_number == SIGILL || signal_number == SIGBUS || signal_number == SIGFPE || signal_number == SIGSEGV); @@ -453,6 +434,71 @@ void HandleUnexpectedSignalCommon(int signal_number, } } +void HandleUnexpectedSignalCommon(int signal_number, + siginfo_t* info, + void* raw_context, + bool handle_timeout_signal, + bool dump_on_stderr) { + // Local _static_ storing the currently handled signal (or -1). + static int handling_unexpected_signal = -1; + + // Whether the dump code should be run under the unexpected-signal lock. For diagnostics we + // allow recursive unexpected-signals in certain cases - avoid a deadlock. + bool grab_lock = true; + + if (handling_unexpected_signal != -1) { + LogHelper::LogLineLowStack(__FILE__, + __LINE__, + ::android::base::FATAL_WITHOUT_ABORT, + "HandleUnexpectedSignal reentered\n"); + // Print the signal number. Don't use any standard functions, just some arithmetic. Just best + // effort, with a minimal buffer. + if (0 < signal_number && signal_number < 100) { + char buf[] = { ' ', + 'S', + static_cast<char>('0' + (signal_number / 10)), + static_cast<char>('0' + (signal_number % 10)), + '\n', + 0 }; + LogHelper::LogLineLowStack(__FILE__, + __LINE__, + ::android::base::FATAL_WITHOUT_ABORT, + buf); + } + if (handle_timeout_signal) { + if (IsTimeoutSignal(signal_number)) { + // Ignore a recursive timeout. + return; + } + } + // If we were handling a timeout signal, try to go on. Otherwise hard-exit. + // This relies on the expectation that we'll only ever get one timeout signal. + if (!handle_timeout_signal || handling_unexpected_signal != GetTimeoutSignal()) { + _exit(1); + } + grab_lock = false; // The "outer" handling instance already holds the lock. + } + handling_unexpected_signal = signal_number; + + gAborting++; // set before taking any locks + + if (grab_lock) { + MutexLock mu(Thread::Current(), *Locks::unexpected_signal_lock_); + + HandleUnexpectedSignalCommonDump(signal_number, + info, + raw_context, + handle_timeout_signal, + dump_on_stderr); + } else { + HandleUnexpectedSignalCommonDump(signal_number, + info, + raw_context, + handle_timeout_signal, + dump_on_stderr); + } +} + #if defined(__APPLE__) #pragma GCC diagnostic pop #endif diff --git a/runtime/stack_map.cc b/runtime/stack_map.cc index 250ff2af1a1..9c7b6875cf4 100644 --- a/runtime/stack_map.cc +++ b/runtime/stack_map.cc @@ -19,7 +19,7 @@ #include <stdint.h> #include "art_method.h" -#include "indenter.h" +#include "base/indenter.h" #include "scoped_thread_state_change-inl.h" namespace art { diff --git a/runtime/stack_map.h b/runtime/stack_map.h index bde34624374..38397643b6b 100644 --- a/runtime/stack_map.h +++ b/runtime/stack_map.h @@ -20,12 +20,12 @@ #include <limits> #include "arch/code_offset.h" +#include "base/bit_memory_region.h" #include "base/bit_utils.h" #include "base/bit_vector.h" #include "base/leb128.h" -#include "bit_memory_region.h" +#include "base/memory_region.h" #include "dex/dex_file_types.h" -#include "memory_region.h" #include "method_info.h" namespace art { diff --git a/runtime/subtype_check.h b/runtime/subtype_check.h index 3b1d5f8c4a7..1fe62e8f461 100644 --- a/runtime/subtype_check.h +++ b/runtime/subtype_check.h @@ -286,6 +286,17 @@ struct SubtypeCheck { return SubtypeCheckInfo::kUninitialized; } + // Retrieve the state of this class's SubtypeCheckInfo. + // + // Cost: O(Depth(Class)). + // + // Returns: The precise SubtypeCheckInfo::State. + static SubtypeCheckInfo::State GetState(ClassPtr klass) + REQUIRES(Locks::subtype_check_lock_) + REQUIRES_SHARED(Locks::mutator_lock_) { + return GetSubtypeCheckInfo(klass).GetState(); + } + // Retrieve the path to root bitstring as a plain uintN_t value that is amenable to // be used by a fast check "encoded_src & mask_target == encoded_target". // @@ -308,8 +319,9 @@ struct SubtypeCheck { static BitString::StorageType GetEncodedPathToRootForTarget(ClassPtr klass) REQUIRES(Locks::subtype_check_lock_) REQUIRES_SHARED(Locks::mutator_lock_) { - DCHECK_EQ(SubtypeCheckInfo::kAssigned, GetSubtypeCheckInfo(klass).GetState()); - return GetSubtypeCheckInfo(klass).GetEncodedPathToRoot(); + SubtypeCheckInfo sci = GetSubtypeCheckInfo(klass); + DCHECK_EQ(SubtypeCheckInfo::kAssigned, sci.GetState()); + return sci.GetEncodedPathToRoot(); } // Retrieve the path to root bitstring mask as a plain uintN_t value that is amenable to @@ -321,8 +333,9 @@ struct SubtypeCheck { static BitString::StorageType GetEncodedPathToRootMask(ClassPtr klass) REQUIRES(Locks::subtype_check_lock_) REQUIRES_SHARED(Locks::mutator_lock_) { - DCHECK_EQ(SubtypeCheckInfo::kAssigned, GetSubtypeCheckInfo(klass).GetState()); - return GetSubtypeCheckInfo(klass).GetEncodedPathToRootMask(); + SubtypeCheckInfo sci = GetSubtypeCheckInfo(klass); + DCHECK_EQ(SubtypeCheckInfo::kAssigned, sci.GetState()); + return sci.GetEncodedPathToRootMask(); } // Is the source class a subclass of the target? diff --git a/runtime/thread-inl.h b/runtime/thread-inl.h index 2f6f50e31ee..e34f32e0bff 100644 --- a/runtime/thread-inl.h +++ b/runtime/thread-inl.h @@ -251,6 +251,7 @@ inline ThreadState Thread::TransitionFromSuspendedToRunnable() { union StateAndFlags new_state_and_flags; new_state_and_flags.as_int = old_state_and_flags.as_int; new_state_and_flags.as_struct.state = kRunnable; + // CAS the value with a memory barrier. if (LIKELY(tls32_.state_and_flags.as_atomic_int.CompareAndSetWeakAcquire( old_state_and_flags.as_int, diff --git a/runtime/thread.cc b/runtime/thread.cc index ea6c071fa7a..f6ac64f7bdd 100644 --- a/runtime/thread.cc +++ b/runtime/thread.cc @@ -1280,7 +1280,7 @@ bool Thread::ModifySuspendCountInternal(Thread* self, AtomicClearFlag(kSuspendRequest); } else { // Two bits might be set simultaneously. - tls32_.state_and_flags.as_atomic_int.FetchAndBitwiseOrSequentiallyConsistent(flags); + tls32_.state_and_flags.as_atomic_int.fetch_or(flags, std::memory_order_seq_cst); TriggerSuspend(); } return true; @@ -1318,7 +1318,7 @@ bool Thread::PassActiveSuspendBarriers(Thread* self) { if (pending_threads != nullptr) { bool done = false; do { - int32_t cur_val = pending_threads->LoadRelaxed(); + int32_t cur_val = pending_threads->load(std::memory_order_relaxed); CHECK_GT(cur_val, 0) << "Unexpected value for PassActiveSuspendBarriers(): " << cur_val; // Reduce value by 1. done = pending_threads->CompareAndSetWeakRelaxed(cur_val, cur_val - 1); @@ -1562,7 +1562,7 @@ Closure* Thread::GetFlipFunction() { Atomic<Closure*>* atomic_func = reinterpret_cast<Atomic<Closure*>*>(&tlsPtr_.flip_function); Closure* func; do { - func = atomic_func->LoadRelaxed(); + func = atomic_func->load(std::memory_order_relaxed); if (func == nullptr) { return nullptr; } @@ -1574,7 +1574,7 @@ Closure* Thread::GetFlipFunction() { void Thread::SetFlipFunction(Closure* function) { CHECK(function != nullptr); Atomic<Closure*>* atomic_func = reinterpret_cast<Atomic<Closure*>*>(&tlsPtr_.flip_function); - atomic_func->StoreSequentiallyConsistent(function); + atomic_func->store(function, std::memory_order_seq_cst); } void Thread::FullSuspendCheck() { @@ -2106,7 +2106,7 @@ Thread::Thread(bool daemon) "art::Thread has a size which is not a multiple of 4."); tls32_.state_and_flags.as_struct.flags = 0; tls32_.state_and_flags.as_struct.state = kNative; - tls32_.interrupted.StoreRelaxed(false); + tls32_.interrupted.store(false, std::memory_order_relaxed); memset(&tlsPtr_.held_mutexes[0], 0, sizeof(tlsPtr_.held_mutexes)); std::fill(tlsPtr_.rosalloc_runs, tlsPtr_.rosalloc_runs + kNumRosAllocThreadLocalSizeBracketsInThread, @@ -2401,24 +2401,24 @@ bool Thread::IsJWeakCleared(jweak obj) const { bool Thread::Interrupted() { DCHECK_EQ(Thread::Current(), this); // No other thread can concurrently reset the interrupted flag. - bool interrupted = tls32_.interrupted.LoadSequentiallyConsistent(); + bool interrupted = tls32_.interrupted.load(std::memory_order_seq_cst); if (interrupted) { - tls32_.interrupted.StoreSequentiallyConsistent(false); + tls32_.interrupted.store(false, std::memory_order_seq_cst); } return interrupted; } // Implements java.lang.Thread.isInterrupted. bool Thread::IsInterrupted() { - return tls32_.interrupted.LoadSequentiallyConsistent(); + return tls32_.interrupted.load(std::memory_order_seq_cst); } void Thread::Interrupt(Thread* self) { MutexLock mu(self, *wait_mutex_); - if (tls32_.interrupted.LoadSequentiallyConsistent()) { + if (tls32_.interrupted.load(std::memory_order_seq_cst)) { return; } - tls32_.interrupted.StoreSequentiallyConsistent(true); + tls32_.interrupted.store(true, std::memory_order_seq_cst); NotifyLocked(self); } @@ -3021,7 +3021,8 @@ void Thread::ThrowNewWrappedException(const char* exception_class_descriptor, // If we couldn't allocate the exception, throw the pre-allocated out of memory exception. if (exception == nullptr) { - SetException(Runtime::Current()->GetPreAllocatedOutOfMemoryError()); + Dump(LOG_STREAM(WARNING)); // The pre-allocated OOME has no stack, so help out and log one. + SetException(Runtime::Current()->GetPreAllocatedOutOfMemoryErrorWhenThrowingException()); return; } @@ -3101,7 +3102,7 @@ void Thread::ThrowOutOfMemoryError(const char* msg) { tls32_.throwing_OutOfMemoryError = false; } else { Dump(LOG_STREAM(WARNING)); // The pre-allocated OOME has no stack, so help out and log one. - SetException(Runtime::Current()->GetPreAllocatedOutOfMemoryError()); + SetException(Runtime::Current()->GetPreAllocatedOutOfMemoryErrorWhenThrowingOOME()); } } diff --git a/runtime/thread.h b/runtime/thread.h index 9adae96a9ce..22b77eea64c 100644 --- a/runtime/thread.h +++ b/runtime/thread.h @@ -556,7 +556,7 @@ class Thread { bool IsInterrupted(); void Interrupt(Thread* self) REQUIRES(!*wait_mutex_); void SetInterrupted(bool i) { - tls32_.interrupted.StoreSequentiallyConsistent(i); + tls32_.interrupted.store(i, std::memory_order_seq_cst); } void Notify() REQUIRES(!*wait_mutex_); @@ -1110,11 +1110,11 @@ class Thread { } void AtomicSetFlag(ThreadFlag flag) { - tls32_.state_and_flags.as_atomic_int.FetchAndBitwiseOrSequentiallyConsistent(flag); + tls32_.state_and_flags.as_atomic_int.fetch_or(flag, std::memory_order_seq_cst); } void AtomicClearFlag(ThreadFlag flag) { - tls32_.state_and_flags.as_atomic_int.FetchAndBitwiseAndSequentiallyConsistent(-1 ^ flag); + tls32_.state_and_flags.as_atomic_int.fetch_and(-1 ^ flag, std::memory_order_seq_cst); } void ResetQuickAllocEntryPointsForThread(bool is_marking); diff --git a/runtime/thread_list.cc b/runtime/thread_list.cc index 8095ef57c70..44af867d602 100644 --- a/runtime/thread_list.cc +++ b/runtime/thread_list.cc @@ -732,7 +732,7 @@ void ThreadList::SuspendAllInternal(Thread* self, if (reason == SuspendReason::kForDebugger) { ++debug_suspend_all_count_; } - pending_threads.StoreRelaxed(list_.size() - num_ignored); + pending_threads.store(list_.size() - num_ignored, std::memory_order_relaxed); // Increment everybody's suspend count (except those that should be ignored). for (const auto& thread : list_) { if (thread == ignore1 || thread == ignore2) { @@ -748,7 +748,7 @@ void ThreadList::SuspendAllInternal(Thread* self, if (thread->IsSuspended()) { // Only clear the counter for the current thread. thread->ClearSuspendBarrier(&pending_threads); - pending_threads.FetchAndSubSequentiallyConsistent(1); + pending_threads.fetch_sub(1, std::memory_order_seq_cst); } } } @@ -761,7 +761,7 @@ void ThreadList::SuspendAllInternal(Thread* self, #endif const uint64_t start_time = NanoTime(); while (true) { - int32_t cur_val = pending_threads.LoadRelaxed(); + int32_t cur_val = pending_threads.load(std::memory_order_relaxed); if (LIKELY(cur_val > 0)) { #if ART_USE_FUTEXES if (futex(pending_threads.Address(), FUTEX_WAIT, cur_val, &wait_timeout, nullptr, 0) != 0) { diff --git a/runtime/thread_pool.h b/runtime/thread_pool.h index a465e110557..2784953d69d 100644 --- a/runtime/thread_pool.h +++ b/runtime/thread_pool.h @@ -21,8 +21,8 @@ #include <vector> #include "barrier.h" +#include "base/mem_map.h" #include "base/mutex.h" -#include "mem_map.h" namespace art { diff --git a/runtime/thread_pool_test.cc b/runtime/thread_pool_test.cc index 895a108af0f..d7842002ee2 100644 --- a/runtime/thread_pool_test.cc +++ b/runtime/thread_pool_test.cc @@ -71,7 +71,7 @@ TEST_F(ThreadPoolTest, CheckRun) { // Wait for tasks to complete. thread_pool.Wait(self, true, false); // Make sure that we finished all the work. - EXPECT_EQ(num_tasks, count.LoadSequentiallyConsistent()); + EXPECT_EQ(num_tasks, count.load(std::memory_order_seq_cst)); } TEST_F(ThreadPoolTest, StopStart) { @@ -84,7 +84,7 @@ TEST_F(ThreadPoolTest, StopStart) { } usleep(200); // Check that no threads started prematurely. - EXPECT_EQ(0, count.LoadSequentiallyConsistent()); + EXPECT_EQ(0, count.load(std::memory_order_seq_cst)); // Signal the threads to start processing tasks. thread_pool.StartWorkers(self); usleep(200); @@ -93,7 +93,7 @@ TEST_F(ThreadPoolTest, StopStart) { thread_pool.AddTask(self, new CountTask(&bad_count)); usleep(200); // Ensure that the task added after the workers were stopped doesn't get run. - EXPECT_EQ(0, bad_count.LoadSequentiallyConsistent()); + EXPECT_EQ(0, bad_count.load(std::memory_order_seq_cst)); // Allow tasks to finish up and delete themselves. thread_pool.StartWorkers(self); thread_pool.Wait(self, false, false); @@ -157,7 +157,7 @@ TEST_F(ThreadPoolTest, RecursiveTest) { thread_pool.AddTask(self, new TreeTask(&thread_pool, &count, depth)); thread_pool.StartWorkers(self); thread_pool.Wait(self, true, false); - EXPECT_EQ((1 << depth) - 1, count.LoadSequentiallyConsistent()); + EXPECT_EQ((1 << depth) - 1, count.load(std::memory_order_seq_cst)); } class PeerTask : public Task { diff --git a/runtime/trace.cc b/runtime/trace.cc index 0f321b6591b..292cac6d0ab 100644 --- a/runtime/trace.cc +++ b/runtime/trace.cc @@ -319,8 +319,74 @@ void* Trace::RunSamplingThread(void* arg) { return nullptr; } -void Trace::Start(const char* trace_filename, int trace_fd, size_t buffer_size, int flags, - TraceOutputMode output_mode, TraceMode trace_mode, int interval_us) { +void Trace::Start(const char* trace_filename, + size_t buffer_size, + int flags, + TraceOutputMode output_mode, + TraceMode trace_mode, + int interval_us) { + std::unique_ptr<File> file(OS::CreateEmptyFileWriteOnly(trace_filename)); + if (file == nullptr) { + std::string msg = android::base::StringPrintf("Unable to open trace file '%s'", trace_filename); + PLOG(ERROR) << msg; + ScopedObjectAccess soa(Thread::Current()); + Thread::Current()->ThrowNewException("Ljava/lang/RuntimeException;", msg.c_str()); + return; + } + Start(std::move(file), buffer_size, flags, output_mode, trace_mode, interval_us); +} + +void Trace::Start(int trace_fd, + size_t buffer_size, + int flags, + TraceOutputMode output_mode, + TraceMode trace_mode, + int interval_us) { + if (trace_fd < 0) { + std::string msg = android::base::StringPrintf("Unable to start tracing with invalid fd %d", + trace_fd); + LOG(ERROR) << msg; + ScopedObjectAccess soa(Thread::Current()); + Thread::Current()->ThrowNewException("Ljava/lang/RuntimeException;", msg.c_str()); + return; + } + std::unique_ptr<File> file(new File(trace_fd, "tracefile")); + Start(std::move(file), buffer_size, flags, output_mode, trace_mode, interval_us); +} + +void Trace::StartDDMS(size_t buffer_size, + int flags, + TraceMode trace_mode, + int interval_us) { + Start(std::unique_ptr<File>(), + buffer_size, + flags, + TraceOutputMode::kDDMS, + trace_mode, + interval_us); +} + +void Trace::Start(std::unique_ptr<File>&& trace_file_in, + size_t buffer_size, + int flags, + TraceOutputMode output_mode, + TraceMode trace_mode, + int interval_us) { + // We own trace_file now and are responsible for closing it. To account for error situations, use + // a specialized unique_ptr to ensure we close it on the way out (if it hasn't been passed to a + // Trace instance). + auto deleter = [](File* file) { + if (file != nullptr) { + file->MarkUnchecked(); // Don't deal with flushing requirements. + int result ATTRIBUTE_UNUSED = file->Close(); + delete file; + } + }; + std::unique_ptr<File, decltype(deleter)> trace_file(trace_file_in.release(), deleter); + if (trace_file != nullptr) { + trace_file->DisableAutoClose(); + } + Thread* self = Thread::Current(); { MutexLock mu(self, *Locks::trace_lock_); @@ -338,23 +404,6 @@ void Trace::Start(const char* trace_filename, int trace_fd, size_t buffer_size, return; } - // Open trace file if not going directly to ddms. - std::unique_ptr<File> trace_file; - if (output_mode != TraceOutputMode::kDDMS) { - if (trace_fd < 0) { - trace_file.reset(OS::CreateEmptyFileWriteOnly(trace_filename)); - } else { - trace_file.reset(new File(trace_fd, "tracefile")); - trace_file->DisableAutoClose(); - } - if (trace_file.get() == nullptr) { - PLOG(ERROR) << "Unable to open trace file '" << trace_filename << "'"; - ScopedObjectAccess soa(self); - ThrowRuntimeException("Unable to open trace file '%s'", trace_filename); - return; - } - } - Runtime* runtime = Runtime::Current(); // Enable count of allocs if specified in the flags. @@ -372,8 +421,7 @@ void Trace::Start(const char* trace_filename, int trace_fd, size_t buffer_size, LOG(ERROR) << "Trace already in progress, ignoring this request"; } else { enable_stats = (flags && kTraceCountAllocs) != 0; - the_trace_ = new Trace(trace_file.release(), trace_filename, buffer_size, flags, output_mode, - trace_mode); + the_trace_ = new Trace(trace_file.release(), buffer_size, flags, output_mode, trace_mode); if (trace_mode == TraceMode::kSampling) { CHECK_PTHREAD_CALL(pthread_create, (&sampling_pthread_, nullptr, &RunSamplingThread, reinterpret_cast<void*>(interval_us)), @@ -422,24 +470,30 @@ void Trace::StopTracing(bool finish_tracing, bool flush_file) { if (the_trace != nullptr) { stop_alloc_counting = (the_trace->flags_ & Trace::kTraceCountAllocs) != 0; + // Stop the trace sources adding more entries to the trace buffer and synchronise stores. + { + gc::ScopedGCCriticalSection gcs(self, + gc::kGcCauseInstrumentation, + gc::kCollectorTypeInstrumentation); + ScopedSuspendAll ssa(__FUNCTION__); + + if (the_trace->trace_mode_ == TraceMode::kSampling) { + MutexLock mu(self, *Locks::thread_list_lock_); + runtime->GetThreadList()->ForEach(ClearThreadStackTraceAndClockBase, nullptr); + } else { + runtime->GetInstrumentation()->DisableMethodTracing(kTracerInstrumentationKey); + runtime->GetInstrumentation()->RemoveListener( + the_trace, instrumentation::Instrumentation::kMethodEntered | + instrumentation::Instrumentation::kMethodExited | + instrumentation::Instrumentation::kMethodUnwind); + } + } + // At this point, code may read buf_ as it's writers are shutdown + // and the ScopedSuspendAll above has ensured all stores to buf_ + // are now visible. if (finish_tracing) { the_trace->FinishTracing(); } - gc::ScopedGCCriticalSection gcs(self, - gc::kGcCauseInstrumentation, - gc::kCollectorTypeInstrumentation); - ScopedSuspendAll ssa(__FUNCTION__); - - if (the_trace->trace_mode_ == TraceMode::kSampling) { - MutexLock mu(self, *Locks::thread_list_lock_); - runtime->GetThreadList()->ForEach(ClearThreadStackTraceAndClockBase, nullptr); - } else { - runtime->GetInstrumentation()->DisableMethodTracing(kTracerInstrumentationKey); - runtime->GetInstrumentation()->RemoveListener( - the_trace, instrumentation::Instrumentation::kMethodEntered | - instrumentation::Instrumentation::kMethodExited | - instrumentation::Instrumentation::kMethodUnwind); - } if (the_trace->trace_file_.get() != nullptr) { // Do not try to erase, so flush and close explicitly. if (flush_file) { @@ -595,16 +649,21 @@ TracingMode Trace::GetMethodTracingMode() { static constexpr size_t kMinBufSize = 18U; // Trace header is up to 18B. -Trace::Trace(File* trace_file, const char* trace_name, size_t buffer_size, int flags, - TraceOutputMode output_mode, TraceMode trace_mode) +Trace::Trace(File* trace_file, + size_t buffer_size, + int flags, + TraceOutputMode output_mode, + TraceMode trace_mode) : trace_file_(trace_file), buf_(new uint8_t[std::max(kMinBufSize, buffer_size)]()), flags_(flags), trace_output_mode_(output_mode), trace_mode_(trace_mode), clock_source_(default_clock_source_), buffer_size_(std::max(kMinBufSize, buffer_size)), - start_time_(MicroTime()), clock_overhead_ns_(GetClockOverheadNanoSeconds()), cur_offset_(0), + start_time_(MicroTime()), clock_overhead_ns_(GetClockOverheadNanoSeconds()), overflow_(false), interval_us_(0), streaming_lock_(nullptr), unique_methods_lock_(new Mutex("unique methods lock", kTracingUniqueMethodsLock)) { + CHECK(trace_file != nullptr || output_mode == TraceOutputMode::kDDMS); + uint16_t trace_version = GetTraceVersion(clock_source_); if (output_mode == TraceOutputMode::kStreaming) { trace_version |= 0xF0U; @@ -621,11 +680,9 @@ Trace::Trace(File* trace_file, const char* trace_name, size_t buffer_size, int f } static_assert(18 <= kMinBufSize, "Minimum buffer size not large enough for trace header"); - // Update current offset. - cur_offset_.StoreRelaxed(kTraceHeaderLength); + cur_offset_.store(kTraceHeaderLength, std::memory_order_relaxed); if (output_mode == TraceOutputMode::kStreaming) { - streaming_file_name_ = trace_name; streaming_lock_ = new Mutex("tracing lock", LockLevel::kTracingStreamingLock); seen_threads_.reset(new ThreadIDBitSet()); } @@ -659,13 +716,13 @@ void Trace::DumpBuf(uint8_t* buf, size_t buf_size, TraceClockSource clock_source void Trace::FinishTracing() { size_t final_offset = 0; - std::set<ArtMethod*> visited_methods; if (trace_output_mode_ == TraceOutputMode::kStreaming) { // Clean up. + MutexLock mu(Thread::Current(), *streaming_lock_); STLDeleteValues(&seen_methods_); } else { - final_offset = cur_offset_.LoadRelaxed(); + final_offset = cur_offset_.load(std::memory_order_relaxed); GetVisitedMethods(final_offset, &visited_methods); } @@ -707,7 +764,8 @@ void Trace::FinishTracing() { std::string header(os.str()); if (trace_output_mode_ == TraceOutputMode::kStreaming) { - MutexLock mu(Thread::Current(), *streaming_lock_); // To serialize writing. + // Protect access to buf_ and satisfy sanitizer for calls to WriteBuf / FlushBuf. + MutexLock mu(Thread::Current(), *streaming_lock_); // Write a special token to mark the end of trace records and the start of // trace summary. uint8_t buf[7]; @@ -892,7 +950,8 @@ std::string Trace::GetMethodLine(ArtMethod* method) { } void Trace::WriteToBuf(const uint8_t* src, size_t src_size) { - int32_t old_offset = cur_offset_.LoadRelaxed(); + // Updates to cur_offset_ are done under the streaming_lock_ here as in streaming mode. + int32_t old_offset = cur_offset_.load(std::memory_order_relaxed); int32_t new_offset = old_offset + static_cast<int32_t>(src_size); if (dchecked_integral_cast<size_t>(new_offset) > buffer_size_) { // Flush buffer. @@ -905,46 +964,59 @@ void Trace::WriteToBuf(const uint8_t* src, size_t src_size) { if (!trace_file_->WriteFully(src, src_size)) { PLOG(WARNING) << "Failed streaming a tracing event."; } - cur_offset_.StoreRelease(0); // Buffer is empty now. + cur_offset_.store(0, std::memory_order_relaxed); // Buffer is empty now. return; } old_offset = 0; new_offset = static_cast<int32_t>(src_size); } - cur_offset_.StoreRelease(new_offset); + cur_offset_.store(new_offset, std::memory_order_relaxed); // Fill in data. memcpy(buf_.get() + old_offset, src, src_size); } void Trace::FlushBuf() { - int32_t offset = cur_offset_.LoadRelaxed(); + // Updates to cur_offset_ are done under the streaming_lock_ here as in streaming mode. + int32_t offset = cur_offset_.load(std::memory_order_relaxed); if (!trace_file_->WriteFully(buf_.get(), offset)) { PLOG(WARNING) << "Failed flush the remaining data in streaming."; } - cur_offset_.StoreRelease(0); + cur_offset_.store(0, std::memory_order_relaxed); } void Trace::LogMethodTraceEvent(Thread* thread, ArtMethod* method, instrumentation::Instrumentation::InstrumentationEvent event, uint32_t thread_clock_diff, uint32_t wall_clock_diff) { + // This method is called in both tracing modes (method and + // sampling). In sampling mode, this method is only called by the + // sampling thread. In method tracing mode, it can be called + // concurrently. + // Ensure we always use the non-obsolete version of the method so that entry/exit events have the // same pointer value. method = method->GetNonObsoleteMethod(); + // Advance cur_offset_ atomically. int32_t new_offset; int32_t old_offset = 0; - // We do a busy loop here trying to acquire the next offset. + // In the non-streaming case, we do a busy loop here trying to get + // an offset to write our record and advance cur_offset_ for the + // next use. if (trace_output_mode_ != TraceOutputMode::kStreaming) { + // Although multiple threads can call this method concurrently, + // the compare_exchange_weak here is still atomic (by definition). + // A succeeding update is visible to other cores when they pass + // through this point. + old_offset = cur_offset_.load(std::memory_order_relaxed); // Speculative read do { - old_offset = cur_offset_.LoadRelaxed(); new_offset = old_offset + GetRecordSize(clock_source_); if (static_cast<size_t>(new_offset) > buffer_size_) { overflow_ = true; return; } - } while (!cur_offset_.CompareAndSetWeakSequentiallyConsistent(old_offset, new_offset)); + } while (!cur_offset_.compare_exchange_weak(old_offset, new_offset, std::memory_order_relaxed)); } TraceAction action = kTraceMethodEnter; @@ -964,7 +1036,14 @@ void Trace::LogMethodTraceEvent(Thread* thread, ArtMethod* method, uint32_t method_value = EncodeTraceMethodAndAction(method, action); - // Write data + // Write data into the tracing buffer (if not streaming) or into a + // small buffer on the stack (if streaming) which we'll put into the + // tracing buffer below. + // + // These writes to the tracing buffer are synchronised with the + // future reads that (only) occur under FinishTracing(). The callers + // of FinishTracing() acquire locks and (implicitly) synchronise + // the buffer memory. uint8_t* ptr; static constexpr size_t kPacketSize = 14U; // The maximum size of data in a packet. uint8_t stack_buf[kPacketSize]; // Space to store a packet when in streaming mode. diff --git a/runtime/trace.h b/runtime/trace.h index 86b8d00d515..b242d1596c4 100644 --- a/runtime/trace.h +++ b/runtime/trace.h @@ -33,6 +33,10 @@ #include "globals.h" #include "instrumentation.h" +namespace unix_file { +class FdFile; +} // namespace unix_file + namespace art { class ArtField; @@ -48,9 +52,10 @@ using ThreadIDBitSet = std::bitset<kMaxThreadIdNumber>; enum TracingMode { kTracingInactive, - kMethodTracingActive, - kSampleProfilingActive, + kMethodTracingActive, // Trace activity synchronous with method progress. + kSampleProfilingActive, // Trace activity captured by sampling thread. }; +std::ostream& operator<<(std::ostream& os, const TracingMode& rhs); // File format: // header @@ -94,6 +99,9 @@ enum TraceAction { kTraceMethodActionMask = 0x03, // two bits }; +// Class for recording event traces. Trace data is either collected +// synchronously during execution (TracingMode::kMethodTracingActive), +// or by a separate sampling thread (TracingMode::kSampleProfilingActive). class Trace FINAL : public instrumentation::InstrumentationListener { public: enum TraceFlag { @@ -115,10 +123,37 @@ class Trace FINAL : public instrumentation::InstrumentationListener { static void SetDefaultClockSource(TraceClockSource clock_source); - static void Start(const char* trace_filename, int trace_fd, size_t buffer_size, int flags, - TraceOutputMode output_mode, TraceMode trace_mode, int interval_us) + static void Start(const char* trace_filename, + size_t buffer_size, + int flags, + TraceOutputMode output_mode, + TraceMode trace_mode, + int interval_us) + REQUIRES(!Locks::mutator_lock_, !Locks::thread_list_lock_, !Locks::thread_suspend_count_lock_, + !Locks::trace_lock_); + static void Start(int trace_fd, + size_t buffer_size, + int flags, + TraceOutputMode output_mode, + TraceMode trace_mode, + int interval_us) + REQUIRES(!Locks::mutator_lock_, !Locks::thread_list_lock_, !Locks::thread_suspend_count_lock_, + !Locks::trace_lock_); + static void Start(std::unique_ptr<unix_file::FdFile>&& file, + size_t buffer_size, + int flags, + TraceOutputMode output_mode, + TraceMode trace_mode, + int interval_us) + REQUIRES(!Locks::mutator_lock_, !Locks::thread_list_lock_, !Locks::thread_suspend_count_lock_, + !Locks::trace_lock_); + static void StartDDMS(size_t buffer_size, + int flags, + TraceMode trace_mode, + int interval_us) REQUIRES(!Locks::mutator_lock_, !Locks::thread_list_lock_, !Locks::thread_suspend_count_lock_, !Locks::trace_lock_); + static void Pause() REQUIRES(!Locks::trace_lock_, !Locks::thread_list_lock_); static void Resume() REQUIRES(!Locks::trace_lock_); @@ -212,8 +247,11 @@ class Trace FINAL : public instrumentation::InstrumentationListener { static bool IsTracingEnabled() REQUIRES(!Locks::trace_lock_); private: - Trace(File* trace_file, const char* trace_name, size_t buffer_size, int flags, - TraceOutputMode output_mode, TraceMode trace_mode); + Trace(File* trace_file, + size_t buffer_size, + int flags, + TraceOutputMode output_mode, + TraceMode trace_mode); // The sampling interval in microseconds is passed as an argument. static void* RunSamplingThread(void* arg) REQUIRES(!Locks::trace_lock_); @@ -282,7 +320,10 @@ class Trace FINAL : public instrumentation::InstrumentationListener { // File to write trace data out to, null if direct to ddms. std::unique_ptr<File> trace_file_; - // Buffer to store trace data. + // Buffer to store trace data. In streaming mode, this is protected + // by the streaming_lock_. In non-streaming mode, reserved regions + // are atomically allocated (using cur_offset_) for log entries to + // be written. std::unique_ptr<uint8_t[]> buf_; // Flags enabling extra tracing of things such as alloc counts. @@ -305,7 +346,27 @@ class Trace FINAL : public instrumentation::InstrumentationListener { // Clock overhead. const uint32_t clock_overhead_ns_; - // Offset into buf_. + // Offset into buf_. The field is atomic to allow multiple writers + // to concurrently reserve space in the buffer. The newly written + // buffer contents are not read without some other form of thread + // synchronization, such as suspending all potential writers or + // acquiring *streaming_lock_. Reading cur_offset_ is thus never + // used to ensure visibility of any other objects, and all accesses + // are memory_order_relaxed. + // + // All accesses to buf_ in streaming mode occur whilst holding the + // streaming lock. In streaming mode, the buffer may be written out + // so cur_offset_ can move forwards and backwards. + // + // When not in streaming mode, the buf_ writes can come from + // multiple threads when the trace mode is kMethodTracing. When + // trace mode is kSampling, writes only come from the sampling + // thread. + // + // Reads to the buffer happen after the event sources writing to the + // buffer have been shutdown and all stores have completed. The + // stores are made visible in StopTracing() when execution leaves + // the ScopedSuspendAll block. AtomicInteger cur_offset_; // Did we overflow the buffer recording traces? @@ -318,10 +379,9 @@ class Trace FINAL : public instrumentation::InstrumentationListener { int interval_us_; // Streaming mode data. - std::string streaming_file_name_; Mutex* streaming_lock_; - std::map<const DexFile*, DexIndexBitSet*> seen_methods_; - std::unique_ptr<ThreadIDBitSet> seen_threads_; + std::map<const DexFile*, DexIndexBitSet*> seen_methods_ GUARDED_BY(streaming_lock_); + std::unique_ptr<ThreadIDBitSet> seen_threads_ GUARDED_BY(streaming_lock_); // Bijective map from ArtMethod* to index. // Map from ArtMethod* to index in unique_methods_; diff --git a/runtime/vdex_file.h b/runtime/vdex_file.h index 326fcbc1fe4..b7f28f0d03d 100644 --- a/runtime/vdex_file.h +++ b/runtime/vdex_file.h @@ -22,9 +22,9 @@ #include "base/array_ref.h" #include "base/macros.h" +#include "base/mem_map.h" #include "base/os.h" #include "dex/compact_offset_table.h" -#include "mem_map.h" #include "quicken_info.h" namespace art { diff --git a/runtime/verifier/method_verifier.cc b/runtime/verifier/method_verifier.cc index dfb98b60836..bc29aaca6da 100644 --- a/runtime/verifier/method_verifier.cc +++ b/runtime/verifier/method_verifier.cc @@ -25,6 +25,7 @@ #include "base/aborting.h" #include "base/enums.h" #include "base/leb128.h" +#include "base/indenter.h" #include "base/logging.h" // For VLOG. #include "base/mutex-inl.h" #include "base/stl_util.h" @@ -41,7 +42,6 @@ #include "experimental_flags.h" #include "gc/accounting/card_table-inl.h" #include "handle_scope-inl.h" -#include "indenter.h" #include "intern_table.h" #include "mirror/class-inl.h" #include "mirror/class.h" @@ -4075,14 +4075,19 @@ bool MethodVerifier::CheckCallSite(uint32_t call_site_idx) { if (it.Size() < 3) { Fail(VERIFY_ERROR_BAD_CLASS_HARD) << "Call site #" << call_site_idx << " has too few arguments: " - << it.Size() << "< 3"; + << it.Size() << " < 3"; return false; } // Get and check the first argument: the method handle (index range // checked by the dex file verifier). uint32_t method_handle_idx = static_cast<uint32_t>(it.GetJavaValue().i); - it.Next(); + if (method_handle_idx > dex_file_->NumMethodHandles()) { + Fail(VERIFY_ERROR_BAD_CLASS_HARD) << "Call site id #" << call_site_idx + << " method handle index invalid " << method_handle_idx + << " >= " << dex_file_->NumMethodHandles(); + return false; + } const DexFile::MethodHandleItem& mh = dex_file_->GetMethodHandle(method_handle_idx); if (mh.method_handle_type_ != static_cast<uint16_t>(DexFile::MethodHandleType::kInvokeStatic)) { @@ -4091,93 +4096,6 @@ bool MethodVerifier::CheckCallSite(uint32_t call_site_idx) { << mh.method_handle_type_; return false; } - - // Skip the second argument, the name to resolve, as checked by the - // dex file verifier. - it.Next(); - - // Skip the third argument, the method type expected, as checked by - // the dex file verifier. - it.Next(); - - // Check the bootstrap method handle and remaining arguments. - const DexFile::MethodId& method_id = dex_file_->GetMethodId(mh.field_or_method_idx_); - uint32_t length; - const char* shorty = dex_file_->GetMethodShorty(method_id, &length); - - if (it.Size() < length - 1) { - Fail(VERIFY_ERROR_BAD_CLASS_HARD) << "Call site #" << call_site_idx - << " too few arguments for bootstrap method: " - << it.Size() << " < " << (length - 1); - return false; - } - - // Check the return type and first 3 arguments are references - // (CallSite, Lookup, String, MethodType). If they are not of the - // expected types (or subtypes), it will trigger a - // WrongMethodTypeException during execution. - if (shorty[0] != 'L') { - Fail(VERIFY_ERROR_BAD_CLASS_HARD) << "Call site #" << call_site_idx - << " bootstrap return type is not a reference"; - return false; - } - - for (uint32_t i = 1; i < 4; ++i) { - if (shorty[i] != 'L') { - Fail(VERIFY_ERROR_BAD_CLASS_HARD) << "Call site #" << call_site_idx - << " bootstrap method argument " << (i - 1) - << " is not a reference"; - return false; - } - } - - // Check the optional arguments. - for (uint32_t i = 4; i < length; ++i, it.Next()) { - bool match = false; - switch (it.GetValueType()) { - case EncodedArrayValueIterator::ValueType::kBoolean: - case EncodedArrayValueIterator::ValueType::kByte: - case EncodedArrayValueIterator::ValueType::kShort: - case EncodedArrayValueIterator::ValueType::kChar: - case EncodedArrayValueIterator::ValueType::kInt: - // These all fit within one register and encoders do not seem - // too exacting on the encoding type they use (ie using - // integer for all of these). - match = (strchr("ZBCSI", shorty[i]) != nullptr); - break; - case EncodedArrayValueIterator::ValueType::kLong: - match = ('J' == shorty[i]); - break; - case EncodedArrayValueIterator::ValueType::kFloat: - match = ('F' == shorty[i]); - break; - case EncodedArrayValueIterator::ValueType::kDouble: - match = ('D' == shorty[i]); - break; - case EncodedArrayValueIterator::ValueType::kMethodType: - case EncodedArrayValueIterator::ValueType::kMethodHandle: - case EncodedArrayValueIterator::ValueType::kString: - case EncodedArrayValueIterator::ValueType::kType: - case EncodedArrayValueIterator::ValueType::kNull: - match = ('L' == shorty[i]); - break; - case EncodedArrayValueIterator::ValueType::kField: - case EncodedArrayValueIterator::ValueType::kMethod: - case EncodedArrayValueIterator::ValueType::kEnum: - case EncodedArrayValueIterator::ValueType::kArray: - case EncodedArrayValueIterator::ValueType::kAnnotation: - // Unreachable based on current EncodedArrayValueIterator::Next(). - UNREACHABLE(); - } - - if (!match) { - Fail(VERIFY_ERROR_BAD_CLASS_HARD) << "Call site #" << call_site_idx - << " bootstrap method argument " << (i - 1) - << " expected " << shorty[i] - << " got value type: " << it.GetValueType(); - return false; - } - } return true; } diff --git a/runtime/verifier/register_line.h b/runtime/verifier/register_line.h index 18ad6b5d0c5..168eb7bb635 100644 --- a/runtime/verifier/register_line.h +++ b/runtime/verifier/register_line.h @@ -22,6 +22,7 @@ #include <android-base/logging.h> +#include "base/mutex.h" #include "base/safe_map.h" #include "base/scoped_arena_containers.h" diff --git a/runtime/verifier/verifier_deps.cc b/runtime/verifier/verifier_deps.cc index 4772e538aa1..fe839f7312e 100644 --- a/runtime/verifier/verifier_deps.cc +++ b/runtime/verifier/verifier_deps.cc @@ -20,11 +20,11 @@ #include "art_field-inl.h" #include "art_method-inl.h" +#include "base/indenter.h" #include "base/leb128.h" #include "base/stl_util.h" #include "compiler_callbacks.h" #include "dex/dex_file-inl.h" -#include "indenter.h" #include "mirror/class-inl.h" #include "mirror/class_loader.h" #include "obj_ptr-inl.h" diff --git a/runtime/well_known_classes.cc b/runtime/well_known_classes.cc index 742e713774e..f5d112c30b4 100644 --- a/runtime/well_known_classes.cc +++ b/runtime/well_known_classes.cc @@ -58,10 +58,6 @@ jclass WellKnownClasses::java_lang_IllegalAccessError; jclass WellKnownClasses::java_lang_NoClassDefFoundError; jclass WellKnownClasses::java_lang_Object; jclass WellKnownClasses::java_lang_OutOfMemoryError; -jclass WellKnownClasses::java_lang_reflect_Constructor; -jclass WellKnownClasses::java_lang_reflect_Executable; -jclass WellKnownClasses::java_lang_reflect_Field; -jclass WellKnownClasses::java_lang_reflect_Method; jclass WellKnownClasses::java_lang_reflect_Parameter; jclass WellKnownClasses::java_lang_reflect_Parameter__array; jclass WellKnownClasses::java_lang_reflect_Proxy; @@ -145,7 +141,6 @@ jfieldID WellKnownClasses::java_lang_Throwable_detailMessage; jfieldID WellKnownClasses::java_lang_Throwable_stackTrace; jfieldID WellKnownClasses::java_lang_Throwable_stackState; jfieldID WellKnownClasses::java_lang_Throwable_suppressedExceptions; -jfieldID WellKnownClasses::java_lang_reflect_Executable_artMethod; jfieldID WellKnownClasses::java_lang_reflect_Proxy_h; jfieldID WellKnownClasses::java_nio_ByteBuffer_address; jfieldID WellKnownClasses::java_nio_ByteBuffer_hb; @@ -333,10 +328,6 @@ void WellKnownClasses::Init(JNIEnv* env) { java_lang_invoke_MethodHandle = CacheClass(env, "java/lang/invoke/MethodHandle"); java_lang_invoke_VarHandle = CacheClass(env, "java/lang/invoke/VarHandle"); java_lang_NoClassDefFoundError = CacheClass(env, "java/lang/NoClassDefFoundError"); - java_lang_reflect_Constructor = CacheClass(env, "java/lang/reflect/Constructor"); - java_lang_reflect_Executable = CacheClass(env, "java/lang/reflect/Executable"); - java_lang_reflect_Field = CacheClass(env, "java/lang/reflect/Field"); - java_lang_reflect_Method = CacheClass(env, "java/lang/reflect/Method"); java_lang_reflect_Parameter = CacheClass(env, "java/lang/reflect/Parameter"); java_lang_reflect_Parameter__array = CacheClass(env, "[Ljava/lang/reflect/Parameter;"); java_lang_reflect_Proxy = CacheClass(env, "java/lang/reflect/Proxy"); @@ -412,7 +403,6 @@ void WellKnownClasses::Init(JNIEnv* env) { java_lang_Throwable_stackTrace = CacheField(env, java_lang_Throwable, false, "stackTrace", "[Ljava/lang/StackTraceElement;"); java_lang_Throwable_stackState = CacheField(env, java_lang_Throwable, false, "backtrace", "Ljava/lang/Object;"); java_lang_Throwable_suppressedExceptions = CacheField(env, java_lang_Throwable, false, "suppressedExceptions", "Ljava/util/List;"); - java_lang_reflect_Executable_artMethod = CacheField(env, java_lang_reflect_Executable, false, "artMethod", "J"); java_nio_ByteBuffer_address = CacheField(env, java_nio_ByteBuffer, false, "address", "J"); java_nio_ByteBuffer_hb = CacheField(env, java_nio_ByteBuffer, false, "hb", "[B"); java_nio_ByteBuffer_isReadOnly = CacheField(env, java_nio_ByteBuffer, false, "isReadOnly", "Z"); @@ -484,10 +474,6 @@ void WellKnownClasses::Clear() { java_lang_NoClassDefFoundError = nullptr; java_lang_Object = nullptr; java_lang_OutOfMemoryError = nullptr; - java_lang_reflect_Constructor = nullptr; - java_lang_reflect_Executable = nullptr; - java_lang_reflect_Field = nullptr; - java_lang_reflect_Method = nullptr; java_lang_reflect_Parameter = nullptr; java_lang_reflect_Parameter__array = nullptr; java_lang_reflect_Proxy = nullptr; @@ -551,7 +537,7 @@ void WellKnownClasses::Clear() { dalvik_system_DexFile_fileName = nullptr; dalvik_system_DexPathList_dexElements = nullptr; dalvik_system_DexPathList__Element_dexFile = nullptr; - java_lang_reflect_Executable_artMethod = nullptr; + dalvik_system_VMRuntime_nonSdkApiUsageConsumer = nullptr; java_lang_reflect_Proxy_h = nullptr; java_lang_Thread_daemon = nullptr; java_lang_Thread_group = nullptr; diff --git a/runtime/well_known_classes.h b/runtime/well_known_classes.h index 7b1a2943d21..25c07b27ded 100644 --- a/runtime/well_known_classes.h +++ b/runtime/well_known_classes.h @@ -68,10 +68,6 @@ struct WellKnownClasses { static jclass java_lang_NoClassDefFoundError; static jclass java_lang_Object; static jclass java_lang_OutOfMemoryError; - static jclass java_lang_reflect_Constructor; - static jclass java_lang_reflect_Executable; - static jclass java_lang_reflect_Field; - static jclass java_lang_reflect_Method; static jclass java_lang_reflect_Parameter; static jclass java_lang_reflect_Parameter__array; static jclass java_lang_reflect_Proxy; @@ -138,7 +134,6 @@ struct WellKnownClasses { static jfieldID dalvik_system_DexPathList_dexElements; static jfieldID dalvik_system_DexPathList__Element_dexFile; static jfieldID dalvik_system_VMRuntime_nonSdkApiUsageConsumer; - static jfieldID java_lang_reflect_Executable_artMethod; static jfieldID java_lang_reflect_Proxy_h; static jfieldID java_lang_Thread_daemon; static jfieldID java_lang_Thread_group; diff --git a/sigchainlib/Android.bp b/sigchainlib/Android.bp index 1f490cd3b94..a151d7a6bca 100644 --- a/sigchainlib/Android.bp +++ b/sigchainlib/Android.bp @@ -35,7 +35,7 @@ cc_library { }, android: { - shared_libs: ["liblog"], + whole_static_libs: ["libasync_safe"], }, }, // Sigchainlib is whole-statically linked into binaries. For Android.mk-based binaries, @@ -56,7 +56,7 @@ cc_library_static { srcs: ["sigchain_dummy.cc"], target: { android: { - shared_libs: ["liblog"], + whole_static_libs: ["libasync_safe"], }, }, } diff --git a/sigchainlib/log.h b/sigchainlib/log.h new file mode 100644 index 00000000000..d689930c1e9 --- /dev/null +++ b/sigchainlib/log.h @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_SIGCHAINLIB_LOG_H_ +#define ART_SIGCHAINLIB_LOG_H_ + +#if defined(__ANDROID__) + +#include <async_safe/log.h> + +#define log(...) async_safe_format_log(ANDROID_LOG_ERROR, "libsigchain", __VA_ARGS__) +#define fatal async_safe_fatal + +#else + +#include <stdarg.h> +#include <stdio.h> + +static void log(const char* format, ...) { + va_list ap; + va_start(ap, format); + vprintf(format, ap); + va_end(ap); + + printf("\n"); +} + +#define fatal(...) log(__VA_ARGS__); abort() + +#endif + +#endif // ART_SIGCHAINLIB_LOG_H_ diff --git a/sigchainlib/sigchain.cc b/sigchainlib/sigchain.cc index 61346b14874..2e5f46ca69a 100644 --- a/sigchainlib/sigchain.cc +++ b/sigchainlib/sigchain.cc @@ -14,13 +14,6 @@ * limitations under the License. */ -#ifdef ART_TARGET_ANDROID -#include <android/log.h> -#else -#include <stdarg.h> -#include <iostream> -#endif - #include <dlfcn.h> #include <errno.h> #include <pthread.h> @@ -35,6 +28,7 @@ #include <type_traits> #include <utility> +#include "log.h" #include "sigchain.h" #if defined(__APPLE__) @@ -65,22 +59,7 @@ // doesn't have SA_RESTART, and raise the signal to avoid restarting syscalls that are // expected to be interrupted? -static void log(const char* format, ...) { - char buf[256]; - va_list ap; - va_start(ap, format); - vsnprintf(buf, sizeof(buf), format, ap); -#ifdef ART_TARGET_ANDROID - __android_log_write(ANDROID_LOG_ERROR, "libsigchain", buf); -#else - std::cout << buf << "\n"; -#endif - va_end(ap); -} - -#define fatal(...) log(__VA_ARGS__); abort() - -#if defined(__BIONIC__) && !defined(__LP64__) +#if defined(__BIONIC__) && !defined(__LP64__) && !defined(__mips__) static int sigismember(const sigset64_t* sigset, int signum) { return sigismember64(sigset, signum); } @@ -223,7 +202,9 @@ class SignalChain { SigactionType result; result.sa_flags = action_.sa_flags; result.sa_handler = action_.sa_handler; +#if defined(SA_RESTORER) result.sa_restorer = action_.sa_restorer; +#endif memcpy(&result.sa_mask, &action_.sa_mask, std::min(sizeof(action_.sa_mask), sizeof(result.sa_mask))); return result; @@ -237,7 +218,9 @@ class SignalChain { } else { action_.sa_flags = new_action->sa_flags; action_.sa_handler = new_action->sa_handler; +#if defined(SA_RESTORER) action_.sa_restorer = new_action->sa_restorer; +#endif sigemptyset(&action_.sa_mask); memcpy(&action_.sa_mask, &new_action->sa_mask, std::min(sizeof(action_.sa_mask), sizeof(new_action->sa_mask))); diff --git a/sigchainlib/sigchain_dummy.cc b/sigchainlib/sigchain_dummy.cc index edce965e33b..c2745309876 100644 --- a/sigchainlib/sigchain_dummy.cc +++ b/sigchainlib/sigchain_dummy.cc @@ -17,13 +17,7 @@ #include <stdio.h> #include <stdlib.h> -#ifdef ART_TARGET_ANDROID -#include <android/log.h> -#else -#include <stdarg.h> -#include <iostream> -#endif - +#include "log.h" #include "sigchain.h" #define ATTRIBUTE_UNUSED __attribute__((__unused__)) @@ -33,19 +27,6 @@ #pragma GCC diagnostic ignored "-Wunknown-pragmas" #pragma GCC diagnostic ignored "-Wmissing-noreturn" -static void log(const char* format, ...) { - char buf[256]; - va_list ap; - va_start(ap, format); - vsnprintf(buf, sizeof(buf), format, ap); -#ifdef ART_TARGET_ANDROID - __android_log_write(ANDROID_LOG_ERROR, "libsigchain", buf); -#else - std::cout << buf << "\n"; -#endif - va_end(ap); -} - namespace art { extern "C" void EnsureFrontOfChain(int signal ATTRIBUTE_UNUSED) { diff --git a/sigchainlib/sigchain_test.cc b/sigchainlib/sigchain_test.cc index 9f338ad3a2c..1d1e54f1279 100644 --- a/sigchainlib/sigchain_test.cc +++ b/sigchainlib/sigchain_test.cc @@ -50,7 +50,7 @@ static int sigismember64(sigset64_t* set, int member) { static int RealSigprocmask(int how, const sigset64_t* new_sigset, sigset64_t* old_sigset) { // glibc's sigset_t is overly large, so sizeof(*new_sigset) doesn't work. - return syscall(__NR_rt_sigprocmask, how, new_sigset, old_sigset, 8); + return syscall(__NR_rt_sigprocmask, how, new_sigset, old_sigset, NSIG/8); } class SigchainTest : public ::testing::Test { diff --git a/test/004-SignalTest/signaltest.cc b/test/004-SignalTest/signaltest.cc index 67118d5e63c..49fe369b619 100644 --- a/test/004-SignalTest/signaltest.cc +++ b/test/004-SignalTest/signaltest.cc @@ -102,10 +102,14 @@ static struct sigaction oldaction; bool compare_sigaction(const struct sigaction* lhs, const struct sigaction* rhs) { // bionic's definition of `struct sigaction` has internal padding bytes, so we can't just do a // naive memcmp of the entire struct. +#if defined(SA_RESTORER) + if (lhs->sa_restorer != rhs->sa_restorer) { + return false; + } +#endif return memcmp(&lhs->sa_mask, &rhs->sa_mask, sizeof(lhs->sa_mask)) == 0 && lhs->sa_sigaction == rhs->sa_sigaction && - lhs->sa_flags == rhs->sa_flags && - lhs->sa_restorer == rhs->sa_restorer; + lhs->sa_flags == rhs->sa_flags; } extern "C" JNIEXPORT void JNICALL Java_Main_initSignalTest(JNIEnv*, jclass) { diff --git a/test/036-finalizer/src/Main.java b/test/036-finalizer/src/Main.java index 9bfd74f91b5..be7ae4a8c2a 100644 --- a/test/036-finalizer/src/Main.java +++ b/test/036-finalizer/src/Main.java @@ -71,16 +71,18 @@ public class Main { return s[0]; } - private static void printWeakReference(WeakReference<FinalizerTest> wimp) { + public static void main(String[] args) { + WeakReference<FinalizerTest> wimp = makeRef(); // Reference ft so we are sure the WeakReference cannot be cleared. + // Note: This is very fragile. It was previously in a helper function but that + // doesn't work for JIT-on-first-use with --gcstress where the object would be + // collected when JIT internally allocates an array. Also adding a scope around + // the keepLive lifetime somehow keeps a non-null `keepLive` around and makes + // the test fail (even when keeping the `null` assignment). b/76454261 FinalizerTest keepLive = wimp.get(); System.out.println("wimp: " + wimpString(wimp)); Reference.reachabilityFence(keepLive); - } - - public static void main(String[] args) { - WeakReference<FinalizerTest> wimp = makeRef(); - printWeakReference(wimp); + keepLive = null; // Clear the reference. /* this will try to collect and finalize ft */ System.out.println("gc"); diff --git a/test/115-native-bridge/nativebridge.cc b/test/115-native-bridge/nativebridge.cc index 3b352096df0..a74f7638bd3 100644 --- a/test/115-native-bridge/nativebridge.cc +++ b/test/115-native-bridge/nativebridge.cc @@ -229,7 +229,9 @@ static jint trampoline_Java_Main_testSignal(JNIEnv*, jclass) { struct sigaction64 tmp2; sigemptyset64(&tmp2.sa_mask); tmp2.sa_sigaction = test_sigaction_handler; +#if defined(SA_RESTORER) tmp2.sa_restorer = nullptr; +#endif sigaction64(SIGSEGV, &tmp2, nullptr); sigaction64(SIGILL, &tmp2, nullptr); diff --git a/test/121-modifiers/classes/A$B.class b/test/121-modifiers/classes/A$B.class Binary files differdeleted file mode 100644 index bd7ebfe11db..00000000000 --- a/test/121-modifiers/classes/A$B.class +++ /dev/null diff --git a/test/121-modifiers/classes/A$C.class b/test/121-modifiers/classes/A$C.class Binary files differdeleted file mode 100644 index 3ae872e3560..00000000000 --- a/test/121-modifiers/classes/A$C.class +++ /dev/null diff --git a/test/121-modifiers/classes/A.class b/test/121-modifiers/classes/A.class Binary files differdeleted file mode 100644 index d89d029796a..00000000000 --- a/test/121-modifiers/classes/A.class +++ /dev/null diff --git a/test/121-modifiers/classes/Inf.class b/test/121-modifiers/classes/Inf.class Binary files differdeleted file mode 100644 index e8dd68029df..00000000000 --- a/test/121-modifiers/classes/Inf.class +++ /dev/null diff --git a/test/121-modifiers/classes/Main.class b/test/121-modifiers/classes/Main.class Binary files differdeleted file mode 100644 index e044074269b..00000000000 --- a/test/121-modifiers/classes/Main.class +++ /dev/null diff --git a/test/121-modifiers/classes/NonInf.class b/test/121-modifiers/classes/NonInf.class Binary files differdeleted file mode 100644 index 0f1e826fb74..00000000000 --- a/test/121-modifiers/classes/NonInf.class +++ /dev/null diff --git a/test/121-modifiers/info.txt b/test/121-modifiers/info.txt index 335df53f3d5..7dba1133d1e 100644 --- a/test/121-modifiers/info.txt +++ b/test/121-modifiers/info.txt @@ -10,9 +10,9 @@ Finally, compile with jack/jill or dx, and run baksmali. javac Inf.java NonInf.java Main.java javac -cp asm.jar:asm-tree.jar:. Asm.java java -cp asm.jar:asm-tree.jar:. Asm -mv Inf.out classes/Inf.class -mv NonInf.out classes/NonInf.class -mv Main.class A.class A\$B.class A\$C.class classes/ +mv Inf.out classes_tmp/Inf.class +mv NonInf.out classes_tmp/NonInf.class +mv Main.class A.class A\$B.class A\$C.class classes_tmp/ dx --debug --dex --output=classes.dex classes baksmali disassemble classes.dex mv out/*.smali smali/ diff --git a/test/151-OpenFileLimit/src/Main.java b/test/151-OpenFileLimit/src/Main.java index de5890cbfa5..9b16090fbba 100644 --- a/test/151-OpenFileLimit/src/Main.java +++ b/test/151-OpenFileLimit/src/Main.java @@ -38,7 +38,8 @@ public class Main { if (e.getMessage().contains("Too many open files")) { System.out.println("Message includes \"Too many open files\""); } else { - System.out.println(e.getMessage()); + System.out.println("Unexpected exception:"); + e.printStackTrace(); } } diff --git a/test/161-final-abstract-class/smali/Main.smali b/test/161-final-abstract-class/smali/Main.smali new file mode 100644 index 00000000000..588854cf528 --- /dev/null +++ b/test/161-final-abstract-class/smali/Main.smali @@ -0,0 +1,214 @@ +# Created with baksmali. + +# Java file for reference. + +# import java.lang.reflect.InvocationTargetException; +# import java.lang.reflect.Method; +# +# public class Main { +# public static void main(String[] args) { +# try { +# // Make sure that the abstract final class is marked as erroneous. +# Class.forName("AbstractFinal"); +# System.out.println("UNREACHABLE!"); +# } catch (VerifyError expected) { +# } catch (Throwable t) { +# t.printStackTrace(System.out); +# } +# try { +# // Verification of TestClass.test() used to crash when processing +# // the final abstract (erroneous) class. +# Class<?> tc = Class.forName("TestClass"); +# Method test = tc.getDeclaredMethod("test"); +# test.invoke(null); +# System.out.println("UNREACHABLE!"); +# } catch (InvocationTargetException ite) { +# if (ite.getCause() instanceof InstantiationError) { +# System.out.println( +# ite.getCause().getClass().getName() + ": " + ite.getCause().getMessage()); +# } else { +# ite.printStackTrace(System.out); +# } +# } catch (Throwable t) { +# t.printStackTrace(System.out); +# } +# } +# } + +.class public LMain; +.super Ljava/lang/Object; +.source "Main.java" + + +# direct methods +.method public constructor <init>()V + .registers 1 + + .line 20 + invoke-direct {p0}, Ljava/lang/Object;-><init>()V + + return-void +.end method + +.method public static main([Ljava/lang/String;)V + .registers 4 + + .line 24 + :try_start_0 + const-string p0, "AbstractFinal" + + invoke-static {p0}, Ljava/lang/Class;->forName(Ljava/lang/String;)Ljava/lang/Class; + + .line 25 + sget-object p0, Ljava/lang/System;->out:Ljava/io/PrintStream; + + const-string v0, "UNREACHABLE!" + + invoke-virtual {p0, v0}, Ljava/io/PrintStream;->println(Ljava/lang/String;)V + :try_end_c + .catch Ljava/lang/VerifyError; {:try_start_0 .. :try_end_c} :catch_14 + .catch Ljava/lang/Throwable; {:try_start_0 .. :try_end_c} :catch_d + + goto :goto_15 + + .line 27 + :catch_d + move-exception p0 + + .line 28 + sget-object v0, Ljava/lang/System;->out:Ljava/io/PrintStream; + + invoke-virtual {p0, v0}, Ljava/lang/Throwable;->printStackTrace(Ljava/io/PrintStream;)V + + goto :goto_16 + + .line 26 + :catch_14 + move-exception p0 + + .line 29 + :goto_15 + nop + + .line 33 + :goto_16 + :try_start_16 + const-string p0, "TestClass" + + invoke-static {p0}, Ljava/lang/Class;->forName(Ljava/lang/String;)Ljava/lang/Class; + + move-result-object p0 + + .line 34 + const-string v0, "test" + + const/4 v1, 0x0 + + new-array v2, v1, [Ljava/lang/Class; + + invoke-virtual {p0, v0, v2}, Ljava/lang/Class;->getDeclaredMethod(Ljava/lang/String;[Ljava/lang/Class;)Ljava/lang/reflect/Method; + + move-result-object p0 + + .line 35 + const/4 v0, 0x0 + + new-array v1, v1, [Ljava/lang/Object; + + invoke-virtual {p0, v0, v1}, Ljava/lang/reflect/Method;->invoke(Ljava/lang/Object;[Ljava/lang/Object;)Ljava/lang/Object; + + .line 36 + sget-object p0, Ljava/lang/System;->out:Ljava/io/PrintStream; + + const-string v0, "UNREACHABLE!" + + invoke-virtual {p0, v0}, Ljava/io/PrintStream;->println(Ljava/lang/String;)V + :try_end_32 + .catch Ljava/lang/reflect/InvocationTargetException; {:try_start_16 .. :try_end_32} :catch_3a + .catch Ljava/lang/Throwable; {:try_start_16 .. :try_end_32} :catch_33 + + goto :goto_76 + + .line 44 + :catch_33 + move-exception p0 + + .line 45 + sget-object v0, Ljava/lang/System;->out:Ljava/io/PrintStream; + + invoke-virtual {p0, v0}, Ljava/lang/Throwable;->printStackTrace(Ljava/io/PrintStream;)V + + goto :goto_77 + + .line 37 + :catch_3a + move-exception p0 + + .line 38 + invoke-virtual {p0}, Ljava/lang/reflect/InvocationTargetException;->getCause()Ljava/lang/Throwable; + + move-result-object v0 + + instance-of v0, v0, Ljava/lang/InstantiationError; + + if-eqz v0, :cond_71 + + .line 39 + sget-object v0, Ljava/lang/System;->out:Ljava/io/PrintStream; + + new-instance v1, Ljava/lang/StringBuilder; + + invoke-direct {v1}, Ljava/lang/StringBuilder;-><init>()V + + .line 40 + invoke-virtual {p0}, Ljava/lang/reflect/InvocationTargetException;->getCause()Ljava/lang/Throwable; + + move-result-object v2 + + invoke-virtual {v2}, Ljava/lang/Object;->getClass()Ljava/lang/Class; + + move-result-object v2 + + invoke-virtual {v2}, Ljava/lang/Class;->getName()Ljava/lang/String; + + move-result-object v2 + + invoke-virtual {v1, v2}, Ljava/lang/StringBuilder;->append(Ljava/lang/String;)Ljava/lang/StringBuilder; + + const-string v2, ": " + + invoke-virtual {v1, v2}, Ljava/lang/StringBuilder;->append(Ljava/lang/String;)Ljava/lang/StringBuilder; + + invoke-virtual {p0}, Ljava/lang/reflect/InvocationTargetException;->getCause()Ljava/lang/Throwable; + + move-result-object p0 + + invoke-virtual {p0}, Ljava/lang/Throwable;->getMessage()Ljava/lang/String; + + move-result-object p0 + + invoke-virtual {v1, p0}, Ljava/lang/StringBuilder;->append(Ljava/lang/String;)Ljava/lang/StringBuilder; + + invoke-virtual {v1}, Ljava/lang/StringBuilder;->toString()Ljava/lang/String; + + move-result-object p0 + + .line 39 + invoke-virtual {v0, p0}, Ljava/io/PrintStream;->println(Ljava/lang/String;)V + + goto :goto_76 + + .line 42 + :cond_71 + sget-object v0, Ljava/lang/System;->out:Ljava/io/PrintStream; + + invoke-virtual {p0, v0}, Ljava/lang/reflect/InvocationTargetException;->printStackTrace(Ljava/io/PrintStream;)V + + .line 46 + :goto_76 + nop + + .line 47 + :goto_77 + return-void +.end method diff --git a/test/161-final-abstract-class/src/Main.java b/test/161-final-abstract-class/src/Main.java deleted file mode 100644 index 24524902262..00000000000 --- a/test/161-final-abstract-class/src/Main.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (C) 2017 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; - -public class Main { - public static void main(String[] args) { - try { - // Make sure that the abstract final class is marked as erroneous. - Class.forName("AbstractFinal"); - System.out.println("UNREACHABLE!"); - } catch (VerifyError expected) { - } catch (Throwable t) { - t.printStackTrace(System.out); - } - try { - // Verification of TestClass.test() used to crash when processing - // the final abstract (erroneous) class. - Class<?> tc = Class.forName("TestClass"); - Method test = tc.getDeclaredMethod("test"); - test.invoke(null); - System.out.println("UNREACHABLE!"); - } catch (InvocationTargetException ite) { - if (ite.getCause() instanceof InstantiationError) { - System.out.println( - ite.getCause().getClass().getName() + ": " + ite.getCause().getMessage()); - } else { - ite.printStackTrace(System.out); - } - } catch (Throwable t) { - t.printStackTrace(System.out); - } - } -} diff --git a/test/1940-ddms-ext/check b/test/166-bad-interface-super/build index d2c03841fc9..d85147f17bd 100755..100644 --- a/test/1940-ddms-ext/check +++ b/test/166-bad-interface-super/build @@ -1,12 +1,12 @@ #!/bin/bash # -# Copyright (C) 2017 The Android Open Source Project +# Copyright 2018 The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -14,8 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Need to pull out the describeException ouput since that won't be there on -# device. -sed -e '/\t.*$/d' "$2" | sed -e '/java.lang.ArrayIndexOutOfBoundsException:.*$/d' > "$2.tmp" +# See b/65168732 +export USE_D8=false -./default-check "$1" "$2.tmp" +./default-build "$@" diff --git a/test/1929-exception-catch-exception/expected.txt b/test/1929-exception-catch-exception/expected.txt index bc5608ac4e9..a82b732eda8 100644 --- a/test/1929-exception-catch-exception/expected.txt +++ b/test/1929-exception-catch-exception/expected.txt @@ -1,11 +1,11 @@ Test "art.Test1929$DoThrowClass": Running breakpoint with handler "art.Test1929$DoNothingHandler" -main: public static void art.Test1929.run() throws java.lang.Exception @ line = 283 caught class art.Test1929$TestException: doThrow +main: public static void art.Test1929.run() throws java.lang.Exception @ line = 298 caught class art.Test1929$TestException: doThrow Current Stack: private static native art.StackTrace$StackFrameData[] art.StackTrace.nativeGetStackTrace(java.lang.Thread) @ line = -1 public static art.StackTrace$StackFrameData[] art.StackTrace.GetStackTrace(java.lang.Thread) @ line = 61 private static void art.Test1929.PrintStack() @ line = 52 public static void art.Test1929.ExceptionCatchEvent(java.lang.Thread,java.lang.reflect.Executable,long,java.lang.Throwable) @ line = 65 - public static void art.Test1929.run() throws java.lang.Exception @ line = 283 + public static void art.Test1929.run() throws java.lang.Exception @ line = 298 Test "art.Test1929$DoThrowClass": Caught error art.Test1929$TestException:"doThrow" with handler "art.Test1929$DoNothingHandler" Test "art.Test1929$DoThrowClass": Finished running with handler "art.Test1929$DoNothingHandler" Test "art.Test1929$DoThrowCatchBaseTestException": Running breakpoint with handler "art.Test1929$DoNothingHandler" @@ -17,71 +17,71 @@ main: public static void art.Test1929.throwCatchBaseTestException() @ line = 140 public static void art.Test1929.ExceptionCatchEvent(java.lang.Thread,java.lang.reflect.Executable,long,java.lang.Throwable) @ line = 65 public static void art.Test1929.throwCatchBaseTestException() @ line = 140 public void art.Test1929$DoThrowCatchBaseTestException.run() @ line = 149 - public static void art.Test1929.run() throws java.lang.Exception @ line = 280 + public static void art.Test1929.run() throws java.lang.Exception @ line = 295 Doing nothing! Caught art.Test1929$TestException: "throwCatchBaseTestException" Test "art.Test1929$DoThrowCatchBaseTestException": No error caught with handler "art.Test1929$DoNothingHandler" Test "art.Test1929$DoThrowCatchBaseTestException": Finished running with handler "art.Test1929$DoNothingHandler" Test "art.Test1929$DoThrowCatchBaseTestExceptionTwice": Running breakpoint with handler "art.Test1929$DoNothingHandler" -main: public static void art.Test1929$Impl.throwCatchBaseTestExceptionTwiceImpl() @ line = 161 caught class art.Test1929$TestException: throwCatchBaseTestExceptionTwice +main: public static void art.Test1929$Impl.throwCatchBaseTestExceptionTwiceImpl() @ line = 157 caught class art.Test1929$TestException: throwCatchBaseTestExceptionTwice Current Stack: private static native art.StackTrace$StackFrameData[] art.StackTrace.nativeGetStackTrace(java.lang.Thread) @ line = -1 public static art.StackTrace$StackFrameData[] art.StackTrace.GetStackTrace(java.lang.Thread) @ line = 61 private static void art.Test1929.PrintStack() @ line = 52 public static void art.Test1929.ExceptionCatchEvent(java.lang.Thread,java.lang.reflect.Executable,long,java.lang.Throwable) @ line = 65 - public static void art.Test1929$Impl.throwCatchBaseTestExceptionTwiceImpl() @ line = 161 - public static void art.Test1929.throwCatchBaseTestExceptionTwice() @ line = 197 - public void art.Test1929$DoThrowCatchBaseTestExceptionTwice.run() @ line = 201 - public static void art.Test1929.run() throws java.lang.Exception @ line = 280 + public static void art.Test1929$Impl.throwCatchBaseTestExceptionTwiceImpl() @ line = 157 + public static void art.Test1929.throwCatchBaseTestExceptionTwice() @ line = 203 + public void art.Test1929$DoThrowCatchBaseTestExceptionTwice.run() @ line = 210 + public static void art.Test1929.run() throws java.lang.Exception @ line = 295 Caught art.Test1929$TestException: "throwCatchBaseTestExceptionTwice" Test "art.Test1929$DoThrowCatchBaseTestExceptionTwice": No error caught with handler "art.Test1929$DoNothingHandler" Test "art.Test1929$DoThrowCatchBaseTestExceptionTwice": Finished running with handler "art.Test1929$DoNothingHandler" Test "art.Test1929$DoThrowCatchTestException": Running breakpoint with handler "art.Test1929$DoNothingHandler" -main: public static void art.Test1929.throwCatchTestException() @ line = 207 caught class art.Test1929$TestException: throwCatchTestException +main: public static void art.Test1929.throwCatchTestException() @ line = 216 caught class art.Test1929$TestException: throwCatchTestException Current Stack: private static native art.StackTrace$StackFrameData[] art.StackTrace.nativeGetStackTrace(java.lang.Thread) @ line = -1 public static art.StackTrace$StackFrameData[] art.StackTrace.GetStackTrace(java.lang.Thread) @ line = 61 private static void art.Test1929.PrintStack() @ line = 52 public static void art.Test1929.ExceptionCatchEvent(java.lang.Thread,java.lang.reflect.Executable,long,java.lang.Throwable) @ line = 65 - public static void art.Test1929.throwCatchTestException() @ line = 207 - public void art.Test1929$DoThrowCatchTestException.run() @ line = 216 - public static void art.Test1929.run() throws java.lang.Exception @ line = 280 + public static void art.Test1929.throwCatchTestException() @ line = 216 + public void art.Test1929$DoThrowCatchTestException.run() @ line = 225 + public static void art.Test1929.run() throws java.lang.Exception @ line = 295 Doing nothing! Caught art.Test1929$TestException: "throwCatchTestException" Test "art.Test1929$DoThrowCatchTestException": No error caught with handler "art.Test1929$DoNothingHandler" Test "art.Test1929$DoThrowCatchTestException": Finished running with handler "art.Test1929$DoNothingHandler" Test "art.Test1929$DoThrowCatchTestExceptionTwice": Running breakpoint with handler "art.Test1929$DoNothingHandler" -main: public static void art.Test1929$Impl.throwCatchTestExceptionTwiceImpl() @ line = 179 caught class art.Test1929$TestException: throwCatchTestExceptionTwice +main: public static void art.Test1929$Impl.throwCatchTestExceptionTwiceImpl() @ line = 175 caught class art.Test1929$TestException: throwCatchTestExceptionTwice Current Stack: private static native art.StackTrace$StackFrameData[] art.StackTrace.nativeGetStackTrace(java.lang.Thread) @ line = -1 public static art.StackTrace$StackFrameData[] art.StackTrace.GetStackTrace(java.lang.Thread) @ line = 61 private static void art.Test1929.PrintStack() @ line = 52 public static void art.Test1929.ExceptionCatchEvent(java.lang.Thread,java.lang.reflect.Executable,long,java.lang.Throwable) @ line = 65 - public static void art.Test1929$Impl.throwCatchTestExceptionTwiceImpl() @ line = 179 - public static void art.Test1929.throwCatchTestExceptionTwice() @ line = 222 - public void art.Test1929$DoThrowCatchTestExceptionTwice.run() @ line = 226 - public static void art.Test1929.run() throws java.lang.Exception @ line = 280 + public static void art.Test1929$Impl.throwCatchTestExceptionTwiceImpl() @ line = 175 + public static void art.Test1929.throwCatchTestExceptionTwice() @ line = 234 + public void art.Test1929$DoThrowCatchTestExceptionTwice.run() @ line = 241 + public static void art.Test1929.run() throws java.lang.Exception @ line = 295 Caught art.Test1929$TestException: "throwCatchTestExceptionTwice" Test "art.Test1929$DoThrowCatchTestExceptionTwice": No error caught with handler "art.Test1929$DoNothingHandler" Test "art.Test1929$DoThrowCatchTestExceptionTwice": Finished running with handler "art.Test1929$DoNothingHandler" Test "art.Test1929$DoThrowCatchTestExceptionNoRethrow": Running breakpoint with handler "art.Test1929$DoNothingHandler" -main: public static void art.Test1929.run() throws java.lang.Exception @ line = 283 caught class art.Test1929$TestException: throwCatchTestExceptionNoRethrow +main: public static void art.Test1929.run() throws java.lang.Exception @ line = 298 caught class art.Test1929$TestException: throwCatchTestExceptionNoRethrow Current Stack: private static native art.StackTrace$StackFrameData[] art.StackTrace.nativeGetStackTrace(java.lang.Thread) @ line = -1 public static art.StackTrace$StackFrameData[] art.StackTrace.GetStackTrace(java.lang.Thread) @ line = 61 private static void art.Test1929.PrintStack() @ line = 52 public static void art.Test1929.ExceptionCatchEvent(java.lang.Thread,java.lang.reflect.Executable,long,java.lang.Throwable) @ line = 65 - public static void art.Test1929.run() throws java.lang.Exception @ line = 283 + public static void art.Test1929.run() throws java.lang.Exception @ line = 298 Test "art.Test1929$DoThrowCatchTestExceptionNoRethrow": Caught error art.Test1929$TestException:"throwCatchTestExceptionNoRethrow" with handler "art.Test1929$DoNothingHandler" Test "art.Test1929$DoThrowCatchTestExceptionNoRethrow": Finished running with handler "art.Test1929$DoNothingHandler" Test "art.Test1929$DoThrowClass": Running breakpoint with handler "art.Test1929$ThrowCatchBase" -main: public static void art.Test1929.run() throws java.lang.Exception @ line = 283 caught class art.Test1929$TestException: doThrow +main: public static void art.Test1929.run() throws java.lang.Exception @ line = 298 caught class art.Test1929$TestException: doThrow Current Stack: private static native art.StackTrace$StackFrameData[] art.StackTrace.nativeGetStackTrace(java.lang.Thread) @ line = -1 public static art.StackTrace$StackFrameData[] art.StackTrace.GetStackTrace(java.lang.Thread) @ line = 61 private static void art.Test1929.PrintStack() @ line = 52 public static void art.Test1929.ExceptionCatchEvent(java.lang.Thread,java.lang.reflect.Executable,long,java.lang.Throwable) @ line = 65 - public static void art.Test1929.run() throws java.lang.Exception @ line = 283 + public static void art.Test1929.run() throws java.lang.Exception @ line = 298 Test "art.Test1929$DoThrowClass": Caught error art.Test1929$TestException:"doThrow" with handler "art.Test1929$ThrowCatchBase" Test "art.Test1929$DoThrowClass": Finished running with handler "art.Test1929$ThrowCatchBase" Test "art.Test1929$DoThrowCatchBaseTestException": Running breakpoint with handler "art.Test1929$ThrowCatchBase" @@ -93,73 +93,73 @@ main: public static void art.Test1929.throwCatchBaseTestException() @ line = 140 public static void art.Test1929.ExceptionCatchEvent(java.lang.Thread,java.lang.reflect.Executable,long,java.lang.Throwable) @ line = 65 public static void art.Test1929.throwCatchBaseTestException() @ line = 140 public void art.Test1929$DoThrowCatchBaseTestException.run() @ line = 149 - public static void art.Test1929.run() throws java.lang.Exception @ line = 280 + public static void art.Test1929.run() throws java.lang.Exception @ line = 295 Throwing BaseTestException and catching it! Caught art.Test1929$BaseTestException: "ThrowBaseHandler during throw from public static void art.Test1929.throwCatchBaseTestException() @ line = 140" Caught art.Test1929$TestException: "throwCatchBaseTestException" Test "art.Test1929$DoThrowCatchBaseTestException": No error caught with handler "art.Test1929$ThrowCatchBase" Test "art.Test1929$DoThrowCatchBaseTestException": Finished running with handler "art.Test1929$ThrowCatchBase" Test "art.Test1929$DoThrowCatchBaseTestExceptionTwice": Running breakpoint with handler "art.Test1929$ThrowCatchBase" -main: public static void art.Test1929$Impl.throwCatchBaseTestExceptionTwiceImpl() @ line = 161 caught class art.Test1929$TestException: throwCatchBaseTestExceptionTwice +main: public static void art.Test1929$Impl.throwCatchBaseTestExceptionTwiceImpl() @ line = 157 caught class art.Test1929$TestException: throwCatchBaseTestExceptionTwice Current Stack: private static native art.StackTrace$StackFrameData[] art.StackTrace.nativeGetStackTrace(java.lang.Thread) @ line = -1 public static art.StackTrace$StackFrameData[] art.StackTrace.GetStackTrace(java.lang.Thread) @ line = 61 private static void art.Test1929.PrintStack() @ line = 52 public static void art.Test1929.ExceptionCatchEvent(java.lang.Thread,java.lang.reflect.Executable,long,java.lang.Throwable) @ line = 65 - public static void art.Test1929$Impl.throwCatchBaseTestExceptionTwiceImpl() @ line = 161 - public static void art.Test1929.throwCatchBaseTestExceptionTwice() @ line = 197 - public void art.Test1929$DoThrowCatchBaseTestExceptionTwice.run() @ line = 201 - public static void art.Test1929.run() throws java.lang.Exception @ line = 280 + public static void art.Test1929$Impl.throwCatchBaseTestExceptionTwiceImpl() @ line = 157 + public static void art.Test1929.throwCatchBaseTestExceptionTwice() @ line = 203 + public void art.Test1929$DoThrowCatchBaseTestExceptionTwice.run() @ line = 210 + public static void art.Test1929.run() throws java.lang.Exception @ line = 295 Caught art.Test1929$TestException: "throwCatchBaseTestExceptionTwice" Test "art.Test1929$DoThrowCatchBaseTestExceptionTwice": No error caught with handler "art.Test1929$ThrowCatchBase" Test "art.Test1929$DoThrowCatchBaseTestExceptionTwice": Finished running with handler "art.Test1929$ThrowCatchBase" Test "art.Test1929$DoThrowCatchTestException": Running breakpoint with handler "art.Test1929$ThrowCatchBase" -main: public static void art.Test1929.throwCatchTestException() @ line = 207 caught class art.Test1929$TestException: throwCatchTestException +main: public static void art.Test1929.throwCatchTestException() @ line = 216 caught class art.Test1929$TestException: throwCatchTestException Current Stack: private static native art.StackTrace$StackFrameData[] art.StackTrace.nativeGetStackTrace(java.lang.Thread) @ line = -1 public static art.StackTrace$StackFrameData[] art.StackTrace.GetStackTrace(java.lang.Thread) @ line = 61 private static void art.Test1929.PrintStack() @ line = 52 public static void art.Test1929.ExceptionCatchEvent(java.lang.Thread,java.lang.reflect.Executable,long,java.lang.Throwable) @ line = 65 - public static void art.Test1929.throwCatchTestException() @ line = 207 - public void art.Test1929$DoThrowCatchTestException.run() @ line = 216 - public static void art.Test1929.run() throws java.lang.Exception @ line = 280 + public static void art.Test1929.throwCatchTestException() @ line = 216 + public void art.Test1929$DoThrowCatchTestException.run() @ line = 225 + public static void art.Test1929.run() throws java.lang.Exception @ line = 295 Throwing BaseTestException and catching it! -Caught art.Test1929$BaseTestException: "ThrowBaseHandler during throw from public static void art.Test1929.throwCatchTestException() @ line = 207" +Caught art.Test1929$BaseTestException: "ThrowBaseHandler during throw from public static void art.Test1929.throwCatchTestException() @ line = 216" Caught art.Test1929$TestException: "throwCatchTestException" Test "art.Test1929$DoThrowCatchTestException": No error caught with handler "art.Test1929$ThrowCatchBase" Test "art.Test1929$DoThrowCatchTestException": Finished running with handler "art.Test1929$ThrowCatchBase" Test "art.Test1929$DoThrowCatchTestExceptionTwice": Running breakpoint with handler "art.Test1929$ThrowCatchBase" -main: public static void art.Test1929$Impl.throwCatchTestExceptionTwiceImpl() @ line = 179 caught class art.Test1929$TestException: throwCatchTestExceptionTwice +main: public static void art.Test1929$Impl.throwCatchTestExceptionTwiceImpl() @ line = 175 caught class art.Test1929$TestException: throwCatchTestExceptionTwice Current Stack: private static native art.StackTrace$StackFrameData[] art.StackTrace.nativeGetStackTrace(java.lang.Thread) @ line = -1 public static art.StackTrace$StackFrameData[] art.StackTrace.GetStackTrace(java.lang.Thread) @ line = 61 private static void art.Test1929.PrintStack() @ line = 52 public static void art.Test1929.ExceptionCatchEvent(java.lang.Thread,java.lang.reflect.Executable,long,java.lang.Throwable) @ line = 65 - public static void art.Test1929$Impl.throwCatchTestExceptionTwiceImpl() @ line = 179 - public static void art.Test1929.throwCatchTestExceptionTwice() @ line = 222 - public void art.Test1929$DoThrowCatchTestExceptionTwice.run() @ line = 226 - public static void art.Test1929.run() throws java.lang.Exception @ line = 280 + public static void art.Test1929$Impl.throwCatchTestExceptionTwiceImpl() @ line = 175 + public static void art.Test1929.throwCatchTestExceptionTwice() @ line = 234 + public void art.Test1929$DoThrowCatchTestExceptionTwice.run() @ line = 241 + public static void art.Test1929.run() throws java.lang.Exception @ line = 295 Caught art.Test1929$TestException: "throwCatchTestExceptionTwice" Test "art.Test1929$DoThrowCatchTestExceptionTwice": No error caught with handler "art.Test1929$ThrowCatchBase" Test "art.Test1929$DoThrowCatchTestExceptionTwice": Finished running with handler "art.Test1929$ThrowCatchBase" Test "art.Test1929$DoThrowCatchTestExceptionNoRethrow": Running breakpoint with handler "art.Test1929$ThrowCatchBase" -main: public static void art.Test1929.run() throws java.lang.Exception @ line = 283 caught class art.Test1929$TestException: throwCatchTestExceptionNoRethrow +main: public static void art.Test1929.run() throws java.lang.Exception @ line = 298 caught class art.Test1929$TestException: throwCatchTestExceptionNoRethrow Current Stack: private static native art.StackTrace$StackFrameData[] art.StackTrace.nativeGetStackTrace(java.lang.Thread) @ line = -1 public static art.StackTrace$StackFrameData[] art.StackTrace.GetStackTrace(java.lang.Thread) @ line = 61 private static void art.Test1929.PrintStack() @ line = 52 public static void art.Test1929.ExceptionCatchEvent(java.lang.Thread,java.lang.reflect.Executable,long,java.lang.Throwable) @ line = 65 - public static void art.Test1929.run() throws java.lang.Exception @ line = 283 + public static void art.Test1929.run() throws java.lang.Exception @ line = 298 Test "art.Test1929$DoThrowCatchTestExceptionNoRethrow": Caught error art.Test1929$TestException:"throwCatchTestExceptionNoRethrow" with handler "art.Test1929$ThrowCatchBase" Test "art.Test1929$DoThrowCatchTestExceptionNoRethrow": Finished running with handler "art.Test1929$ThrowCatchBase" Test "art.Test1929$DoThrowClass": Running breakpoint with handler "art.Test1929$ThrowBaseTestExceptionHandler" -main: public static void art.Test1929.run() throws java.lang.Exception @ line = 283 caught class art.Test1929$TestException: doThrow +main: public static void art.Test1929.run() throws java.lang.Exception @ line = 298 caught class art.Test1929$TestException: doThrow Current Stack: private static native art.StackTrace$StackFrameData[] art.StackTrace.nativeGetStackTrace(java.lang.Thread) @ line = -1 public static art.StackTrace$StackFrameData[] art.StackTrace.GetStackTrace(java.lang.Thread) @ line = 61 private static void art.Test1929.PrintStack() @ line = 52 public static void art.Test1929.ExceptionCatchEvent(java.lang.Thread,java.lang.reflect.Executable,long,java.lang.Throwable) @ line = 65 - public static void art.Test1929.run() throws java.lang.Exception @ line = 283 + public static void art.Test1929.run() throws java.lang.Exception @ line = 298 Test "art.Test1929$DoThrowClass": Caught error art.Test1929$TestException:"doThrow" with handler "art.Test1929$ThrowBaseTestExceptionHandler" Test "art.Test1929$DoThrowClass": Finished running with handler "art.Test1929$ThrowBaseTestExceptionHandler" Test "art.Test1929$DoThrowCatchBaseTestException": Running breakpoint with handler "art.Test1929$ThrowBaseTestExceptionHandler" @@ -171,69 +171,69 @@ main: public static void art.Test1929.throwCatchBaseTestException() @ line = 140 public static void art.Test1929.ExceptionCatchEvent(java.lang.Thread,java.lang.reflect.Executable,long,java.lang.Throwable) @ line = 65 public static void art.Test1929.throwCatchBaseTestException() @ line = 140 public void art.Test1929$DoThrowCatchBaseTestException.run() @ line = 149 - public static void art.Test1929.run() throws java.lang.Exception @ line = 280 + public static void art.Test1929.run() throws java.lang.Exception @ line = 295 Throwing BaseTestException! Test "art.Test1929$DoThrowCatchBaseTestException": Caught error art.Test1929$BaseTestException:"ThrowBaseHandler during throw from public static void art.Test1929.throwCatchBaseTestException() @ line = 140" with handler "art.Test1929$ThrowBaseTestExceptionHandler" Test "art.Test1929$DoThrowCatchBaseTestException": Finished running with handler "art.Test1929$ThrowBaseTestExceptionHandler" Test "art.Test1929$DoThrowCatchBaseTestExceptionTwice": Running breakpoint with handler "art.Test1929$ThrowBaseTestExceptionHandler" -main: public static void art.Test1929$Impl.throwCatchBaseTestExceptionTwiceImpl() @ line = 161 caught class art.Test1929$TestException: throwCatchBaseTestExceptionTwice +main: public static void art.Test1929$Impl.throwCatchBaseTestExceptionTwiceImpl() @ line = 157 caught class art.Test1929$TestException: throwCatchBaseTestExceptionTwice Current Stack: private static native art.StackTrace$StackFrameData[] art.StackTrace.nativeGetStackTrace(java.lang.Thread) @ line = -1 public static art.StackTrace$StackFrameData[] art.StackTrace.GetStackTrace(java.lang.Thread) @ line = 61 private static void art.Test1929.PrintStack() @ line = 52 public static void art.Test1929.ExceptionCatchEvent(java.lang.Thread,java.lang.reflect.Executable,long,java.lang.Throwable) @ line = 65 - public static void art.Test1929$Impl.throwCatchBaseTestExceptionTwiceImpl() @ line = 161 - public static void art.Test1929.throwCatchBaseTestExceptionTwice() @ line = 197 - public void art.Test1929$DoThrowCatchBaseTestExceptionTwice.run() @ line = 201 - public static void art.Test1929.run() throws java.lang.Exception @ line = 280 + public static void art.Test1929$Impl.throwCatchBaseTestExceptionTwiceImpl() @ line = 157 + public static void art.Test1929.throwCatchBaseTestExceptionTwice() @ line = 203 + public void art.Test1929$DoThrowCatchBaseTestExceptionTwice.run() @ line = 210 + public static void art.Test1929.run() throws java.lang.Exception @ line = 295 Caught art.Test1929$TestException: "throwCatchBaseTestExceptionTwice" Test "art.Test1929$DoThrowCatchBaseTestExceptionTwice": No error caught with handler "art.Test1929$ThrowBaseTestExceptionHandler" Test "art.Test1929$DoThrowCatchBaseTestExceptionTwice": Finished running with handler "art.Test1929$ThrowBaseTestExceptionHandler" Test "art.Test1929$DoThrowCatchTestException": Running breakpoint with handler "art.Test1929$ThrowBaseTestExceptionHandler" -main: public static void art.Test1929.throwCatchTestException() @ line = 207 caught class art.Test1929$TestException: throwCatchTestException +main: public static void art.Test1929.throwCatchTestException() @ line = 216 caught class art.Test1929$TestException: throwCatchTestException Current Stack: private static native art.StackTrace$StackFrameData[] art.StackTrace.nativeGetStackTrace(java.lang.Thread) @ line = -1 public static art.StackTrace$StackFrameData[] art.StackTrace.GetStackTrace(java.lang.Thread) @ line = 61 private static void art.Test1929.PrintStack() @ line = 52 public static void art.Test1929.ExceptionCatchEvent(java.lang.Thread,java.lang.reflect.Executable,long,java.lang.Throwable) @ line = 65 - public static void art.Test1929.throwCatchTestException() @ line = 207 - public void art.Test1929$DoThrowCatchTestException.run() @ line = 216 - public static void art.Test1929.run() throws java.lang.Exception @ line = 280 + public static void art.Test1929.throwCatchTestException() @ line = 216 + public void art.Test1929$DoThrowCatchTestException.run() @ line = 225 + public static void art.Test1929.run() throws java.lang.Exception @ line = 295 Throwing BaseTestException! -Test "art.Test1929$DoThrowCatchTestException": Caught error art.Test1929$BaseTestException:"ThrowBaseHandler during throw from public static void art.Test1929.throwCatchTestException() @ line = 207" with handler "art.Test1929$ThrowBaseTestExceptionHandler" +Test "art.Test1929$DoThrowCatchTestException": Caught error art.Test1929$BaseTestException:"ThrowBaseHandler during throw from public static void art.Test1929.throwCatchTestException() @ line = 216" with handler "art.Test1929$ThrowBaseTestExceptionHandler" Test "art.Test1929$DoThrowCatchTestException": Finished running with handler "art.Test1929$ThrowBaseTestExceptionHandler" Test "art.Test1929$DoThrowCatchTestExceptionTwice": Running breakpoint with handler "art.Test1929$ThrowBaseTestExceptionHandler" -main: public static void art.Test1929$Impl.throwCatchTestExceptionTwiceImpl() @ line = 179 caught class art.Test1929$TestException: throwCatchTestExceptionTwice +main: public static void art.Test1929$Impl.throwCatchTestExceptionTwiceImpl() @ line = 175 caught class art.Test1929$TestException: throwCatchTestExceptionTwice Current Stack: private static native art.StackTrace$StackFrameData[] art.StackTrace.nativeGetStackTrace(java.lang.Thread) @ line = -1 public static art.StackTrace$StackFrameData[] art.StackTrace.GetStackTrace(java.lang.Thread) @ line = 61 private static void art.Test1929.PrintStack() @ line = 52 public static void art.Test1929.ExceptionCatchEvent(java.lang.Thread,java.lang.reflect.Executable,long,java.lang.Throwable) @ line = 65 - public static void art.Test1929$Impl.throwCatchTestExceptionTwiceImpl() @ line = 179 - public static void art.Test1929.throwCatchTestExceptionTwice() @ line = 222 - public void art.Test1929$DoThrowCatchTestExceptionTwice.run() @ line = 226 - public static void art.Test1929.run() throws java.lang.Exception @ line = 280 + public static void art.Test1929$Impl.throwCatchTestExceptionTwiceImpl() @ line = 175 + public static void art.Test1929.throwCatchTestExceptionTwice() @ line = 234 + public void art.Test1929$DoThrowCatchTestExceptionTwice.run() @ line = 241 + public static void art.Test1929.run() throws java.lang.Exception @ line = 295 Caught art.Test1929$TestException: "throwCatchTestExceptionTwice" Test "art.Test1929$DoThrowCatchTestExceptionTwice": No error caught with handler "art.Test1929$ThrowBaseTestExceptionHandler" Test "art.Test1929$DoThrowCatchTestExceptionTwice": Finished running with handler "art.Test1929$ThrowBaseTestExceptionHandler" Test "art.Test1929$DoThrowCatchTestExceptionNoRethrow": Running breakpoint with handler "art.Test1929$ThrowBaseTestExceptionHandler" -main: public static void art.Test1929.run() throws java.lang.Exception @ line = 283 caught class art.Test1929$TestException: throwCatchTestExceptionNoRethrow +main: public static void art.Test1929.run() throws java.lang.Exception @ line = 298 caught class art.Test1929$TestException: throwCatchTestExceptionNoRethrow Current Stack: private static native art.StackTrace$StackFrameData[] art.StackTrace.nativeGetStackTrace(java.lang.Thread) @ line = -1 public static art.StackTrace$StackFrameData[] art.StackTrace.GetStackTrace(java.lang.Thread) @ line = 61 private static void art.Test1929.PrintStack() @ line = 52 public static void art.Test1929.ExceptionCatchEvent(java.lang.Thread,java.lang.reflect.Executable,long,java.lang.Throwable) @ line = 65 - public static void art.Test1929.run() throws java.lang.Exception @ line = 283 + public static void art.Test1929.run() throws java.lang.Exception @ line = 298 Test "art.Test1929$DoThrowCatchTestExceptionNoRethrow": Caught error art.Test1929$TestException:"throwCatchTestExceptionNoRethrow" with handler "art.Test1929$ThrowBaseTestExceptionHandler" Test "art.Test1929$DoThrowCatchTestExceptionNoRethrow": Finished running with handler "art.Test1929$ThrowBaseTestExceptionHandler" Test "art.Test1929$DoThrowClass": Running breakpoint with handler "art.Test1929$ThrowTestExceptionNoRethrowHandler" -main: public static void art.Test1929.run() throws java.lang.Exception @ line = 283 caught class art.Test1929$TestException: doThrow +main: public static void art.Test1929.run() throws java.lang.Exception @ line = 298 caught class art.Test1929$TestException: doThrow Current Stack: private static native art.StackTrace$StackFrameData[] art.StackTrace.nativeGetStackTrace(java.lang.Thread) @ line = -1 public static art.StackTrace$StackFrameData[] art.StackTrace.GetStackTrace(java.lang.Thread) @ line = 61 private static void art.Test1929.PrintStack() @ line = 52 public static void art.Test1929.ExceptionCatchEvent(java.lang.Thread,java.lang.reflect.Executable,long,java.lang.Throwable) @ line = 65 - public static void art.Test1929.run() throws java.lang.Exception @ line = 283 + public static void art.Test1929.run() throws java.lang.Exception @ line = 298 Test "art.Test1929$DoThrowClass": Caught error art.Test1929$TestException:"doThrow" with handler "art.Test1929$ThrowTestExceptionNoRethrowHandler" Test "art.Test1929$DoThrowClass": Finished running with handler "art.Test1929$ThrowTestExceptionNoRethrowHandler" Test "art.Test1929$DoThrowCatchBaseTestException": Running breakpoint with handler "art.Test1929$ThrowTestExceptionNoRethrowHandler" @@ -245,58 +245,58 @@ main: public static void art.Test1929.throwCatchBaseTestException() @ line = 140 public static void art.Test1929.ExceptionCatchEvent(java.lang.Thread,java.lang.reflect.Executable,long,java.lang.Throwable) @ line = 65 public static void art.Test1929.throwCatchBaseTestException() @ line = 140 public void art.Test1929$DoThrowCatchBaseTestException.run() @ line = 149 - public static void art.Test1929.run() throws java.lang.Exception @ line = 280 + public static void art.Test1929.run() throws java.lang.Exception @ line = 295 Throwing TestExceptionNoRethrow! Test "art.Test1929$DoThrowCatchBaseTestException": Caught error art.Test1929$TestExceptionNoRethrow:"ThrowTestExceptionNoRethrowHandler during throw from public static void art.Test1929.throwCatchBaseTestException() @ line = 140" with handler "art.Test1929$ThrowTestExceptionNoRethrowHandler" Test "art.Test1929$DoThrowCatchBaseTestException": Finished running with handler "art.Test1929$ThrowTestExceptionNoRethrowHandler" Test "art.Test1929$DoThrowCatchBaseTestExceptionTwice": Running breakpoint with handler "art.Test1929$ThrowTestExceptionNoRethrowHandler" -main: public static void art.Test1929$Impl.throwCatchBaseTestExceptionTwiceImpl() @ line = 161 caught class art.Test1929$TestException: throwCatchBaseTestExceptionTwice +main: public static void art.Test1929$Impl.throwCatchBaseTestExceptionTwiceImpl() @ line = 157 caught class art.Test1929$TestException: throwCatchBaseTestExceptionTwice Current Stack: private static native art.StackTrace$StackFrameData[] art.StackTrace.nativeGetStackTrace(java.lang.Thread) @ line = -1 public static art.StackTrace$StackFrameData[] art.StackTrace.GetStackTrace(java.lang.Thread) @ line = 61 private static void art.Test1929.PrintStack() @ line = 52 public static void art.Test1929.ExceptionCatchEvent(java.lang.Thread,java.lang.reflect.Executable,long,java.lang.Throwable) @ line = 65 - public static void art.Test1929$Impl.throwCatchBaseTestExceptionTwiceImpl() @ line = 161 - public static void art.Test1929.throwCatchBaseTestExceptionTwice() @ line = 197 - public void art.Test1929$DoThrowCatchBaseTestExceptionTwice.run() @ line = 201 - public static void art.Test1929.run() throws java.lang.Exception @ line = 280 + public static void art.Test1929$Impl.throwCatchBaseTestExceptionTwiceImpl() @ line = 157 + public static void art.Test1929.throwCatchBaseTestExceptionTwice() @ line = 203 + public void art.Test1929$DoThrowCatchBaseTestExceptionTwice.run() @ line = 210 + public static void art.Test1929.run() throws java.lang.Exception @ line = 295 Caught art.Test1929$TestException: "throwCatchBaseTestExceptionTwice" Test "art.Test1929$DoThrowCatchBaseTestExceptionTwice": No error caught with handler "art.Test1929$ThrowTestExceptionNoRethrowHandler" Test "art.Test1929$DoThrowCatchBaseTestExceptionTwice": Finished running with handler "art.Test1929$ThrowTestExceptionNoRethrowHandler" Test "art.Test1929$DoThrowCatchTestException": Running breakpoint with handler "art.Test1929$ThrowTestExceptionNoRethrowHandler" -main: public static void art.Test1929.throwCatchTestException() @ line = 207 caught class art.Test1929$TestException: throwCatchTestException +main: public static void art.Test1929.throwCatchTestException() @ line = 216 caught class art.Test1929$TestException: throwCatchTestException Current Stack: private static native art.StackTrace$StackFrameData[] art.StackTrace.nativeGetStackTrace(java.lang.Thread) @ line = -1 public static art.StackTrace$StackFrameData[] art.StackTrace.GetStackTrace(java.lang.Thread) @ line = 61 private static void art.Test1929.PrintStack() @ line = 52 public static void art.Test1929.ExceptionCatchEvent(java.lang.Thread,java.lang.reflect.Executable,long,java.lang.Throwable) @ line = 65 - public static void art.Test1929.throwCatchTestException() @ line = 207 - public void art.Test1929$DoThrowCatchTestException.run() @ line = 216 - public static void art.Test1929.run() throws java.lang.Exception @ line = 280 + public static void art.Test1929.throwCatchTestException() @ line = 216 + public void art.Test1929$DoThrowCatchTestException.run() @ line = 225 + public static void art.Test1929.run() throws java.lang.Exception @ line = 295 Throwing TestExceptionNoRethrow! -Test "art.Test1929$DoThrowCatchTestException": Caught error art.Test1929$TestExceptionNoRethrow:"ThrowTestExceptionNoRethrowHandler during throw from public static void art.Test1929.throwCatchTestException() @ line = 207" with handler "art.Test1929$ThrowTestExceptionNoRethrowHandler" +Test "art.Test1929$DoThrowCatchTestException": Caught error art.Test1929$TestExceptionNoRethrow:"ThrowTestExceptionNoRethrowHandler during throw from public static void art.Test1929.throwCatchTestException() @ line = 216" with handler "art.Test1929$ThrowTestExceptionNoRethrowHandler" Test "art.Test1929$DoThrowCatchTestException": Finished running with handler "art.Test1929$ThrowTestExceptionNoRethrowHandler" Test "art.Test1929$DoThrowCatchTestExceptionTwice": Running breakpoint with handler "art.Test1929$ThrowTestExceptionNoRethrowHandler" -main: public static void art.Test1929$Impl.throwCatchTestExceptionTwiceImpl() @ line = 179 caught class art.Test1929$TestException: throwCatchTestExceptionTwice +main: public static void art.Test1929$Impl.throwCatchTestExceptionTwiceImpl() @ line = 175 caught class art.Test1929$TestException: throwCatchTestExceptionTwice Current Stack: private static native art.StackTrace$StackFrameData[] art.StackTrace.nativeGetStackTrace(java.lang.Thread) @ line = -1 public static art.StackTrace$StackFrameData[] art.StackTrace.GetStackTrace(java.lang.Thread) @ line = 61 private static void art.Test1929.PrintStack() @ line = 52 public static void art.Test1929.ExceptionCatchEvent(java.lang.Thread,java.lang.reflect.Executable,long,java.lang.Throwable) @ line = 65 - public static void art.Test1929$Impl.throwCatchTestExceptionTwiceImpl() @ line = 179 - public static void art.Test1929.throwCatchTestExceptionTwice() @ line = 222 - public void art.Test1929$DoThrowCatchTestExceptionTwice.run() @ line = 226 - public static void art.Test1929.run() throws java.lang.Exception @ line = 280 + public static void art.Test1929$Impl.throwCatchTestExceptionTwiceImpl() @ line = 175 + public static void art.Test1929.throwCatchTestExceptionTwice() @ line = 234 + public void art.Test1929$DoThrowCatchTestExceptionTwice.run() @ line = 241 + public static void art.Test1929.run() throws java.lang.Exception @ line = 295 Caught art.Test1929$TestException: "throwCatchTestExceptionTwice" Test "art.Test1929$DoThrowCatchTestExceptionTwice": No error caught with handler "art.Test1929$ThrowTestExceptionNoRethrowHandler" Test "art.Test1929$DoThrowCatchTestExceptionTwice": Finished running with handler "art.Test1929$ThrowTestExceptionNoRethrowHandler" Test "art.Test1929$DoThrowCatchTestExceptionNoRethrow": Running breakpoint with handler "art.Test1929$ThrowTestExceptionNoRethrowHandler" -main: public static void art.Test1929.run() throws java.lang.Exception @ line = 283 caught class art.Test1929$TestException: throwCatchTestExceptionNoRethrow +main: public static void art.Test1929.run() throws java.lang.Exception @ line = 298 caught class art.Test1929$TestException: throwCatchTestExceptionNoRethrow Current Stack: private static native art.StackTrace$StackFrameData[] art.StackTrace.nativeGetStackTrace(java.lang.Thread) @ line = -1 public static art.StackTrace$StackFrameData[] art.StackTrace.GetStackTrace(java.lang.Thread) @ line = 61 private static void art.Test1929.PrintStack() @ line = 52 public static void art.Test1929.ExceptionCatchEvent(java.lang.Thread,java.lang.reflect.Executable,long,java.lang.Throwable) @ line = 65 - public static void art.Test1929.run() throws java.lang.Exception @ line = 283 + public static void art.Test1929.run() throws java.lang.Exception @ line = 298 Test "art.Test1929$DoThrowCatchTestExceptionNoRethrow": Caught error art.Test1929$TestException:"throwCatchTestExceptionNoRethrow" with handler "art.Test1929$ThrowTestExceptionNoRethrowHandler" Test "art.Test1929$DoThrowCatchTestExceptionNoRethrow": Finished running with handler "art.Test1929$ThrowTestExceptionNoRethrowHandler" diff --git a/test/1929-exception-catch-exception/src/art/Test1929.java b/test/1929-exception-catch-exception/src/art/Test1929.java index 07d2087a0f5..e2deb3f85f7 100644 --- a/test/1929-exception-catch-exception/src/art/Test1929.java +++ b/test/1929-exception-catch-exception/src/art/Test1929.java @@ -152,49 +152,58 @@ public class Test1929 { // dx/d8/jack all do an optimization around catch blocks that (while legal) breaks assumptions // this test relies on so we have the actual implementation be corrected smali. This does work // for RI however. - public static final class Impl { - private Impl() {} - public static void throwCatchBaseTestExceptionTwiceImpl() { - try { - try { - throw new TestException("throwCatchBaseTestExceptionTwice"); - } catch (BaseTestException t) { - System.out.println("Caught " + t.getClass().getName() + ": \"" + t.getMessage() + "\""); - if (PRINT_FULL_EXCEPTION) { - t.printStackTrace(System.out); - } - } - } catch (BaseTestException t) { - System.out.println("2nd Caught " + t.getClass().getName() + ": \"" + t.getMessage() + "\""); - if (PRINT_FULL_EXCEPTION) { - t.printStackTrace(System.out); - } - } - } - public static void throwCatchTestExceptionTwiceImpl() { - try { - try { - throw new TestException("throwCatchTestExceptionTwice"); - } catch (TestException t) { - System.out.println("Caught " + t.getClass().getName() + ": \"" + t.getMessage() + "\""); - if (PRINT_FULL_EXCEPTION) { - t.printStackTrace(System.out); - } - } - } catch (TestException t) { - System.out.println("2nd Caught " + t.getClass().getName() + ": \"" + t.getMessage() + "\""); - if (PRINT_FULL_EXCEPTION) { - t.printStackTrace(System.out); - } - } - } - } + // For reference: + + // public static final class Impl { + // private Impl() {} + // public static void throwCatchBaseTestExceptionTwiceImpl() { + // try { + // try { + // throw new TestException("throwCatchBaseTestExceptionTwice"); + // } catch (BaseTestException t) { + // System.out.println("Caught " + t.getClass().getName() + ": \"" + t.getMessage() + "\""); + // if (PRINT_FULL_EXCEPTION) { + // t.printStackTrace(System.out); + // } + // } + // } catch (BaseTestException t) { + // System.out.println("2nd Caught " + t.getClass().getName() + ": \"" + t.getMessage() + "\""); + // if (PRINT_FULL_EXCEPTION) { + // t.printStackTrace(System.out); + // } + // } + // } + + // public static void throwCatchTestExceptionTwiceImpl() { + // try { + // try { + // throw new TestException("throwCatchTestExceptionTwice"); + // } catch (TestException t) { + // System.out.println("Caught " + t.getClass().getName() + ": \"" + t.getMessage() + "\""); + // if (PRINT_FULL_EXCEPTION) { + // t.printStackTrace(System.out); + // } + // } + // } catch (TestException t) { + // System.out.println("2nd Caught " + t.getClass().getName() + ": \"" + t.getMessage() + "\""); + // if (PRINT_FULL_EXCEPTION) { + // t.printStackTrace(System.out); + // } + // } + // } + // } public static void throwCatchBaseTestExceptionTwice() { // The implementation of this has to change depending upon the runtime slightly due to compiler // optimizations present in DX/D8/Jack. - Impl.throwCatchBaseTestExceptionTwiceImpl(); + try { + Class<?> Impl = Class.forName("art.Test1929$Impl"); + Method m = Impl.getMethod("throwCatchBaseTestExceptionTwiceImpl"); + m.invoke(null); + } catch (Exception e) { + e.printStackTrace(System.out); + } } public static class DoThrowCatchBaseTestExceptionTwice implements Runnable { @@ -219,7 +228,13 @@ public class Test1929 { public static void throwCatchTestExceptionTwice() { // The implementation of this has to change depending upon the runtime slightly due to compiler // optimizations present in DX/D8/Jack. - Impl.throwCatchTestExceptionTwiceImpl(); + try { + Class<?> Impl = Class.forName("art.Test1929$Impl"); + Method m = Impl.getMethod("throwCatchTestExceptionTwiceImpl"); + m.invoke(null); + } catch (Exception e) { + e.printStackTrace(System.out); + } } public static class DoThrowCatchTestExceptionTwice implements Runnable { diff --git a/test/1934-jvmti-signal-thread/src/art/Test1934.java b/test/1934-jvmti-signal-thread/src/art/Test1934.java index 2a3f8db5f86..308f17b961b 100644 --- a/test/1934-jvmti-signal-thread/src/art/Test1934.java +++ b/test/1934-jvmti-signal-thread/src/art/Test1934.java @@ -25,6 +25,8 @@ public class Test1934 { public static final boolean PRINT_STACK_TRACE = false; public static void run() throws Exception { + ensureClassesLoaded(); + System.out.println("Interrupt before start"); testInterruptBeforeStart(); @@ -53,6 +55,22 @@ public class Test1934 { testStopInNative(); } + private static void ensureInitialized(Class c) { + try { + Class.forName(c.getName()); + } catch (Exception e) { + throw new Error("Failed to initialize " + c, e); + } + } + + private static void ensureClassesLoaded() { + // Depending on timing this class might (or might not) be loaded during testing of Stop. If it + // is and the StopThread occurs inside of it we will get a ExceptionInInitializerError which is + // not what we are looking for. In order to avoid this ever happening simply initialize the + // class that can cause it early. + ensureInitialized(java.util.concurrent.locks.LockSupport.class); + } + public static void testStopBeforeStart() throws Exception { final Throwable[] out_err = new Throwable[] { null, }; final Object tst = new Object(); diff --git a/test/1935-get-set-current-frame-jit/src/Main.java b/test/1935-get-set-current-frame-jit/src/Main.java index 378aaf7a944..70e94c439f1 100644 --- a/test/1935-get-set-current-frame-jit/src/Main.java +++ b/test/1935-get-set-current-frame-jit/src/Main.java @@ -74,9 +74,11 @@ public class Main { if (hasJit()) { boolean inOsr = Main.isInOsrCode("run"); if (expectOsr && !inOsr) { - throw new Error("Expected to be in OSR but was not."); + throw new Error( + "Expected to be in OSR but was not. interpreter: " + Main.isInterpreted()); } else if (!expectOsr && inOsr) { - throw new Error("Expected not to be in OSR but was."); + throw new Error( + "Expected not to be in OSR but was. interpreter: " + Main.isInterpreted()); } } reportValue(TARGET); diff --git a/test/305-other-fault-handler/fault_handler.cc b/test/305-other-fault-handler/fault_handler.cc index f04832613b5..a0831ca8c27 100644 --- a/test/305-other-fault-handler/fault_handler.cc +++ b/test/305-other-fault-handler/fault_handler.cc @@ -23,9 +23,9 @@ #include <stdint.h> #include <sys/mman.h> +#include "base/mem_map.h" #include "fault_handler.h" #include "globals.h" -#include "mem_map.h" namespace art { diff --git a/test/411-optimizing-arith-mul/info.txt b/test/411-optimizing-arith-mul/info.txt deleted file mode 100644 index 10155512f09..00000000000 --- a/test/411-optimizing-arith-mul/info.txt +++ /dev/null @@ -1 +0,0 @@ -Tests for basic arithmethic operations. diff --git a/test/411-optimizing-arith-mul/expected.txt b/test/411-optimizing-arith/expected.txt index e69de29bb2d..e69de29bb2d 100644 --- a/test/411-optimizing-arith-mul/expected.txt +++ b/test/411-optimizing-arith/expected.txt diff --git a/test/411-optimizing-arith/info.txt b/test/411-optimizing-arith/info.txt new file mode 100644 index 00000000000..42be5d564f9 --- /dev/null +++ b/test/411-optimizing-arith/info.txt @@ -0,0 +1,7 @@ +Tests for basic arithmethic operations: + - multiply, + - subtract, + - negate, + - division, + - modulo (rem), + - shifts. diff --git a/test/417-optimizing-arith-div/src/Main.java b/test/411-optimizing-arith/src/DivTest.java index 68e89b3eb23..7696d0a806b 100644 --- a/test/417-optimizing-arith-div/src/Main.java +++ b/test/411-optimizing-arith/src/DivTest.java @@ -16,7 +16,7 @@ // Note that $opt$ is a marker for the optimizing compiler to test // it does compile the method. -public class Main { +public class DivTest { public static void expectEquals(int expected, int result) { if (expected != result) { @@ -98,11 +98,7 @@ public class Main { } } - public static void main(String[] args) { - div(); - } - - public static void div() { + public static void main() { divInt(); divLong(); divFloat(); diff --git a/runtime/base/dumpable-inl.h b/test/411-optimizing-arith/src/Main.java index 9d7fc39093b..e1a43d3b576 100644 --- a/runtime/base/dumpable-inl.h +++ b/test/411-optimizing-arith/src/Main.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2016 The Android Open Source Project + * Copyright (C) 2018 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,22 +14,13 @@ * limitations under the License. */ -#ifndef ART_RUNTIME_BASE_DUMPABLE_INL_H_ -#define ART_RUNTIME_BASE_DUMPABLE_INL_H_ - -#include "base/dumpable.h" -#include "base/mutex.h" -#include "thread-current-inl.h" - -namespace art { - -template<typename T> -inline std::ostream& operator<<(std::ostream& os, const MutatorLockedDumpable<T>& rhs) { - Locks::mutator_lock_->AssertSharedHeld(Thread::Current()); - rhs.Dump(os); - return os; +public class Main { + public static void main(String args[]) { + MulTest.main(); + SubTest.main(); + NegTest.main(); + DivTest.main(); + RemTest.main(); + ShiftsTest.main(); + } } - -} // namespace art - -#endif // ART_RUNTIME_BASE_DUMPABLE_INL_H_ diff --git a/test/411-optimizing-arith-mul/src/Main.java b/test/411-optimizing-arith/src/MulTest.java index 60e418e1e53..b9bffca0d1a 100644 --- a/test/411-optimizing-arith-mul/src/Main.java +++ b/test/411-optimizing-arith/src/MulTest.java @@ -16,7 +16,7 @@ // Note that $opt$ is a marker for the optimizing compiler to test // it does compile the method. -public class Main { +public class MulTest { public static void expectEquals(int expected, int result) { if (expected != result) { @@ -72,11 +72,7 @@ public class Main { } } - public static void main(String[] args) { - mul(); - } - - public static void mul() { + public static void main() { mulInt(); mulLong(); mulFloat(); @@ -129,9 +125,12 @@ public class Main { expectEquals(Float.NEGATIVE_INFINITY, $opt$Mul(-2F, 3.40282346638528860e+38F)); expectEquals(Float.NEGATIVE_INFINITY, $opt$Mul(2F, Float.NEGATIVE_INFINITY)); expectEquals(Float.POSITIVE_INFINITY, $opt$Mul(-2F, Float.NEGATIVE_INFINITY)); - expectEquals(Float.NEGATIVE_INFINITY, $opt$Mul(Float.POSITIVE_INFINITY, Float.NEGATIVE_INFINITY)); - expectEquals(Float.POSITIVE_INFINITY, $opt$Mul(Float.POSITIVE_INFINITY, Float.POSITIVE_INFINITY)); - expectEquals(Float.POSITIVE_INFINITY, $opt$Mul(Float.NEGATIVE_INFINITY, Float.NEGATIVE_INFINITY)); + expectEquals(Float.NEGATIVE_INFINITY, + $opt$Mul(Float.POSITIVE_INFINITY, Float.NEGATIVE_INFINITY)); + expectEquals(Float.POSITIVE_INFINITY, + $opt$Mul(Float.POSITIVE_INFINITY, Float.POSITIVE_INFINITY)); + expectEquals(Float.POSITIVE_INFINITY, + $opt$Mul(Float.NEGATIVE_INFINITY, Float.NEGATIVE_INFINITY)); } private static void mulDouble() { diff --git a/test/415-optimizing-arith-neg/src/Main.java b/test/411-optimizing-arith/src/NegTest.java index c53b639d40a..83047269bb2 100644 --- a/test/415-optimizing-arith-neg/src/Main.java +++ b/test/411-optimizing-arith/src/NegTest.java @@ -17,7 +17,7 @@ // Note that $opt$ is a marker for the optimizing compiler to test // it does compile the method, and that $noinline$ is a marker to // test that it does not inline it. -public class Main { +public class NegTest { public static void assertEquals(int expected, int result) { if (expected != result) { @@ -67,7 +67,7 @@ public class Main { } } - public static void main(String[] args) { + public static void main() { negInt(); $opt$noinline$InplaceNegOneInt(1); @@ -169,55 +169,29 @@ public class Main { } - static boolean doThrow = false; - private static void $opt$noinline$InplaceNegOneInt(int a) { - if (doThrow) { - // Try defeating inlining. - throw new Error(); - } a = -a; assertEquals(-1, a); } private static void $opt$noinline$InplaceNegOneLong(long a) { - if (doThrow) { - // Try defeating inlining. - throw new Error(); - } a = -a; assertEquals(-1L, a); } private static int $opt$noinline$NegInt(int a){ - if (doThrow) { - // Try defeating inlining. - throw new Error(); - } return -a; } private static long $opt$noinline$NegLong(long a){ - if (doThrow) { - // Try defeating inlining. - throw new Error(); - } return -a; } private static float $opt$noinline$NegFloat(float a){ - if (doThrow) { - // Try defeating inlining. - throw new Error(); - } return -a; } private static double $opt$noinline$NegDouble(double a){ - if (doThrow) { - // Try defeating inlining. - throw new Error(); - } return -a; } } diff --git a/test/428-optimizing-arith-rem/src/Main.java b/test/411-optimizing-arith/src/RemTest.java index 3f77318e6ca..1b31f635697 100644 --- a/test/428-optimizing-arith-rem/src/Main.java +++ b/test/411-optimizing-arith/src/RemTest.java @@ -14,9 +14,9 @@ * limitations under the License. */ -public class Main { +public class RemTest { - public static void main(String[] args) { + public static void main() { remInt(); remLong(); } diff --git a/test/431-optimizing-arith-shifts/src/Main.java b/test/411-optimizing-arith/src/ShiftsTest.java index b7a112f6a30..139ff70bf02 100644 --- a/test/431-optimizing-arith-shifts/src/Main.java +++ b/test/411-optimizing-arith/src/ShiftsTest.java @@ -14,7 +14,7 @@ * limitations under the License. */ -public class Main { +public class ShiftsTest { public static void expectEquals(int expected, int result) { if (expected != result) { @@ -28,7 +28,7 @@ public class Main { } } - public static void main(String[] args) { + public static void main() { testShlInt(); testShlLong(); testShrInt(); diff --git a/test/414-optimizing-arith-sub/src/Main.java b/test/411-optimizing-arith/src/SubTest.java index b4531cdfd43..9c9ea92f20a 100644 --- a/test/414-optimizing-arith-sub/src/Main.java +++ b/test/411-optimizing-arith/src/SubTest.java @@ -16,7 +16,7 @@ // Note that $opt$ is a marker for the optimizing compiler to test // it does compile the method. -public class Main { +public class SubTest { public static void expectEquals(int expected, int result) { if (expected != result) { @@ -70,7 +70,7 @@ public class Main { } } - public static void main(String[] args) { + public static void main() { subInt(); subLong(); subFloat(); diff --git a/test/414-optimizing-arith-sub/expected.txt b/test/414-optimizing-arith-sub/expected.txt deleted file mode 100644 index e69de29bb2d..00000000000 --- a/test/414-optimizing-arith-sub/expected.txt +++ /dev/null diff --git a/test/414-optimizing-arith-sub/info.txt b/test/414-optimizing-arith-sub/info.txt deleted file mode 100644 index 1eaa14887b7..00000000000 --- a/test/414-optimizing-arith-sub/info.txt +++ /dev/null @@ -1 +0,0 @@ -Subtraction tests. diff --git a/test/415-optimizing-arith-neg/expected.txt b/test/415-optimizing-arith-neg/expected.txt deleted file mode 100644 index e69de29bb2d..00000000000 --- a/test/415-optimizing-arith-neg/expected.txt +++ /dev/null diff --git a/test/415-optimizing-arith-neg/info.txt b/test/415-optimizing-arith-neg/info.txt deleted file mode 100644 index 8494aad938c..00000000000 --- a/test/415-optimizing-arith-neg/info.txt +++ /dev/null @@ -1 +0,0 @@ -Tests for arithmetic negation operations. diff --git a/test/417-optimizing-arith-div/expected.txt b/test/417-optimizing-arith-div/expected.txt deleted file mode 100644 index e69de29bb2d..00000000000 --- a/test/417-optimizing-arith-div/expected.txt +++ /dev/null diff --git a/test/417-optimizing-arith-div/info.txt b/test/417-optimizing-arith-div/info.txt deleted file mode 100644 index 1374b0ffb33..00000000000 --- a/test/417-optimizing-arith-div/info.txt +++ /dev/null @@ -1 +0,0 @@ -Tests for division operation. diff --git a/test/428-optimizing-arith-rem/expected.txt b/test/428-optimizing-arith-rem/expected.txt deleted file mode 100644 index e69de29bb2d..00000000000 --- a/test/428-optimizing-arith-rem/expected.txt +++ /dev/null diff --git a/test/428-optimizing-arith-rem/info.txt b/test/428-optimizing-arith-rem/info.txt deleted file mode 100644 index 3e37ffeee84..00000000000 --- a/test/428-optimizing-arith-rem/info.txt +++ /dev/null @@ -1 +0,0 @@ -Tests for modulo (rem) operation. diff --git a/test/431-optimizing-arith-shifts/expected.txt b/test/431-optimizing-arith-shifts/expected.txt deleted file mode 100644 index e69de29bb2d..00000000000 --- a/test/431-optimizing-arith-shifts/expected.txt +++ /dev/null diff --git a/test/431-optimizing-arith-shifts/info.txt b/test/431-optimizing-arith-shifts/info.txt deleted file mode 100644 index 14ff264662f..00000000000 --- a/test/431-optimizing-arith-shifts/info.txt +++ /dev/null @@ -1 +0,0 @@ -Tests for shift operations. diff --git a/test/441-checker-inliner/src/Main.java b/test/441-checker-inliner/src/Main.java index aff4a0717e6..3ccfce4de1d 100644 --- a/test/441-checker-inliner/src/Main.java +++ b/test/441-checker-inliner/src/Main.java @@ -152,10 +152,12 @@ public class Main { /// CHECK-DAG: <<Result:i\d+>> InvokeStaticOrDirect /// CHECK-DAG: Return [<<Result>>] + // + // Intrinsic directly simplified into Abs and evaluated! + // /// CHECK-START: int Main.InlinedIntrinsicsAreStillIntrinsic() inliner (after) - /// CHECK-DAG: <<ConstMinus1:i\d+>> IntConstant -1 - /// CHECK-DAG: <<Result:i\d+>> InvokeStaticOrDirect intrinsic:MathAbsInt - /// CHECK-DAG: Return [<<Result>>] + /// CHECK-DAG: <<Const1:i\d+>> IntConstant 1 + /// CHECK-DAG: Return [<<Const1>>] public static int InlinedIntrinsicsAreStillIntrinsic() { return returnAbs(-1); diff --git a/test/445-checker-licm/src/Main.java b/test/445-checker-licm/src/Main.java index 9635e702787..517aacd9f20 100644 --- a/test/445-checker-licm/src/Main.java +++ b/test/445-checker-licm/src/Main.java @@ -153,14 +153,17 @@ public class Main { return result; } - /// CHECK-START: int Main.invariantBoundIntrinsic(int) licm (before) + /// CHECK-START: int Main.invariantBoundIntrinsic(int) instruction_simplifier (before) /// CHECK-DAG: InvokeStaticOrDirect loop:{{B\d+}} + // + /// CHECK-START: int Main.invariantBoundIntrinsic(int) licm (before) + /// CHECK-DAG: Abs loop:{{B\d+}} /// CHECK-START: int Main.invariantBoundIntrinsic(int) licm (after) - /// CHECK-NOT: InvokeStaticOrDirect loop:{{B\d+}} + /// CHECK-NOT: Abs loop:{{B\d+}} /// CHECK-START: int Main.invariantBoundIntrinsic(int) licm (after) - /// CHECK-DAG: InvokeStaticOrDirect loop:none + /// CHECK-DAG: Abs loop:none public static int invariantBoundIntrinsic(int x) { int result = 0; @@ -172,14 +175,17 @@ public class Main { return result; } - /// CHECK-START: int Main.invariantBodyIntrinsic(int, int) licm (before) + /// CHECK-START: int Main.invariantBodyIntrinsic(int, int) instruction_simplifier (before) /// CHECK-DAG: InvokeStaticOrDirect loop:{{B\d+}} + /// CHECK-START: int Main.invariantBodyIntrinsic(int, int) licm (before) + /// CHECK-DAG: Max loop:{{B\d+}} + /// CHECK-START: int Main.invariantBodyIntrinsic(int, int) licm (after) - /// CHECK-NOT: InvokeStaticOrDirect loop:{{B\d+}} + /// CHECK-NOT: Max loop:{{B\d+}} /// CHECK-START: int Main.invariantBodyIntrinsic(int, int) licm (after) - /// CHECK-DAG: InvokeStaticOrDirect loop:none + /// CHECK-DAG: Max loop:none public static int invariantBodyIntrinsic(int x, int y) { int result = 0; diff --git a/test/449-checker-bce/src/Main.java b/test/449-checker-bce/src/Main.java index 4868355b905..1144366dfb8 100644 --- a/test/449-checker-bce/src/Main.java +++ b/test/449-checker-bce/src/Main.java @@ -1151,8 +1151,7 @@ public class Main { /// CHECK-DAG: <<Array2>> NullCheck [<<Get1>>] loop:<<InnerLoop>> /// CHECK-DAG: <<Len2:i\d+>> ArrayLength [<<Array2>>] loop:<<InnerLoop>> /// CHECK-DAG: <<Bounds2>> BoundsCheck [<<Index2:i\d+>>,<<Len2>>] loop:<<InnerLoop>> - // Note: The ArtMethod* (typed as int or long) is optional after sharpening. - /// CHECK-DAG: InvokeStaticOrDirect [<<Get2>>{{(,[ij]\d+)?}}] loop:<<InnerLoop>> + /// CHECK-DAG: Abs [<<Get2>>] loop:<<InnerLoop>> /// CHECK-DAG: <<Index2>> Phi loop:<<InnerLoop>> /// CHECK-DAG: <<Index1>> Phi loop:<<OuterLoop:B\d+>> /// CHECK-DAG: <<Field1>> StaticFieldGet loop:none @@ -1165,8 +1164,7 @@ public class Main { /// CHECK-DAG: <<Get1:l\d+>> ArrayGet [<<Array1:l\d+>>,<<Index1:i\d+>>] loop:<<OuterLoop>> // Array reference ..[j] still in inner loop, with a direct index. /// CHECK-DAG: <<Get2:i\d+>> ArrayGet [<<Array2:l\d+>>,<<Index2:i\d+>>] loop:<<InnerLoop:B\d+>> - // Note: The ArtMethod* (typed as int or long) is optional after sharpening. - /// CHECK-DAG: InvokeStaticOrDirect [<<Get2>>{{(,[ij]\d+)?}}] loop:<<InnerLoop>> + /// CHECK-DAG: Abs [<<Get2>>] loop:<<InnerLoop>> /// CHECK-DAG: <<Index2>> Phi loop:<<InnerLoop>> /// CHECK-DAG: <<Index1>> Phi loop:<<OuterLoop>> // Synthetic phi. diff --git a/test/455-checker-gvn/src/Main.java b/test/455-checker-gvn/src/Main.java index 6df57fdeefd..74b18398088 100644 --- a/test/455-checker-gvn/src/Main.java +++ b/test/455-checker-gvn/src/Main.java @@ -40,7 +40,7 @@ public class Main { /// CHECK: StaticFieldGet /// CHECK: StaticFieldGet /// CHECK: Mul - /// CHECK: InvokeStaticOrDirect + /// CHECK: Abs /// CHECK: StaticFieldGet /// CHECK: StaticFieldGet /// CHECK: Mul @@ -50,7 +50,7 @@ public class Main { /// CHECK: StaticFieldGet /// CHECK: StaticFieldGet /// CHECK: Mul - /// CHECK: InvokeStaticOrDirect + /// CHECK: Abs /// CHECK-NOT: StaticFieldGet /// CHECK-NOT: StaticFieldGet /// CHECK-NOT: Mul @@ -66,13 +66,13 @@ public class Main { } /// CHECK-START: int Main.directIntrinsic(int) GVN (before) - /// CHECK: InvokeStaticOrDirect - /// CHECK: InvokeStaticOrDirect + /// CHECK: Abs + /// CHECK: Abs /// CHECK: Add /// CHECK-START: int Main.directIntrinsic(int) GVN (after) - /// CHECK: InvokeStaticOrDirect - /// CHECK-NOT: InvokeStaticOrDirect + /// CHECK: Abs + /// CHECK-NOT: Abs /// CHECK: Add public static int directIntrinsic(int x) { diff --git a/test/458-checker-instruct-simplification/src/Main.java b/test/458-checker-instruct-simplification/src/Main.java index 7797f318673..444b4557ce4 100644 --- a/test/458-checker-instruct-simplification/src/Main.java +++ b/test/458-checker-instruct-simplification/src/Main.java @@ -2572,7 +2572,7 @@ public class Main { /// CHECK-DAG: <<Const1:i\d+>> IntConstant 1 /// CHECK-DAG: <<Const255:i\d+>> IntConstant 255 /// CHECK-DAG: <<Select:i\d+>> Select [<<Const0>>,<<Const1>>,<<Arg>>] - /// CHECK-DAG: <<And:i\d+>> And [<<Const255>>,<<Select>>] + /// CHECK-DAG: <<And:i\d+>> And [<<Select>>,<<Const255>>] /// CHECK-DAG: <<Conv:b\d+>> TypeConversion [<<And>>] /// CHECK-DAG: Return [<<Conv>>] diff --git a/test/461-get-reference-vreg/get_reference_vreg_jni.cc b/test/461-get-reference-vreg/get_reference_vreg_jni.cc index b2cad678299..7eb3fe5ae6a 100644 --- a/test/461-get-reference-vreg/get_reference_vreg_jni.cc +++ b/test/461-get-reference-vreg/get_reference_vreg_jni.cc @@ -37,21 +37,21 @@ class TestVisitor : public StackVisitor { ArtMethod* m = GetMethod(); std::string m_name(m->GetName()); - if (m_name.compare("testThisWithInstanceCall") == 0) { + if (m_name.compare("$noinline$testThisWithInstanceCall") == 0) { found_method_index_ = 1; uint32_t value = 0; CHECK(GetVReg(m, 1, kReferenceVReg, &value)); CHECK_EQ(reinterpret_cast<mirror::Object*>(value), this_value_); CHECK_EQ(GetThisObject(), this_value_); - } else if (m_name.compare("testThisWithStaticCall") == 0) { + } else if (m_name.compare("$noinline$testThisWithStaticCall") == 0) { found_method_index_ = 2; uint32_t value = 0; CHECK(GetVReg(m, 1, kReferenceVReg, &value)); - } else if (m_name.compare("testParameter") == 0) { + } else if (m_name.compare("$noinline$testParameter") == 0) { found_method_index_ = 3; uint32_t value = 0; CHECK(GetVReg(m, 1, kReferenceVReg, &value)); - } else if (m_name.compare("testObjectInScope") == 0) { + } else if (m_name.compare("$noinline$testObjectInScope") == 0) { found_method_index_ = 4; uint32_t value = 0; CHECK(GetVReg(m, 0, kReferenceVReg, &value)); diff --git a/test/461-get-reference-vreg/src/Main.java b/test/461-get-reference-vreg/src/Main.java index f7d43568d5c..a14b0e96ede 100644 --- a/test/461-get-reference-vreg/src/Main.java +++ b/test/461-get-reference-vreg/src/Main.java @@ -18,19 +18,19 @@ public class Main { public Main() { } - int testThisWithInstanceCall() { + int $noinline$testThisWithInstanceCall() { return doNativeCallRef(); } - int testThisWithStaticCall() { + int $noinline$testThisWithStaticCall() { return doStaticNativeCallRef(); } - static int testParameter(Object a) { + static int $noinline$testParameter(Object a) { return doStaticNativeCallRef(); } - static int testObjectInScope() { + static int $noinline$testObjectInScope() { Object a = array[0]; return doStaticNativeCallRef(); } @@ -41,19 +41,19 @@ public class Main { public static void main(String[] args) { System.loadLibrary(args[0]); Main rm = new Main(); - if (rm.testThisWithInstanceCall() != 1) { + if (rm.$noinline$testThisWithInstanceCall() != 1) { throw new Error("Expected 1"); } - if (rm.testThisWithStaticCall() != 2) { + if (rm.$noinline$testThisWithStaticCall() != 2) { throw new Error("Expected 2"); } - if (testParameter(new Object()) != 3) { + if ($noinline$testParameter(new Object()) != 3) { throw new Error("Expected 3"); } - if (testObjectInScope() != 4) { + if ($noinline$testObjectInScope() != 4) { throw new Error("Expected 4"); } } diff --git a/test/551-checker-shifter-operand/build b/test/551-checker-shifter-operand/build new file mode 100644 index 00000000000..d85147f17bd --- /dev/null +++ b/test/551-checker-shifter-operand/build @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Copyright 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See b/65168732 +export USE_D8=false + +./default-build "$@" diff --git a/test/551-checker-shifter-operand/src/Main.java b/test/551-checker-shifter-operand/src/Main.java index 3177ec0a3ce..fb769046773 100644 --- a/test/551-checker-shifter-operand/src/Main.java +++ b/test/551-checker-shifter-operand/src/Main.java @@ -728,40 +728,9 @@ public class Main { /// CHECK: UShr /// CHECK-NOT: UShr // - // Note: simplification followed by GVN exposes the common subexpressions between shifts with larger distance - // `b << 62`, `b << 63` etc. and the equivalent smaller distances. - // - /// CHECK-START: void Main.$opt$validateShiftInt(int, int) GVN (after) - /// CHECK: Shl - /// CHECK: Shl - /// CHECK: Shl - /// CHECK: Shl - /// CHECK: Shl - /// CHECK: Shl - /// CHECK: Shl - /// CHECK: Shl - /// CHECK: Shl - /// CHECK-NOT: Shl - /// CHECK: Shr - /// CHECK: Shr - /// CHECK: Shr - /// CHECK: Shr - /// CHECK: Shr - /// CHECK: Shr - /// CHECK: Shr - /// CHECK: Shr - /// CHECK: Shr - /// CHECK-NOT: Shl - /// CHECK: UShr - /// CHECK: UShr - /// CHECK: UShr - /// CHECK: UShr - /// CHECK: UShr - /// CHECK: UShr - /// CHECK: UShr - /// CHECK: UShr - /// CHECK: UShr - /// CHECK-NOT: UShr + // Note: running extra simplification before GVN would expose the common subexpressions between + // shifts with larger distance `b << 62`, `b << 63` etc. and the equivalent smaller distances. + // TODO: b/78171933 // /// CHECK-START-ARM: void Main.$opt$validateShiftInt(int, int) instruction_simplifier_arm (after) /// CHECK: DataProcWithShifterOp @@ -791,6 +760,12 @@ public class Main { /// CHECK: DataProcWithShifterOp /// CHECK: DataProcWithShifterOp /// CHECK: DataProcWithShifterOp + /// CHECK: DataProcWithShifterOp + /// CHECK: DataProcWithShifterOp + /// CHECK: DataProcWithShifterOp + /// CHECK: DataProcWithShifterOp + /// CHECK: DataProcWithShifterOp + /// CHECK: DataProcWithShifterOp /// CHECK-NOT: DataProcWithShifterOp /// CHECK-START-ARM: void Main.$opt$validateShiftInt(int, int) instruction_simplifier_arm (after) @@ -826,6 +801,12 @@ public class Main { /// CHECK: DataProcWithShifterOp /// CHECK: DataProcWithShifterOp /// CHECK: DataProcWithShifterOp + /// CHECK: DataProcWithShifterOp + /// CHECK: DataProcWithShifterOp + /// CHECK: DataProcWithShifterOp + /// CHECK: DataProcWithShifterOp + /// CHECK: DataProcWithShifterOp + /// CHECK: DataProcWithShifterOp /// CHECK-NOT: DataProcWithShifterOp /// CHECK-START-ARM64: void Main.$opt$validateShiftInt(int, int) instruction_simplifier_arm64 (after) diff --git a/test/552-checker-sharpening/src/Main.java b/test/552-checker-sharpening/src/Main.java index 3173afdfcd8..17707e12783 100644 --- a/test/552-checker-sharpening/src/Main.java +++ b/test/552-checker-sharpening/src/Main.java @@ -141,7 +141,7 @@ public class Main { /// CHECK-START-{ARM,ARM64,MIPS,MIPS64,X86,X86_64}: java.lang.String Main.$noinline$getBootImageString() builder (after) // Note: load kind depends on PIC/non-PIC - /// CHECK: LoadString load_kind:{{BootImageAddress|BootImageInternTable}} + /// CHECK: LoadString load_kind:{{BootImageAddress|BootImageRelRo}} public static String $noinline$getBootImageString() { // Prevent inlining to avoid the string comparison being optimized away. @@ -169,7 +169,7 @@ public class Main { /// CHECK-START-{ARM,ARM64,MIPS,MIPS64,X86,X86_64}: java.lang.Class Main.$noinline$getStringClass() builder (after) // Note: load kind depends on PIC/non-PIC - /// CHECK: LoadClass load_kind:{{BootImageAddress|BootImageClassTable}} class_name:java.lang.String + /// CHECK: LoadClass load_kind:{{BootImageAddress|BootImageRelRo}} class_name:java.lang.String public static Class<?> $noinline$getStringClass() { // Prevent inlining to avoid the string comparison being optimized away. @@ -195,6 +195,32 @@ public class Main { return Other.class; } + /// CHECK-START-{ARM,ARM64,MIPS,MIPS64,X86,X86_64}: java.lang.String Main.$noinline$toHexString(int) builder (after) + /// CHECK: InvokeStaticOrDirect method_load_kind:RuntimeCall + + /// CHECK-START-{ARM,ARM64,MIPS,MIPS64,X86,X86_64}: java.lang.String Main.$noinline$toHexString(int) sharpening (after) + // Note: load kind depends on PIC/non-PIC + /// CHECK: InvokeStaticOrDirect method_load_kind:{{BootImageRelRo|DirectAddress}} + public static String $noinline$toHexString(int value) { + return Integer.toString(value, 16); + } + + /// CHECK-START-{ARM,ARM64,MIPS,MIPS64,X86,X86_64}: java.lang.String Main.$noinline$toHexStringIndirect(int) builder (after) + /// CHECK: InvokeStaticOrDirect method_load_kind:RuntimeCall + + /// CHECK-START-{ARM,ARM64,MIPS,MIPS64,X86,X86_64}: java.lang.String Main.$noinline$toHexStringIndirect(int) sharpening (after) + /// CHECK: InvokeStaticOrDirect method_load_kind:BssEntry + + /// CHECK-START-X86: java.lang.String Main.$noinline$toHexStringIndirect(int) pc_relative_fixups_x86 (before) + /// CHECK-NOT: X86ComputeBaseMethodAddress + + /// CHECK-START-X86: java.lang.String Main.$noinline$toHexStringIndirect(int) pc_relative_fixups_x86 (after) + /// CHECK-DAG: X86ComputeBaseMethodAddress + /// CHECK-DAG: InvokeStaticOrDirect method_load_kind:BssEntry + public static String $noinline$toHexStringIndirect(int value) { + return $noinline$toHexString(value); + } + public static void main(String[] args) { assertIntEquals(1, testSimple(1)); assertIntEquals(1, testDiamond(false, 1)); @@ -208,6 +234,8 @@ public class Main { assertStringEquals("non-boot-image-string", $noinline$getNonBootImageString()); assertClassEquals(String.class, $noinline$getStringClass()); assertClassEquals(Other.class, $noinline$getOtherClass()); + assertStringEquals("12345678", $noinline$toHexString(0x12345678)); + assertStringEquals("76543210", $noinline$toHexStringIndirect(0x76543210)); } } diff --git a/test/562-checker-no-intermediate/src/Main.java b/test/562-checker-no-intermediate/src/Main.java index 104ba8bc061..2b199181426 100644 --- a/test/562-checker-no-intermediate/src/Main.java +++ b/test/562-checker-no-intermediate/src/Main.java @@ -18,7 +18,7 @@ public class Main { /** * Check that the intermediate address computation is not reordered or merged - * across the call to Math.abs(). + * across a method call. */ /// CHECK-START-ARM: void Main.main(java.lang.String[]) instruction_simplifier_arm (before) @@ -26,7 +26,7 @@ public class Main { /// CHECK-DAG: <<Array:l\d+>> NullCheck /// CHECK-DAG: <<Index:i\d+>> BoundsCheck /// CHECK-DAG: <<ArrayGet:i\d+>> ArrayGet [<<Array>>,<<Index>>] - /// CHECK-DAG: <<AbsM42:i\d+>> InvokeStaticOrDirect [<<ConstM42>>] intrinsic:MathAbsInt + /// CHECK-DAG: <<AbsM42:i\d+>> InvokeStaticOrDirect [<<ConstM42>>{{(,[ij]\d+)?}}] method_name:Main.$noinline$abs /// CHECK-DAG: <<Add:i\d+>> Add [<<ArrayGet>>,<<AbsM42>>] /// CHECK-DAG: ArraySet [<<Array>>,<<Index>>,<<Add>>] @@ -37,7 +37,7 @@ public class Main { /// CHECK-DAG: <<Index:i\d+>> BoundsCheck /// CHECK-DAG: <<Address1:i\d+>> IntermediateAddress [<<Array>>,<<DataOffset>>] /// CHECK-DAG: <<ArrayGet:i\d+>> ArrayGet [<<Address1>>,<<Index>>] - /// CHECK-DAG: <<AbsM42:i\d+>> InvokeStaticOrDirect [<<ConstM42>>] intrinsic:MathAbsInt + /// CHECK-DAG: <<AbsM42:i\d+>> InvokeStaticOrDirect [<<ConstM42>>{{(,[ij]\d+)?}}] method_name:Main.$noinline$abs /// CHECK-DAG: <<Add:i\d+>> Add [<<ArrayGet>>,<<AbsM42>>] /// CHECK-DAG: <<Address2:i\d+>> IntermediateAddress [<<Array>>,<<DataOffset>>] /// CHECK-DAG: ArraySet [<<Address2>>,<<Index>>,<<Add>>] @@ -49,7 +49,7 @@ public class Main { /// CHECK-DAG: <<Index:i\d+>> BoundsCheck /// CHECK-DAG: <<Address1:i\d+>> IntermediateAddress [<<Array>>,<<DataOffset>>] /// CHECK-DAG: <<ArrayGet:i\d+>> ArrayGet [<<Address1>>,<<Index>>] - /// CHECK-DAG: <<AbsM42:i\d+>> InvokeStaticOrDirect [<<ConstM42>>] intrinsic:MathAbsInt + /// CHECK-DAG: <<AbsM42:i\d+>> InvokeStaticOrDirect [<<ConstM42>>{{(,[ij]\d+)?}}] method_name:Main.$noinline$abs /// CHECK-DAG: <<Add:i\d+>> Add [<<ArrayGet>>,<<AbsM42>>] /// CHECK-DAG: <<Address2:i\d+>> IntermediateAddress [<<Array>>,<<DataOffset>>] /// CHECK-DAG: ArraySet [<<Address2>>,<<Index>>,<<Add>>] @@ -60,7 +60,7 @@ public class Main { /// CHECK-DAG: <<Array:l\d+>> NullCheck /// CHECK-DAG: <<Index:i\d+>> BoundsCheck /// CHECK-DAG: <<ArrayGet:i\d+>> ArrayGet [<<Array>>,<<Index>>] - /// CHECK-DAG: <<AbsM42:i\d+>> InvokeStaticOrDirect [<<ConstM42>>] intrinsic:MathAbsInt + /// CHECK-DAG: <<AbsM42:i\d+>> InvokeStaticOrDirect [<<ConstM42>>{{(,[ij]\d+)?}}] method_name:Main.$noinline$abs /// CHECK-DAG: <<Add:i\d+>> Add [<<ArrayGet>>,<<AbsM42>>] /// CHECK-DAG: ArraySet [<<Array>>,<<Index>>,<<Add>>] @@ -71,7 +71,7 @@ public class Main { /// CHECK-DAG: <<Index:i\d+>> BoundsCheck /// CHECK-DAG: <<Address1:i\d+>> IntermediateAddress [<<Array>>,<<DataOffset>>] /// CHECK-DAG: <<ArrayGet:i\d+>> ArrayGet [<<Address1>>,<<Index>>] - /// CHECK-DAG: <<AbsM42:i\d+>> InvokeStaticOrDirect [<<ConstM42>>] intrinsic:MathAbsInt + /// CHECK-DAG: <<AbsM42:i\d+>> InvokeStaticOrDirect [<<ConstM42>>{{(,[ij]\d+)?}}] method_name:Main.$noinline$abs /// CHECK-DAG: <<Add:i\d+>> Add [<<ArrayGet>>,<<AbsM42>>] /// CHECK-DAG: <<Address2:i\d+>> IntermediateAddress [<<Array>>,<<DataOffset>>] /// CHECK-DAG: ArraySet [<<Address2>>,<<Index>>,<<Add>>] @@ -83,13 +83,17 @@ public class Main { /// CHECK-DAG: <<Index:i\d+>> BoundsCheck /// CHECK-DAG: <<Address1:i\d+>> IntermediateAddress [<<Array>>,<<DataOffset>>] /// CHECK-DAG: <<ArrayGet:i\d+>> ArrayGet [<<Address1>>,<<Index>>] - /// CHECK-DAG: <<AbsM42:i\d+>> InvokeStaticOrDirect [<<ConstM42>>] intrinsic:MathAbsInt + /// CHECK-DAG: <<AbsM42:i\d+>> InvokeStaticOrDirect [<<ConstM42>>{{(,[ij]\d+)?}}] method_name:Main.$noinline$abs /// CHECK-DAG: <<Add:i\d+>> Add [<<ArrayGet>>,<<AbsM42>>] /// CHECK-DAG: <<Address2:i\d+>> IntermediateAddress [<<Array>>,<<DataOffset>>] /// CHECK-DAG: ArraySet [<<Address2>>,<<Index>>,<<Add>>] public static void main(String[] args) { - array[index] += Math.abs(-42); + array[index] += $noinline$abs(-42); + } + + public static int $noinline$abs(int value) { + return Math.abs(value); } static int index = 0; diff --git a/test/565-checker-doublenegbitwise/src/Main.java b/test/565-checker-doublenegbitwise/src/Main.java index 816cafc2ba9..80358cd2bad 100644 --- a/test/565-checker-doublenegbitwise/src/Main.java +++ b/test/565-checker-doublenegbitwise/src/Main.java @@ -101,13 +101,13 @@ public class Main { /// CHECK-DAG: <<Const1:i\d+>> IntConstant 1 /// CHECK-DAG: <<Select1:i\d+>> Select [<<Const1>>,<<Const0>>,<<P1>>] /// CHECK-DAG: <<Select2:i\d+>> Select [<<Const1>>,<<Const0>>,<<P2>>] - /// CHECK-DAG: <<And:i\d+>> And [<<Select2>>,<<Select1>>] + /// CHECK-DAG: <<And:i\d+>> And [<<Select1>>,<<Select2>>] /// CHECK-DAG: Return [<<And>>] /// CHECK-START: boolean Main.$opt$noinline$booleanAndToOr(boolean, boolean) instruction_simplifier$after_inlining (after) /// CHECK-DAG: <<Cond1:z\d+>> ParameterValue /// CHECK-DAG: <<Cond2:z\d+>> ParameterValue - /// CHECK-DAG: <<Or:i\d+>> Or [<<Cond2>>,<<Cond1>>] + /// CHECK-DAG: <<Or:i\d+>> Or [<<Cond1>>,<<Cond2>>] /// CHECK-DAG: <<BooleanNot:z\d+>> BooleanNot [<<Or>>] /// CHECK-DAG: Return [<<BooleanNot>>] @@ -172,13 +172,13 @@ public class Main { /// CHECK-DAG: <<Const1:i\d+>> IntConstant 1 /// CHECK-DAG: <<Select1:i\d+>> Select [<<Const1>>,<<Const0>>,<<P1>>] /// CHECK-DAG: <<Select2:i\d+>> Select [<<Const1>>,<<Const0>>,<<P2>>] - /// CHECK-DAG: <<Or:i\d+>> Or [<<Select2>>,<<Select1>>] + /// CHECK-DAG: <<Or:i\d+>> Or [<<Select1>>,<<Select2>>] /// CHECK-DAG: Return [<<Or>>] /// CHECK-START: boolean Main.$opt$noinline$booleanOrToAnd(boolean, boolean) instruction_simplifier$after_inlining (after) /// CHECK-DAG: <<Cond1:z\d+>> ParameterValue /// CHECK-DAG: <<Cond2:z\d+>> ParameterValue - /// CHECK-DAG: <<And:i\d+>> And [<<Cond2>>,<<Cond1>>] + /// CHECK-DAG: <<And:i\d+>> And [<<Cond1>>,<<Cond2>>] /// CHECK-DAG: <<BooleanNot:z\d+>> BooleanNot [<<And>>] /// CHECK-DAG: Return [<<BooleanNot>>] @@ -282,13 +282,13 @@ public class Main { /// CHECK-DAG: <<Const1:i\d+>> IntConstant 1 /// CHECK-DAG: <<Select1:i\d+>> Select [<<Const1>>,<<Const0>>,<<P1>>] /// CHECK-DAG: <<Select2:i\d+>> Select [<<Const1>>,<<Const0>>,<<P2>>] - /// CHECK-DAG: <<Xor:i\d+>> Xor [<<Select2>>,<<Select1>>] + /// CHECK-DAG: <<Xor:i\d+>> Xor [<<Select1>>,<<Select2>>] /// CHECK-DAG: Return [<<Xor>>] /// CHECK-START: boolean Main.$opt$noinline$booleanNotXorToXor(boolean, boolean) instruction_simplifier$after_inlining (after) /// CHECK-DAG: <<Cond1:z\d+>> ParameterValue /// CHECK-DAG: <<Cond2:z\d+>> ParameterValue - /// CHECK-DAG: <<Xor:i\d+>> Xor [<<Cond2>>,<<Cond1>>] + /// CHECK-DAG: <<Xor:i\d+>> Xor [<<Cond1>>,<<Cond2>>] /// CHECK-DAG: Return [<<Xor>>] /// CHECK-START: boolean Main.$opt$noinline$booleanNotXorToXor(boolean, boolean) instruction_simplifier$after_bce (after) diff --git a/test/570-checker-osr/src/Main.java b/test/570-checker-osr/src/Main.java index 4de563469dc..78f3c32d2f7 100644 --- a/test/570-checker-osr/src/Main.java +++ b/test/570-checker-osr/src/Main.java @@ -61,51 +61,71 @@ public class Main { } public static int $noinline$returnInt() { - if (doThrow) throw new Error(""); + // If we are running in non-JIT mode, or were unlucky enough to get this method + // already JITted, skip the wait for OSR code. + boolean interpreting = isInInterpreter("$noinline$returnInt"); int i = 0; for (; i < 100000; ++i) { } - while (!isInOsrCode("$noinline$returnInt")) {} + if (interpreting) { + while (!isInOsrCode("$noinline$returnInt")) {} + } System.out.println(i); return 53; } public static float $noinline$returnFloat() { - if (doThrow) throw new Error(""); + // If we are running in non-JIT mode, or were unlucky enough to get this method + // already JITted, skip the wait for OSR code. + boolean interpreting = isInInterpreter("$noinline$returnFloat"); int i = 0; for (; i < 200000; ++i) { } - while (!isInOsrCode("$noinline$returnFloat")) {} + if (interpreting) { + while (!isInOsrCode("$noinline$returnFloat")) {} + } System.out.println(i); return 42.2f; } public static double $noinline$returnDouble() { - if (doThrow) throw new Error(""); + // If we are running in non-JIT mode, or were unlucky enough to get this method + // already JITted, skip the wait for OSR code. + boolean interpreting = isInInterpreter("$noinline$returnDouble"); int i = 0; for (; i < 300000; ++i) { } - while (!isInOsrCode("$noinline$returnDouble")) {} + if (interpreting) { + while (!isInOsrCode("$noinline$returnDouble")) {} + } System.out.println(i); return Double.longBitsToDouble(0xF000000000001111L); } public static long $noinline$returnLong() { - if (doThrow) throw new Error(""); + // If we are running in non-JIT mode, or were unlucky enough to get this method + // already JITted, skip the wait for OSR code. + boolean interpreting = isInInterpreter("$noinline$returnLong"); int i = 0; for (; i < 400000; ++i) { } - while (!isInOsrCode("$noinline$returnLong")) {} + if (interpreting) { + while (!isInOsrCode("$noinline$returnLong")) {} + } System.out.println(i); return 0xFFFF000000001111L; } public static void $noinline$deopt() { - if (doThrow) throw new Error(""); + // If we are running in non-JIT mode, or were unlucky enough to get this method + // already JITted, skip the wait for OSR code. + boolean interpreting = isInInterpreter("$noinline$deopt"); int i = 0; for (; i < 100000; ++i) { } - while (!isInOsrCode("$noinline$deopt")) {} + if (interpreting) { + while (!isInOsrCode("$noinline$deopt")) {} + } DeoptimizationController.startDeoptimization(); } @@ -242,7 +262,6 @@ public class Main { public static void $opt$noinline$testOsrInlineLoop(String[] args) { // Regression test for inlining a method with a loop to a method without a loop in OSR mode. - if (doThrow) throw new Error(); assertIntEquals(12, $opt$inline$testRemoveSuspendCheck(12, 5)); // Since we cannot have a loop directly in this method, we need to force the OSR // compilation from native code. @@ -280,8 +299,6 @@ public class Main { public static native boolean isInInterpreter(String methodName); public static native void ensureHasProfilingInfo(String methodName); public static native void ensureHasOsrCode(String methodName); - - public static boolean doThrow = false; } class SubMain extends Main { diff --git a/test/597-deopt-invoke-stub/run b/test/597-deopt-invoke-stub/run index 53b7c4cc719..990c30e70cc 100644 --- a/test/597-deopt-invoke-stub/run +++ b/test/597-deopt-invoke-stub/run @@ -16,6 +16,6 @@ # In order to test deoptimizing at quick-to-interpreter bridge, # we want to run in debuggable mode with jit compilation. -# We also bump up the jit threshold to 10 to make sure that the method +# We also bump up the jit threshold to 10000 to make sure that the method # that should be interpreted is not compiled. -exec ${RUN} --jit --runtime-option -Xjitthreshold:10000 -Xcompiler-option --debuggable "${@}" +exec ${RUN} "${@}" --jit --runtime-option -Xjitthreshold:10000 -Xcompiler-option --debuggable diff --git a/test/616-cha-unloading/cha_unload.cc b/test/616-cha-unloading/cha_unload.cc index 4be3456e3d7..b17be6bd07b 100644 --- a/test/616-cha-unloading/cha_unload.cc +++ b/test/616-cha-unloading/cha_unload.cc @@ -19,6 +19,7 @@ #include <iostream> #include "art_method.h" +#include "class_linker.h" #include "jit/jit.h" #include "linear_alloc.h" #include "nativehelper/ScopedUtfChars.h" @@ -29,6 +30,22 @@ namespace art { namespace { +class FindPointerAllocatorVisitor : public AllocatorVisitor { + public: + explicit FindPointerAllocatorVisitor(void* ptr) : is_found(false), ptr_(ptr) {} + + bool Visit(LinearAlloc* alloc) + REQUIRES_SHARED(Locks::classlinker_classes_lock_, Locks::mutator_lock_) OVERRIDE { + is_found = alloc->Contains(ptr_); + return !is_found; + } + + bool is_found; + + private: + void* ptr_; +}; + extern "C" JNIEXPORT jlong JNICALL Java_Main_getArtMethod(JNIEnv* env, jclass, jobject java_method) { @@ -40,13 +57,30 @@ extern "C" JNIEXPORT jlong JNICALL Java_Main_getArtMethod(JNIEnv* env, extern "C" JNIEXPORT void JNICALL Java_Main_reuseArenaOfMethod(JNIEnv*, jclass, jlong art_method) { - // Create a new allocation and use it to request a specified amount of arenas. - // Hopefully one of them is a reused one, the one that covers the art_method pointer. + void* ptr = reinterpret_cast<void*>(static_cast<uintptr_t>(art_method)); + + ReaderMutexLock mu(Thread::Current(), *Locks::mutator_lock_); + ReaderMutexLock mu2(Thread::Current(), *Locks::classlinker_classes_lock_); + // Check if the arena was already implicitly reused by boot classloader. + if (Runtime::Current()->GetLinearAlloc()->Contains(ptr)) { + return; + } + + // Check if the arena was already implicitly reused by some other classloader. + FindPointerAllocatorVisitor visitor(ptr); + Runtime::Current()->GetClassLinker()->VisitAllocators(&visitor); + if (visitor.is_found) { + return; + } + + // The arena was not reused yet. Do it explicitly. + // Create a new allocation and use it to request new arenas until one of them is + // a reused one that covers the art_method pointer. std::unique_ptr<LinearAlloc> alloc(Runtime::Current()->CreateLinearAlloc()); do { - // Ask for a byte - it's sufficient to get an arena and not have issues with size. + // Ask for a byte - it's sufficient to get an arena. alloc->Alloc(Thread::Current(), 1); - } while (!alloc->Contains(reinterpret_cast<void*>(static_cast<uintptr_t>(art_method)))); + } while (!alloc->Contains(ptr)); } } // namespace diff --git a/test/623-checker-loop-regressions/src/Main.java b/test/623-checker-loop-regressions/src/Main.java index 4e2b241fd71..ff6e335b7f8 100644 --- a/test/623-checker-loop-regressions/src/Main.java +++ b/test/623-checker-loop-regressions/src/Main.java @@ -584,6 +584,18 @@ public class Main { s24 + s25 + s26 + s27 + s28 + s29 + s30 + s31; } + public static int reductionIntoReplication() { + int[] a = { 1, 2, 3, 4 }; + int x = 0; + for (int i = 0; i < 4; i++) { + x += a[i]; + } + for (int i = 0; i < 4; i++) { + a[i] = x; + } + return a[3]; + } + public static void main(String[] args) { System.loadLibrary(args[0]); @@ -767,6 +779,8 @@ public class Main { expectEquals(85800, reduction32Values(a1, a2, a3, a4)); } + expectEquals(10, reductionIntoReplication()); + System.out.println("passed"); } diff --git a/test/626-checker-arm64-scratch-register/build b/test/626-checker-arm64-scratch-register/build new file mode 100644 index 00000000000..d85147f17bd --- /dev/null +++ b/test/626-checker-arm64-scratch-register/build @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Copyright 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See b/65168732 +export USE_D8=false + +./default-build "$@" diff --git a/test/631-checker-fp-abs/src/Main.java b/test/631-checker-fp-abs/src/Main.java index 0f85dc68650..2db93b8248b 100644 --- a/test/631-checker-fp-abs/src/Main.java +++ b/test/631-checker-fp-abs/src/Main.java @@ -23,6 +23,9 @@ */ public class Main { + private final static boolean isDalvik = + System.getProperty("java.vm.name").equals("Dalvik"); + private static final int SPQUIET = 1 << 22; private static final long DPQUIET = 1L << 51; @@ -73,13 +76,16 @@ public class Main { // A few NaN numbers. int[] spnans = { - 0x7f800001, + 0x7f800001, // signaling 0x7fa00000, - 0x7fc00000, + 0x7fbfffff, + 0x7fc00000, // quiet + 0x7fc00001, 0x7fffffff, - 0xff800001, + 0xff800001, // signaling 0xffa00000, - 0xffc00000, + 0xffbfffff, + 0xffc00000, // quiet 0xffffffff }; for (int i = 0; i < spnans.length; i++) { @@ -142,6 +148,13 @@ public class Main { // We allow that an expected NaN result has become quiet. private static void expectEqualsNaN32(int expected, int result) { if (expected != result && (expected | SPQUIET) != result) { + if (!isDalvik) { + // If not on ART, relax the expected value more towards just + // "spec compliance" and allow sign bit to remain set for NaN. + if (expected == (result & Integer.MAX_VALUE)) { + return; + } + } throw new Error("Expected: 0x" + Integer.toHexString(expected) + ", found: 0x" + Integer.toHexString(result)); } @@ -157,6 +170,13 @@ public class Main { // We allow that an expected NaN result has become quiet. private static void expectEqualsNaN64(long expected, long result) { if (expected != result && (expected | DPQUIET) != result) { + if (!isDalvik) { + // If not on ART, relax the expected value more towards just + // "spec compliance" and allow sign bit to remain set for NaN. + if (expected == (result & Long.MAX_VALUE)) { + return; + } + } throw new Error("Expected: 0x" + Long.toHexString(expected) + ", found: 0x" + Long.toHexString(result)); } diff --git a/test/645-checker-abs-simd/src/Main.java b/test/645-checker-abs-simd/src/Main.java index d498bda0527..819304a9e96 100644 --- a/test/645-checker-abs-simd/src/Main.java +++ b/test/645-checker-abs-simd/src/Main.java @@ -19,22 +19,25 @@ */ public class Main { + private final static boolean isDalvik = + System.getProperty("java.vm.name").equals("Dalvik"); + private static final int SPQUIET = 1 << 22; private static final long DPQUIET = 1L << 51; /// CHECK-START: void Main.doitByte(byte[]) loop_optimization (before) - /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none - /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none - /// CHECK-DAG: InvokeStaticOrDirect intrinsic:MathAbsInt loop:<<Loop>> outer_loop:none - /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: Abs loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none // /// CHECK-START-{ARM,ARM64,MIPS64}: void Main.doitByte(byte[]) loop_optimization (after) - /// CHECK-DAG: VecLoad loop:<<Loop1:B\d+>> outer_loop:none - /// CHECK-DAG: VecAbs loop:<<Loop1>> outer_loop:none - /// CHECK-DAG: VecStore loop:<<Loop1>> outer_loop:none - /// CHECK-DAG: ArrayGet loop:<<Loop2:B\d+>> outer_loop:none - /// CHECK-DAG: InvokeStaticOrDirect intrinsic:MathAbsInt loop:<<Loop2>> outer_loop:none - /// CHECK-DAG: ArraySet loop:<<Loop2>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop1:B\d+>> outer_loop:none + /// CHECK-DAG: VecAbs loop:<<Loop1>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop1>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop2:B\d+>> outer_loop:none + /// CHECK-DAG: Abs loop:<<Loop2>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop2>> outer_loop:none // /// CHECK-EVAL: "<<Loop1>>" != "<<Loop2>>" // @@ -45,10 +48,7 @@ public class Main { } /// CHECK-START: void Main.doitChar(char[]) loop_optimization (before) - /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none - /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none - /// CHECK-DAG: InvokeStaticOrDirect intrinsic:MathAbsInt loop:<<Loop>> outer_loop:none - /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + /// CHECK-NOT: Abs // /// CHECK-START: void Main.doitChar(char[]) loop_optimization (after) /// CHECK-NOT: VecAbs @@ -60,18 +60,18 @@ public class Main { } /// CHECK-START: void Main.doitShort(short[]) loop_optimization (before) - /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none - /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none - /// CHECK-DAG: InvokeStaticOrDirect intrinsic:MathAbsInt loop:<<Loop>> outer_loop:none - /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: Abs loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none // /// CHECK-START-{ARM,ARM64,MIPS64}: void Main.doitShort(short[]) loop_optimization (after) - /// CHECK-DAG: VecLoad loop:<<Loop1:B\d+>> outer_loop:none - /// CHECK-DAG: VecAbs loop:<<Loop1>> outer_loop:none - /// CHECK-DAG: VecStore loop:<<Loop1>> outer_loop:none - /// CHECK-DAG: ArrayGet loop:<<Loop2:B\d+>> outer_loop:none - /// CHECK-DAG: InvokeStaticOrDirect intrinsic:MathAbsInt loop:<<Loop2>> outer_loop:none - /// CHECK-DAG: ArraySet loop:<<Loop2>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop1:B\d+>> outer_loop:none + /// CHECK-DAG: VecAbs loop:<<Loop1>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop1>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop2:B\d+>> outer_loop:none + /// CHECK-DAG: Abs loop:<<Loop2>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop2>> outer_loop:none // /// CHECK-EVAL: "<<Loop1>>" != "<<Loop2>>" // @@ -82,18 +82,18 @@ public class Main { } /// CHECK-START: void Main.doitCastedChar(char[]) loop_optimization (before) - /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none - /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none - /// CHECK-DAG: InvokeStaticOrDirect intrinsic:MathAbsInt loop:<<Loop>> outer_loop:none - /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: Abs loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none // /// CHECK-START-ARM64: void Main.doitCastedChar(char[]) loop_optimization (after) - /// CHECK-DAG: VecLoad loop:<<Loop1:B\d+>> outer_loop:none - /// CHECK-DAG: VecAbs loop:<<Loop1>> outer_loop:none - /// CHECK-DAG: VecStore loop:<<Loop1>> outer_loop:none - /// CHECK-DAG: ArrayGet loop:<<Loop2:B\d+>> outer_loop:none - /// CHECK-DAG: InvokeStaticOrDirect intrinsic:MathAbsInt loop:<<Loop2>> outer_loop:none - /// CHECK-DAG: ArraySet loop:<<Loop2>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop1:B\d+>> outer_loop:none + /// CHECK-DAG: VecAbs loop:<<Loop1>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop1>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop2:B\d+>> outer_loop:none + /// CHECK-DAG: Abs loop:<<Loop2>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop2>> outer_loop:none // /// CHECK-EVAL: "<<Loop1>>" != "<<Loop2>>" // @@ -104,18 +104,18 @@ public class Main { } /// CHECK-START: void Main.doitInt(int[]) loop_optimization (before) - /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none - /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none - /// CHECK-DAG: InvokeStaticOrDirect intrinsic:MathAbsInt loop:<<Loop>> outer_loop:none - /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: Abs loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none // /// CHECK-START-{ARM,ARM64,MIPS64}: void Main.doitInt(int[]) loop_optimization (after) - /// CHECK-DAG: VecLoad loop:<<Loop1:B\d+>> outer_loop:none - /// CHECK-DAG: VecAbs loop:<<Loop1>> outer_loop:none - /// CHECK-DAG: VecStore loop:<<Loop1>> outer_loop:none - /// CHECK-DAG: ArrayGet loop:<<Loop2:B\d+>> outer_loop:none - /// CHECK-DAG: InvokeStaticOrDirect intrinsic:MathAbsInt loop:<<Loop2>> outer_loop:none - /// CHECK-DAG: ArraySet loop:<<Loop2>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop1:B\d+>> outer_loop:none + /// CHECK-DAG: VecAbs loop:<<Loop1>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop1>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop2:B\d+>> outer_loop:none + /// CHECK-DAG: Abs loop:<<Loop2>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop2>> outer_loop:none // /// CHECK-EVAL: "<<Loop1>>" != "<<Loop2>>" // @@ -126,18 +126,18 @@ public class Main { } /// CHECK-START: void Main.doitLong(long[]) loop_optimization (before) - /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none - /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none - /// CHECK-DAG: InvokeStaticOrDirect intrinsic:MathAbsLong loop:<<Loop>> outer_loop:none - /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: Abs loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none // /// CHECK-START-{ARM64,MIPS64}: void Main.doitLong(long[]) loop_optimization (after) - /// CHECK-DAG: VecLoad loop:<<Loop1:B\d+>> outer_loop:none - /// CHECK-DAG: VecAbs loop:<<Loop1>> outer_loop:none - /// CHECK-DAG: VecStore loop:<<Loop1>> outer_loop:none - /// CHECK-DAG: ArrayGet loop:<<Loop2:B\d+>> outer_loop:none - /// CHECK-DAG: InvokeStaticOrDirect intrinsic:MathAbsLong loop:<<Loop2>> outer_loop:none - /// CHECK-DAG: ArraySet loop:<<Loop2>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop1:B\d+>> outer_loop:none + /// CHECK-DAG: VecAbs loop:<<Loop1>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop1>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop2:B\d+>> outer_loop:none + /// CHECK-DAG: Abs loop:<<Loop2>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop2>> outer_loop:none // /// CHECK-EVAL: "<<Loop1>>" != "<<Loop2>>" // @@ -148,18 +148,18 @@ public class Main { } /// CHECK-START: void Main.doitFloat(float[]) loop_optimization (before) - /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none - /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none - /// CHECK-DAG: InvokeStaticOrDirect intrinsic:MathAbsFloat loop:<<Loop>> outer_loop:none - /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: Abs loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none // /// CHECK-START-{ARM64,MIPS64}: void Main.doitFloat(float[]) loop_optimization (after) - /// CHECK-DAG: VecLoad loop:<<Loop1:B\d+>> outer_loop:none - /// CHECK-DAG: VecAbs loop:<<Loop1>> outer_loop:none - /// CHECK-DAG: VecStore loop:<<Loop1>> outer_loop:none - /// CHECK-DAG: ArrayGet loop:<<Loop2:B\d+>> outer_loop:none - /// CHECK-DAG: InvokeStaticOrDirect intrinsic:MathAbsFloat loop:<<Loop2>> outer_loop:none - /// CHECK-DAG: ArraySet loop:<<Loop2>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop1:B\d+>> outer_loop:none + /// CHECK-DAG: VecAbs loop:<<Loop1>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop1>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop2:B\d+>> outer_loop:none + /// CHECK-DAG: Abs loop:<<Loop2>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop2>> outer_loop:none // /// CHECK-EVAL: "<<Loop1>>" != "<<Loop2>>" // @@ -170,18 +170,18 @@ public class Main { } /// CHECK-START: void Main.doitDouble(double[]) loop_optimization (before) - /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none - /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none - /// CHECK-DAG: InvokeStaticOrDirect intrinsic:MathAbsDouble loop:<<Loop>> outer_loop:none - /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: Abs loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop>> outer_loop:none // /// CHECK-START-{ARM64,MIPS64}: void Main.doitDouble(double[]) loop_optimization (after) - /// CHECK-DAG: VecLoad loop:<<Loop1:B\d+>> outer_loop:none - /// CHECK-DAG: VecAbs loop:<<Loop1>> outer_loop:none - /// CHECK-DAG: VecStore loop:<<Loop1>> outer_loop:none - /// CHECK-DAG: ArrayGet loop:<<Loop2:B\d+>> outer_loop:none - /// CHECK-DAG: InvokeStaticOrDirect intrinsic:MathAbsDouble loop:<<Loop2>> outer_loop:none - /// CHECK-DAG: ArraySet loop:<<Loop2>> outer_loop:none + /// CHECK-DAG: VecLoad loop:<<Loop1:B\d+>> outer_loop:none + /// CHECK-DAG: VecAbs loop:<<Loop1>> outer_loop:none + /// CHECK-DAG: VecStore loop:<<Loop1>> outer_loop:none + /// CHECK-DAG: ArrayGet loop:<<Loop2:B\d+>> outer_loop:none + /// CHECK-DAG: Abs loop:<<Loop2>> outer_loop:none + /// CHECK-DAG: ArraySet loop:<<Loop2>> outer_loop:none // /// CHECK-EVAL: "<<Loop1>>" != "<<Loop2>>" // @@ -378,6 +378,13 @@ public class Main { // We allow that an expected NaN result has become quiet. private static void expectEqualsNaN32(int expected, int result) { if (expected != result && (expected | SPQUIET) != result) { + if (!isDalvik) { + // If not on ART, relax the expected value more towards just + // "spec compliance" and allow sign bit to remain set for NaN. + if (expected == (result & Integer.MAX_VALUE)) { + return; + } + } throw new Error("Expected: 0x" + Integer.toHexString(expected) + ", found: 0x" + Integer.toHexString(result)); } @@ -386,6 +393,13 @@ public class Main { // We allow that an expected NaN result has become quiet. private static void expectEqualsNaN64(long expected, long result) { if (expected != result && (expected | DPQUIET) != result) { + if (!isDalvik) { + // If not on ART, relax the expected value more towards just + // "spec compliance" and allow sign bit to remain set for NaN. + if (expected == (result & Long.MAX_VALUE)) { + return; + } + } throw new Error("Expected: 0x" + Long.toHexString(expected) + ", found: 0x" + Long.toHexString(result)); } diff --git a/test/646-checker-hadd-alt-char/build b/test/646-checker-hadd-alt-char/build new file mode 100644 index 00000000000..d85147f17bd --- /dev/null +++ b/test/646-checker-hadd-alt-char/build @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Copyright 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See b/65168732 +export USE_D8=false + +./default-build "$@" diff --git a/test/646-checker-hadd-alt-short/build b/test/646-checker-hadd-alt-short/build new file mode 100644 index 00000000000..d85147f17bd --- /dev/null +++ b/test/646-checker-hadd-alt-short/build @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Copyright 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See b/65168732 +export USE_D8=false + +./default-build "$@" diff --git a/test/646-checker-hadd-char/build b/test/646-checker-hadd-char/build new file mode 100644 index 00000000000..d85147f17bd --- /dev/null +++ b/test/646-checker-hadd-char/build @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Copyright 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See b/65168732 +export USE_D8=false + +./default-build "$@" diff --git a/test/646-checker-hadd-short/build b/test/646-checker-hadd-short/build new file mode 100644 index 00000000000..d85147f17bd --- /dev/null +++ b/test/646-checker-hadd-short/build @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Copyright 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See b/65168732 +export USE_D8=false + +./default-build "$@" diff --git a/test/646-checker-hadd-short/src/Main.java b/test/646-checker-hadd-short/src/Main.java index 85c2fcaf220..c09da8125ba 100644 --- a/test/646-checker-hadd-short/src/Main.java +++ b/test/646-checker-hadd-short/src/Main.java @@ -26,6 +26,10 @@ public class Main { static short[] sB2 = new short[M]; static short[] sBo = new short[M]; + private static int $inline$mone() { + return -1; + } + /// CHECK-START: void Main.halving_add_signed(short[], short[], short[]) loop_optimization (before) /// CHECK-DAG: <<I1:i\d+>> IntConstant 1 loop:none /// CHECK-DAG: <<Phi:i\d+>> Phi loop:<<Loop:B\d+>> outer_loop:none @@ -184,6 +188,35 @@ public class Main { } } + /// CHECK-START: void Main.rounding_halving_add_signed_alt3(short[], short[], short[]) loop_optimization (before) + /// CHECK-DAG: <<I1:i\d+>> IntConstant 1 loop:none + /// CHECK-DAG: <<M1:i\d+>> IntConstant -1 loop:none + /// CHECK-DAG: <<I9:i\d+>> IntConstant 9 loop:none + /// CHECK-DAG: <<M9:i\d+>> IntConstant -9 loop:none + /// CHECK-DAG: <<Phi:i\d+>> Phi loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Get1:s\d+>> ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Get2:s\d+>> ArrayGet loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Add1:i\d+>> Add [<<Get1>>,<<I9>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Add2:i\d+>> Add [<<Get2>>,<<M9>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Add3:i\d+>> Add [<<Add1>>,<<Add2>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Sub:i\d+>> Sub [<<Add3>>,<<M1>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Shr:i\d+>> Shr [<<Sub>>,<<I1>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Cnv:s\d+>> TypeConversion [<<Shr>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet [{{l\d+}},<<Phi>>,<<Cnv>>] loop:<<Loop>> outer_loop:none + // + /// CHECK-START-{ARM,ARM64,MIPS64}: void Main.rounding_halving_add_signed_alt3(short[], short[], short[]) loop_optimization (after) + /// CHECK-DAG: <<Get1:d\d+>> VecLoad loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Get2:d\d+>> VecLoad loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<HAdd:d\d+>> VecHalvingAdd [<<Get1>>,<<Get2>>] packed_type:Int16 rounded:true loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore [{{l\d+}},{{i\d+}},<<HAdd>>] loop:<<Loop>> outer_loop:none + private static void rounding_halving_add_signed_alt3(short[] b1, short[] b2, short[] bo) { + int min_length = Math.min(bo.length, Math.min(b1.length, b2.length)); + for (int i = 0; i < min_length; i++) { + // Computations that cancel to adding 1 also do not confuse recognition. + bo[i] = (short) (((b1[i] + 9) + (b2[i] - 9) - $inline$mone()) >> 1); + } + } + /// CHECK-START: void Main.rounding_halving_add_unsigned(short[], short[], short[]) instruction_simplifier (before) /// CHECK-DAG: <<I1:i\d+>> IntConstant 1 loop:none /// CHECK-DAG: <<UMAX:i\d+>> IntConstant 65535 loop:none @@ -366,6 +399,11 @@ public class Main { short e = (short) ((sB1[i] + sB2[i] + 1) >> 1); expectEquals(e, sBo[i]); } + rounding_halving_add_signed_alt3(sB1, sB2, sBo); + for (int i = 0; i < M; i++) { + short e = (short) ((sB1[i] + sB2[i] + 1) >> 1); + expectEquals(e, sBo[i]); + } rounding_halving_add_unsigned(sB1, sB2, sBo); for (int i = 0; i < M; i++) { short e = (short) (((sB1[i] & 0xffff) + (sB2[i] & 0xffff) + 1) >> 1); diff --git a/test/651-checker-char-simd-minmax/info.txt b/test/651-checker-char-simd-minmax/info.txt deleted file mode 100644 index 73af1242c08..00000000000 --- a/test/651-checker-char-simd-minmax/info.txt +++ /dev/null @@ -1 +0,0 @@ -Functional tests on min/max SIMD vectorization. diff --git a/test/651-checker-double-simd-minmax/info.txt b/test/651-checker-double-simd-minmax/info.txt deleted file mode 100644 index 73af1242c08..00000000000 --- a/test/651-checker-double-simd-minmax/info.txt +++ /dev/null @@ -1 +0,0 @@ -Functional tests on min/max SIMD vectorization. diff --git a/test/651-checker-float-simd-minmax/expected.txt b/test/651-checker-float-simd-minmax/expected.txt deleted file mode 100644 index b0aad4deb5b..00000000000 --- a/test/651-checker-float-simd-minmax/expected.txt +++ /dev/null @@ -1 +0,0 @@ -passed diff --git a/test/651-checker-float-simd-minmax/info.txt b/test/651-checker-float-simd-minmax/info.txt deleted file mode 100644 index 73af1242c08..00000000000 --- a/test/651-checker-float-simd-minmax/info.txt +++ /dev/null @@ -1 +0,0 @@ -Functional tests on min/max SIMD vectorization. diff --git a/test/651-checker-int-simd-minmax/expected.txt b/test/651-checker-int-simd-minmax/expected.txt deleted file mode 100644 index b0aad4deb5b..00000000000 --- a/test/651-checker-int-simd-minmax/expected.txt +++ /dev/null @@ -1 +0,0 @@ -passed diff --git a/test/651-checker-int-simd-minmax/info.txt b/test/651-checker-int-simd-minmax/info.txt deleted file mode 100644 index 73af1242c08..00000000000 --- a/test/651-checker-int-simd-minmax/info.txt +++ /dev/null @@ -1 +0,0 @@ -Functional tests on min/max SIMD vectorization. diff --git a/test/651-checker-long-simd-minmax/expected.txt b/test/651-checker-long-simd-minmax/expected.txt deleted file mode 100644 index b0aad4deb5b..00000000000 --- a/test/651-checker-long-simd-minmax/expected.txt +++ /dev/null @@ -1 +0,0 @@ -passed diff --git a/test/651-checker-long-simd-minmax/info.txt b/test/651-checker-long-simd-minmax/info.txt deleted file mode 100644 index 73af1242c08..00000000000 --- a/test/651-checker-long-simd-minmax/info.txt +++ /dev/null @@ -1 +0,0 @@ -Functional tests on min/max SIMD vectorization. diff --git a/test/651-checker-short-simd-minmax/expected.txt b/test/651-checker-short-simd-minmax/expected.txt deleted file mode 100644 index b0aad4deb5b..00000000000 --- a/test/651-checker-short-simd-minmax/expected.txt +++ /dev/null @@ -1 +0,0 @@ -passed diff --git a/test/651-checker-short-simd-minmax/info.txt b/test/651-checker-short-simd-minmax/info.txt deleted file mode 100644 index 73af1242c08..00000000000 --- a/test/651-checker-short-simd-minmax/info.txt +++ /dev/null @@ -1 +0,0 @@ -Functional tests on min/max SIMD vectorization. diff --git a/test/651-checker-simd-minmax/build b/test/651-checker-simd-minmax/build new file mode 100644 index 00000000000..d85147f17bd --- /dev/null +++ b/test/651-checker-simd-minmax/build @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Copyright 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See b/65168732 +export USE_D8=false + +./default-build "$@" diff --git a/test/651-checker-simd-minmax/expected.txt b/test/651-checker-simd-minmax/expected.txt new file mode 100644 index 00000000000..f362c45b776 --- /dev/null +++ b/test/651-checker-simd-minmax/expected.txt @@ -0,0 +1,7 @@ +ByteSimdMinMax passed +CharSimdMinMax passed +ShortSimdMinMax passed +IntSimdMinMax passed +LongSimdMinMax passed +DoubleSimdMinMax passed +FloatSimdMinMax passed diff --git a/test/651-checker-byte-simd-minmax/info.txt b/test/651-checker-simd-minmax/info.txt index 73af1242c08..73af1242c08 100644 --- a/test/651-checker-byte-simd-minmax/info.txt +++ b/test/651-checker-simd-minmax/info.txt diff --git a/test/651-checker-byte-simd-minmax/src/Main.java b/test/651-checker-simd-minmax/src/ByteSimdMinMax.java index 45949ae90a0..fff15faeaac 100644 --- a/test/651-checker-byte-simd-minmax/src/Main.java +++ b/test/651-checker-simd-minmax/src/ByteSimdMinMax.java @@ -17,17 +17,17 @@ /** * Tests for MIN/MAX vectorization. */ -public class Main { +public class ByteSimdMinMax { - /// CHECK-START: void Main.doitMin(byte[], byte[], byte[]) loop_optimization (before) + /// CHECK-START: void ByteSimdMinMax.doitMin(byte[], byte[], byte[]) loop_optimization (before) /// CHECK-DAG: <<Phi:i\d+>> Phi loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get1:b\d+>> ArrayGet loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get2:b\d+>> ArrayGet loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Min:i\d+>> InvokeStaticOrDirect [<<Get1>>,<<Get2>>] intrinsic:MathMinIntInt loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Min:i\d+>> Min [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Cnv:b\d+>> TypeConversion [<<Min>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: ArraySet [{{l\d+}},<<Phi>>,<<Cnv>>] loop:<<Loop>> outer_loop:none // - /// CHECK-START-{ARM,ARM64,MIPS64}: void Main.doitMin(byte[], byte[], byte[]) loop_optimization (after) + /// CHECK-START-{ARM,ARM64,MIPS64}: void ByteSimdMinMax.doitMin(byte[], byte[], byte[]) loop_optimization (after) /// CHECK-DAG: <<Get1:d\d+>> VecLoad loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get2:d\d+>> VecLoad loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Min:d\d+>> VecMin [<<Get1>>,<<Get2>>] packed_type:Int8 loop:<<Loop>> outer_loop:none @@ -39,7 +39,7 @@ public class Main { } } - /// CHECK-START: void Main.doitMinUnsigned(byte[], byte[], byte[]) instruction_simplifier (before) + /// CHECK-START: void ByteSimdMinMax.doitMinUnsigned(byte[], byte[], byte[]) instruction_simplifier (before) /// CHECK-DAG: <<I255:i\d+>> IntConstant 255 loop:none /// CHECK-DAG: <<Phi:i\d+>> Phi loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get1:b\d+>> ArrayGet loop:<<Loop>> outer_loop:none @@ -50,15 +50,15 @@ public class Main { /// CHECK-DAG: <<Cnv:b\d+>> TypeConversion [<<Min>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: ArraySet [{{l\d+}},{{i\d+}},<<Cnv>>] loop:<<Loop>> outer_loop:none // - /// CHECK-START: void Main.doitMinUnsigned(byte[], byte[], byte[]) loop_optimization (before) + /// CHECK-START: void ByteSimdMinMax.doitMinUnsigned(byte[], byte[], byte[]) loop_optimization (before) /// CHECK-DAG: <<Phi:i\d+>> Phi loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get1:a\d+>> ArrayGet loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get2:a\d+>> ArrayGet loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Min:i\d+>> InvokeStaticOrDirect [<<Get1>>,<<Get2>>] intrinsic:MathMinIntInt loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Min:i\d+>> Min [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Cnv:b\d+>> TypeConversion [<<Min>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: ArraySet [{{l\d+}},<<Phi>>,<<Cnv>>] loop:<<Loop>> outer_loop:none // - /// CHECK-START-{ARM,ARM64,MIPS64}: void Main.doitMinUnsigned(byte[], byte[], byte[]) loop_optimization (after) + /// CHECK-START-{ARM,ARM64,MIPS64}: void ByteSimdMinMax.doitMinUnsigned(byte[], byte[], byte[]) loop_optimization (after) /// CHECK-DAG: <<Get1:d\d+>> VecLoad loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get2:d\d+>> VecLoad loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Min:d\d+>> VecMin [<<Get1>>,<<Get2>>] packed_type:Uint8 loop:<<Loop>> outer_loop:none @@ -70,15 +70,15 @@ public class Main { } } - /// CHECK-START: void Main.doitMax(byte[], byte[], byte[]) loop_optimization (before) + /// CHECK-START: void ByteSimdMinMax.doitMax(byte[], byte[], byte[]) loop_optimization (before) /// CHECK-DAG: <<Phi:i\d+>> Phi loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get1:b\d+>> ArrayGet loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get2:b\d+>> ArrayGet loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Max:i\d+>> InvokeStaticOrDirect [<<Get1>>,<<Get2>>] intrinsic:MathMaxIntInt loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Max:i\d+>> Max [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Cnv:b\d+>> TypeConversion [<<Max>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: ArraySet [{{l\d+}},<<Phi>>,<<Cnv>>] loop:<<Loop>> outer_loop:none // - /// CHECK-START-{ARM,ARM64,MIPS64}: void Main.doitMax(byte[], byte[], byte[]) loop_optimization (after) + /// CHECK-START-{ARM,ARM64,MIPS64}: void ByteSimdMinMax.doitMax(byte[], byte[], byte[]) loop_optimization (after) /// CHECK-DAG: <<Get1:d\d+>> VecLoad loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get2:d\d+>> VecLoad loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Max:d\d+>> VecMax [<<Get1>>,<<Get2>>] packed_type:Int8 loop:<<Loop>> outer_loop:none @@ -90,7 +90,7 @@ public class Main { } } - /// CHECK-START: void Main.doitMaxUnsigned(byte[], byte[], byte[]) instruction_simplifier (before) + /// CHECK-START: void ByteSimdMinMax.doitMaxUnsigned(byte[], byte[], byte[]) instruction_simplifier (before) /// CHECK-DAG: <<I255:i\d+>> IntConstant 255 loop:none /// CHECK-DAG: <<Phi:i\d+>> Phi loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get1:b\d+>> ArrayGet loop:<<Loop>> outer_loop:none @@ -101,15 +101,15 @@ public class Main { /// CHECK-DAG: <<Cnv:b\d+>> TypeConversion [<<Max>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: ArraySet [{{l\d+}},{{i\d+}},<<Cnv>>] loop:<<Loop>> outer_loop:none // - /// CHECK-START: void Main.doitMaxUnsigned(byte[], byte[], byte[]) loop_optimization (before) + /// CHECK-START: void ByteSimdMinMax.doitMaxUnsigned(byte[], byte[], byte[]) loop_optimization (before) /// CHECK-DAG: <<Phi:i\d+>> Phi loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get1:a\d+>> ArrayGet loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get2:a\d+>> ArrayGet loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Max:i\d+>> InvokeStaticOrDirect [<<Get1>>,<<Get2>>] intrinsic:MathMaxIntInt loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Max:i\d+>> Max [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Cnv:b\d+>> TypeConversion [<<Max>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: ArraySet [{{l\d+}},<<Phi>>,<<Cnv>>] loop:<<Loop>> outer_loop:none // - /// CHECK-START-{ARM,ARM64,MIPS64}: void Main.doitMaxUnsigned(byte[], byte[], byte[]) loop_optimization (after) + /// CHECK-START-{ARM,ARM64,MIPS64}: void ByteSimdMinMax.doitMaxUnsigned(byte[], byte[], byte[]) loop_optimization (after) /// CHECK-DAG: <<Get1:d\d+>> VecLoad loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get2:d\d+>> VecLoad loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Max:d\d+>> VecMax [<<Get1>>,<<Get2>>] packed_type:Uint8 loop:<<Loop>> outer_loop:none @@ -121,15 +121,15 @@ public class Main { } } - /// CHECK-START: void Main.doitMin100(byte[], byte[]) loop_optimization (before) + /// CHECK-START: void ByteSimdMinMax.doitMin100(byte[], byte[]) loop_optimization (before) /// CHECK-DAG: <<I100:i\d+>> IntConstant 100 loop:none /// CHECK-DAG: <<Phi:i\d+>> Phi loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get:b\d+>> ArrayGet loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Min:i\d+>> InvokeStaticOrDirect [<<Get>>,<<I100>>] intrinsic:MathMinIntInt loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Min:i\d+>> Min [<<Get>>,<<I100>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Cnv:b\d+>> TypeConversion [<<Min>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: ArraySet [{{l\d+}},<<Phi>>,<<Cnv>>] loop:<<Loop>> outer_loop:none // - /// CHECK-START-{ARM64,MIPS64}: void Main.doitMin100(byte[], byte[]) loop_optimization (after) + /// CHECK-START-{ARM,ARM64,MIPS64}: void ByteSimdMinMax.doitMin100(byte[], byte[]) loop_optimization (after) /// CHECK-DAG: <<I100:i\d+>> IntConstant 100 loop:none /// CHECK-DAG: <<Repl:d\d+>> VecReplicateScalar [<<I100>>] loop:none /// CHECK-DAG: <<Get:d\d+>> VecLoad loop:<<Loop:B\d+>> outer_loop:none @@ -142,7 +142,63 @@ public class Main { } } - public static void main(String[] args) { + /// CHECK-START-{ARM,ARM64,MIPS64}: void ByteSimdMinMax.doitMinMax(byte[], byte[]) loop_optimization (after) + /// CHECK-DAG: <<I11:i\d+>> IntConstant -11 loop:none + /// CHECK-DAG: <<I23:i\d+>> IntConstant 23 loop:none + /// CHECK-DAG: <<Rpl1:d\d+>> VecReplicateScalar [<<I23>>] loop:none + /// CHECK-DAG: <<Rpl2:d\d+>> VecReplicateScalar [<<I11>>] loop:none + /// CHECK-DAG: <<Get:d\d+>> VecLoad loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Min:d\d+>> VecMin [<<Get>>,<<Rpl1>>] packed_type:Int8 loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Max:d\d+>> VecMax [<<Min>>,<<Rpl2>>] packed_type:Int8 loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore [{{l\d+}},{{i\d+}},<<Max>>] loop:<<Loop>> outer_loop:none + private static void doitMinMax(byte[] x, byte[] y) { + int n = Math.min(x.length, y.length); + for (int i = 0; i < n; i++) { + x[i] = (byte) Math.max(-11, Math.min(y[i], 23)); + } + } + + /// CHECK-START-{ARM,ARM64,MIPS64}: void ByteSimdMinMax.doitMinMaxUnsigned(byte[], byte[]) loop_optimization (after) + /// CHECK-DAG: <<I11:i\d+>> IntConstant 11 loop:none + /// CHECK-DAG: <<I23:i\d+>> IntConstant 23 loop:none + /// CHECK-DAG: <<Rpl1:d\d+>> VecReplicateScalar [<<I23>>] loop:none + /// CHECK-DAG: <<Rpl2:d\d+>> VecReplicateScalar [<<I11>>] loop:none + /// CHECK-DAG: <<Get:d\d+>> VecLoad loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Min:d\d+>> VecMin [<<Get>>,<<Rpl1>>] packed_type:Uint8 loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Max:d\d+>> VecMax [<<Min>>,<<Rpl2>>] packed_type:Uint8 loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore [{{l\d+}},{{i\d+}},<<Max>>] loop:<<Loop>> outer_loop:none + private static void doitMinMaxUnsigned(byte[] x, byte[] y) { + int n = Math.min(x.length, y.length); + for (int i = 0; i < n; i++) { + x[i] = (byte) Math.max(11, Math.min(y[i] & 0xff, 23)); + } + } + + /// CHECK-START-{ARM,ARM64}: void ByteSimdMinMax.doitMinAlt(byte[], byte[], byte[]) loop_optimization (after) + /// CHECK-DAG: <<Get1:d\d+>> VecLoad loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Get2:d\d+>> VecLoad loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Min:d\d+>> VecMin [<<Get1>>,<<Get2>>] packed_type:Int8 loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore [{{l\d+}},{{i\d+}},<<Min>>] loop:<<Loop>> outer_loop:none + private static void doitMinAlt(byte[] x, byte[] y, byte[] z) { + int n = Math.min(x.length, Math.min(y.length, z.length)); + for (int i = 0; i < n; ++i) { + x[i] = y[i] < z[i] ? y[i] : z[i]; + } + } + + /// CHECK-START-{ARM,ARM64,MIPS64}: void ByteSimdMinMax.doitMaxAlt(byte[], byte[], byte[]) loop_optimization (after) + /// CHECK-DAG: <<Get1:d\d+>> VecLoad loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Get2:d\d+>> VecLoad loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Max:d\d+>> VecMax [<<Get1>>,<<Get2>>] packed_type:Int8 loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore [{{l\d+}},{{i\d+}},<<Max>>] loop:<<Loop>> outer_loop:none + private static void doitMaxAlt(byte[] x, byte[] y, byte[] z) { + int n = Math.min(x.length, Math.min(y.length, z.length)); + for (int i = 0; i < n; ++i) { + x[i] = y[i] > z[i] ? y[i] : z[i]; + } + } + + public static void main() { // Initialize cross-values for all possible values. int total = 256 * 256; byte[] x = new byte[total]; @@ -184,8 +240,29 @@ public class Main { byte expected = (byte) Math.min(y[i], 100); expectEquals(expected, x[i]); } - - System.out.println("passed"); + doitMinMax(x, y); + for (int i = 0; i < total; i++) { + int s = y[i]; + byte expected = (byte) (s < -11 ? -11 : (s > 23 ? 23 : s)); + expectEquals(expected, x[i]); + } + doitMinMaxUnsigned(x, y); + for (int i = 0; i < total; i++) { + int u = y[i] & 0xff; + byte expected = (byte) (u < 11 ? 11 : (u > 23 ? 23 : u)); + expectEquals(expected, x[i]); + } + doitMinAlt(x, y, z); + for (int i = 0; i < total; i++) { + byte expected = (byte) Math.min(y[i], z[i]); + expectEquals(expected, x[i]); + } + doitMaxAlt(x, y, z); + for (int i = 0; i < total; i++) { + byte expected = (byte) Math.max(y[i], z[i]); + expectEquals(expected, x[i]); + } + System.out.println("ByteSimdMinMax passed"); } private static void expectEquals(byte expected, byte result) { diff --git a/test/651-checker-char-simd-minmax/src/Main.java b/test/651-checker-simd-minmax/src/CharSimdMinMax.java index 9b056094a33..30169c45912 100644 --- a/test/651-checker-char-simd-minmax/src/Main.java +++ b/test/651-checker-simd-minmax/src/CharSimdMinMax.java @@ -17,17 +17,17 @@ /** * Tests for MIN/MAX vectorization. */ -public class Main { +public class CharSimdMinMax { - /// CHECK-START: void Main.doitMin(char[], char[], char[]) loop_optimization (before) + /// CHECK-START: void CharSimdMinMax.doitMin(char[], char[], char[]) loop_optimization (before) /// CHECK-DAG: <<Phi:i\d+>> Phi loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get1:c\d+>> ArrayGet loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get2:c\d+>> ArrayGet loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Min:i\d+>> InvokeStaticOrDirect [<<Get1>>,<<Get2>>] intrinsic:MathMinIntInt loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Min:i\d+>> Min [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Cnv:c\d+>> TypeConversion [<<Min>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: ArraySet [{{l\d+}},<<Phi>>,<<Cnv>>] loop:<<Loop>> outer_loop:none // - /// CHECK-START-{ARM,ARM64,MIPS64}: void Main.doitMin(char[], char[], char[]) loop_optimization (after) + /// CHECK-START-{ARM,ARM64,MIPS64}: void CharSimdMinMax.doitMin(char[], char[], char[]) loop_optimization (after) /// CHECK-DAG: <<Get1:d\d+>> VecLoad loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get2:d\d+>> VecLoad loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Min:d\d+>> VecMin [<<Get1>>,<<Get2>>] packed_type:Uint16 loop:<<Loop>> outer_loop:none @@ -39,15 +39,15 @@ public class Main { } } - /// CHECK-START: void Main.doitMax(char[], char[], char[]) loop_optimization (before) + /// CHECK-START: void CharSimdMinMax.doitMax(char[], char[], char[]) loop_optimization (before) /// CHECK-DAG: <<Phi:i\d+>> Phi loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get1:c\d+>> ArrayGet loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get2:c\d+>> ArrayGet loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Max:i\d+>> InvokeStaticOrDirect [<<Get1>>,<<Get2>>] intrinsic:MathMaxIntInt loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Max:i\d+>> Max [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Cnv:c\d+>> TypeConversion [<<Max>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: ArraySet [{{l\d+}},<<Phi>>,<<Cnv>>] loop:<<Loop>> outer_loop:none // - /// CHECK-START-{ARM,ARM64,MIPS64}: void Main.doitMax(char[], char[], char[]) loop_optimization (after) + /// CHECK-START-{ARM,ARM64,MIPS64}: void CharSimdMinMax.doitMax(char[], char[], char[]) loop_optimization (after) /// CHECK-DAG: <<Get1:d\d+>> VecLoad loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get2:d\d+>> VecLoad loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Max:d\d+>> VecMax [<<Get1>>,<<Get2>>] packed_type:Uint16 loop:<<Loop>> outer_loop:none @@ -59,15 +59,15 @@ public class Main { } } - /// CHECK-START: void Main.doitMin100(char[], char[]) loop_optimization (before) + /// CHECK-START: void CharSimdMinMax.doitMin100(char[], char[]) loop_optimization (before) /// CHECK-DAG: <<I100:i\d+>> IntConstant 100 loop:none /// CHECK-DAG: <<Phi:i\d+>> Phi loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get:c\d+>> ArrayGet loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Min:i\d+>> InvokeStaticOrDirect [<<Get>>,<<I100>>] intrinsic:MathMinIntInt loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Min:i\d+>> Min [<<Get>>,<<I100>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Cnv:c\d+>> TypeConversion [<<Min>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: ArraySet [{{l\d+}},<<Phi>>,<<Cnv>>] loop:<<Loop>> outer_loop:none // - /// CHECK-START-{ARM64,MIPS64}: void Main.doitMin100(char[], char[]) loop_optimization (after) + /// CHECK-START-{ARM64,MIPS64}: void CharSimdMinMax.doitMin100(char[], char[]) loop_optimization (after) /// CHECK-DAG: <<I100:i\d+>> IntConstant 100 loop:none /// CHECK-DAG: <<Repl:d\d+>> VecReplicateScalar [<<I100>>] loop:none /// CHECK-DAG: <<Get:d\d+>> VecLoad loop:<<Loop:B\d+>> outer_loop:none @@ -80,7 +80,7 @@ public class Main { } } - public static void main(String[] args) { + public static void main() { char[] interesting = { 0x0000, 0x0001, 0x007f, 0x0080, 0x0081, 0x00ff, 0x0100, 0x0101, 0x017f, 0x0180, 0x0181, 0x01ff, @@ -121,7 +121,7 @@ public class Main { expectEquals(expected, x[i]); } - System.out.println("passed"); + System.out.println("CharSimdMinMax passed"); } private static void expectEquals(char expected, char result) { diff --git a/test/651-checker-double-simd-minmax/src/Main.java b/test/651-checker-simd-minmax/src/DoubleSimdMinMax.java index 6b12e7e63c0..da20594db86 100644 --- a/test/651-checker-double-simd-minmax/src/Main.java +++ b/test/651-checker-simd-minmax/src/DoubleSimdMinMax.java @@ -17,19 +17,19 @@ /** * Tests for MIN/MAX vectorization. */ -public class Main { +public class DoubleSimdMinMax { - /// CHECK-START: void Main.doitMin(double[], double[], double[]) loop_optimization (before) + /// CHECK-START: void DoubleSimdMinMax.doitMin(double[], double[], double[]) loop_optimization (before) /// CHECK-DAG: <<Phi:i\d+>> Phi loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get1:d\d+>> ArrayGet loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get2:d\d+>> ArrayGet loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Min:d\d+>> InvokeStaticOrDirect [<<Get1>>,<<Get2>>] intrinsic:MathMinDoubleDouble loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Min:d\d+>> Min [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: ArraySet [{{l\d+}},<<Phi>>,<<Min>>] loop:<<Loop>> outer_loop:none // // TODO x86: 0.0 vs -0.0? // TODO MIPS64: min(x, NaN)? // - /// CHECK-START-ARM64: void Main.doitMin(double[], double[], double[]) loop_optimization (after) + /// CHECK-START-ARM64: void DoubleSimdMinMax.doitMin(double[], double[], double[]) loop_optimization (after) /// CHECK-DAG: <<Get1:d\d+>> VecLoad loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get2:d\d+>> VecLoad loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Min:d\d+>> VecMin [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none @@ -41,17 +41,17 @@ public class Main { } } - /// CHECK-START: void Main.doitMax(double[], double[], double[]) loop_optimization (before) + /// CHECK-START: void DoubleSimdMinMax.doitMax(double[], double[], double[]) loop_optimization (before) /// CHECK-DAG: <<Phi:i\d+>> Phi loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get1:d\d+>> ArrayGet loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get2:d\d+>> ArrayGet loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Max:d\d+>> InvokeStaticOrDirect [<<Get1>>,<<Get2>>] intrinsic:MathMaxDoubleDouble loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Max:d\d+>> Max [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: ArraySet [{{l\d+}},<<Phi>>,<<Max>>] loop:<<Loop>> outer_loop:none // // TODO x86: 0.0 vs -0.0? // TODO MIPS64: max(x, NaN)? // - /// CHECK-START-ARM64: void Main.doitMax(double[], double[], double[]) loop_optimization (after) + /// CHECK-START-ARM64: void DoubleSimdMinMax.doitMax(double[], double[], double[]) loop_optimization (after) /// CHECK-DAG: <<Get1:d\d+>> VecLoad loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get2:d\d+>> VecLoad loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Max:d\d+>> VecMax [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none @@ -63,7 +63,7 @@ public class Main { } } - public static void main(String[] args) { + public static void main() { double[] interesting = { -0.0f, +0.0f, @@ -109,7 +109,7 @@ public class Main { expectEquals(expected, x[i]); } - System.out.println("passed"); + System.out.println("DoubleSimdMinMax passed"); } private static void expectEquals(double expected, double result) { diff --git a/test/651-checker-float-simd-minmax/src/Main.java b/test/651-checker-simd-minmax/src/FloatSimdMinMax.java index 278a9c9367b..645081248a7 100644 --- a/test/651-checker-float-simd-minmax/src/Main.java +++ b/test/651-checker-simd-minmax/src/FloatSimdMinMax.java @@ -17,19 +17,19 @@ /** * Tests for MIN/MAX vectorization. */ -public class Main { +public class FloatSimdMinMax { - /// CHECK-START: void Main.doitMin(float[], float[], float[]) loop_optimization (before) + /// CHECK-START: void FloatSimdMinMax.doitMin(float[], float[], float[]) loop_optimization (before) /// CHECK-DAG: <<Phi:i\d+>> Phi loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get1:f\d+>> ArrayGet loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get2:f\d+>> ArrayGet loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Min:f\d+>> InvokeStaticOrDirect [<<Get1>>,<<Get2>>] intrinsic:MathMinFloatFloat loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Min:f\d+>> Min [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: ArraySet [{{l\d+}},<<Phi>>,<<Min>>] loop:<<Loop>> outer_loop:none // // TODO x86: 0.0 vs -0.0? // TODO MIPS64: min(x, NaN)? // - /// CHECK-START-ARM64: void Main.doitMin(float[], float[], float[]) loop_optimization (after) + /// CHECK-START-ARM64: void FloatSimdMinMax.doitMin(float[], float[], float[]) loop_optimization (after) /// CHECK-DAG: <<Get1:d\d+>> VecLoad loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get2:d\d+>> VecLoad loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Min:d\d+>> VecMin [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none @@ -41,17 +41,17 @@ public class Main { } } - /// CHECK-START: void Main.doitMax(float[], float[], float[]) loop_optimization (before) + /// CHECK-START: void FloatSimdMinMax.doitMax(float[], float[], float[]) loop_optimization (before) /// CHECK-DAG: <<Phi:i\d+>> Phi loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get1:f\d+>> ArrayGet loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get2:f\d+>> ArrayGet loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Max:f\d+>> InvokeStaticOrDirect [<<Get1>>,<<Get2>>] intrinsic:MathMaxFloatFloat loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Max:f\d+>> Max [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: ArraySet [{{l\d+}},<<Phi>>,<<Max>>] loop:<<Loop>> outer_loop:none // // TODO x86: 0.0 vs -0.0? // TODO MIPS64: max(x, NaN)? // - /// CHECK-START-ARM64: void Main.doitMax(float[], float[], float[]) loop_optimization (after) + /// CHECK-START-ARM64: void FloatSimdMinMax.doitMax(float[], float[], float[]) loop_optimization (after) /// CHECK-DAG: <<Get1:d\d+>> VecLoad loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get2:d\d+>> VecLoad loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Max:d\d+>> VecMax [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none @@ -63,7 +63,7 @@ public class Main { } } - public static void main(String[] args) { + public static void main() { float[] interesting = { -0.0f, +0.0f, @@ -109,7 +109,7 @@ public class Main { expectEquals(expected, x[i]); } - System.out.println("passed"); + System.out.println("FloatSimdMinMax passed"); } private static void expectEquals(float expected, float result) { diff --git a/test/651-checker-int-simd-minmax/src/Main.java b/test/651-checker-simd-minmax/src/IntSimdMinMax.java index cfa0ae7dca6..ad888431754 100644 --- a/test/651-checker-int-simd-minmax/src/Main.java +++ b/test/651-checker-simd-minmax/src/IntSimdMinMax.java @@ -17,16 +17,16 @@ /** * Tests for MIN/MAX vectorization. */ -public class Main { +public class IntSimdMinMax { - /// CHECK-START: void Main.doitMin(int[], int[], int[]) loop_optimization (before) + /// CHECK-START: void IntSimdMinMax.doitMin(int[], int[], int[]) loop_optimization (before) /// CHECK-DAG: <<Phi:i\d+>> Phi loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get1:i\d+>> ArrayGet loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get2:i\d+>> ArrayGet loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Min:i\d+>> InvokeStaticOrDirect [<<Get1>>,<<Get2>>] intrinsic:MathMinIntInt loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Min:i\d+>> Min [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: ArraySet [{{l\d+}},<<Phi>>,<<Min>>] loop:<<Loop>> outer_loop:none // - /// CHECK-START-{ARM,ARM64,MIPS64}: void Main.doitMin(int[], int[], int[]) loop_optimization (after) + /// CHECK-START-{ARM,ARM64,MIPS64}: void IntSimdMinMax.doitMin(int[], int[], int[]) loop_optimization (after) /// CHECK-DAG: <<Get1:d\d+>> VecLoad loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get2:d\d+>> VecLoad loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Min:d\d+>> VecMin [<<Get1>>,<<Get2>>] packed_type:Int32 loop:<<Loop>> outer_loop:none @@ -38,14 +38,14 @@ public class Main { } } - /// CHECK-START: void Main.doitMax(int[], int[], int[]) loop_optimization (before) + /// CHECK-START: void IntSimdMinMax.doitMax(int[], int[], int[]) loop_optimization (before) /// CHECK-DAG: <<Phi:i\d+>> Phi loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get1:i\d+>> ArrayGet loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get2:i\d+>> ArrayGet loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Max:i\d+>> InvokeStaticOrDirect [<<Get1>>,<<Get2>>] intrinsic:MathMaxIntInt loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Max:i\d+>> Max [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: ArraySet [{{l\d+}},<<Phi>>,<<Max>>] loop:<<Loop>> outer_loop:none // - /// CHECK-START-{ARM,ARM64,MIPS64}: void Main.doitMax(int[], int[], int[]) loop_optimization (after) + /// CHECK-START-{ARM,ARM64,MIPS64}: void IntSimdMinMax.doitMax(int[], int[], int[]) loop_optimization (after) /// CHECK-DAG: <<Get1:d\d+>> VecLoad loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get2:d\d+>> VecLoad loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Max:d\d+>> VecMax [<<Get1>>,<<Get2>>] packed_type:Int32 loop:<<Loop>> outer_loop:none @@ -57,7 +57,39 @@ public class Main { } } - public static void main(String[] args) { + /// CHECK-START-{ARM,ARM64}: int IntSimdMinMax.findMin(int[]) loop_optimization (after) + /// CHECK-DAG: <<Rep:d\d+>> VecReplicateScalar loop:none + /// CHECK-DAG: <<VPhi:d\d+>> Phi [<<Rep>>,<<Max:d\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Get:d\d+>> VecLoad [{{l\d+}},{{i\d+}}] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Max>> VecMin [<<Get>>,<<VPhi>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Red:d\d+>> VecReduce [<<VPhi>>] loop:none + /// CHECK-DAG: VecExtractScalar [<<Red>>] loop:none + private static int findMin(int[] a) { + int x = Integer.MAX_VALUE; + for (int i = 0; i < a.length; i++) { + if (a[i] < x) + x = a[i]; + } + return x; + } + + /// CHECK-START-{ARM,ARM64}: int IntSimdMinMax.findMax(int[]) loop_optimization (after) + /// CHECK-DAG: <<Rep:d\d+>> VecReplicateScalar loop:none + /// CHECK-DAG: <<VPhi:d\d+>> Phi [<<Rep>>,<<Max:d\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Get:d\d+>> VecLoad [{{l\d+}},{{i\d+}}] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Max>> VecMax [<<Get>>,<<VPhi>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Red:d\d+>> VecReduce [<<VPhi>>] loop:none + /// CHECK-DAG: VecExtractScalar [<<Red>>] loop:none + private static int findMax(int[] a) { + int x = Integer.MIN_VALUE; + for (int i = 0; i < a.length; i++) { + if (a[i] > x) + x = a[i]; + } + return x; + } + + public static void main() { int[] interesting = { 0x00000000, 0x00000001, 0x00007fff, 0x00008000, 0x00008001, 0x0000ffff, 0x00010000, 0x00010001, 0x00017fff, 0x00018000, 0x00018001, 0x0001ffff, @@ -92,8 +124,14 @@ public class Main { int expected = Math.max(y[i], z[i]); expectEquals(expected, x[i]); } + expectEquals(Integer.MIN_VALUE, findMin(x)); + expectEquals(Integer.MAX_VALUE, findMax(x)); + expectEquals(Integer.MIN_VALUE, findMin(y)); + expectEquals(Integer.MAX_VALUE, findMax(y)); + expectEquals(Integer.MIN_VALUE, findMin(z)); + expectEquals(Integer.MAX_VALUE, findMax(z)); - System.out.println("passed"); + System.out.println("IntSimdMinMax passed"); } private static void expectEquals(int expected, int result) { diff --git a/test/651-checker-long-simd-minmax/src/Main.java b/test/651-checker-simd-minmax/src/LongSimdMinMax.java index 458cb8bf1b2..bb0c6047edf 100644 --- a/test/651-checker-long-simd-minmax/src/Main.java +++ b/test/651-checker-simd-minmax/src/LongSimdMinMax.java @@ -17,21 +17,21 @@ /** * Tests for MIN/MAX vectorization. */ -public class Main { +public class LongSimdMinMax { - /// CHECK-START: void Main.doitMin(long[], long[], long[]) loop_optimization (before) + /// CHECK-START: void LongSimdMinMax.doitMin(long[], long[], long[]) loop_optimization (before) /// CHECK-DAG: <<Phi:i\d+>> Phi loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get1:j\d+>> ArrayGet loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get2:j\d+>> ArrayGet loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Min:j\d+>> InvokeStaticOrDirect [<<Get1>>,<<Get2>>] intrinsic:MathMinLongLong loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Min:j\d+>> Min [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: ArraySet [{{l\d+}},<<Phi>>,<<Min>>] loop:<<Loop>> outer_loop:none // // Not directly supported for longs. // - /// CHECK-START-ARM64: void Main.doitMin(long[], long[], long[]) loop_optimization (after) + /// CHECK-START-ARM64: void LongSimdMinMax.doitMin(long[], long[], long[]) loop_optimization (after) /// CHECK-NOT: VecMin // - /// CHECK-START-MIPS64: void Main.doitMin(long[], long[], long[]) loop_optimization (after) + /// CHECK-START-MIPS64: void LongSimdMinMax.doitMin(long[], long[], long[]) loop_optimization (after) /// CHECK-DAG: <<Get1:d\d+>> VecLoad loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get2:d\d+>> VecLoad loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Min:d\d+>> VecMin [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none @@ -44,19 +44,19 @@ public class Main { } } - /// CHECK-START: void Main.doitMax(long[], long[], long[]) loop_optimization (before) + /// CHECK-START: void LongSimdMinMax.doitMax(long[], long[], long[]) loop_optimization (before) /// CHECK-DAG: <<Phi:i\d+>> Phi loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get1:j\d+>> ArrayGet loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get2:j\d+>> ArrayGet loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Max:j\d+>> InvokeStaticOrDirect [<<Get1>>,<<Get2>>] intrinsic:MathMaxLongLong loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Max:j\d+>> Max [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: ArraySet [{{l\d+}},<<Phi>>,<<Max>>] loop:<<Loop>> outer_loop:none // // Not directly supported for longs. // - /// CHECK-START-ARM64: void Main.doitMax(long[], long[], long[]) loop_optimization (after) + /// CHECK-START-ARM64: void LongSimdMinMax.doitMax(long[], long[], long[]) loop_optimization (after) /// CHECK-NOT: VecMax // - /// CHECK-START-MIPS64: void Main.doitMax(long[], long[], long[]) loop_optimization (after) + /// CHECK-START-MIPS64: void LongSimdMinMax.doitMax(long[], long[], long[]) loop_optimization (after) /// CHECK-DAG: <<Get1:d\d+>> VecLoad loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get2:d\d+>> VecLoad loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Max:d\d+>> VecMax [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none @@ -68,7 +68,7 @@ public class Main { } } - public static void main(String[] args) { + public static void main() { long[] interesting = { 0x0000000000000000L, 0x0000000000000001L, 0x000000007fffffffL, 0x0000000080000000L, 0x0000000080000001L, 0x00000000ffffffffL, @@ -110,7 +110,7 @@ public class Main { expectEquals(expected, x[i]); } - System.out.println("passed"); + System.out.println("LongSimdMinMax passed"); } private static void expectEquals(long expected, long result) { diff --git a/test/651-checker-simd-minmax/src/Main.java b/test/651-checker-simd-minmax/src/Main.java new file mode 100644 index 00000000000..9134dd1edd4 --- /dev/null +++ b/test/651-checker-simd-minmax/src/Main.java @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class Main { + public static void main(String[] args) { + ByteSimdMinMax.main(); + CharSimdMinMax.main(); + ShortSimdMinMax.main(); + IntSimdMinMax.main(); + LongSimdMinMax.main(); + DoubleSimdMinMax.main(); + FloatSimdMinMax.main(); + } +} diff --git a/test/651-checker-short-simd-minmax/src/Main.java b/test/651-checker-simd-minmax/src/ShortSimdMinMax.java index 5f10adab79b..9075d8007c8 100644 --- a/test/651-checker-short-simd-minmax/src/Main.java +++ b/test/651-checker-simd-minmax/src/ShortSimdMinMax.java @@ -17,17 +17,17 @@ /** * Tests for MIN/MAX vectorization. */ -public class Main { +public class ShortSimdMinMax { - /// CHECK-START: void Main.doitMin(short[], short[], short[]) loop_optimization (before) + /// CHECK-START: void ShortSimdMinMax.doitMin(short[], short[], short[]) loop_optimization (before) /// CHECK-DAG: <<Phi:i\d+>> Phi loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get1:s\d+>> ArrayGet loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get2:s\d+>> ArrayGet loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Min:i\d+>> InvokeStaticOrDirect [<<Get1>>,<<Get2>>] intrinsic:MathMinIntInt loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Min:i\d+>> Min [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Cnv:s\d+>> TypeConversion [<<Min>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: ArraySet [{{l\d+}},<<Phi>>,<<Cnv>>] loop:<<Loop>> outer_loop:none // - /// CHECK-START-{ARM,ARM64,MIPS64}: void Main.doitMin(short[], short[], short[]) loop_optimization (after) + /// CHECK-START-{ARM,ARM64,MIPS64}: void ShortSimdMinMax.doitMin(short[], short[], short[]) loop_optimization (after) /// CHECK-DAG: <<Get1:d\d+>> VecLoad loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get2:d\d+>> VecLoad loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Min:d\d+>> VecMin [<<Get1>>,<<Get2>>] packed_type:Int16 loop:<<Loop>> outer_loop:none @@ -39,7 +39,7 @@ public class Main { } } - /// CHECK-START: void Main.doitMinUnsigned(short[], short[], short[]) instruction_simplifier (before) + /// CHECK-START: void ShortSimdMinMax.doitMinUnsigned(short[], short[], short[]) instruction_simplifier (before) /// CHECK-DAG: <<IMAX:i\d+>> IntConstant 65535 loop:none /// CHECK-DAG: <<Phi:i\d+>> Phi loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get1:s\d+>> ArrayGet loop:<<Loop>> outer_loop:none @@ -50,15 +50,15 @@ public class Main { /// CHECK-DAG: <<Cnv:s\d+>> TypeConversion [<<Min>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: ArraySet [{{l\d+}},{{i\d+}},<<Cnv>>] loop:<<Loop>> outer_loop:none // - /// CHECK-START: void Main.doitMinUnsigned(short[], short[], short[]) loop_optimization (before) + /// CHECK-START: void ShortSimdMinMax.doitMinUnsigned(short[], short[], short[]) loop_optimization (before) /// CHECK-DAG: <<Phi:i\d+>> Phi loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get1:c\d+>> ArrayGet loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get2:c\d+>> ArrayGet loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Min:i\d+>> InvokeStaticOrDirect [<<Get1>>,<<Get2>>] intrinsic:MathMinIntInt loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Min:i\d+>> Min [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Cnv:s\d+>> TypeConversion [<<Min>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: ArraySet [{{l\d+}},<<Phi>>,<<Cnv>>] loop:<<Loop>> outer_loop:none // - /// CHECK-START-{ARM,ARM64,MIPS64}: void Main.doitMinUnsigned(short[], short[], short[]) loop_optimization (after) + /// CHECK-START-{ARM,ARM64,MIPS64}: void ShortSimdMinMax.doitMinUnsigned(short[], short[], short[]) loop_optimization (after) /// CHECK-DAG: <<Get1:d\d+>> VecLoad loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get2:d\d+>> VecLoad loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Min:d\d+>> VecMin [<<Get1>>,<<Get2>>] packed_type:Uint16 loop:<<Loop>> outer_loop:none @@ -70,15 +70,15 @@ public class Main { } } - /// CHECK-START: void Main.doitMax(short[], short[], short[]) loop_optimization (before) + /// CHECK-START: void ShortSimdMinMax.doitMax(short[], short[], short[]) loop_optimization (before) /// CHECK-DAG: <<Phi:i\d+>> Phi loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get1:s\d+>> ArrayGet loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get2:s\d+>> ArrayGet loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Max:i\d+>> InvokeStaticOrDirect [<<Get1>>,<<Get2>>] intrinsic:MathMaxIntInt loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Max:i\d+>> Max [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Cnv:s\d+>> TypeConversion [<<Max>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: ArraySet [{{l\d+}},<<Phi>>,<<Cnv>>] loop:<<Loop>> outer_loop:none // - /// CHECK-START-{ARM,ARM64,MIPS64}: void Main.doitMax(short[], short[], short[]) loop_optimization (after) + /// CHECK-START-{ARM,ARM64,MIPS64}: void ShortSimdMinMax.doitMax(short[], short[], short[]) loop_optimization (after) /// CHECK-DAG: <<Get1:d\d+>> VecLoad loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get2:d\d+>> VecLoad loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Max:d\d+>> VecMax [<<Get1>>,<<Get2>>] packed_type:Int16 loop:<<Loop>> outer_loop:none @@ -90,7 +90,7 @@ public class Main { } } - /// CHECK-START: void Main.doitMaxUnsigned(short[], short[], short[]) instruction_simplifier (before) + /// CHECK-START: void ShortSimdMinMax.doitMaxUnsigned(short[], short[], short[]) instruction_simplifier (before) /// CHECK-DAG: <<IMAX:i\d+>> IntConstant 65535 loop:none /// CHECK-DAG: <<Phi:i\d+>> Phi loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get1:s\d+>> ArrayGet loop:<<Loop>> outer_loop:none @@ -101,15 +101,15 @@ public class Main { /// CHECK-DAG: <<Cnv:s\d+>> TypeConversion [<<Max>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: ArraySet [{{l\d+}},{{i\d+}},<<Cnv>>] loop:<<Loop>> outer_loop:none // - /// CHECK-START: void Main.doitMaxUnsigned(short[], short[], short[]) loop_optimization (before) + /// CHECK-START: void ShortSimdMinMax.doitMaxUnsigned(short[], short[], short[]) loop_optimization (before) /// CHECK-DAG: <<Phi:i\d+>> Phi loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get1:c\d+>> ArrayGet loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get2:c\d+>> ArrayGet loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Max:i\d+>> InvokeStaticOrDirect [<<Get1>>,<<Get2>>] intrinsic:MathMaxIntInt loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Max:i\d+>> Max [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Cnv:s\d+>> TypeConversion [<<Max>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: ArraySet [{{l\d+}},<<Phi>>,<<Cnv>>] loop:<<Loop>> outer_loop:none // - /// CHECK-START-{ARM,ARM64,MIPS64}: void Main.doitMaxUnsigned(short[], short[], short[]) loop_optimization (after) + /// CHECK-START-{ARM,ARM64,MIPS64}: void ShortSimdMinMax.doitMaxUnsigned(short[], short[], short[]) loop_optimization (after) /// CHECK-DAG: <<Get1:d\d+>> VecLoad loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get2:d\d+>> VecLoad loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Max:d\d+>> VecMax [<<Get1>>,<<Get2>>] packed_type:Uint16 loop:<<Loop>> outer_loop:none @@ -121,15 +121,15 @@ public class Main { } } - /// CHECK-START: void Main.doitMin100(short[], short[]) loop_optimization (before) + /// CHECK-START: void ShortSimdMinMax.doitMin100(short[], short[]) loop_optimization (before) /// CHECK-DAG: <<I100:i\d+>> IntConstant 100 loop:none /// CHECK-DAG: <<Phi:i\d+>> Phi loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Get:s\d+>> ArrayGet loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Min:i\d+>> InvokeStaticOrDirect [<<Get>>,<<I100>>] intrinsic:MathMinIntInt loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Min:i\d+>> Min [<<Get>>,<<I100>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Cnv:s\d+>> TypeConversion [<<Min>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: ArraySet [{{l\d+}},<<Phi>>,<<Cnv>>] loop:<<Loop>> outer_loop:none // - /// CHECK-START-{ARM64,MIPS64}: void Main.doitMin100(short[], short[]) loop_optimization (after) + /// CHECK-START-{ARM,ARM64,MIPS64}: void ShortSimdMinMax.doitMin100(short[], short[]) loop_optimization (after) /// CHECK-DAG: <<I100:i\d+>> IntConstant 100 loop:none /// CHECK-DAG: <<Repl:d\d+>> VecReplicateScalar [<<I100>>] loop:none /// CHECK-DAG: <<Get:d\d+>> VecLoad loop:<<Loop:B\d+>> outer_loop:none @@ -142,7 +142,39 @@ public class Main { } } - public static void main(String[] args) { + /// CHECK-START-{ARM,ARM64,MIPS64}: void ShortSimdMinMax.doitMinMax(short[], short[]) loop_optimization (after) + /// CHECK-DAG: <<I11:i\d+>> IntConstant -1111 loop:none + /// CHECK-DAG: <<I23:i\d+>> IntConstant 2323 loop:none + /// CHECK-DAG: <<Rpl1:d\d+>> VecReplicateScalar [<<I23>>] loop:none + /// CHECK-DAG: <<Rpl2:d\d+>> VecReplicateScalar [<<I11>>] loop:none + /// CHECK-DAG: <<Get:d\d+>> VecLoad loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Min:d\d+>> VecMin [<<Get>>,<<Rpl1>>] packed_type:Int16 loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Max:d\d+>> VecMax [<<Min>>,<<Rpl2>>] packed_type:Int16 loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore [{{l\d+}},{{i\d+}},<<Max>>] loop:<<Loop>> outer_loop:none + private static void doitMinMax(short[] x, short[] y) { + int n = Math.min(x.length, y.length); + for (int i = 0; i < n; i++) { + x[i] = (short) Math.max(-1111, Math.min(y[i], 2323)); + } + } + + /// CHECK-START-{ARM,ARM64,MIPS64}: void ShortSimdMinMax.doitMinMaxUnsigned(short[], short[]) loop_optimization (after) + /// CHECK-DAG: <<I11:i\d+>> IntConstant 1111 loop:none + /// CHECK-DAG: <<I23:i\d+>> IntConstant 2323 loop:none + /// CHECK-DAG: <<Rpl1:d\d+>> VecReplicateScalar [<<I23>>] loop:none + /// CHECK-DAG: <<Rpl2:d\d+>> VecReplicateScalar [<<I11>>] loop:none + /// CHECK-DAG: <<Get:d\d+>> VecLoad loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Min:d\d+>> VecMin [<<Get>>,<<Rpl1>>] packed_type:Uint16 loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Max:d\d+>> VecMax [<<Min>>,<<Rpl2>>] packed_type:Uint16 loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore [{{l\d+}},{{i\d+}},<<Max>>] loop:<<Loop>> outer_loop:none + private static void doitMinMaxUnsigned(short[] x, short[] y) { + int n = Math.min(x.length, y.length); + for (int i = 0; i < n; i++) { + x[i] = (short) Math.max(1111, Math.min(y[i] & 0xffff, 2323)); + } + } + + public static void main() { short[] interesting = { (short) 0x0000, (short) 0x0001, (short) 0x007f, (short) 0x0080, (short) 0x0081, (short) 0x00ff, @@ -198,8 +230,20 @@ public class Main { short expected = (short) Math.min(y[i], 100); expectEquals(expected, x[i]); } + doitMinMax(x, y); + for (int i = 0; i < total; i++) { + int s = y[i]; + short expected = (short) (s < -1111 ? -1111 : (s > 2323 ? 2323 : s)); + expectEquals(expected, x[i]); + } + doitMinMaxUnsigned(x, y); + for (int i = 0; i < total; i++) { + int u = y[i] & 0xffff; + short expected = (short) (u < 1111 ? 1111 : (u > 2323 ? 2323 : u)); + expectEquals(expected, x[i]); + } - System.out.println("passed"); + System.out.println("ShortSimdMinMax passed"); } private static void expectEquals(short expected, short result) { diff --git a/test/655-jit-clinit/src/Main.java b/test/655-jit-clinit/src/Main.java index 2fb8f2a86e6..7fe5084b3fa 100644 --- a/test/655-jit-clinit/src/Main.java +++ b/test/655-jit-clinit/src/Main.java @@ -20,7 +20,7 @@ public class Main { if (!hasJit()) { return; } - Foo.hotMethod(); + Foo.$noinline$hotMethod(); } public native static boolean hasJitCompiledEntrypoint(Class<?> cls, String methodName); @@ -28,7 +28,7 @@ public class Main { } class Foo { - static void hotMethod() { + static void $noinline$hotMethod() { for (int i = 0; i < array.length; ++i) { array[i] = array; } @@ -36,8 +36,8 @@ class Foo { static { array = new Object[10000]; - while (!Main.hasJitCompiledEntrypoint(Foo.class, "hotMethod")) { - Foo.hotMethod(); + while (!Main.hasJitCompiledEntrypoint(Foo.class, "$noinline$hotMethod")) { + Foo.$noinline$hotMethod(); try { // Sleep to give a chance for the JIT to compile `hotMethod`. Thread.sleep(100); diff --git a/test/660-checker-sad-byte/src/Main.java b/test/660-checker-sad-byte/src/Main.java index 6bcd046bfc2..cd7fbcbdb9b 100644 --- a/test/660-checker-sad-byte/src/Main.java +++ b/test/660-checker-sad-byte/src/Main.java @@ -24,7 +24,7 @@ public class Main { /// CHECK-DAG: Return [<<Select>>] // /// CHECK-START: int Main.sad1(byte, byte) instruction_simplifier$after_inlining (after) - /// CHECK-DAG: <<Intrin:i\d+>> InvokeStaticOrDirect intrinsic:MathAbsInt + /// CHECK-DAG: <<Intrin:i\d+>> Abs /// CHECK-DAG: Return [<<Intrin>>] static int sad1(byte x, byte y) { return x >= y ? x - y : y - x; @@ -35,7 +35,7 @@ public class Main { /// CHECK-DAG: Return [<<Select>>] // /// CHECK-START: int Main.sad2(byte, byte) instruction_simplifier$after_inlining (after) - /// CHECK-DAG: <<Intrin:i\d+>> InvokeStaticOrDirect intrinsic:MathAbsInt + /// CHECK-DAG: <<Intrin:i\d+>> Abs /// CHECK-DAG: Return [<<Intrin>>] static int sad2(byte x, byte y) { int diff = x - y; @@ -48,7 +48,7 @@ public class Main { /// CHECK-DAG: Return [<<Select>>] // /// CHECK-START: int Main.sad3(byte, byte) instruction_simplifier$after_inlining (after) - /// CHECK-DAG: <<Intrin:i\d+>> InvokeStaticOrDirect intrinsic:MathAbsInt + /// CHECK-DAG: <<Intrin:i\d+>> Abs /// CHECK-DAG: Return [<<Intrin>>] static int sad3(byte x, byte y) { int diff = x - y; @@ -60,7 +60,7 @@ public class Main { /// CHECK-DAG: Return [<<Select>>] // /// CHECK-START: int Main.sad3Alt(byte, byte) instruction_simplifier$after_inlining (after) - /// CHECK-DAG: <<Intrin:i\d+>> InvokeStaticOrDirect intrinsic:MathAbsInt + /// CHECK-DAG: <<Intrin:i\d+>> Abs /// CHECK-DAG: Return [<<Intrin>>] static int sad3Alt(byte x, byte y) { int diff = x - y; @@ -72,7 +72,7 @@ public class Main { /// CHECK-DAG: Return [<<Select>>] // /// CHECK-START: long Main.sadL1(byte, byte) instruction_simplifier$after_inlining (after) - /// CHECK-DAG: <<Intrin:j\d+>> InvokeStaticOrDirect intrinsic:MathAbsLong + /// CHECK-DAG: <<Intrin:j\d+>> Abs /// CHECK-DAG: Return [<<Intrin>>] static long sadL1(byte x, byte y) { long xl = x; @@ -85,7 +85,7 @@ public class Main { /// CHECK-DAG: Return [<<Select>>] // /// CHECK-START: long Main.sadL2(byte, byte) instruction_simplifier$after_inlining (after) - /// CHECK-DAG: <<Intrin:j\d+>> InvokeStaticOrDirect intrinsic:MathAbsLong + /// CHECK-DAG: <<Intrin:j\d+>> Abs /// CHECK-DAG: Return [<<Intrin>>] static long sadL2(byte x, byte y) { long diff = x - y; @@ -98,7 +98,7 @@ public class Main { /// CHECK-DAG: Return [<<Select>>] // /// CHECK-START: long Main.sadL3(byte, byte) instruction_simplifier$after_inlining (after) - /// CHECK-DAG: <<Intrin:j\d+>> InvokeStaticOrDirect intrinsic:MathAbsLong + /// CHECK-DAG: <<Intrin:j\d+>> Abs /// CHECK-DAG: Return [<<Intrin>>] static long sadL3(byte x, byte y) { long diff = x - y; @@ -110,7 +110,7 @@ public class Main { /// CHECK-DAG: Return [<<Select>>] // /// CHECK-START: long Main.sadL3Alt(byte, byte) instruction_simplifier$after_inlining (after) - /// CHECK-DAG: <<Intrin:j\d+>> InvokeStaticOrDirect intrinsic:MathAbsLong + /// CHECK-DAG: <<Intrin:j\d+>> Abs /// CHECK-DAG: Return [<<Intrin>>] static long sadL3Alt(byte x, byte y) { long diff = x - y; diff --git a/test/660-checker-sad-char/src/Main.java b/test/660-checker-sad-char/src/Main.java index 3033509cdfe..ecf748ae078 100644 --- a/test/660-checker-sad-char/src/Main.java +++ b/test/660-checker-sad-char/src/Main.java @@ -24,7 +24,7 @@ public class Main { /// CHECK-DAG: Return [<<Select>>] // /// CHECK-START: int Main.sad1(char, char) instruction_simplifier$after_inlining (after) - /// CHECK-DAG: <<Intrin:i\d+>> InvokeStaticOrDirect intrinsic:MathAbsInt + /// CHECK-DAG: <<Intrin:i\d+>> Abs /// CHECK-DAG: Return [<<Intrin>>] static int sad1(char x, char y) { return x >= y ? x - y : y - x; @@ -35,7 +35,7 @@ public class Main { /// CHECK-DAG: Return [<<Select>>] // /// CHECK-START: int Main.sad2(char, char) instruction_simplifier$after_inlining (after) - /// CHECK-DAG: <<Intrin:i\d+>> InvokeStaticOrDirect intrinsic:MathAbsInt + /// CHECK-DAG: <<Intrin:i\d+>> Abs /// CHECK-DAG: Return [<<Intrin>>] static int sad2(char x, char y) { int diff = x - y; @@ -48,7 +48,7 @@ public class Main { /// CHECK-DAG: Return [<<Select>>] // /// CHECK-START: int Main.sad3(char, char) instruction_simplifier$after_inlining (after) - /// CHECK-DAG: <<Intrin:i\d+>> InvokeStaticOrDirect intrinsic:MathAbsInt + /// CHECK-DAG: <<Intrin:i\d+>> Abs /// CHECK-DAG: Return [<<Intrin>>] static int sad3(char x, char y) { int diff = x - y; @@ -60,7 +60,7 @@ public class Main { /// CHECK-DAG: Return [<<Select>>] // /// CHECK-START: int Main.sad3Alt(char, char) instruction_simplifier$after_inlining (after) - /// CHECK-DAG: <<Intrin:i\d+>> InvokeStaticOrDirect intrinsic:MathAbsInt + /// CHECK-DAG: <<Intrin:i\d+>> Abs /// CHECK-DAG: Return [<<Intrin>>] static int sad3Alt(char x, char y) { int diff = x - y; @@ -72,7 +72,7 @@ public class Main { /// CHECK-DAG: Return [<<Select>>] // /// CHECK-START: long Main.sadL1(char, char) instruction_simplifier$after_inlining (after) - /// CHECK-DAG: <<Intrin:j\d+>> InvokeStaticOrDirect intrinsic:MathAbsLong + /// CHECK-DAG: <<Intrin:j\d+>> Abs /// CHECK-DAG: Return [<<Intrin>>] static long sadL1(char x, char y) { long xl = x; @@ -85,7 +85,7 @@ public class Main { /// CHECK-DAG: Return [<<Select>>] // /// CHECK-START: long Main.sadL2(char, char) instruction_simplifier$after_inlining (after) - /// CHECK-DAG: <<Intrin:j\d+>> InvokeStaticOrDirect intrinsic:MathAbsLong + /// CHECK-DAG: <<Intrin:j\d+>> Abs /// CHECK-DAG: Return [<<Intrin>>] static long sadL2(char x, char y) { long diff = x - y; @@ -98,7 +98,7 @@ public class Main { /// CHECK-DAG: Return [<<Select>>] // /// CHECK-START: long Main.sadL3(char, char) instruction_simplifier$after_inlining (after) - /// CHECK-DAG: <<Intrin:j\d+>> InvokeStaticOrDirect intrinsic:MathAbsLong + /// CHECK-DAG: <<Intrin:j\d+>> Abs /// CHECK-DAG: Return [<<Intrin>>] static long sadL3(char x, char y) { long diff = x - y; @@ -110,7 +110,7 @@ public class Main { /// CHECK-DAG: Return [<<Select>>] // /// CHECK-START: long Main.sadL3Alt(char, char) instruction_simplifier$after_inlining (after) - /// CHECK-DAG: <<Intrin:j\d+>> InvokeStaticOrDirect intrinsic:MathAbsLong + /// CHECK-DAG: <<Intrin:j\d+>> Abs /// CHECK-DAG: Return [<<Intrin>>] static long sadL3Alt(char x, char y) { long diff = x - y; diff --git a/test/660-checker-sad-int/src/Main.java b/test/660-checker-sad-int/src/Main.java index 42ecea68dac..280dd66a51a 100644 --- a/test/660-checker-sad-int/src/Main.java +++ b/test/660-checker-sad-int/src/Main.java @@ -28,7 +28,7 @@ public class Main { /// CHECK-DAG: Return [<<Select>>] // /// CHECK-START: int Main.sad1(int, int) instruction_simplifier$after_inlining (after) - /// CHECK-NOT: InvokeStaticOrDirect intrinsic:MathAbsInt + /// CHECK-NOT: Abs // // NOTE: for direct 32-bit operands, this is not an ABS. static int sad1(int x, int y) { @@ -40,7 +40,7 @@ public class Main { /// CHECK-DAG: Return [<<Select>>] // /// CHECK-START: int Main.sad2(int, int) instruction_simplifier$after_inlining (after) - /// CHECK-DAG: <<Intrin:i\d+>> InvokeStaticOrDirect intrinsic:MathAbsInt + /// CHECK-DAG: <<Intrin:i\d+>> Abs /// CHECK-DAG: Return [<<Intrin>>] static int sad2(int x, int y) { int diff = x - y; @@ -53,7 +53,7 @@ public class Main { /// CHECK-DAG: Return [<<Select>>] // /// CHECK-START: int Main.sad3(int, int) instruction_simplifier$after_inlining (after) - /// CHECK-DAG: <<Intrin:i\d+>> InvokeStaticOrDirect intrinsic:MathAbsInt + /// CHECK-DAG: <<Intrin:i\d+>> Abs /// CHECK-DAG: Return [<<Intrin>>] static int sad3(int x, int y) { int diff = x - y; @@ -65,7 +65,7 @@ public class Main { /// CHECK-DAG: Return [<<Select>>] // /// CHECK-START: int Main.sad3Alt(int, int) instruction_simplifier$after_inlining (after) - /// CHECK-DAG: <<Intrin:i\d+>> InvokeStaticOrDirect intrinsic:MathAbsInt + /// CHECK-DAG: <<Intrin:i\d+>> Abs /// CHECK-DAG: Return [<<Intrin>>] static int sad3Alt(int x, int y) { int diff = x - y; @@ -77,7 +77,7 @@ public class Main { /// CHECK-DAG: Return [<<Select>>] // /// CHECK-START: long Main.sadL1(int, int) instruction_simplifier$after_inlining (after) - /// CHECK-DAG: <<Intrin:j\d+>> InvokeStaticOrDirect intrinsic:MathAbsLong + /// CHECK-DAG: <<Intrin:j\d+>> Abs /// CHECK-DAG: Return [<<Intrin>>] static long sadL1(int x, int y) { long xl = x; @@ -90,7 +90,7 @@ public class Main { /// CHECK-DAG: Return [<<Select>>] // /// CHECK-START: long Main.sadL2(int, int) instruction_simplifier$after_inlining (after) - /// CHECK-DAG: <<Intrin:j\d+>> InvokeStaticOrDirect intrinsic:MathAbsLong + /// CHECK-DAG: <<Intrin:j\d+>> Abs /// CHECK-DAG: Return [<<Intrin>>] static long sadL2(int x, int y) { long diff = x - y; @@ -103,7 +103,7 @@ public class Main { /// CHECK-DAG: Return [<<Select>>] // /// CHECK-START: long Main.sadL3(int, int) instruction_simplifier$after_inlining (after) - /// CHECK-DAG: <<Intrin:j\d+>> InvokeStaticOrDirect intrinsic:MathAbsLong + /// CHECK-DAG: <<Intrin:j\d+>> Abs /// CHECK-DAG: Return [<<Intrin>>] static long sadL3(int x, int y) { long diff = x - y; @@ -115,7 +115,7 @@ public class Main { /// CHECK-DAG: Return [<<Select>>] // /// CHECK-START: long Main.sadL3Alt(int, int) instruction_simplifier$after_inlining (after) - /// CHECK-DAG: <<Intrin:j\d+>> InvokeStaticOrDirect intrinsic:MathAbsLong + /// CHECK-DAG: <<Intrin:j\d+>> Abs /// CHECK-DAG: Return [<<Intrin>>] static long sadL3Alt(int x, int y) { long diff = x - y; diff --git a/test/660-checker-sad-long/src/Main.java b/test/660-checker-sad-long/src/Main.java index d2e32acffc4..ca0f4b71dd4 100644 --- a/test/660-checker-sad-long/src/Main.java +++ b/test/660-checker-sad-long/src/Main.java @@ -28,7 +28,7 @@ public class Main { /// CHECK-DAG: Return [<<Select>>] // /// CHECK-START: long Main.sad1(long, long) instruction_simplifier$after_inlining (after) - /// CHECK-NOT: InvokeStaticOrDirect intrinsic:MathAbsLong + /// CHECK-NOT: Abs // // NOTE: for direct 64-bit operands, this is not an ABS. static long sad1(long x, long y) { @@ -40,7 +40,7 @@ public class Main { /// CHECK-DAG: Return [<<Select>>] // /// CHECK-START: long Main.sad2(long, long) instruction_simplifier$after_inlining (after) - /// CHECK-DAG: <<Intrin:j\d+>> InvokeStaticOrDirect intrinsic:MathAbsLong + /// CHECK-DAG: <<Intrin:j\d+>> Abs /// CHECK-DAG: Return [<<Intrin>>] static long sad2(long x, long y) { long diff = x - y; @@ -53,7 +53,7 @@ public class Main { /// CHECK-DAG: Return [<<Select>>] // /// CHECK-START: long Main.sad3(long, long) instruction_simplifier$after_inlining (after) - /// CHECK-DAG: <<Intrin:j\d+>> InvokeStaticOrDirect intrinsic:MathAbsLong + /// CHECK-DAG: <<Intrin:j\d+>> Abs /// CHECK-DAG: Return [<<Intrin>>] static long sad3(long x, long y) { long diff = x - y; @@ -65,7 +65,7 @@ public class Main { /// CHECK-DAG: Return [<<Select>>] // /// CHECK-START: long Main.sad3Alt(long, long) instruction_simplifier$after_inlining (after) - /// CHECK-DAG: <<Intrin:j\d+>> InvokeStaticOrDirect intrinsic:MathAbsLong + /// CHECK-DAG: <<Intrin:j\d+>> Abs /// CHECK-DAG: Return [<<Intrin>>] static long sad3Alt(long x, long y) { long diff = x - y; diff --git a/test/660-checker-sad-short/src/Main.java b/test/660-checker-sad-short/src/Main.java index 182fe8a029f..b712a146f43 100644 --- a/test/660-checker-sad-short/src/Main.java +++ b/test/660-checker-sad-short/src/Main.java @@ -24,7 +24,7 @@ public class Main { /// CHECK-DAG: Return [<<Select>>] // /// CHECK-START: int Main.sad1(short, short) instruction_simplifier$after_inlining (after) - /// CHECK-DAG: <<Intrin:i\d+>> InvokeStaticOrDirect intrinsic:MathAbsInt + /// CHECK-DAG: <<Intrin:i\d+>> Abs /// CHECK-DAG: Return [<<Intrin>>] static int sad1(short x, short y) { return x >= y ? x - y : y - x; @@ -35,7 +35,7 @@ public class Main { /// CHECK-DAG: Return [<<Select>>] // /// CHECK-START: int Main.sad2(short, short) instruction_simplifier$after_inlining (after) - /// CHECK-DAG: <<Intrin:i\d+>> InvokeStaticOrDirect intrinsic:MathAbsInt + /// CHECK-DAG: <<Intrin:i\d+>> Abs /// CHECK-DAG: Return [<<Intrin>>] static int sad2(short x, short y) { int diff = x - y; @@ -48,7 +48,7 @@ public class Main { /// CHECK-DAG: Return [<<Select>>] // /// CHECK-START: int Main.sad3(short, short) instruction_simplifier$after_inlining (after) - /// CHECK-DAG: <<Intrin:i\d+>> InvokeStaticOrDirect intrinsic:MathAbsInt + /// CHECK-DAG: <<Intrin:i\d+>> Abs /// CHECK-DAG: Return [<<Intrin>>] static int sad3(short x, short y) { int diff = x - y; @@ -60,7 +60,7 @@ public class Main { /// CHECK-DAG: Return [<<Select>>] // /// CHECK-START: int Main.sad3Alt(short, short) instruction_simplifier$after_inlining (after) - /// CHECK-DAG: <<Intrin:i\d+>> InvokeStaticOrDirect intrinsic:MathAbsInt + /// CHECK-DAG: <<Intrin:i\d+>> Abs /// CHECK-DAG: Return [<<Intrin>>] static int sad3Alt(short x, short y) { int diff = x - y; @@ -72,7 +72,7 @@ public class Main { /// CHECK-DAG: Return [<<Select>>] // /// CHECK-START: long Main.sadL1(short, short) instruction_simplifier$after_inlining (after) - /// CHECK-DAG: <<Intrin:j\d+>> InvokeStaticOrDirect intrinsic:MathAbsLong + /// CHECK-DAG: <<Intrin:j\d+>> Abs /// CHECK-DAG: Return [<<Intrin>>] static long sadL1(short x, short y) { long xl = x; @@ -85,7 +85,7 @@ public class Main { /// CHECK-DAG: Return [<<Select>>] // /// CHECK-START: long Main.sadL2(short, short) instruction_simplifier$after_inlining (after) - /// CHECK-DAG: <<Intrin:j\d+>> InvokeStaticOrDirect intrinsic:MathAbsLong + /// CHECK-DAG: <<Intrin:j\d+>> Abs /// CHECK-DAG: Return [<<Intrin>>] static long sadL2(short x, short y) { long diff = x - y; @@ -98,7 +98,7 @@ public class Main { /// CHECK-DAG: Return [<<Select>>] // /// CHECK-START: long Main.sadL3(short, short) instruction_simplifier$after_inlining (after) - /// CHECK-DAG: <<Intrin:j\d+>> InvokeStaticOrDirect intrinsic:MathAbsLong + /// CHECK-DAG: <<Intrin:j\d+>> Abs /// CHECK-DAG: Return [<<Intrin>>] static long sadL3(short x, short y) { long diff = x - y; @@ -110,7 +110,7 @@ public class Main { /// CHECK-DAG: Return [<<Select>>] // /// CHECK-START: long Main.sadL3Alt(short, short) instruction_simplifier$after_inlining (after) - /// CHECK-DAG: <<Intrin:j\d+>> InvokeStaticOrDirect intrinsic:MathAbsLong + /// CHECK-DAG: <<Intrin:j\d+>> Abs /// CHECK-DAG: Return [<<Intrin>>] static long sadL3Alt(short x, short y) { long diff = x - y; diff --git a/test/660-checker-simd-sad-byte/build b/test/660-checker-simd-sad-byte/build new file mode 100644 index 00000000000..d85147f17bd --- /dev/null +++ b/test/660-checker-simd-sad-byte/build @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Copyright 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See b/65168732 +export USE_D8=false + +./default-build "$@" diff --git a/test/660-checker-simd-sad-byte/src/Main.java b/test/660-checker-simd-sad-byte/src/Main.java index 594948b7f9d..778d55c3ced 100644 --- a/test/660-checker-simd-sad-byte/src/Main.java +++ b/test/660-checker-simd-sad-byte/src/Main.java @@ -95,7 +95,7 @@ public class Main { /// CHECK-DAG: <<Get1:b\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get2:b\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Sub:i\d+>> Sub [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Intrin:i\d+>> InvokeStaticOrDirect [<<Sub>>] intrinsic:MathAbsInt loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Intrin:i\d+>> Abs [<<Sub>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi2>>,<<Intrin>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi1>>,<<Cons1>>] loop:<<Loop>> outer_loop:none // @@ -126,7 +126,7 @@ public class Main { /// CHECK-DAG: <<Get1:b\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get2:b\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Sub:i\d+>> Sub [<<Get2>>,<<Get1>>] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Intrin:i\d+>> InvokeStaticOrDirect [<<Sub>>] intrinsic:MathAbsInt loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Intrin:i\d+>> Abs [<<Sub>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi2>>,<<Intrin>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi1>>,<<Cons1>>] loop:<<Loop>> outer_loop:none // @@ -159,7 +159,7 @@ public class Main { /// CHECK-DAG: <<Get1:b\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get2:b\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Sub:i\d+>> Sub [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Intrin:i\d+>> InvokeStaticOrDirect [<<Sub>>] intrinsic:MathAbsInt loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Intrin:i\d+>> Abs [<<Sub>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi2>>,<<Intrin>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi1>>,<<Cons1>>] loop:<<Loop>> outer_loop:none // @@ -197,7 +197,7 @@ public class Main { /// CHECK-DAG: <<Cnv1:j\d+>> TypeConversion [<<Get1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Cnv2:j\d+>> TypeConversion [<<Get2>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Sub:j\d+>> Sub [<<Cnv1>>,<<Cnv2>>] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Intrin:j\d+>> InvokeStaticOrDirect [<<Sub>>] intrinsic:MathAbsLong loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Intrin:j\d+>> Abs [<<Sub>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi2>>,<<Intrin>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi1>>,<<Cons1>>] loop:<<Loop>> outer_loop:none // @@ -234,7 +234,7 @@ public class Main { /// CHECK-DAG: <<Cnv1:j\d+>> TypeConversion [<<Get1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Cnv2:j\d+>> TypeConversion [<<Get2>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Sub:j\d+>> Sub [<<Cnv1>>,<<Cnv2>>] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Intrin:j\d+>> InvokeStaticOrDirect [<<Sub>>] intrinsic:MathAbsLong loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Intrin:j\d+>> Abs [<<Sub>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi2>>,<<Intrin>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi1>>,<<Cons1>>] loop:<<Loop>> outer_loop:none // diff --git a/test/660-checker-simd-sad-char/build b/test/660-checker-simd-sad-char/build new file mode 100644 index 00000000000..d85147f17bd --- /dev/null +++ b/test/660-checker-simd-sad-char/build @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Copyright 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See b/65168732 +export USE_D8=false + +./default-build "$@" diff --git a/test/660-checker-simd-sad-char/src/Main.java b/test/660-checker-simd-sad-char/src/Main.java index ba226149b45..91c92f11790 100644 --- a/test/660-checker-simd-sad-char/src/Main.java +++ b/test/660-checker-simd-sad-char/src/Main.java @@ -64,7 +64,7 @@ public class Main { /// CHECK-DAG: <<Get1:c\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get2:c\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Sub:i\d+>> Sub [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Intrin:i\d+>> InvokeStaticOrDirect [<<Sub>>] intrinsic:MathAbsInt loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Intrin:i\d+>> Abs [<<Sub>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi2>>,<<Intrin>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi1>>,<<Cons1>>] loop:<<Loop>> outer_loop:none // @@ -87,7 +87,7 @@ public class Main { /// CHECK-DAG: <<Get1:c\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get2:c\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Sub:i\d+>> Sub [<<Get2>>,<<Get1>>] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Intrin:i\d+>> InvokeStaticOrDirect [<<Sub>>] intrinsic:MathAbsInt loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Intrin:i\d+>> Abs [<<Sub>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi2>>,<<Intrin>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi1>>,<<Cons1>>] loop:<<Loop>> outer_loop:none // @@ -112,7 +112,7 @@ public class Main { /// CHECK-DAG: <<Get1:c\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get2:c\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Sub:i\d+>> Sub [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Intrin:i\d+>> InvokeStaticOrDirect [<<Sub>>] intrinsic:MathAbsInt loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Intrin:i\d+>> Abs [<<Sub>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi2>>,<<Intrin>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi1>>,<<Cons1>>] loop:<<Loop>> outer_loop:none // @@ -142,7 +142,7 @@ public class Main { /// CHECK-DAG: <<Cnv1:j\d+>> TypeConversion [<<Get1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Cnv2:j\d+>> TypeConversion [<<Get2>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Sub:j\d+>> Sub [<<Cnv1>>,<<Cnv2>>] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Intrin:j\d+>> InvokeStaticOrDirect [<<Sub>>] intrinsic:MathAbsLong loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Intrin:j\d+>> Abs [<<Sub>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi2>>,<<Intrin>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi1>>,<<Cons1>>] loop:<<Loop>> outer_loop:none // @@ -170,7 +170,7 @@ public class Main { /// CHECK-DAG: <<Cnv1:j\d+>> TypeConversion [<<Get1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Cnv2:j\d+>> TypeConversion [<<Get2>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Sub:j\d+>> Sub [<<Cnv1>>,<<Cnv2>>] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Intrin:j\d+>> InvokeStaticOrDirect [<<Sub>>] intrinsic:MathAbsLong loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Intrin:j\d+>> Abs [<<Sub>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi2>>,<<Intrin>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi1>>,<<Cons1>>] loop:<<Loop>> outer_loop:none // diff --git a/test/660-checker-simd-sad-int/build b/test/660-checker-simd-sad-int/build new file mode 100644 index 00000000000..d85147f17bd --- /dev/null +++ b/test/660-checker-simd-sad-int/build @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Copyright 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See b/65168732 +export USE_D8=false + +./default-build "$@" diff --git a/test/660-checker-simd-sad-int/src/Main.java b/test/660-checker-simd-sad-int/src/Main.java index aa8431c1f53..29415fd2cf6 100644 --- a/test/660-checker-simd-sad-int/src/Main.java +++ b/test/660-checker-simd-sad-int/src/Main.java @@ -27,7 +27,7 @@ public class Main { /// CHECK-DAG: <<Get1:i\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get2:i\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Sub:i\d+>> Sub [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Intrin:i\d+>> InvokeStaticOrDirect [<<Sub>>] intrinsic:MathAbsInt loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Intrin:i\d+>> Abs [<<Sub>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi2>>,<<Intrin>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi1>>,<<Cons1>>] loop:<<Loop>> outer_loop:none // @@ -84,7 +84,7 @@ public class Main { /// CHECK-DAG: <<Get1:i\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get2:i\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Sub:i\d+>> Sub [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Intrin:i\d+>> InvokeStaticOrDirect [<<Sub>>] intrinsic:MathAbsInt loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Intrin:i\d+>> Abs [<<Sub>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi2>>,<<Intrin>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi1>>,<<Cons1>>] loop:<<Loop>> outer_loop:none // @@ -120,7 +120,7 @@ public class Main { /// CHECK-DAG: <<Cnv1:j\d+>> TypeConversion [<<Get1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Cnv2:j\d+>> TypeConversion [<<Get2>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Sub:j\d+>> Sub [<<Cnv1>>,<<Cnv2>>] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Intrin:j\d+>> InvokeStaticOrDirect [<<Sub>>] intrinsic:MathAbsLong loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Intrin:j\d+>> Abs [<<Sub>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi2>>,<<Intrin>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi1>>,<<Cons1>>] loop:<<Loop>> outer_loop:none // @@ -157,7 +157,7 @@ public class Main { /// CHECK-DAG: <<Cnv1:j\d+>> TypeConversion [<<Get1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Cnv2:j\d+>> TypeConversion [<<Get2>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Sub:j\d+>> Sub [<<Cnv1>>,<<Cnv2>>] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Intrin:j\d+>> InvokeStaticOrDirect [<<Sub>>] intrinsic:MathAbsLong loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Intrin:j\d+>> Abs [<<Sub>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi2>>,<<Intrin>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi1>>,<<Cons1>>] loop:<<Loop>> outer_loop:none // diff --git a/test/660-checker-simd-sad-long/src/Main.java b/test/660-checker-simd-sad-long/src/Main.java index 8281812b5bf..360e7234ebb 100644 --- a/test/660-checker-simd-sad-long/src/Main.java +++ b/test/660-checker-simd-sad-long/src/Main.java @@ -28,7 +28,7 @@ public class Main { /// CHECK-DAG: <<Get1:j\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get2:j\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Sub:j\d+>> Sub [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Intrin:j\d+>> InvokeStaticOrDirect [<<Sub>>] intrinsic:MathAbsLong loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Intrin:j\d+>> Abs [<<Sub>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi2>>,<<Intrin>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi1>>,<<Cons1>>] loop:<<Loop>> outer_loop:none // @@ -90,7 +90,7 @@ public class Main { /// CHECK-DAG: <<Get1:j\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get2:j\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Sub:j\d+>> Sub [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Intrin:j\d+>> InvokeStaticOrDirect [<<Sub>>] intrinsic:MathAbsLong loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Intrin:j\d+>> Abs [<<Sub>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi2>>,<<Intrin>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi1>>,<<Cons1>>] loop:<<Loop>> outer_loop:none // @@ -127,7 +127,7 @@ public class Main { /// CHECK-DAG: <<Get1:j\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get2:j\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Sub:j\d+>> Sub [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Intrin:j\d+>> InvokeStaticOrDirect [<<Sub>>] intrinsic:MathAbsLong loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Intrin:j\d+>> Abs [<<Sub>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi2>>,<<Intrin>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi1>>,<<Cons1>>] loop:<<Loop>> outer_loop:none // diff --git a/test/660-checker-simd-sad-short/build b/test/660-checker-simd-sad-short/build new file mode 100644 index 00000000000..d85147f17bd --- /dev/null +++ b/test/660-checker-simd-sad-short/build @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Copyright 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See b/65168732 +export USE_D8=false + +./default-build "$@" diff --git a/test/660-checker-simd-sad-short/src/Main.java b/test/660-checker-simd-sad-short/src/Main.java index 16bcabac95a..77c9e53e0cf 100644 --- a/test/660-checker-simd-sad-short/src/Main.java +++ b/test/660-checker-simd-sad-short/src/Main.java @@ -19,6 +19,10 @@ */ public class Main { + private static int $inline$seven() { + return 7; + } + // TODO: lower precision still coming, b/64091002 private static short sadShort2Short(short[] s1, short[] s2) { @@ -62,7 +66,7 @@ public class Main { /// CHECK-DAG: <<Get1:s\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get2:s\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Sub:i\d+>> Sub [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Intrin:i\d+>> InvokeStaticOrDirect [<<Sub>>] intrinsic:MathAbsInt loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Intrin:i\d+>> Abs [<<Sub>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi2>>,<<Intrin>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi1>>,<<Cons1>>] loop:<<Loop>> outer_loop:none // @@ -93,7 +97,7 @@ public class Main { /// CHECK-DAG: <<Get1:s\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get2:s\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Sub:i\d+>> Sub [<<Get2>>,<<Get1>>] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Intrin:i\d+>> InvokeStaticOrDirect [<<Sub>>] intrinsic:MathAbsInt loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Intrin:i\d+>> Abs [<<Sub>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi2>>,<<Intrin>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi1>>,<<Cons1>>] loop:<<Loop>> outer_loop:none // @@ -126,7 +130,7 @@ public class Main { /// CHECK-DAG: <<Get1:s\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get2:s\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Sub:i\d+>> Sub [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Intrin:i\d+>> InvokeStaticOrDirect [<<Sub>>] intrinsic:MathAbsInt loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Intrin:i\d+>> Abs [<<Sub>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi2>>,<<Intrin>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi1>>,<<Cons1>>] loop:<<Loop>> outer_loop:none // @@ -153,6 +157,102 @@ public class Main { return sad; } + /// CHECK-START: int Main.sadShort2IntConstant1(short[]) loop_optimization (before) + /// CHECK-DAG: <<Cons0:i\d+>> IntConstant 0 loop:none + /// CHECK-DAG: <<Cons1:i\d+>> IntConstant 1 loop:none + /// CHECK-DAG: <<Cons:i\d+>> IntConstant -7 loop:none + /// CHECK-DAG: <<Phi1:i\d+>> Phi [<<Cons0>>,{{i\d+}}] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Phi2:i\d+>> Phi [<<Cons0>>,{{i\d+}}] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Get1:s\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Add:i\d+>> Add [<<Get1>>,<<Cons>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Intrin:i\d+>> Abs [<<Add>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: Add [<<Phi2>>,<<Intrin>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: Add [<<Phi1>>,<<Cons1>>] loop:<<Loop>> outer_loop:none + // + /// CHECK-START-{ARM64,MIPS64}: int Main.sadShort2IntConstant1(short[]) loop_optimization (after) + /// CHECK-DAG: <<Cons0:i\d+>> IntConstant 0 loop:none + /// CHECK-DAG: <<Cons1:i\d+>> IntConstant 1 loop:none + /// CHECK-DAG: <<Cons:i\d+>> IntConstant 7 loop:none + /// CHECK-DAG: <<Cons8:i\d+>> IntConstant 8 loop:none + /// CHECK-DAG: <<Rep:d\d+>> VecReplicateScalar [<<Cons>>] loop:none + /// CHECK-DAG: <<Set:d\d+>> VecSetScalars [<<Cons0>>] loop:none + /// CHECK-DAG: <<Phi1:i\d+>> Phi [<<Cons0>>,{{i\d+}}] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Phi2:d\d+>> Phi [<<Set>>,{{d\d+}}] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Load1:d\d+>> VecLoad [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<SAD:d\d+>> VecSADAccumulate [<<Phi2>>,<<Load1>>,<<Rep>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: Add [<<Phi1>>,<<Cons8>>] loop:<<Loop>> outer_loop:none + private static int sadShort2IntConstant1(short[] s) { + int sad = 0; + for (int i = 0; i < s.length; i++) { + sad += Math.abs(s[i] - 7); // s[i] + -7 + } + return sad; + } + + /// CHECK-START: int Main.sadShort2IntConstant2(short[]) loop_optimization (before) + /// CHECK-DAG: <<Cons0:i\d+>> IntConstant 0 loop:none + /// CHECK-DAG: <<Cons1:i\d+>> IntConstant 1 loop:none + /// CHECK-DAG: <<Cons:i\d+>> IntConstant 7 loop:none + /// CHECK-DAG: <<Phi1:i\d+>> Phi [<<Cons0>>,{{i\d+}}] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Phi2:i\d+>> Phi [<<Cons0>>,{{i\d+}}] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Get1:s\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Sub:i\d+>> Sub [<<Get1>>,<<Cons>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Intrin:i\d+>> Abs [<<Sub>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: Add [<<Phi2>>,<<Intrin>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: Add [<<Phi1>>,<<Cons1>>] loop:<<Loop>> outer_loop:none + // + /// CHECK-START-{ARM64,MIPS64}: int Main.sadShort2IntConstant2(short[]) loop_optimization (after) + /// CHECK-DAG: <<Cons0:i\d+>> IntConstant 0 loop:none + /// CHECK-DAG: <<Cons1:i\d+>> IntConstant 1 loop:none + /// CHECK-DAG: <<Cons:i\d+>> IntConstant 7 loop:none + /// CHECK-DAG: <<Cons8:i\d+>> IntConstant 8 loop:none + /// CHECK-DAG: <<Rep:d\d+>> VecReplicateScalar [<<Cons>>] loop:none + /// CHECK-DAG: <<Set:d\d+>> VecSetScalars [<<Cons0>>] loop:none + /// CHECK-DAG: <<Phi1:i\d+>> Phi [<<Cons0>>,{{i\d+}}] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Phi2:d\d+>> Phi [<<Set>>,{{d\d+}}] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Load1:d\d+>> VecLoad [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<SAD:d\d+>> VecSADAccumulate [<<Phi2>>,<<Load1>>,<<Rep>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: Add [<<Phi1>>,<<Cons8>>] loop:<<Loop>> outer_loop:none + private static int sadShort2IntConstant2(short[] s) { + int sad = 0; + for (int i = 0; i < s.length; i++) { + sad += Math.abs(s[i] - $inline$seven()); // s[i] - 7 + } + return sad; + } + + /// CHECK-START: int Main.sadShort2IntConstant3(short[]) loop_optimization (before) + /// CHECK-DAG: <<Cons0:i\d+>> IntConstant 0 loop:none + /// CHECK-DAG: <<Cons1:i\d+>> IntConstant 1 loop:none + /// CHECK-DAG: <<Cons:i\d+>> IntConstant 7 loop:none + /// CHECK-DAG: <<Phi1:i\d+>> Phi [<<Cons0>>,{{i\d+}}] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Phi2:i\d+>> Phi [<<Cons0>>,{{i\d+}}] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Get1:s\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Add:i\d+>> Add [<<Get1>>,<<Cons>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Intrin:i\d+>> Abs [<<Add>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: Add [<<Phi2>>,<<Intrin>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: Add [<<Phi1>>,<<Cons1>>] loop:<<Loop>> outer_loop:none + // + /// CHECK-START-{ARM64,MIPS64}: int Main.sadShort2IntConstant3(short[]) loop_optimization (after) + /// CHECK-DAG: <<Cons0:i\d+>> IntConstant 0 loop:none + /// CHECK-DAG: <<Cons1:i\d+>> IntConstant 1 loop:none + /// CHECK-DAG: <<Cons:i\d+>> IntConstant -7 loop:none + /// CHECK-DAG: <<Cons8:i\d+>> IntConstant 8 loop:none + /// CHECK-DAG: <<Rep:d\d+>> VecReplicateScalar [<<Cons>>] loop:none + /// CHECK-DAG: <<Set:d\d+>> VecSetScalars [<<Cons0>>] loop:none + /// CHECK-DAG: <<Phi1:i\d+>> Phi [<<Cons0>>,{{i\d+}}] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Phi2:d\d+>> Phi [<<Set>>,{{d\d+}}] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Load1:d\d+>> VecLoad [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<SAD:d\d+>> VecSADAccumulate [<<Phi2>>,<<Load1>>,<<Rep>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: Add [<<Phi1>>,<<Cons8>>] loop:<<Loop>> outer_loop:none + private static int sadShort2IntConstant3(short[] s) { + int sad = 0; + for (int i = 0; i < s.length; i++) { + sad += Math.abs(s[i] + $inline$seven()); // hidden s[i] - (-7) + } + return sad; + } + /// CHECK-START: long Main.sadShort2Long(short[], short[]) loop_optimization (before) /// CHECK-DAG: <<Cons0:i\d+>> IntConstant 0 loop:none /// CHECK-DAG: <<Cons1:i\d+>> IntConstant 1 loop:none @@ -164,7 +264,7 @@ public class Main { /// CHECK-DAG: <<Cnv1:j\d+>> TypeConversion [<<Get1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Cnv2:j\d+>> TypeConversion [<<Get2>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Sub:j\d+>> Sub [<<Cnv1>>,<<Cnv2>>] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Intrin:j\d+>> InvokeStaticOrDirect [<<Sub>>] intrinsic:MathAbsLong loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Intrin:j\d+>> Abs [<<Sub>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi2>>,<<Intrin>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi1>>,<<Cons1>>] loop:<<Loop>> outer_loop:none // @@ -201,7 +301,7 @@ public class Main { /// CHECK-DAG: <<Cnv1:j\d+>> TypeConversion [<<Get1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Cnv2:j\d+>> TypeConversion [<<Get2>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Sub:j\d+>> Sub [<<Cnv1>>,<<Cnv2>>] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Intrin:j\d+>> InvokeStaticOrDirect [<<Sub>>] intrinsic:MathAbsLong loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Intrin:j\d+>> Abs [<<Sub>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi2>>,<<Intrin>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi1>>,<<Cons1>>] loop:<<Loop>> outer_loop:none // @@ -243,6 +343,9 @@ public class Main { expectEquals(65535, sadShort2IntAlt(s2, s1)); expectEquals(65535, sadShort2IntAlt2(s1, s2)); expectEquals(65535, sadShort2IntAlt2(s2, s1)); + expectEquals(32880, sadShort2IntConstant1(s1)); + expectEquals(32880, sadShort2IntConstant2(s1)); + expectEquals(32866, sadShort2IntConstant3(s1)); expectEquals(65535L, sadShort2Long(s1, s2)); expectEquals(65535L, sadShort2Long(s2, s1)); expectEquals(65536L, sadShort2LongAt1(s1, s2)); @@ -279,6 +382,9 @@ public class Main { expectEquals(1291788, sadShort2Int(s1, s2)); expectEquals(1291788, sadShort2IntAlt(s1, s2)); expectEquals(1291788, sadShort2IntAlt2(s1, s2)); + expectEquals(823907, sadShort2IntConstant1(s1)); + expectEquals(823907, sadShort2IntConstant2(s1)); + expectEquals(823953, sadShort2IntConstant3(s1)); expectEquals(1291788L, sadShort2Long(s1, s2)); expectEquals(1291789L, sadShort2LongAt1(s1, s2)); diff --git a/test/660-checker-simd-sad-short2/build b/test/660-checker-simd-sad-short2/build new file mode 100644 index 00000000000..d85147f17bd --- /dev/null +++ b/test/660-checker-simd-sad-short2/build @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Copyright 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See b/65168732 +export USE_D8=false + +./default-build "$@" diff --git a/test/660-checker-simd-sad-short2/src/Main.java b/test/660-checker-simd-sad-short2/src/Main.java index 274892d0010..a1f98297c5d 100644 --- a/test/660-checker-simd-sad-short2/src/Main.java +++ b/test/660-checker-simd-sad-short2/src/Main.java @@ -61,10 +61,10 @@ public class Main { /// CHECK-DAG: <<Cons1:i\d+>> IntConstant 1 loop:none /// CHECK-DAG: <<Phi1:i\d+>> Phi [<<Cons0>>,{{i\d+}}] loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Phi2:i\d+>> Phi [<<Cons0>>,{{i\d+}}] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<BC1:i\d+>> BoundsCheck [<<Phi1>>,{{i\d+}}] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<BC2:i\d+>> BoundsCheck [<<Phi1>>,{{i\d+}}] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Get1:c\d+>> ArrayGet [{{l\d+}},<<BC1>>] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Get2:c\d+>> ArrayGet [{{l\d+}},<<BC2>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<BC1:i\d+>> BoundsCheck [<<Phi1>>,{{i\d+}}] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<BC2:i\d+>> BoundsCheck [<<Phi1>>,{{i\d+}}] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Get1:c\d+>> ArrayGet [{{l\d+}},<<BC1>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Get2:c\d+>> ArrayGet [{{l\d+}},<<BC2>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Cnv1:s\d+>> TypeConversion [<<Get1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Cnv2:s\d+>> TypeConversion [<<Get2>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Sub:i\d+>> Sub [<<Cnv1>>,<<Cnv2>>] loop:<<Loop>> outer_loop:none @@ -80,7 +80,7 @@ public class Main { /// CHECK-DAG: <<Get1:s\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get2:s\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Sub:i\d+>> Sub [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Intrin:i\d+>> InvokeStaticOrDirect [<<Sub>>] intrinsic:MathAbsInt loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Intrin:i\d+>> Abs [<<Sub>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi2>>,<<Intrin>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi1>>,<<Cons1>>] loop:<<Loop>> outer_loop:none // @@ -130,7 +130,7 @@ public class Main { /// CHECK-DAG: <<Get2:s\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Cnv1:s\d+>> TypeConversion [<<Get1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Sub:i\d+>> Sub [<<Get2>>,<<Cnv1>>] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Intrin:i\d+>> InvokeStaticOrDirect [<<Sub>>] intrinsic:MathAbsInt loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Intrin:i\d+>> Abs [<<Sub>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi2>>,<<Intrin>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi1>>,<<Cons1>>] loop:<<Loop>> outer_loop:none // @@ -182,7 +182,7 @@ public class Main { /// CHECK-DAG: <<Get2:s\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Cnv1:s\d+>> TypeConversion [<<Get1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Sub:i\d+>> Sub [<<Cnv1>>,<<Get2>>] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Intrin:i\d+>> InvokeStaticOrDirect [<<Sub>>] intrinsic:MathAbsInt loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Intrin:i\d+>> Abs [<<Sub>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi2>>,<<Intrin>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi1>>,<<Cons1>>] loop:<<Loop>> outer_loop:none // @@ -215,8 +215,8 @@ public class Main { /// CHECK-DAG: <<ConsL:j\d+>> LongConstant 0 loop:none /// CHECK-DAG: <<Phi1:i\d+>> Phi [<<Cons0>>,{{i\d+}}] loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Phi2:j\d+>> Phi [<<ConsL>>,{{j\d+}}] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<BC1:\i\d+>> BoundsCheck [<<Phi1>>,{{i\d+}}] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<BC2:\i\d+>> BoundsCheck [<<Phi1>>,{{i\d+}}] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<BC1:\i\d+>> BoundsCheck [<<Phi1>>,{{i\d+}}] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<BC2:\i\d+>> BoundsCheck [<<Phi1>>,{{i\d+}}] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get1:c\d+>> ArrayGet [{{l\d+}},<<BC1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get2:c\d+>> ArrayGet [{{l\d+}},<<BC2>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Cnv1:s\d+>> TypeConversion [<<Get1>>] loop:<<Loop>> outer_loop:none @@ -239,7 +239,7 @@ public class Main { /// CHECK-DAG: <<Cnv1:j\d+>> TypeConversion [<<Get1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Cnv2:j\d+>> TypeConversion [<<Get2>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Sub:j\d+>> Sub [<<Cnv1>>,<<Cnv2>>] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Intrin:j\d+>> InvokeStaticOrDirect [<<Sub>>] intrinsic:MathAbsLong loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Intrin:j\d+>> Abs [<<Sub>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi2>>,<<Intrin>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi1>>,<<Cons1>>] loop:<<Loop>> outer_loop:none // @@ -271,8 +271,8 @@ public class Main { /// CHECK-DAG: <<ConsL:j\d+>> LongConstant 1 loop:none /// CHECK-DAG: <<Phi1:i\d+>> Phi [<<Cons0>>,{{i\d+}}] loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Phi2:j\d+>> Phi [<<ConsL>>,{{j\d+}}] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<BC1:\i\d+>> BoundsCheck [<<Phi1>>,{{i\d+}}] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<BC2:\i\d+>> BoundsCheck [<<Phi1>>,{{i\d+}}] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<BC1:\i\d+>> BoundsCheck [<<Phi1>>,{{i\d+}}] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<BC2:\i\d+>> BoundsCheck [<<Phi1>>,{{i\d+}}] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get1:c\d+>> ArrayGet [{{l\d+}},<<BC1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get2:c\d+>> ArrayGet [{{l\d+}},<<BC2>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Cnv1:s\d+>> TypeConversion [<<Get1>>] loop:<<Loop>> outer_loop:none @@ -295,7 +295,7 @@ public class Main { /// CHECK-DAG: <<Cnv1:j\d+>> TypeConversion [<<Get1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Cnv2:j\d+>> TypeConversion [<<Get2>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Sub:j\d+>> Sub [<<Cnv1>>,<<Cnv2>>] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Intrin:j\d+>> InvokeStaticOrDirect [<<Sub>>] intrinsic:MathAbsLong loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Intrin:j\d+>> Abs [<<Sub>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi2>>,<<Intrin>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi1>>,<<Cons1>>] loop:<<Loop>> outer_loop:none // diff --git a/test/660-checker-simd-sad-short3/build b/test/660-checker-simd-sad-short3/build new file mode 100644 index 00000000000..d85147f17bd --- /dev/null +++ b/test/660-checker-simd-sad-short3/build @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Copyright 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See b/65168732 +export USE_D8=false + +./default-build "$@" diff --git a/test/660-checker-simd-sad-short3/src/Main.java b/test/660-checker-simd-sad-short3/src/Main.java index 5016b658e56..877a5362cee 100644 --- a/test/660-checker-simd-sad-short3/src/Main.java +++ b/test/660-checker-simd-sad-short3/src/Main.java @@ -29,7 +29,7 @@ public class Main { /// CHECK-DAG: <<Phi2:i\d+>> Phi [<<Cons0>>,{{i\d+}}] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get:s\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Sub:i\d+>> Sub [<<Get>>,<<Param>>] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Intrin:i\d+>> InvokeStaticOrDirect [<<Sub>>] intrinsic:MathAbsInt loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Intrin:i\d+>> Abs [<<Sub>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi2>>,<<Intrin>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi1>>,<<Cons1>>] loop:<<Loop>> outer_loop:none // @@ -60,7 +60,7 @@ public class Main { /// CHECK-DAG: <<Phi2:i\d+>> Phi [<<Cons0>>,{{i\d+}}] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get:s\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Sub:i\d+>> Sub [<<Param>>,<<Get>>] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Intrin:i\d+>> InvokeStaticOrDirect [<<Sub>>] intrinsic:MathAbsInt loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Intrin:i\d+>> Abs [<<Sub>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi2>>,<<Intrin>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi1>>,<<Cons1>>] loop:<<Loop>> outer_loop:none // @@ -91,7 +91,7 @@ public class Main { /// CHECK-DAG: <<Phi2:i\d+>> Phi [<<Cons0>>,{{i\d+}}] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get:s\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Add:i\d+>> Add [<<Get>>,<<ConsI>>] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Intrin:i\d+>> InvokeStaticOrDirect [<<Add>>] intrinsic:MathAbsInt loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Intrin:i\d+>> Abs [<<Add>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi2>>,<<Intrin>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi1>>,<<Cons1>>] loop:<<Loop>> outer_loop:none // @@ -122,7 +122,7 @@ public class Main { /// CHECK-DAG: <<Phi2:i\d+>> Phi [<<Cons0>>,{{i\d+}}] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get:s\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Sub:i\d+>> Sub [<<ConsI>>,<<Get>>] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Intrin:i\d+>> InvokeStaticOrDirect [<<Sub>>] intrinsic:MathAbsInt loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Intrin:i\d+>> Abs [<<Sub>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi2>>,<<Intrin>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi1>>,<<Cons1>>] loop:<<Loop>> outer_loop:none // @@ -153,7 +153,7 @@ public class Main { /// CHECK-DAG: <<Phi2:i\d+>> Phi [<<Cons0>>,{{i\d+}}] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get:s\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Sub:i\d+>> Sub [<<Get>>,<<Conv>>] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Intrin:i\d+>> InvokeStaticOrDirect [<<Sub>>] intrinsic:MathAbsInt loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Intrin:i\d+>> Abs [<<Sub>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi2>>,<<Intrin>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi1>>,<<Cons1>>] loop:<<Loop>> outer_loop:none // @@ -185,7 +185,7 @@ public class Main { /// CHECK-DAG: <<Phi2:i\d+>> Phi [<<Cons0>>,{{i\d+}}] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get:s\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Sub:i\d+>> Sub [<<Conv>>,<<Get>>] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Intrin:i\d+>> InvokeStaticOrDirect [<<Sub>>] intrinsic:MathAbsInt loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Intrin:i\d+>> Abs [<<Sub>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi2>>,<<Intrin>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi1>>,<<Cons1>>] loop:<<Loop>> outer_loop:none // @@ -219,7 +219,7 @@ public class Main { /// CHECK-DAG: <<Add:i\d+>> [<<Get>>,<<ConsI>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Conv:s\d+>> TypeConversion [<<Add>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Sub:i\d+>> Sub [<<Get>>,<<Conv>>] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Intrin:i\d+>> InvokeStaticOrDirect [<<Sub>>] intrinsic:MathAbsInt loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Intrin:i\d+>> Abs [<<Sub>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi2>>,<<Intrin>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi1>>,<<Cons1>>] loop:<<Loop>> outer_loop:none // @@ -254,7 +254,7 @@ public class Main { /// CHECK-DAG: <<Add:i\d+>> [<<Get>>,<<ConsI>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Conv:s\d+>> TypeConversion [<<Add>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Sub:i\d+>> Sub [<<Conv>>,<<Get>>] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: <<Intrin:i\d+>> InvokeStaticOrDirect [<<Sub>>] intrinsic:MathAbsInt loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Intrin:i\d+>> Abs [<<Sub>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi2>>,<<Intrin>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi1>>,<<Cons1>>] loop:<<Loop>> outer_loop:none // diff --git a/test/661-checker-simd-reduc/build b/test/661-checker-simd-reduc/build new file mode 100644 index 00000000000..d85147f17bd --- /dev/null +++ b/test/661-checker-simd-reduc/build @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Copyright 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See b/65168732 +export USE_D8=false + +./default-build "$@" diff --git a/test/661-checker-simd-reduc/src/Main.java b/test/661-checker-simd-reduc/src/Main.java index 3a0a0495c4a..fcd50a6d153 100644 --- a/test/661-checker-simd-reduc/src/Main.java +++ b/test/661-checker-simd-reduc/src/Main.java @@ -378,7 +378,7 @@ public class Main { /// CHECK-DAG: <<Phi1:i\d+>> Phi [<<Cons0>>,{{i\d+}}] loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Phi2:i\d+>> Phi [<<ConsM>>,{{i\d+}}] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get:i\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: InvokeStaticOrDirect [<<Phi2>>,<<Get>>] intrinsic:MathMinIntInt loop:<<Loop>> outer_loop:none + /// CHECK-DAG: Min [<<Phi2>>,<<Get>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi1>>,<<Cons1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Return [<<Phi2>>] loop:none // @@ -438,7 +438,7 @@ public class Main { /// CHECK-DAG: <<Phi1:i\d+>> Phi [<<Cons0>>,{{i\d+}}] loop:<<Loop:B\d+>> outer_loop:none /// CHECK-DAG: <<Phi2:i\d+>> Phi [<<ConsM>>,{{i\d+}}] loop:<<Loop>> outer_loop:none /// CHECK-DAG: <<Get:i\d+>> ArrayGet [{{l\d+}},<<Phi1>>] loop:<<Loop>> outer_loop:none - /// CHECK-DAG: InvokeStaticOrDirect [<<Phi2>>,<<Get>>] intrinsic:MathMaxIntInt loop:<<Loop>> outer_loop:none + /// CHECK-DAG: Max [<<Phi2>>,<<Get>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Add [<<Phi1>>,<<Cons1>>] loop:<<Loop>> outer_loop:none /// CHECK-DAG: Return [<<Phi2>>] loop:none // diff --git a/test/672-checker-throw-method/build b/test/672-checker-throw-method/build new file mode 100644 index 00000000000..d85147f17bd --- /dev/null +++ b/test/672-checker-throw-method/build @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Copyright 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See b/65168732 +export USE_D8=false + +./default-build "$@" diff --git a/test/673-checker-throw-vmethod/build b/test/673-checker-throw-vmethod/build new file mode 100644 index 00000000000..d85147f17bd --- /dev/null +++ b/test/673-checker-throw-vmethod/build @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Copyright 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See b/65168732 +export USE_D8=false + +./default-build "$@" diff --git a/test/678-checker-simd-saturation/build b/test/678-checker-simd-saturation/build new file mode 100644 index 00000000000..d85147f17bd --- /dev/null +++ b/test/678-checker-simd-saturation/build @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Copyright 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See b/65168732 +export USE_D8=false + +./default-build "$@" diff --git a/test/651-checker-byte-simd-minmax/expected.txt b/test/678-checker-simd-saturation/expected.txt index b0aad4deb5b..b0aad4deb5b 100644 --- a/test/651-checker-byte-simd-minmax/expected.txt +++ b/test/678-checker-simd-saturation/expected.txt diff --git a/test/678-checker-simd-saturation/info.txt b/test/678-checker-simd-saturation/info.txt new file mode 100644 index 00000000000..ab7a80241d4 --- /dev/null +++ b/test/678-checker-simd-saturation/info.txt @@ -0,0 +1 @@ +Functional tests on saturation arithmetic vectorization. diff --git a/test/678-checker-simd-saturation/src/Main.java b/test/678-checker-simd-saturation/src/Main.java new file mode 100644 index 00000000000..7a22ca175d2 --- /dev/null +++ b/test/678-checker-simd-saturation/src/Main.java @@ -0,0 +1,784 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Functional tests for saturation aritmethic vectorization. + */ +public class Main { + + static final int $inline$p15() { + return 15; + } + + static final int $inline$m15() { + return -15; + } + + // + // Direct min-max. + // + + /// CHECK-START: void Main.satAddUByte(byte[], byte[], byte[]) loop_optimization (before) + /// CHECK-DAG: <<Clip:i\d+>> IntConstant 255 loop:none + /// CHECK-DAG: <<Get1:a\d+>> ArrayGet [{{l\d+}},<<Phi:i\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Get2:a\d+>> ArrayGet [{{l\d+}},<<Phi>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Add:i\d+>> Add [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Min:i\d+>> Min [<<Add>>,<<Clip>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Conv:b\d+>> TypeConversion [<<Min>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet [{{l\d+}},<<Phi>>,<<Conv>>] loop:<<Loop>> outer_loop:none + // + /// CHECK-START-{ARM,ARM64}: void Main.satAddUByte(byte[], byte[], byte[]) loop_optimization (after) + /// CHECK-DAG: <<Get1:d\d+>> VecLoad [{{l\d+}},<<Phi:i\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Get2:d\d+>> VecLoad [{{l\d+}},<<Phi>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Add:d\d+>> VecSaturationAdd [<<Get1>>,<<Get2>>] packed_type:Uint8 loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore [{{l\d+}},<<Phi>>,<<Add>>] loop:<<Loop>> outer_loop:none + public static void satAddUByte(byte[] a, byte[] b, byte[] c) { + int n = Math.min(a.length, Math.min(b.length, c.length)); + for (int i = 0; i < n; i++) { + c[i] = (byte) Math.min((a[i] & 0xff) + (b[i] & 0xff), 255); + } + } + + /// CHECK-START: void Main.satAddSByte(byte[], byte[], byte[]) loop_optimization (before) + /// CHECK-DAG: <<Clp1:i\d+>> IntConstant -128 loop:none + /// CHECK-DAG: <<Clp2:i\d+>> IntConstant 127 loop:none + /// CHECK-DAG: <<Get1:b\d+>> ArrayGet [{{l\d+}},<<Phi:i\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Get2:b\d+>> ArrayGet [{{l\d+}},<<Phi>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Add:i\d+>> Add [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Min:i\d+>> Min [<<Add>>,<<Clp2>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Max:i\d+>> Max [<<Min>>,<<Clp1>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Conv:b\d+>> TypeConversion [<<Max>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet [{{l\d+}},<<Phi>>,<<Conv>>] loop:<<Loop>> outer_loop:none + // + /// CHECK-START-{ARM,ARM64}: void Main.satAddSByte(byte[], byte[], byte[]) loop_optimization (after) + /// CHECK-DAG: <<Get1:d\d+>> VecLoad [{{l\d+}},<<Phi:i\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Get2:d\d+>> VecLoad [{{l\d+}},<<Phi>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Add:d\d+>> VecSaturationAdd [<<Get1>>,<<Get2>>] packed_type:Int8 loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore [{{l\d+}},<<Phi>>,<<Add>>] loop:<<Loop>> outer_loop:none + public static void satAddSByte(byte[] a, byte[] b, byte[] c) { + int n = Math.min(a.length, Math.min(b.length, c.length)); + for (int i = 0; i < n; i++) { + c[i] = (byte) Math.max(Math.min(a[i] + b[i], 127), -128); + } + } + + /// CHECK-START: void Main.satAddUShort(short[], short[], short[]) loop_optimization (before) + /// CHECK-DAG: <<Clip:i\d+>> IntConstant 65535 loop:none + /// CHECK-DAG: <<Get1:c\d+>> ArrayGet [{{l\d+}},<<Phi:i\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Get2:c\d+>> ArrayGet [{{l\d+}},<<Phi>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Add:i\d+>> Add [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Min:i\d+>> Min [<<Add>>,<<Clip>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Conv:s\d+>> TypeConversion [<<Min>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet [{{l\d+}},<<Phi>>,<<Conv>>] loop:<<Loop>> outer_loop:none + // + /// CHECK-START-{ARM,ARM64}: void Main.satAddUShort(short[], short[], short[]) loop_optimization (after) + /// CHECK-DAG: <<Get1:d\d+>> VecLoad [{{l\d+}},<<Phi:i\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Get2:d\d+>> VecLoad [{{l\d+}},<<Phi>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Add:d\d+>> VecSaturationAdd [<<Get1>>,<<Get2>>] packed_type:Uint16 loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore [{{l\d+}},<<Phi>>,<<Add>>] loop:<<Loop>> outer_loop:none + public static void satAddUShort(short[] a, short[] b, short[] c) { + int n = Math.min(a.length, Math.min(b.length, c.length)); + for (int i = 0; i < n; i++) { + c[i] = (short) Math.min((a[i] & 0xffff) + (b[i] & 0xffff), 65535); + } + } + + /// CHECK-START: void Main.satAddSShort(short[], short[], short[]) loop_optimization (before) + /// CHECK-DAG: <<Clp1:i\d+>> IntConstant -32768 loop:none + /// CHECK-DAG: <<Clp2:i\d+>> IntConstant 32767 loop:none + /// CHECK-DAG: <<Get1:s\d+>> ArrayGet [{{l\d+}},<<Phi:i\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Get2:s\d+>> ArrayGet [{{l\d+}},<<Phi>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Add:i\d+>> Add [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Min:i\d+>> Min [<<Add>>,<<Clp2>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Max:i\d+>> Max [<<Min>>,<<Clp1>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Conv:s\d+>> TypeConversion [<<Max>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet [{{l\d+}},<<Phi>>,<<Conv>>] loop:<<Loop>> outer_loop:none + // + /// CHECK-START-{ARM,ARM64}: void Main.satAddSShort(short[], short[], short[]) loop_optimization (after) + /// CHECK-DAG: <<Get1:d\d+>> VecLoad [{{l\d+}},<<Phi:i\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Get2:d\d+>> VecLoad [{{l\d+}},<<Phi>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Add:d\d+>> VecSaturationAdd [<<Get1>>,<<Get2>>] packed_type:Int16 loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore [{{l\d+}},<<Phi>>,<<Add>>] loop:<<Loop>> outer_loop:none + public static void satAddSShort(short[] a, short[] b, short[] c) { + int n = Math.min(a.length, Math.min(b.length, c.length)); + for (int i = 0; i < n; i++) { + c[i] = (short) Math.max(Math.min(a[i] + b[i], 32767), -32768); + } + } + + /// CHECK-START: void Main.satSubUByte(byte[], byte[], byte[]) loop_optimization (before) + /// CHECK-DAG: <<Clip:i\d+>> IntConstant 0 loop:none + /// CHECK-DAG: <<Get1:a\d+>> ArrayGet [{{l\d+}},<<Phi:i\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Get2:a\d+>> ArrayGet [{{l\d+}},<<Phi>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Sub:i\d+>> Sub [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Max:i\d+>> Max [<<Sub>>,<<Clip>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Conv:b\d+>> TypeConversion [<<Max>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet [{{l\d+}},<<Phi>>,<<Conv>>] loop:<<Loop>> outer_loop:none + // + /// CHECK-START-{ARM,ARM64}: void Main.satSubUByte(byte[], byte[], byte[]) loop_optimization (after) + /// CHECK-DAG: <<Get1:d\d+>> VecLoad [{{l\d+}},<<Phi:i\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Get2:d\d+>> VecLoad [{{l\d+}},<<Phi>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Sub:d\d+>> VecSaturationSub [<<Get1>>,<<Get2>>] packed_type:Uint8 loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore [{{l\d+}},<<Phi>>,<<Sub>>] loop:<<Loop>> outer_loop:none + public static void satSubUByte(byte[] a, byte[] b, byte[] c) { + int n = Math.min(a.length, Math.min(b.length, c.length)); + for (int i = 0; i < n; i++) { + c[i] = (byte) Math.max((a[i] & 0xff) - (b[i] & 0xff), 0); + } + } + + /// CHECK-START: void Main.satSubSByte(byte[], byte[], byte[]) loop_optimization (before) + /// CHECK-DAG: <<Clp1:i\d+>> IntConstant -128 loop:none + /// CHECK-DAG: <<Clp2:i\d+>> IntConstant 127 loop:none + /// CHECK-DAG: <<Get1:b\d+>> ArrayGet [{{l\d+}},<<Phi:i\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Get2:b\d+>> ArrayGet [{{l\d+}},<<Phi>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Sub:i\d+>> Sub [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Min:i\d+>> Min [<<Sub>>,<<Clp2>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Max:i\d+>> Max [<<Min>>,<<Clp1>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Conv:b\d+>> TypeConversion [<<Max>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet [{{l\d+}},<<Phi>>,<<Conv>>] loop:<<Loop>> outer_loop:none + // + /// CHECK-START-{ARM,ARM64}: void Main.satSubSByte(byte[], byte[], byte[]) loop_optimization (after) + /// CHECK-DAG: <<Get1:d\d+>> VecLoad [{{l\d+}},<<Phi:i\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Get2:d\d+>> VecLoad [{{l\d+}},<<Phi>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Sub:d\d+>> VecSaturationSub [<<Get1>>,<<Get2>>] packed_type:Int8 loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore [{{l\d+}},<<Phi>>,<<Sub>>] loop:<<Loop>> outer_loop:none + public static void satSubSByte(byte[] a, byte[] b, byte[] c) { + int n = Math.min(a.length, Math.min(b.length, c.length)); + for (int i = 0; i < n; i++) { + c[i] = (byte) Math.max(Math.min(a[i] - b[i], 127), -128); + } + } + + /// CHECK-START: void Main.satSubUShort(short[], short[], short[]) loop_optimization (before) + /// CHECK-DAG: <<Clip:i\d+>> IntConstant 0 loop:none + /// CHECK-DAG: <<Get1:c\d+>> ArrayGet [{{l\d+}},<<Phi:i\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Get2:c\d+>> ArrayGet [{{l\d+}},<<Phi>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Sub:i\d+>> Sub [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Max:i\d+>> Max [<<Sub>>,<<Clip>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Conv:s\d+>> TypeConversion [<<Max>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet [{{l\d+}},<<Phi>>,<<Conv>>] loop:<<Loop>> outer_loop:none + // + /// CHECK-START-{ARM,ARM64}: void Main.satSubUShort(short[], short[], short[]) loop_optimization (after) + /// CHECK-DAG: <<Get1:d\d+>> VecLoad [{{l\d+}},<<Phi:i\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Get2:d\d+>> VecLoad [{{l\d+}},<<Phi>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Sub:d\d+>> VecSaturationSub [<<Get1>>,<<Get2>>] packed_type:Uint16 loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore [{{l\d+}},<<Phi>>,<<Sub>>] loop:<<Loop>> outer_loop:none + public static void satSubUShort(short[] a, short[] b, short[] c) { + int n = Math.min(a.length, Math.min(b.length, c.length)); + for (int i = 0; i < n; i++) { + c[i] = (short) Math.max((a[i] & 0xffff) - (b[i] & 0xffff), 0); + } + } + + /// CHECK-START: void Main.satSubSShort(short[], short[], short[]) loop_optimization (before) + /// CHECK-DAG: <<Clp1:i\d+>> IntConstant -32768 loop:none + /// CHECK-DAG: <<Clp2:i\d+>> IntConstant 32767 loop:none + /// CHECK-DAG: <<Get1:s\d+>> ArrayGet [{{l\d+}},<<Phi:i\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Get2:s\d+>> ArrayGet [{{l\d+}},<<Phi>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Sub:i\d+>> Sub [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Min:i\d+>> Min [<<Sub>>,<<Clp2>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Max:i\d+>> Max [<<Min>>,<<Clp1>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Conv:s\d+>> TypeConversion [<<Max>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet [{{l\d+}},<<Phi>>,<<Conv>>] loop:<<Loop>> outer_loop:none + // + /// CHECK-START-{ARM,ARM64}: void Main.satSubSShort(short[], short[], short[]) loop_optimization (after) + /// CHECK-DAG: <<Get1:d\d+>> VecLoad [{{l\d+}},<<Phi:i\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Get2:d\d+>> VecLoad [{{l\d+}},<<Phi>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Sub:d\d+>> VecSaturationSub [<<Get1>>,<<Get2>>] packed_type:Int16 loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore [{{l\d+}},<<Phi>>,<<Sub>>] loop:<<Loop>> outer_loop:none + public static void satSubSShort(short[] a, short[] b, short[] c) { + int n = Math.min(a.length, Math.min(b.length, c.length)); + for (int i = 0; i < n; i++) { + c[i] = (short) Math.max(Math.min(a[i] - b[i], 32767), -32768); + } + } + + // + // Single clipping signed 8-bit saturation. + // + + /// CHECK-START-{ARM,ARM64}: void Main.satAddPConstSByte(byte[], byte[]) loop_optimization (after) + /// CHECK-DAG: <<Get1:d\d+>> VecReplicateScalar loop:none + /// CHECK-DAG: <<Get2:d\d+>> VecLoad [{{l\d+}},<<Phi:i\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Add:d\d+>> VecSaturationAdd [<<Get2>>,<<Get1>>] packed_type:Int8 loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore [{{l\d+}},<<Phi>>,<<Add>>] loop:<<Loop>> outer_loop:none + public static void satAddPConstSByte(byte[] a, byte[] b) { + int n = Math.min(a.length, b.length); + for (int i = 0; i < n; i++) { + b[i] = (byte) Math.min(a[i] + 15, 127); + } + } + + /// CHECK-START-{ARM,ARM64}: void Main.satAddNConstSByte(byte[], byte[]) loop_optimization (after) + /// CHECK-DAG: <<Get1:d\d+>> VecReplicateScalar loop:none + /// CHECK-DAG: <<Get2:d\d+>> VecLoad [{{l\d+}},<<Phi:i\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Add:d\d+>> VecSaturationAdd [<<Get2>>,<<Get1>>] packed_type:Int8 loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore [{{l\d+}},<<Phi>>,<<Add>>] loop:<<Loop>> outer_loop:none + public static void satAddNConstSByte(byte[] a, byte[] b) { + int n = Math.min(a.length, b.length); + for (int i = 0; i < n; i++) { + b[i] = (byte) Math.max(a[i] - 15, -128); + } + } + + /// CHECK-START-{ARM,ARM64}: void Main.satSubPConstSByte(byte[], byte[]) loop_optimization (after) + /// CHECK-DAG: <<Get1:d\d+>> VecReplicateScalar loop:none + /// CHECK-DAG: <<Get2:d\d+>> VecLoad [{{l\d+}},<<Phi:i\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Sub:d\d+>> VecSaturationSub [<<Get1>>,<<Get2>>] packed_type:Int8 loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore [{{l\d+}},<<Phi>>,<<Sub>>] loop:<<Loop>> outer_loop:none + public static void satSubPConstSByte(byte[] a, byte[] b) { + int n = Math.min(a.length, b.length); + for (int i = 0; i < n; i++) { + b[i] = (byte) Math.min(15 - a[i], 127); + } + } + + /// CHECK-START-{ARM,ARM64}: void Main.satSubNConstSByte(byte[], byte[]) loop_optimization (after) + /// CHECK-DAG: <<Get1:d\d+>> VecReplicateScalar loop:none + /// CHECK-DAG: <<Get2:d\d+>> VecLoad [{{l\d+}},<<Phi:i\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Sub:d\d+>> VecSaturationSub [<<Get1>>,<<Get2>>] packed_type:Int8 loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore [{{l\d+}},<<Phi>>,<<Sub>>] loop:<<Loop>> outer_loop:none + public static void satSubNConstSByte(byte[] a, byte[] b) { + int n = Math.min(a.length, b.length); + for (int i = 0; i < n; i++) { + b[i] = (byte) Math.max(-15 - a[i], -128); + } + } + + // + // Single clipping signed 16-bit saturation. + // + + /// CHECK-START-{ARM,ARM64}: void Main.satAddPConstSShort(short[], short[]) loop_optimization (after) + /// CHECK-DAG: <<Get1:d\d+>> VecReplicateScalar loop:none + /// CHECK-DAG: <<Get2:d\d+>> VecLoad [{{l\d+}},<<Phi:i\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Add:d\d+>> VecSaturationAdd [<<Get2>>,<<Get1>>] packed_type:Int16 loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore [{{l\d+}},<<Phi>>,<<Add>>] loop:<<Loop>> outer_loop:none + public static void satAddPConstSShort(short[] a, short[] b) { + int n = Math.min(a.length, b.length); + for (int i = 0; i < n; i++) { + b[i] = (short) Math.min(a[i] + 15, 32767); + } + } + + /// CHECK-START-{ARM,ARM64}: void Main.satAddNConstSShort(short[], short[]) loop_optimization (after) + /// CHECK-DAG: <<Get1:d\d+>> VecReplicateScalar loop:none + /// CHECK-DAG: <<Get2:d\d+>> VecLoad [{{l\d+}},<<Phi:i\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Add:d\d+>> VecSaturationAdd [<<Get2>>,<<Get1>>] packed_type:Int16 loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore [{{l\d+}},<<Phi>>,<<Add>>] loop:<<Loop>> outer_loop:none + public static void satAddNConstSShort(short[] a, short[] b) { + int n = Math.min(a.length, b.length); + for (int i = 0; i < n; i++) { + b[i] = (short) Math.max(a[i] - 15, -32768); + } + } + + /// CHECK-START-{ARM,ARM64}: void Main.satSubPConstSShort(short[], short[]) loop_optimization (after) + /// CHECK-DAG: <<Get1:d\d+>> VecReplicateScalar loop:none + /// CHECK-DAG: <<Get2:d\d+>> VecLoad [{{l\d+}},<<Phi:i\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Sub:d\d+>> VecSaturationSub [<<Get1>>,<<Get2>>] packed_type:Int16 loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore [{{l\d+}},<<Phi>>,<<Sub>>] loop:<<Loop>> outer_loop:none + public static void satSubPConstSShort(short[] a, short[] b) { + int n = Math.min(a.length, b.length); + for (int i = 0; i < n; i++) { + b[i] = (short) Math.min(15 - a[i], 32767); + } + } + + /// CHECK-START-{ARM,ARM64}: void Main.satSubNConstSShort(short[], short[]) loop_optimization (after) + /// CHECK-DAG: <<Get1:d\d+>> VecReplicateScalar loop:none + /// CHECK-DAG: <<Get2:d\d+>> VecLoad [{{l\d+}},<<Phi:i\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Sub:d\d+>> VecSaturationSub [<<Get1>>,<<Get2>>] packed_type:Int16 loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore [{{l\d+}},<<Phi>>,<<Sub>>] loop:<<Loop>> outer_loop:none + public static void satSubNConstSShort(short[] a, short[] b) { + int n = Math.min(a.length, b.length); + for (int i = 0; i < n; i++) { + b[i] = (short) Math.max(-15 - a[i], -32768); + } + } + + // + // Alternatives 8-bit clipping. + // + + /// CHECK-START-{ARM,ARM64}: void Main.usatAddConst(byte[], byte[]) loop_optimization (after) + /// CHECK-DAG: <<Get1:d\d+>> VecReplicateScalar loop:none + /// CHECK-DAG: <<Get2:d\d+>> VecLoad [{{l\d+}},<<Phi:i\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Add:d\d+>> VecSaturationAdd [<<Get2>>,<<Get1>>] packed_type:Uint8 loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore [{{l\d+}},<<Phi>>,<<Add>>] loop:<<Loop>> outer_loop:none + public static void usatAddConst(byte[] a, byte[] b) { + int n = Math.min(a.length, b.length); + for (int i = 0; i < n; i++) { + b[i] = (byte) Math.min((a[i] & 0xff) + $inline$p15(), 255); + } + } + + /// CHECK-START-{ARM,ARM64}: void Main.usatAddConstAlt(byte[], byte[]) loop_optimization (after) + /// CHECK-DAG: <<Get1:d\d+>> VecReplicateScalar loop:none + /// CHECK-DAG: <<Get2:d\d+>> VecLoad [{{l\d+}},<<Phi:i\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Add:d\d+>> VecSaturationAdd [<<Get2>>,<<Get1>>] packed_type:Uint8 loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore [{{l\d+}},<<Phi>>,<<Add>>] loop:<<Loop>> outer_loop:none + public static void usatAddConstAlt(byte[] a, byte[] b) { + int n = Math.min(a.length, b.length); + for (int i = 0; i < n; i++) { + b[i] = (byte) Math.min((a[i] & 0xff) - $inline$m15(), 255); + } + } + + /// CHECK-START-{ARM,ARM64}: void Main.usatSubConst(byte[], byte[]) loop_optimization (after) + /// CHECK-DAG: <<Get1:d\d+>> VecReplicateScalar loop:none + /// CHECK-DAG: <<Get2:d\d+>> VecLoad [{{l\d+}},<<Phi:i\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Sub:d\d+>> VecSaturationSub [<<Get2>>,<<Get1>>] packed_type:Uint8 loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore [{{l\d+}},<<Phi>>,<<Sub>>] loop:<<Loop>> outer_loop:none + public static void usatSubConst(byte[] a, byte[] b) { + int n = Math.min(a.length, b.length); + for (int i = 0; i < n; i++) { + b[i] = (byte) Math.max((a[i] & 0xff) - $inline$p15(), 0); + } + } + + /// CHECK-START-{ARM,ARM64}: void Main.usatSubConstAlt(byte[], byte[]) loop_optimization (after) + /// CHECK-DAG: <<Get1:d\d+>> VecReplicateScalar loop:none + /// CHECK-DAG: <<Get2:d\d+>> VecLoad [{{l\d+}},<<Phi:i\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Sub:d\d+>> VecSaturationSub [<<Get2>>,<<Get1>>] packed_type:Uint8 loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore [{{l\d+}},<<Phi>>,<<Sub>>] loop:<<Loop>> outer_loop:none + public static void usatSubConstAlt(byte[] a, byte[] b) { + int n = Math.min(a.length, b.length); + for (int i = 0; i < n; i++) { + b[i] = (byte) Math.max((a[i] & 0xff) + $inline$m15(), 0); + } + } + + // + // Alternatives 16-bit clipping. + // + + /// CHECK-START: void Main.satAlt1(short[], short[], short[]) loop_optimization (before) + /// CHECK-DAG: <<Clp1:i\d+>> IntConstant -32768 loop:none + /// CHECK-DAG: <<Clp2:i\d+>> IntConstant 32767 loop:none + /// CHECK-DAG: <<Get1:s\d+>> ArrayGet [{{l\d+}},<<Phi:i\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Get2:s\d+>> ArrayGet [{{l\d+}},<<Phi>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Add:i\d+>> Add [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Min:i\d+>> Min [<<Add>>,<<Clp2>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Max:i\d+>> Max [<<Min>>,<<Clp1>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Conv:s\d+>> TypeConversion [<<Max>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet [{{l\d+}},<<Phi>>,<<Conv>>] loop:<<Loop>> outer_loop:none + // + /// CHECK-START-{ARM,ARM64}: void Main.satAlt1(short[], short[], short[]) loop_optimization (after) + /// CHECK-DAG: <<Get1:d\d+>> VecLoad [{{l\d+}},<<Phi:i\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Get2:d\d+>> VecLoad [{{l\d+}},<<Phi>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Add:d\d+>> VecSaturationAdd [<<Get1>>,<<Get2>>] packed_type:Int16 loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore [{{l\d+}},<<Phi>>,<<Add>>] loop:<<Loop>> outer_loop:none + public static void satAlt1(short[] a, short[] b, short[] c) { + int n = Math.min(a.length, Math.min(b.length, c.length)); + for (int i = 0; i < n; i++) { + int s = a[i] + b[i]; + if (s > 32767) { + s = 32767; + } + if (s < -32768) { + s = -32768; + } + c[i] = (short) s; + } + } + + /// CHECK-START: void Main.satAlt2(short[], short[], short[]) loop_optimization (before) + /// CHECK-DAG: <<Clp1:i\d+>> IntConstant -32768 loop:none + /// CHECK-DAG: <<Clp2:i\d+>> IntConstant 32767 loop:none + /// CHECK-DAG: <<Get1:s\d+>> ArrayGet [{{l\d+}},<<Phi:i\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Get2:s\d+>> ArrayGet [{{l\d+}},<<Phi>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Add:i\d+>> Add [<<Get1>>,<<Get2>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Max:i\d+>> Max [<<Add>>,<<Clp1>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Min:i\d+>> Min [<<Max>>,<<Clp2>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Conv:s\d+>> TypeConversion [<<Min>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: ArraySet [{{l\d+}},<<Phi>>,<<Conv>>] loop:<<Loop>> outer_loop:none + // + /// CHECK-START-{ARM,ARM64}: void Main.satAlt2(short[], short[], short[]) loop_optimization (after) + /// CHECK-DAG: <<Get1:d\d+>> VecLoad [{{l\d+}},<<Phi:i\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Get2:d\d+>> VecLoad [{{l\d+}},<<Phi>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Add:d\d+>> VecSaturationAdd [<<Get1>>,<<Get2>>] packed_type:Int16 loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore [{{l\d+}},<<Phi>>,<<Add>>] loop:<<Loop>> outer_loop:none + public static void satAlt2(short[] a, short[] b, short[] c) { + int n = Math.min(a.length, Math.min(b.length, c.length)); + for (int i = 0; i < n; i++) { + int s = a[i] + b[i]; + if (s > 32767) { + s = 32767; + } else if (s < -32768) { + s = -32768; + } + c[i] = (short) s; + } + } + + /// CHECK-START-{ARM,ARM64}: void Main.satAlt3(short[], short[], short[]) loop_optimization (after) + /// CHECK-DAG: <<Get1:d\d+>> VecLoad [{{l\d+}},<<Phi:i\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Get2:d\d+>> VecLoad [{{l\d+}},<<Phi>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Add:d\d+>> VecSaturationAdd [<<Get1>>,<<Get2>>] packed_type:Int16 loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore [{{l\d+}},<<Phi>>,<<Add>>] loop:<<Loop>> outer_loop:none + public static void satAlt3(short[] a, short[] b, short[] c) { + int n = Math.min(a.length, Math.min(b.length, c.length)); + for (int i = 0; i < n; i++) { + int s = a[i] + b[i]; + s = (s > 32767) ? 32767 : ((s < -32768) ? -32768 : s); + c[i] = (short) s; + } + } + + /// CHECK-START-{ARM,ARM64}: void Main.usatAlt1(short[], short[], short[]) loop_optimization (after) + /// CHECK-DAG: <<Get1:d\d+>> VecLoad [{{l\d+}},<<Phi:i\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Get2:d\d+>> VecLoad [{{l\d+}},<<Phi>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Add:d\d+>> VecSaturationAdd [<<Get1>>,<<Get2>>] packed_type:Uint16 loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore [{{l\d+}},<<Phi>>,<<Add>>] loop:<<Loop>> outer_loop:none + public static void usatAlt1(short[] a, short[] b, short[] c) { + int n = Math.min(a.length, Math.min(b.length, c.length)); + for (int i = 0; i < n; i++) { + int t = (0xffff & a[i]) + (0xffff & b[i]); + c[i] = (short) (t <= 65535 ? t : 65535); + } + } + + /// CHECK-START-{ARM,ARM64}: void Main.usatAlt2(short[], short[], short[]) loop_optimization (after) + /// CHECK-DAG: <<Get1:d\d+>> VecLoad [{{l\d+}},<<Phi:i\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Get2:d\d+>> VecLoad [{{l\d+}},<<Phi>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Add:d\d+>> VecSaturationAdd [<<Get1>>,<<Get2>>] packed_type:Uint16 loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore [{{l\d+}},<<Phi>>,<<Add>>] loop:<<Loop>> outer_loop:none + public static void usatAlt2(short[] a, short[] b, short[] c) { + int n = Math.min(a.length, Math.min(b.length, c.length)); + for (int i = 0; i < n; i++) { + int t = (a[i] & 0xffff) + (b[i] & 0xffff); + c[i] = (short) (t < 65535 ? t : 65535); + } + } + + /// CHECK-START-{ARM,ARM64}: void Main.usatAlt3(short[], short[], short[]) loop_optimization (after) + /// CHECK-DAG: <<Get1:d\d+>> VecLoad [{{l\d+}},<<Phi:i\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Get2:d\d+>> VecLoad [{{l\d+}},<<Phi>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Add:d\d+>> VecSaturationAdd [<<Get2>>,<<Get1>>] packed_type:Uint16 loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore [{{l\d+}},<<Phi>>,<<Add>>] loop:<<Loop>> outer_loop:none + public static void usatAlt3(short[] a, short[] b, short[] c) { + int n = Math.min(a.length, Math.min(b.length, c.length)); + for (int i = 0; i < n; i++) { + int x = (a[i] & 0xffff); + int y = (b[i] & 0xffff); + int t = y + x ; + if (t >= 65535) t = 65535; + c[i] = (short) t; + } + } + + /// CHECK-START-{ARM,ARM64}: void Main.usatAlt4(short[], short[], short[]) loop_optimization (after) + /// CHECK-DAG: <<Get1:d\d+>> VecLoad [{{l\d+}},<<Phi:i\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Get2:d\d+>> VecLoad [{{l\d+}},<<Phi>>] loop:<<Loop>> outer_loop:none + /// CHECK-DAG: <<Add:d\d+>> VecSaturationAdd [<<Get2>>,<<Get1>>] packed_type:Uint16 loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore [{{l\d+}},<<Phi>>,<<Add>>] loop:<<Loop>> outer_loop:none + public static void usatAlt4(short[] a, short[] b, short[] c) { + int n = Math.min(a.length, Math.min(b.length, c.length)); + for (int i = 0; i < n; i++) { + int x = (a[i] & 0xffff); + int y = (b[i] & 0xffff); + int t = y + x ; + if (t > 65535) t = 65535; + c[i] = (short) t; + } + } + + /// CHECK-START-{ARM,ARM64}: void Main.satRedundantClip(short[], short[]) loop_optimization (after) + /// CHECK-DAG: <<Get1:d\d+>> VecReplicateScalar loop:none + /// CHECK-DAG: <<Get2:d\d+>> VecLoad [{{l\d+}},<<Phi:i\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Add:d\d+>> VecSaturationAdd [<<Get2>>,<<Get1>>] packed_type:Int16 loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore [{{l\d+}},<<Phi>>,<<Add>>] loop:<<Loop>> outer_loop:none + public static void satRedundantClip(short[] a, short[] b) { + int n = Math.min(a.length, b.length); + for (int i = 0; i < n; i++) { + // Max clipping redundant. + b[i] = (short) Math.max(Math.min(a[i] + 15, 32767), -32768 + 15); + } + } + + /// CHECK-START: void Main.satNonRedundantClip(short[], short[]) loop_optimization (after) + /// CHECK-NOT: VecSaturationAdd + public static void satNonRedundantClip(short[] a, short[] b) { + int n = Math.min(a.length, b.length); + for (int i = 0; i < n; i++) { + // Max clipping not redundant (one off). + b[i] = (short) Math.max(Math.min(a[i] + 15, 32767), -32768 + 16); + } + } + + /// CHECK-START-{ARM,ARM64}: void Main.usatSubConst(short[], short[]) loop_optimization (after) + /// CHECK-DAG: <<Get1:d\d+>> VecReplicateScalar loop:none + /// CHECK-DAG: <<Get2:d\d+>> VecLoad [{{l\d+}},<<Phi:i\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Sub:d\d+>> VecSaturationSub [<<Get2>>,<<Get1>>] packed_type:Uint16 loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore [{{l\d+}},<<Phi>>,<<Sub>>] loop:<<Loop>> outer_loop:none + public static void usatSubConst(short[] a, short[] b) { + int n = Math.min(a.length, b.length); + for (int i = 0; i < n; i++) { + int t = a[i] & 0xffff; + int s = t - $inline$p15(); + b[i] = (short)(s > 0 ? s : 0); + } + } + + /// CHECK-START-{ARM,ARM64}: void Main.usatSubConstAlt(short[], short[]) loop_optimization (after) + /// CHECK-DAG: <<Get1:d\d+>> VecReplicateScalar loop:none + /// CHECK-DAG: <<Get2:d\d+>> VecLoad [{{l\d+}},<<Phi:i\d+>>] loop:<<Loop:B\d+>> outer_loop:none + /// CHECK-DAG: <<Sub:d\d+>> VecSaturationSub [<<Get2>>,<<Get1>>] packed_type:Uint16 loop:<<Loop>> outer_loop:none + /// CHECK-DAG: VecStore [{{l\d+}},<<Phi>>,<<Sub>>] loop:<<Loop>> outer_loop:none + public static void usatSubConstAlt(short[] a, short[] b) { + int n = Math.min(a.length, b.length); + for (int i = 0; i < n; i++) { + int t = a[i] & 0xffff; + int s = t + $inline$m15(); + b[i] = (short)(s > 0 ? s : 0); + } + } + + // + // Test drivers. + // + + private static void test08Bit() { + // Use cross-values to test all cases. + int n = 256; + int m = n * n; + int k = 0; + byte[] b1 = new byte[m]; + byte[] b2 = new byte[m]; + for (int i = 0; i < n; i++) { + for (int j = 0; j < n; j++) { + b1[k] = (byte) i; + b2[k] = (byte) j; + k++; + } + } + // Tests. + byte[] out = new byte[m]; + satAddUByte(b1, b2, out); + for (int i = 0; i < m; i++) { + byte e = (byte) Math.min((b1[i] & 0xff) + (b2[i] & 0xff), 255); + expectEquals(e, out[i]); + } + satAddSByte( b1, b2, out); + for (int i = 0; i < m; i++) { + byte e = (byte) Math.max(Math.min(b1[i] + b2[i], 127), -128); + expectEquals(e, out[i]); + } + satSubUByte(b1, b2, out); + for (int i = 0; i < m; i++) { + byte e = (byte) Math.max((b1[i] & 0xff) - (b2[i] & 0xff), 0); + expectEquals(e, out[i]); + } + satSubSByte(b1, b2, out); + for (int i = 0; i < m; i++) { + byte e = (byte) Math.max(Math.min(b1[i] - b2[i], 127), -128); + expectEquals(e, out[i]); + } + // Single clipping. + satAddPConstSByte(b1, out); + for (int i = 0; i < m; i++) { + byte e = (byte) Math.min(b1[i] + 15, 127); + expectEquals(e, out[i]); + } + satAddNConstSByte(b1, out); + for (int i = 0; i < m; i++) { + byte e = (byte) Math.max(b1[i] - 15, -128); + expectEquals(e, out[i]); + } + satSubPConstSByte(b1, out); + for (int i = 0; i < m; i++) { + byte e = (byte) Math.min(15 - b1[i], 127); + expectEquals(e, out[i]); + } + satSubNConstSByte(b1, out); + for (int i = 0; i < m; i++) { + byte e = (byte) Math.max(-15 - b1[i], -128); + expectEquals(e, out[i]); + } + // Alternatives. + usatAddConst(b1, out); + for (int i = 0; i < m; i++) { + byte e = (byte) Math.min((b1[i] & 0xff) + 15, 255); + expectEquals(e, out[i]); + } + usatAddConstAlt(b1, out); + for (int i = 0; i < m; i++) { + byte e = (byte) Math.min((b1[i] & 0xff) + 15, 255); + expectEquals(e, out[i]); + } + usatSubConst(b1, out); + for (int i = 0; i < m; i++) { + byte e = (byte) Math.max((b1[i] & 0xff) - 15, 0); + expectEquals(e, out[i]); + } + usatSubConstAlt(b1, out); + for (int i = 0; i < m; i++) { + byte e = (byte) Math.max((b1[i] & 0xff) - 15, 0); + expectEquals(e, out[i]); + } + } + + private static void test16Bit() { + // Use cross-values to test interesting cases. + short[] interesting = { + (short) 0x0000, + (short) 0x0001, + (short) 0x0002, + (short) 0x0003, + (short) 0x0004, + (short) 0x007f, + (short) 0x0080, + (short) 0x00ff, + (short) 0x7f00, + (short) 0x7f7f, + (short) 0x7f80, + (short) 0x7fff, + (short) 0x8000, + (short) 0x807f, + (short) 0x8080, + (short) 0x80ff, + (short) 0xff00, + (short) 0xff7f, + (short) 0xff80, + (short) 0xffff, + }; + int n = interesting.length; + int m = n * n; + short[] s1 = new short[m]; + short[] s2 = new short[m]; + int k = 0; + for (int i = 0; i < n; i++) { + for (int j = 0; j < n; j++) { + s1[k] = interesting[i]; + s2[k] = interesting[j]; + k++; + } + } + // Tests. + short[] out = new short[m]; + satAddUShort(s1, s2, out); + for (int i = 0; i < m; i++) { + short e = (short) Math.min((s1[i] & 0xffff) + (s2[i] & 0xffff), 65535); + expectEquals(e, out[i]); + } + satAddSShort(s1, s2, out); + for (int i = 0; i < m; i++) { + short e = (short) Math.max(Math.min(s1[i] + s2[i], 32767), -32768); + expectEquals(e, out[i]); + } + satSubUShort(s1, s2, out); + for (int i = 0; i < m; i++) { + short e = (short) Math.max((s1[i] & 0xffff) - (s2[i] & 0xffff), 0); + expectEquals(e, out[i]); + } + satSubSShort(s1, s2, out); + for (int i = 0; i < m; i++) { + short e = (short) Math.max(Math.min(s1[i] - s2[i], 32767), -32768); + expectEquals(e, out[i]); + } + // Single clipping. + satAddPConstSShort(s1, out); + for (int i = 0; i < m; i++) { + short e = (short) Math.min(s1[i] + 15, 32767); + expectEquals(e, out[i]); + } + satAddNConstSShort(s1, out); + for (int i = 0; i < m; i++) { + short e = (short) Math.max(s1[i] - 15, -32768); + expectEquals(e, out[i]); + } + satSubPConstSShort(s1, out); + for (int i = 0; i < m; i++) { + short e = (short) Math.min(15 - s1[i], 32767); + expectEquals(e, out[i]); + } + satSubNConstSShort(s1, out); + for (int i = 0; i < m; i++) { + short e = (short) Math.max(-15 - s1[i], -32768); + expectEquals(e, out[i]); + } + // Alternatives. + satAlt1(s1, s2, out); + for (int i = 0; i < m; i++) { + short e = (short) Math.max(Math.min(s1[i] + s2[i], 32767), -32768); + expectEquals(e, out[i]); + } + satAlt2(s1, s2, out); + for (int i = 0; i < m; i++) { + short e = (short) Math.max(Math.min(s1[i] + s2[i], 32767), -32768); + expectEquals(e, out[i]); + } + satAlt3(s1, s2, out); + for (int i = 0; i < m; i++) { + short e = (short) Math.max(Math.min(s1[i] + s2[i], 32767), -32768); + expectEquals(e, out[i]); + } + usatAlt1(s1, s2, out); + for (int i = 0; i < m; i++) { + short e = (short) Math.min((s1[i] & 0xffff) + (s2[i] & 0xffff), 65535); + expectEquals(e, out[i]); + } + usatAlt2(s1, s2, out); + for (int i = 0; i < m; i++) { + short e = (short) Math.min((s1[i] & 0xffff) + (s2[i] & 0xffff), 65535); + expectEquals(e, out[i]); + } + usatAlt3(s1, s2, out); + for (int i = 0; i < m; i++) { + short e = (short) Math.min((s1[i] & 0xffff) + (s2[i] & 0xffff), 65535); + expectEquals(e, out[i]); + } + usatAlt4(s1, s2, out); + for (int i = 0; i < m; i++) { + short e = (short) Math.min((s1[i] & 0xffff) + (s2[i] & 0xffff), 65535); + expectEquals(e, out[i]); + } + satRedundantClip(s1, out); + for (int i = 0; i < m; i++) { + short e = (short) Math.min(s1[i] + 15, 32767); + expectEquals(e, out[i]); + } + satNonRedundantClip(s1, out); + for (int i = 0; i < m; i++) { + short e = (short) Math.max(Math.min(s1[i] + 15, 32767), -32752); + expectEquals(e, out[i]); + } + usatSubConst(s1, out); + for (int i = 0; i < m; i++) { + short e = (short) Math.max((s1[i] & 0xffff) - 15, 0); + expectEquals(e, out[i]); + } + usatSubConstAlt(s1, out); + for (int i = 0; i < m; i++) { + short e = (short) Math.max((s1[i] & 0xffff) - 15, 0); + expectEquals(e, out[i]); + } + } + + public static void main(String[] args) { + test08Bit(); + test16Bit(); + System.out.println("passed"); + } + + private static void expectEquals(int expected, int result) { + if (expected != result) { + throw new Error("Expected: " + expected + ", found: " + result); + } + } +} diff --git a/test/651-checker-char-simd-minmax/expected.txt b/test/679-checker-minmax/expected.txt index b0aad4deb5b..b0aad4deb5b 100644 --- a/test/651-checker-char-simd-minmax/expected.txt +++ b/test/679-checker-minmax/expected.txt diff --git a/test/679-checker-minmax/info.txt b/test/679-checker-minmax/info.txt new file mode 100644 index 00000000000..4f7b9f52c5f --- /dev/null +++ b/test/679-checker-minmax/info.txt @@ -0,0 +1 @@ +Functional tests on detecting min/max. diff --git a/test/679-checker-minmax/src/Main.java b/test/679-checker-minmax/src/Main.java new file mode 100644 index 00000000000..48de1da2913 --- /dev/null +++ b/test/679-checker-minmax/src/Main.java @@ -0,0 +1,597 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Functional tests for detecting min/max. + */ +public class Main { + + // + // Direct intrinsics. + // + + /// CHECK-START: int Main.minI(int) instruction_simplifier (before) + /// CHECK-DAG: <<Par:i\d+>> ParameterValue + /// CHECK-DAG: <<Con:i\d+>> IntConstant 20 + /// CHECK-DAG: <<Min:i\d+>> InvokeStaticOrDirect [<<Par>>,<<Con>>] intrinsic:MathMinIntInt + /// CHECK-DAG: Return [<<Min>>] + // + /// CHECK-START: int Main.minI(int) instruction_simplifier (after) + /// CHECK-DAG: <<Par:i\d+>> ParameterValue + /// CHECK-DAG: <<Con:i\d+>> IntConstant 20 + /// CHECK-DAG: <<Min:i\d+>> Min [<<Par>>,<<Con>>] + /// CHECK-DAG: Return [<<Min>>] + // + /// CHECK-START: int Main.minI(int) instruction_simplifier (after) + /// CHECK-NOT: InvokeStaticOrDirect + public static int minI(int a) { + return Math.min(a, 20); + } + + /// CHECK-START: long Main.minL(long) instruction_simplifier (before) + /// CHECK-DAG: <<Par:j\d+>> ParameterValue + /// CHECK-DAG: <<Con:j\d+>> LongConstant 20 + /// CHECK-DAG: <<Min:j\d+>> InvokeStaticOrDirect [<<Par>>,<<Con>>] intrinsic:MathMinLongLong + /// CHECK-DAG: Return [<<Min>>] + // + /// CHECK-START: long Main.minL(long) instruction_simplifier (after) + /// CHECK-DAG: <<Par:j\d+>> ParameterValue + /// CHECK-DAG: <<Con:j\d+>> LongConstant 20 + /// CHECK-DAG: <<Min:j\d+>> Min [<<Par>>,<<Con>>] + /// CHECK-DAG: Return [<<Min>>] + // + /// CHECK-START: long Main.minL(long) instruction_simplifier (after) + /// CHECK-NOT: InvokeStaticOrDirect + public static long minL(long a) { + return Math.min(a, 20L); + } + + /// CHECK-START: int Main.maxI(int) instruction_simplifier (before) + /// CHECK-DAG: <<Par:i\d+>> ParameterValue + /// CHECK-DAG: <<Con:i\d+>> IntConstant 20 + /// CHECK-DAG: <<Max:i\d+>> InvokeStaticOrDirect [<<Par>>,<<Con>>] intrinsic:MathMaxIntInt + /// CHECK-DAG: Return [<<Max>>] + // + /// CHECK-START: int Main.maxI(int) instruction_simplifier (after) + /// CHECK-DAG: <<Par:i\d+>> ParameterValue + /// CHECK-DAG: <<Con:i\d+>> IntConstant 20 + /// CHECK-DAG: <<Max:i\d+>> Max [<<Par>>,<<Con>>] + /// CHECK-DAG: Return [<<Max>>] + // + /// CHECK-START: int Main.maxI(int) instruction_simplifier (after) + /// CHECK-NOT: InvokeStaticOrDirect + public static int maxI(int a) { + return Math.max(a, 20); + } + + /// CHECK-START: long Main.maxL(long) instruction_simplifier (before) + /// CHECK-DAG: <<Par:j\d+>> ParameterValue + /// CHECK-DAG: <<Con:j\d+>> LongConstant 20 + /// CHECK-DAG: <<Max:j\d+>> InvokeStaticOrDirect [<<Par>>,<<Con>>] intrinsic:MathMaxLongLong + /// CHECK-DAG: Return [<<Max>>] + // + /// CHECK-START: long Main.maxL(long) instruction_simplifier (after) + /// CHECK-DAG: <<Par:j\d+>> ParameterValue + /// CHECK-DAG: <<Con:j\d+>> LongConstant 20 + /// CHECK-DAG: <<Max:j\d+>> Max [<<Par>>,<<Con>>] + /// CHECK-DAG: Return [<<Max>>] + // + /// CHECK-START: long Main.maxL(long) instruction_simplifier (after) + /// CHECK-NOT: InvokeStaticOrDirect + public static long maxL(long a) { + return Math.max(a, 20L); + } + + // + // Different types. + // + + /// CHECK-START: int Main.min1(int, int) instruction_simplifier$after_inlining (before) + /// CHECK-DAG: <<Cnd:z\d+>> GreaterThanOrEqual [<<Op1:i\d+>>,<<Op2:i\d+>>] + /// CHECK-DAG: <<Sel:i\d+>> Select [<<Op1>>,<<Op2>>,<<Cnd>>] + /// CHECK-DAG: Return [<<Sel>>] + // + /// CHECK-START: int Main.min1(int, int) instruction_simplifier$after_inlining (after) + /// CHECK-DAG: <<Min:i\d+>> Min + /// CHECK-DAG: Return [<<Min>>] + // + /// CHECK-START: int Main.min1(int, int) instruction_simplifier$after_inlining (after) + /// CHECK-NOT: Select + public static int min1(int a, int b) { + return a < b ? a : b; + } + + /// CHECK-START: int Main.min2(int, int) instruction_simplifier$after_inlining (before) + /// CHECK-DAG: <<Cnd:z\d+>> GreaterThan [<<Op1:i\d+>>,<<Op2:i\d+>>] + /// CHECK-DAG: <<Sel:i\d+>> Select [<<Op1>>,<<Op2>>,<<Cnd>>] + /// CHECK-DAG: Return [<<Sel>>] + // + /// CHECK-START: int Main.min2(int, int) instruction_simplifier$after_inlining (after) + /// CHECK-DAG: <<Min:i\d+>> Min + /// CHECK-DAG: Return [<<Min>>] + // + /// CHECK-START: int Main.min2(int, int) instruction_simplifier$after_inlining (after) + /// CHECK-NOT: Select + public static int min2(int a, int b) { + return a <= b ? a : b; + } + + /// CHECK-START: int Main.min3(int, int) instruction_simplifier$after_inlining (before) + /// CHECK-DAG: <<Cnd:z\d+>> LessThanOrEqual [<<Op1:i\d+>>,<<Op2:i\d+>>] + /// CHECK-DAG: <<Sel:i\d+>> Select [<<Op2>>,<<Op1>>,<<Cnd>>] + /// CHECK-DAG: Return [<<Sel>>] + // + /// CHECK-START: int Main.min3(int, int) instruction_simplifier$after_inlining (after) + /// CHECK-DAG: <<Min:i\d+>> Min + /// CHECK-DAG: Return [<<Min>>] + // + /// CHECK-START: int Main.min3(int, int) instruction_simplifier$after_inlining (after) + /// CHECK-NOT: Select + public static int min3(int a, int b) { + return a > b ? b : a; + } + + /// CHECK-START: int Main.min4(int, int) instruction_simplifier$after_inlining (before) + /// CHECK-DAG: <<Cnd:z\d+>> LessThan [<<Op1:i\d+>>,<<Op2:i\d+>>] + /// CHECK-DAG: <<Sel:i\d+>> Select [<<Op2>>,<<Op1>>,<<Cnd>>] + /// CHECK-DAG: Return [<<Sel>>] + // + /// CHECK-START: int Main.min4(int, int) instruction_simplifier$after_inlining (after) + /// CHECK-DAG: <<Min:i\d+>> Min + /// CHECK-DAG: Return [<<Min>>] + // + /// CHECK-START: int Main.min4(int, int) instruction_simplifier$after_inlining (after) + /// CHECK-NOT: Select + public static int min4(int a, int b) { + return a >= b ? b : a; + } + + /// CHECK-START: int Main.min5(short, short) instruction_simplifier$after_inlining (before) + /// CHECK-DAG: <<Cnd:z\d+>> LessThan [<<Op1:s\d+>>,<<Op2:s\d+>>] + /// CHECK-DAG: <<Sel:i\d+>> Select [<<Op2>>,<<Op1>>,<<Cnd>>] + /// CHECK-DAG: Return [<<Sel>>] + // + /// CHECK-START: int Main.min5(short, short) instruction_simplifier$after_inlining (after) + /// CHECK-DAG: <<Min:i\d+>> Min + /// CHECK-DAG: Return [<<Min>>] + // + /// CHECK-START: int Main.min5(short, short) instruction_simplifier$after_inlining (after) + /// CHECK-NOT: Select + public static int min5(short a, short b) { + return a >= b ? b : a; + } + + /// CHECK-START: int Main.min6(byte, byte) instruction_simplifier$after_inlining (before) + /// CHECK-DAG: <<Cnd:z\d+>> LessThan [<<Op1:b\d+>>,<<Op2:b\d+>>] + /// CHECK-DAG: <<Sel:i\d+>> Select [<<Op2>>,<<Op1>>,<<Cnd>>] + /// CHECK-DAG: Return [<<Sel>>] + // + /// CHECK-START: int Main.min6(byte, byte) instruction_simplifier$after_inlining (after) + /// CHECK-DAG: <<Min:i\d+>> Min + /// CHECK-DAG: Return [<<Min>>] + // + /// CHECK-START: int Main.min6(byte, byte) instruction_simplifier$after_inlining (after) + /// CHECK-NOT: Select + public static int min6(byte a, byte b) { + return a >= b ? b : a; + } + + /// CHECK-START: long Main.min7(long, long) instruction_simplifier$after_inlining (before) + /// CHECK-DAG: <<Cnd:z\d+>> LessThan [<<Op1:j\d+>>,<<Op2:j\d+>>] + /// CHECK-DAG: <<Sel:j\d+>> Select [<<Op2>>,<<Op1>>,<<Cnd>>] + /// CHECK-DAG: Return [<<Sel>>] + // + /// CHECK-START: long Main.min7(long, long) instruction_simplifier$after_inlining (after) + /// CHECK-DAG: <<Min:j\d+>> Min + /// CHECK-DAG: Return [<<Min>>] + // + /// CHECK-START: long Main.min7(long, long) instruction_simplifier$after_inlining (after) + /// CHECK-NOT: Select + public static long min7(long a, long b) { + return a >= b ? b : a; + } + + /// CHECK-START: int Main.max1(int, int) instruction_simplifier$after_inlining (before) + /// CHECK-DAG: <<Cnd:z\d+>> GreaterThanOrEqual [<<Op1:i\d+>>,<<Op2:i\d+>>] + /// CHECK-DAG: <<Sel:i\d+>> Select [<<Op2>>,<<Op1>>,<<Cnd>>] + /// CHECK-DAG: Return [<<Sel>>] + // + /// CHECK-START: int Main.max1(int, int) instruction_simplifier$after_inlining (after) + /// CHECK-DAG: <<Max:i\d+>> Max + /// CHECK-DAG: Return [<<Max>>] + // + /// CHECK-START: int Main.max1(int, int) instruction_simplifier$after_inlining (after) + /// CHECK-NOT: Select + public static int max1(int a, int b) { + return a < b ? b : a; + } + + /// CHECK-START: int Main.max2(int, int) instruction_simplifier$after_inlining (before) + /// CHECK-DAG: <<Cnd:z\d+>> GreaterThan [<<Op1:i\d+>>,<<Op2:i\d+>>] + /// CHECK-DAG: <<Sel:i\d+>> Select [<<Op2>>,<<Op1>>,<<Cnd>>] + /// CHECK-DAG: Return [<<Sel>>] + // + /// CHECK-START: int Main.max2(int, int) instruction_simplifier$after_inlining (after) + /// CHECK-DAG: <<Max:i\d+>> Max + /// CHECK-DAG: Return [<<Max>>] + // + /// CHECK-START: int Main.max2(int, int) instruction_simplifier$after_inlining (after) + /// CHECK-NOT: Select + public static int max2(int a, int b) { + return a <= b ? b : a; + } + + /// CHECK-START: int Main.max3(int, int) instruction_simplifier$after_inlining (before) + /// CHECK-DAG: <<Cnd:z\d+>> LessThanOrEqual [<<Op1:i\d+>>,<<Op2:i\d+>>] + /// CHECK-DAG: <<Sel:i\d+>> Select [<<Op1>>,<<Op2>>,<<Cnd>>] + /// CHECK-DAG: Return [<<Sel>>] + // + /// CHECK-START: int Main.max3(int, int) instruction_simplifier$after_inlining (after) + /// CHECK-DAG: <<Max:i\d+>> Max + /// CHECK-DAG: Return [<<Max>>] + // + /// CHECK-START: int Main.max3(int, int) instruction_simplifier$after_inlining (after) + /// CHECK-NOT: Select + public static int max3(int a, int b) { + return a > b ? a : b; + } + + /// CHECK-START: int Main.max4(int, int) instruction_simplifier$after_inlining (before) + /// CHECK-DAG: <<Cnd:z\d+>> LessThan [<<Op1:i\d+>>,<<Op2:i\d+>>] + /// CHECK-DAG: <<Sel:i\d+>> Select [<<Op1>>,<<Op2>>,<<Cnd>>] + /// CHECK-DAG: Return [<<Sel>>] + // + /// CHECK-START: int Main.max4(int, int) instruction_simplifier$after_inlining (after) + /// CHECK-DAG: <<Max:i\d+>> Max + /// CHECK-DAG: Return [<<Max>>] + // + /// CHECK-START: int Main.max4(int, int) instruction_simplifier$after_inlining (after) + /// CHECK-NOT: Select + public static int max4(int a, int b) { + return a >= b ? a : b; + } + + /// CHECK-START: int Main.max5(short, short) instruction_simplifier$after_inlining (before) + /// CHECK-DAG: <<Cnd:z\d+>> LessThan [<<Op1:s\d+>>,<<Op2:s\d+>>] + /// CHECK-DAG: <<Sel:i\d+>> Select [<<Op1>>,<<Op2>>,<<Cnd>>] + /// CHECK-DAG: Return [<<Sel>>] + // + /// CHECK-START: int Main.max5(short, short) instruction_simplifier$after_inlining (after) + /// CHECK-DAG: <<Max:i\d+>> Max + /// CHECK-DAG: Return [<<Max>>] + // + /// CHECK-START: int Main.max5(short, short) instruction_simplifier$after_inlining (after) + /// CHECK-NOT: Select + public static int max5(short a, short b) { + return a >= b ? a : b; + } + + /// CHECK-START: int Main.max6(byte, byte) instruction_simplifier$after_inlining (before) + /// CHECK-DAG: <<Cnd:z\d+>> LessThan [<<Op1:b\d+>>,<<Op2:b\d+>>] + /// CHECK-DAG: <<Sel:i\d+>> Select [<<Op1>>,<<Op2>>,<<Cnd>>] + /// CHECK-DAG: Return [<<Sel>>] + // + /// CHECK-START: int Main.max6(byte, byte) instruction_simplifier$after_inlining (after) + /// CHECK-DAG: <<Max:i\d+>> Max + /// CHECK-DAG: Return [<<Max>>] + // + /// CHECK-START: int Main.max6(byte, byte) instruction_simplifier$after_inlining (after) + /// CHECK-NOT: Select + public static int max6(byte a, byte b) { + return a >= b ? a : b; + } + + /// CHECK-START: long Main.max7(long, long) instruction_simplifier$after_inlining (before) + /// CHECK-DAG: <<Cnd:z\d+>> LessThan [<<Op1:j\d+>>,<<Op2:j\d+>>] + /// CHECK-DAG: <<Sel:j\d+>> Select [<<Op1>>,<<Op2>>,<<Cnd>>] + /// CHECK-DAG: Return [<<Sel>>] + // + /// CHECK-START: long Main.max7(long, long) instruction_simplifier$after_inlining (after) + /// CHECK-DAG: <<Max:j\d+>> Max + /// CHECK-DAG: Return [<<Max>>] + // + /// CHECK-START: long Main.max7(long, long) instruction_simplifier$after_inlining (after) + /// CHECK-NOT: Select + public static long max7(long a, long b) { + return a >= b ? a : b; + } + + // + // Complications. + // + + /// CHECK-START: int Main.min0(int[], int[]) instruction_simplifier$after_inlining (before) + /// CHECK-DAG: <<Ar1:i\d+>> ArrayGet [{{l\d+}},{{i\d+}}] + /// CHECK-DAG: <<Ar2:i\d+>> ArrayGet [{{l\d+}},{{i\d+}}] + /// CHECK-DAG: <<Cnd:z\d+>> GreaterThan [<<Ar1>>,<<Ar2>>] + /// CHECK-DAG: <<Sel:i\d+>> Select [<<Ar1>>,<<Ar2>>,<<Cnd>>] + /// CHECK-DAG: Return [<<Sel>>] + // + /// CHECK-START: int Main.min0(int[], int[]) instruction_simplifier$after_inlining (after) + /// CHECK-DAG: <<Min:i\d+>> Min + /// CHECK-DAG: Return [<<Min>>] + // + /// CHECK-START: int Main.min0(int[], int[]) instruction_simplifier$after_inlining (after) + /// CHECK-NOT: Select + public static int min0(int[] a, int[] b) { + // Repeat of array references needs finding the common subexpressions + // prior to doing the select and min/max recognition. + return a[0] <= b[0] ? a[0] : b[0]; + } + + /// CHECK-START: int Main.max0(int[], int[]) instruction_simplifier$after_inlining (before) + /// CHECK-DAG: <<Ar1:i\d+>> ArrayGet [{{l\d+}},{{i\d+}}] + /// CHECK-DAG: <<Ar2:i\d+>> ArrayGet [{{l\d+}},{{i\d+}}] + /// CHECK-DAG: <<Cnd:z\d+>> LessThan [<<Ar1>>,<<Ar2>>] + /// CHECK-DAG: <<Sel:i\d+>> Select [<<Ar1>>,<<Ar2>>,<<Cnd>>] + /// CHECK-DAG: Return [<<Sel>>] + // + /// CHECK-START: int Main.max0(int[], int[]) instruction_simplifier$after_inlining (after) + /// CHECK-DAG: <<Max:i\d+>> Max + /// CHECK-DAG: Return [<<Max>>] + // + /// CHECK-START: int Main.max0(int[], int[]) instruction_simplifier$after_inlining (after) + /// CHECK-NOT: Select + public static int max0(int[] a, int[] b) { + // Repeat of array references needs finding the common subexpressions + // prior to doing the select and min/max recognition. + return a[0] >= b[0] ? a[0] : b[0]; + } + + /// CHECK-START: int Main.minmax1(int) instruction_simplifier$after_inlining (before) + /// CHECK-DAG: <<Par:i\d+>> ParameterValue + /// CHECK-DAG: <<P100:i\d+>> IntConstant 100 + /// CHECK-DAG: <<M100:i\d+>> IntConstant -100 + /// CHECK-DAG: <<Cnd1:z\d+>> LessThanOrEqual [<<Par>>,<<P100>>] + /// CHECK-DAG: <<Sel1:i\d+>> Select [<<P100>>,<<Par>>,<<Cnd1>>] + /// CHECK-DAG: <<Cnd2:z\d+>> GreaterThanOrEqual [<<Sel1>>,<<M100>>] + /// CHECK-DAG: <<Sel2:i\d+>> Select [<<M100>>,<<Sel1>>,<<Cnd2>>] + /// CHECK-DAG: Return [<<Sel2>>] + // + /// CHECK-START: int Main.minmax1(int) instruction_simplifier$after_inlining (after) + /// CHECK-DAG: <<Par:i\d+>> ParameterValue + /// CHECK-DAG: <<P100:i\d+>> IntConstant 100 + /// CHECK-DAG: <<M100:i\d+>> IntConstant -100 + /// CHECK-DAG: <<Min:i\d+>> Min [<<Par>>,<<P100>>] + /// CHECK-DAG: <<Max:i\d+>> Max [<<Min>>,<<M100>>] + /// CHECK-DAG: Return [<<Max>>] + // + /// CHECK-START: int Main.minmax1(int) instruction_simplifier$after_inlining (after) + /// CHECK-NOT: Select + public static int minmax1(int x) { + // Simple if-if gives clean select sequence. + if (x > 100) { + x = 100; + } + if (x < -100) { + x = -100; + } + return x; + } + + /// CHECK-START: int Main.minmax2(int) instruction_simplifier$after_inlining (before) + /// CHECK-DAG: <<Par:i\d+>> ParameterValue + /// CHECK-DAG: <<P100:i\d+>> IntConstant 100 + /// CHECK-DAG: <<M100:i\d+>> IntConstant -100 + /// CHECK-DAG: <<Cnd1:z\d+>> LessThanOrEqual [<<Par>>,<<P100>>] + /// CHECK-DAG: <<Cnd2:z\d+>> GreaterThanOrEqual [<<Par>>,<<M100>>] + /// CHECK-DAG: <<Sel1:i\d+>> Select [<<M100>>,<<Par>>,<<Cnd2>>] + /// CHECK-DAG: <<Sel2:i\d+>> Select [<<P100>>,<<Sel1>>,<<Cnd1>>] + /// CHECK-DAG: Return [<<Sel2>>] + // + /// CHECK-START: int Main.minmax2(int) instruction_simplifier$after_inlining (after) + /// CHECK-DAG: <<Par:i\d+>> ParameterValue + /// CHECK-DAG: <<P100:i\d+>> IntConstant 100 + /// CHECK-DAG: <<M100:i\d+>> IntConstant -100 + /// CHECK-DAG: <<Max:i\d+>> Max [<<Par>>,<<M100>>] + /// CHECK-DAG: <<Min:i\d+>> Min [<<Max>>,<<P100>>] + /// CHECK-DAG: Return [<<Min>>] + // + /// CHECK-START: int Main.minmax2(int) instruction_simplifier$after_inlining (after) + /// CHECK-NOT: Select + public static int minmax2(int x) { + // Simple if-else requires inspecting bounds of resulting selects. + if (x > 100) { + x = 100; + } else if (x < -100) { + x = -100; + } + return x; + } + + /// CHECK-START: int Main.minmax3(int) instruction_simplifier$after_inlining (after) + /// CHECK-DAG: <<Par:i\d+>> ParameterValue + /// CHECK-DAG: <<P100:i\d+>> IntConstant 100 + /// CHECK-DAG: <<M100:i\d+>> IntConstant -100 + /// CHECK-DAG: <<Max:i\d+>> Max [<<Par>>,<<M100>>] + /// CHECK-DAG: <<Min:i\d+>> Min [<<Max>>,<<P100>>] + /// CHECK-DAG: Return [<<Min>>] + // + /// CHECK-START: int Main.minmax3(int) instruction_simplifier$after_inlining (after) + /// CHECK-NOT: Select + public static int minmax3(int x) { + return (x > 100) ? 100 : ((x < -100) ? -100 : x); + } + + /// CHECK-START: int Main.minmax4(int) instruction_simplifier$after_inlining (after) + /// CHECK-DAG: <<Par:i\d+>> ParameterValue + /// CHECK-DAG: <<P100:i\d+>> IntConstant 100 + /// CHECK-DAG: <<M100:i\d+>> IntConstant -100 + /// CHECK-DAG: <<Min:i\d+>> Min [<<Par>>,<<P100>>] + /// CHECK-DAG: <<Max:i\d+>> Max [<<Min>>,<<M100>>] + /// CHECK-DAG: Return [<<Max>>] + // + /// CHECK-START: int Main.minmax4(int) instruction_simplifier$after_inlining (after) + /// CHECK-NOT: Select + public static int minmax4(int x) { + return (x < -100) ? -100 : ((x > 100) ? 100 : x); + } + + /// CHECK-START: int Main.minmaxCSEScalar(int, int) select_generator (after) + /// CHECK-DAG: <<Par1:i\d+>> ParameterValue + /// CHECK-DAG: <<Par2:i\d+>> ParameterValue + /// CHECK-DAG: <<Cnd1:z\d+>> LessThanOrEqual [<<Par1>>,<<Par2>>] + /// CHECK-DAG: <<Sel1:i\d+>> Select [<<Par1>>,<<Par2>>,<<Cnd1>>] + /// CHECK-DAG: <<Cnd2:z\d+>> GreaterThanOrEqual [<<Par1>>,<<Par2>>] + /// CHECK-DAG: <<Sel2:i\d+>> Select [<<Par1>>,<<Par2>>,<<Cnd2>>] + /// CHECK-DAG: <<Add1:i\d+>> Add [<<Sel1>>,<<Sel2>>] + /// CHECK-DAG: <<Add2:i\d+>> Add [<<Sel1>>,<<Add1>>] + /// CHECK-DAG: <<Add3:i\d+>> Add [<<Sel2>>,<<Add2>>] + /// CHECK-DAG: <<Add4:i\d+>> Add [<<Sel1>>,<<Add3>>] + /// CHECK-DAG: <<Add5:i\d+>> Add [<<Sel2>>,<<Add4>>] + /// CHECK-DAG: Return [<<Add5>>] + // + /// CHECK-START: int Main.minmaxCSEScalar(int, int) instruction_simplifier$after_inlining (after) + /// CHECK-DAG: <<Par1:i\d+>> ParameterValue + /// CHECK-DAG: <<Par2:i\d+>> ParameterValue + /// CHECK-DAG: <<Max:i\d+>> Max [<<Par1>>,<<Par2>>] + /// CHECK-DAG: <<Min:i\d+>> Min [<<Par1>>,<<Par2>>] + /// CHECK-DAG: <<Add1:i\d+>> Add [<<Max>>,<<Min>>] + /// CHECK-DAG: <<Add2:i\d+>> Add [<<Max>>,<<Add1>>] + /// CHECK-DAG: <<Add3:i\d+>> Add [<<Min>>,<<Add2>>] + /// CHECK-DAG: <<Add4:i\d+>> Add [<<Max>>,<<Add3>>] + /// CHECK-DAG: <<Add5:i\d+>> Add [<<Min>>,<<Add4>>] + /// CHECK-DAG: Return [<<Add5>>] + public static int minmaxCSEScalar(int x, int y) { + int t1 = (x > y) ? x : y; + int t2 = (x < y) ? x : y; + int t3 = (x > y) ? x : y; + int t4 = (x < y) ? x : y; + int t5 = (x > y) ? x : y; + int t6 = (x < y) ? x : y; + // Make sure min/max is CSEed. + return t1 + t2 + t3 + t4 + t5 + t6; + } + + /// CHECK-START: int Main.minmaxCSEArray(int[], int[]) select_generator (after) + /// CHECK-DAG: <<Arr1:i\d+>> ArrayGet + /// CHECK-DAG: <<Arr2:i\d+>> ArrayGet + /// CHECK-DAG: <<Cnd1:z\d+>> LessThanOrEqual [<<Arr1>>,<<Arr2>>] + /// CHECK-DAG: <<Sel1:i\d+>> Select [<<Arr1>>,<<Arr2>>,<<Cnd1>>] + /// CHECK-DAG: <<Cnd2:z\d+>> GreaterThanOrEqual [<<Arr1>>,<<Arr2>>] + /// CHECK-DAG: <<Sel2:i\d+>> Select [<<Arr1>>,<<Arr2>>,<<Cnd2>>] + /// CHECK-DAG: <<Add1:i\d+>> Add [<<Sel1>>,<<Sel2>>] + /// CHECK-DAG: <<Add2:i\d+>> Add [<<Sel1>>,<<Add1>>] + /// CHECK-DAG: <<Add3:i\d+>> Add [<<Sel2>>,<<Add2>>] + /// CHECK-DAG: <<Add4:i\d+>> Add [<<Sel1>>,<<Add3>>] + /// CHECK-DAG: <<Add5:i\d+>> Add [<<Sel2>>,<<Add4>>] + /// CHECK-DAG: Return [<<Add5>>] + // + /// CHECK-START: int Main.minmaxCSEArray(int[], int[]) instruction_simplifier$after_inlining (after) + /// CHECK-DAG: <<Arr1:i\d+>> ArrayGet + /// CHECK-DAG: <<Arr2:i\d+>> ArrayGet + /// CHECK-DAG: <<Max:i\d+>> Max [<<Arr1>>,<<Arr2>>] + /// CHECK-DAG: <<Min:i\d+>> Min [<<Arr1>>,<<Arr2>>] + /// CHECK-DAG: <<Add1:i\d+>> Add [<<Max>>,<<Min>>] + /// CHECK-DAG: <<Add2:i\d+>> Add [<<Max>>,<<Add1>>] + /// CHECK-DAG: <<Add3:i\d+>> Add [<<Min>>,<<Add2>>] + /// CHECK-DAG: <<Add4:i\d+>> Add [<<Max>>,<<Add3>>] + /// CHECK-DAG: <<Add5:i\d+>> Add [<<Min>>,<<Add4>>] + /// CHECK-DAG: Return [<<Add5>>] + public static int minmaxCSEArray(int[] x, int[] y) { + int t1 = (x[0] > y[0]) ? x[0] : y[0]; + int t2 = (x[0] < y[0]) ? x[0] : y[0]; + int t3 = (x[0] > y[0]) ? x[0] : y[0]; + int t4 = (x[0] < y[0]) ? x[0] : y[0]; + int t5 = (x[0] > y[0]) ? x[0] : y[0]; + int t6 = (x[0] < y[0]) ? x[0] : y[0]; + // Make sure min/max is CSEed. + return t1 + t2 + t3 + t4 + t5 + t6; + } + + /// CHECK-START: int Main.minmaxCSEScalarAndCond(int, int) instruction_simplifier$after_inlining (after) + /// CHECK-DAG: <<Par1:i\d+>> ParameterValue + /// CHECK-DAG: <<Par2:i\d+>> ParameterValue + /// CHECK-DAG: <<Max:i\d+>> Max [<<Par1>>,<<Par2>>] + /// CHECK-DAG: <<Min:i\d+>> Min [<<Par1>>,<<Par2>>] + /// CHECK-DAG: <<Add:i\d+>> Add [<<Max>>,<<Min>>] + /// CHECK-DAG: Return [<<Add>>] + /// CHECK-DAG: <<Add1:i\d+>> Add [<<Max>>,<<Min>>] + /// CHECK-DAG: <<Add2:i\d+>> Add [<<Max>>,<<Add1>>] + /// CHECK-DAG: <<Add3:i\d+>> Add [<<Min>>,<<Add2>>] + /// CHECK-DAG: Return [<<Add3>>] + public static int minmaxCSEScalarAndCond(int x, int y) { + int t1 = (x > y) ? x : y; + int t2 = (x < y) ? x : y; + if (x == y) + return t1 + t2; + int t3 = (x > y) ? x : y; + int t4 = (x < y) ? x : y; + // Make sure min/max is CSEed. + return t1 + t2 + t3 + t4; + } + + public static void main(String[] args) { + // Intrinsics. + expectEquals(10, minI(10)); + expectEquals(20, minI(25)); + expectEquals(10L, minL(10L)); + expectEquals(20L, minL(25L)); + expectEquals(20, maxI(10)); + expectEquals(25, maxI(25)); + expectEquals(20L, maxL(10L)); + expectEquals(25L, maxL(25L)); + // Types. + expectEquals(10, min1(10, 20)); + expectEquals(10, min2(10, 20)); + expectEquals(10, min3(10, 20)); + expectEquals(10, min4(10, 20)); + expectEquals(10, min5((short) 10, (short) 20)); + expectEquals(10, min6((byte) 10, (byte) 20)); + expectEquals(10L, min7(10L, 20L)); + expectEquals(20, max1(10, 20)); + expectEquals(20, max2(10, 20)); + expectEquals(20, max3(10, 20)); + expectEquals(20, max4(10, 20)); + expectEquals(20, max5((short) 10, (short) 20)); + expectEquals(20, max6((byte) 10, (byte) 20)); + expectEquals(20L, max7(10L, 20L)); + // Complications. + int[] a = { 10 }; + int[] b = { 20 }; + expectEquals(10, min0(a, b)); + expectEquals(20, max0(a, b)); + expectEquals(-100, minmax1(-200)); + expectEquals(10, minmax1(10)); + expectEquals(100, minmax1(200)); + expectEquals(-100, minmax2(-200)); + expectEquals(10, minmax2(10)); + expectEquals(100, minmax2(200)); + expectEquals(-100, minmax3(-200)); + expectEquals(10, minmax3(10)); + expectEquals(100, minmax3(200)); + expectEquals(-100, minmax4(-200)); + expectEquals(10, minmax4(10)); + expectEquals(100, minmax4(200)); + expectEquals(90, minmaxCSEScalar(10, 20)); + expectEquals(90, minmaxCSEArray(a, b)); + expectEquals(20, minmaxCSEScalarAndCond(10, 10)); + expectEquals(60, minmaxCSEScalarAndCond(10, 20)); + System.out.println("passed"); + } + + private static void expectEquals(int expected, int result) { + if (expected != result) { + throw new Error("Expected: " + expected + ", found: " + result); + } + } + + private static void expectEquals(long expected, long result) { + if (expected != result) { + throw new Error("Expected: " + expected + ", found: " + result); + } + } +} diff --git a/test/680-checker-deopt-dex-pc-0/expected.txt b/test/680-checker-deopt-dex-pc-0/expected.txt new file mode 100644 index 00000000000..805857dc65d --- /dev/null +++ b/test/680-checker-deopt-dex-pc-0/expected.txt @@ -0,0 +1,2 @@ +JNI_OnLoad called +passed diff --git a/test/680-checker-deopt-dex-pc-0/info.txt b/test/680-checker-deopt-dex-pc-0/info.txt new file mode 100644 index 00000000000..8eae156b220 --- /dev/null +++ b/test/680-checker-deopt-dex-pc-0/info.txt @@ -0,0 +1,2 @@ +Regression test for deoptimization at dex pc 0 causing infinite recursion +for JIT-at-first-use. diff --git a/test/680-checker-deopt-dex-pc-0/src/Main.java b/test/680-checker-deopt-dex-pc-0/src/Main.java new file mode 100644 index 00000000000..64a3cb3da7d --- /dev/null +++ b/test/680-checker-deopt-dex-pc-0/src/Main.java @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class Main { + // We run this test for AOT to verify that there is a HDeoptimize with dex pc 0. + /// CHECK-START: int Main.$noinline$getInt(byte[], int) BCE (after) + /// CHECK: Deoptimize dex_pc:0 + public static int $noinline$getInt(byte[] array, int offset) { + // The aget for `array[offset]` is at dex pc 0, so the Deoptimize + // from dynamic BCE shall also be at dex pc 0. + return ((array[offset ] & 0xFF) << 0) + + ((array[offset + 1] & 0xFF) << 8) + + ((array[offset + 2] & 0xFF) << 16) + + ((array[offset + 3] & 0xFF) << 24); + } + + public static void main(String[] args) { + System.loadLibrary(args[0]); + if (hasJit()) { + byte[] array = { 0, 1, 2, 3 }; + ensureJitCompiled(Main.class, "$noinline$getInt"); + if (!hasJitCompiledEntrypoint(Main.class, "$noinline$getInt")) { + throw new Error("Unexpected entrypoint!"); + } + if ($noinline$getInt(array, 0) != 0x03020100) { + throw new Error(); + } + try { + // The HDeoptimize at dex pc 0 was previously handled poorly as the dex pc 0 + // was used to detect whether we entered the method. This meant that the + // instrumentation would have reported MethodEnteredEvent and we would have + // told JIT that the method was entered. With JIT-on-first-use we would also + // immediatelly recompile the method and run the compiled code leading to + // a an infinite deoptimization recursion, yielding StackOverflowError. + $noinline$getInt(array, 1); + } catch (ArrayIndexOutOfBoundsException ignored) {} + } + System.out.println("passed"); + } + + public static native boolean hasJit(); + public native static boolean hasJitCompiledEntrypoint(Class<?> cls, String methodName); + public native static void ensureJitCompiled(Class<?> cls, String methodName); +} diff --git a/test/651-checker-double-simd-minmax/expected.txt b/test/681-checker-abs/expected.txt index b0aad4deb5b..b0aad4deb5b 100644 --- a/test/651-checker-double-simd-minmax/expected.txt +++ b/test/681-checker-abs/expected.txt diff --git a/test/681-checker-abs/info.txt b/test/681-checker-abs/info.txt new file mode 100644 index 00000000000..d36e76e5044 --- /dev/null +++ b/test/681-checker-abs/info.txt @@ -0,0 +1 @@ +Functional tests on detecting abs. diff --git a/test/681-checker-abs/src/Main.java b/test/681-checker-abs/src/Main.java new file mode 100644 index 00000000000..2b95a8d56e1 --- /dev/null +++ b/test/681-checker-abs/src/Main.java @@ -0,0 +1,328 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Functional tests for detecting abs. + */ +public class Main { + + // + // Intrinsics. + // + + /// CHECK-START: int Main.absI(int) instruction_simplifier (before) + /// CHECK-DAG: <<Par:i\d+>> ParameterValue + /// CHECK-DAG: <<Abs:i\d+>> InvokeStaticOrDirect [<<Par>>] intrinsic:MathAbsInt + /// CHECK-DAG: Return [<<Abs>>] + // + /// CHECK-START: int Main.absI(int) instruction_simplifier (after) + /// CHECK-DAG: <<Par:i\d+>> ParameterValue + /// CHECK-DAG: <<Abs:i\d+>> Abs [<<Par>>] + /// CHECK-DAG: Return [<<Abs>>] + // + /// CHECK-START: int Main.absI(int) instruction_simplifier (after) + /// CHECK-NOT: InvokeStaticOrDirect + public static int absI(int a) { + return Math.abs(a); + } + + /// CHECK-START: long Main.absL(long) instruction_simplifier (before) + /// CHECK-DAG: <<Par:j\d+>> ParameterValue + /// CHECK-DAG: <<Abs:j\d+>> InvokeStaticOrDirect [<<Par>>] intrinsic:MathAbsLong + /// CHECK-DAG: Return [<<Abs>>] + // + /// CHECK-START: long Main.absL(long) instruction_simplifier (after) + /// CHECK-DAG: <<Par:j\d+>> ParameterValue + /// CHECK-DAG: <<Abs:j\d+>> Abs [<<Par>>] + /// CHECK-DAG: Return [<<Abs>>] + // + /// CHECK-START: long Main.absL(long) instruction_simplifier (after) + /// CHECK-NOT: InvokeStaticOrDirect + public static long absL(long a) { + return Math.abs(a); + } + + // + // Types. + // + + /// CHECK-START: int Main.abs1(int) instruction_simplifier$after_inlining (before) + /// CHECK-DAG: <<Par:i\d+>> ParameterValue + /// CHECK-DAG: <<Zer:i\d+>> IntConstant 0 + /// CHECK-DAG: <<Cnd:z\d+>> GreaterThanOrEqual [<<Par>>,<<Zer>>] + /// CHECK-DAG: <<Neg:i\d+>> [<<Par>>] + /// CHECK-DAG: <<Sel:i\d+>> Select [<<Neg>>,<<Par>>,<<Cnd>>] + /// CHECK-DAG: Return [<<Sel>>] + // + /// CHECK-START: int Main.abs1(int) instruction_simplifier$after_inlining (after) + /// CHECK-DAG: <<Par:i\d+>> ParameterValue + /// CHECK-DAG: <<Abs:i\d+>> Abs [<<Par>>] + /// CHECK-DAG: Return [<<Abs>>] + // + /// CHECK-START: int Main.abs1(int) instruction_simplifier$after_inlining (after) + /// CHECK-NOT: Select + public static int abs1(int a) { + return a < 0 ? -a : a; + } + + /// CHECK-START: int Main.abs2(int) instruction_simplifier$after_inlining (before) + /// CHECK-DAG: <<Par:i\d+>> ParameterValue + /// CHECK-DAG: <<Zer:i\d+>> IntConstant 0 + /// CHECK-DAG: <<Cnd:z\d+>> GreaterThan [<<Par>>,<<Zer>>] + /// CHECK-DAG: <<Neg:i\d+>> [<<Par>>] + /// CHECK-DAG: <<Sel:i\d+>> Select [<<Neg>>,<<Par>>,<<Cnd>>] + /// CHECK-DAG: Return [<<Sel>>] + // + /// CHECK-START: int Main.abs2(int) instruction_simplifier$after_inlining (after) + /// CHECK-DAG: <<Par:i\d+>> ParameterValue + /// CHECK-DAG: <<Abs:i\d+>> Abs [<<Par>>] + /// CHECK-DAG: Return [<<Abs>>] + // + /// CHECK-START: int Main.abs2(int) instruction_simplifier$after_inlining (after) + /// CHECK-NOT: Select + public static int abs2(int a) { + return a <= 0 ? -a : a; + } + + /// CHECK-START: int Main.abs3(int) instruction_simplifier$after_inlining (before) + /// CHECK-DAG: <<Par:i\d+>> ParameterValue + /// CHECK-DAG: <<Zer:i\d+>> IntConstant 0 + /// CHECK-DAG: <<Cnd:z\d+>> LessThanOrEqual [<<Par>>,<<Zer>>] + /// CHECK-DAG: <<Neg:i\d+>> [<<Par>>] + /// CHECK-DAG: <<Sel:i\d+>> Select [<<Par>>,<<Neg>>,<<Cnd>>] + /// CHECK-DAG: Return [<<Sel>>] + // + /// CHECK-START: int Main.abs3(int) instruction_simplifier$after_inlining (after) + /// CHECK-DAG: <<Par:i\d+>> ParameterValue + /// CHECK-DAG: <<Abs:i\d+>> Abs [<<Par>>] + /// CHECK-DAG: Return [<<Abs>>] + // + /// CHECK-START: int Main.abs3(int) instruction_simplifier$after_inlining (after) + /// CHECK-NOT: Select + public static int abs3(int a) { + return a > 0 ? a : -a; + } + + /// CHECK-START: int Main.abs4(int) instruction_simplifier$after_inlining (before) + /// CHECK-DAG: <<Par:i\d+>> ParameterValue + /// CHECK-DAG: <<Zer:i\d+>> IntConstant 0 + /// CHECK-DAG: <<Cnd:z\d+>> LessThan [<<Par>>,<<Zer>>] + /// CHECK-DAG: <<Neg:i\d+>> [<<Par>>] + /// CHECK-DAG: <<Sel:i\d+>> Select [<<Par>>,<<Neg>>,<<Cnd>>] + /// CHECK-DAG: Return [<<Sel>>] + // + /// CHECK-START: int Main.abs4(int) instruction_simplifier$after_inlining (after) + /// CHECK-DAG: <<Par:i\d+>> ParameterValue + /// CHECK-DAG: <<Abs:i\d+>> Abs [<<Par>>] + /// CHECK-DAG: Return [<<Abs>>] + // + /// CHECK-START: int Main.abs4(int) instruction_simplifier$after_inlining (after) + /// CHECK-NOT: Select + public static int abs4(int a) { + return a >= 0 ? a : -a; + } + + /// CHECK-START: int Main.abs5(short) instruction_simplifier$after_inlining (before) + /// CHECK-DAG: <<Par:s\d+>> ParameterValue + /// CHECK-DAG: <<Zer:i\d+>> IntConstant 0 + /// CHECK-DAG: <<Cnd:z\d+>> LessThan [<<Par>>,<<Zer>>] + /// CHECK-DAG: <<Neg:i\d+>> [<<Par>>] + /// CHECK-DAG: <<Sel:i\d+>> Select [<<Par>>,<<Neg>>,<<Cnd>>] + /// CHECK-DAG: Return [<<Sel>>] + // + /// CHECK-START: int Main.abs5(short) instruction_simplifier$after_inlining (after) + /// CHECK-DAG: <<Par:s\d+>> ParameterValue + /// CHECK-DAG: <<Abs:i\d+>> Abs [<<Par>>] + /// CHECK-DAG: Return [<<Abs>>] + // + /// CHECK-START: int Main.abs5(short) instruction_simplifier$after_inlining (after) + /// CHECK-NOT: Select + public static int abs5(short a) { + return a >= 0 ? a : -a; + } + + /// CHECK-START: int Main.abs6(byte) instruction_simplifier$after_inlining (before) + /// CHECK-DAG: <<Par:b\d+>> ParameterValue + /// CHECK-DAG: <<Zer:i\d+>> IntConstant 0 + /// CHECK-DAG: <<Cnd:z\d+>> LessThan [<<Par>>,<<Zer>>] + /// CHECK-DAG: <<Neg:i\d+>> [<<Par>>] + /// CHECK-DAG: <<Sel:i\d+>> Select [<<Par>>,<<Neg>>,<<Cnd>>] + /// CHECK-DAG: Return [<<Sel>>] + // + /// CHECK-START: int Main.abs6(byte) instruction_simplifier$after_inlining (after) + /// CHECK-DAG: <<Par:b\d+>> ParameterValue + /// CHECK-DAG: <<Abs:i\d+>> Abs [<<Par>>] + /// CHECK-DAG: Return [<<Abs>>] + // + /// CHECK-START: int Main.abs6(byte) instruction_simplifier$after_inlining (after) + /// CHECK-NOT: Select + public static int abs6(byte a) { + return a >= 0 ? a : -a; + } + + /// CHECK-START: long Main.abs7(long) instruction_simplifier$after_inlining (before) + /// CHECK-DAG: <<Par:j\d+>> ParameterValue + /// CHECK-DAG: <<Zer:j\d+>> LongConstant 0 + /// CHECK-DAG: <<Cnd:z\d+>> LessThan [<<Par>>,<<Zer>>] + /// CHECK-DAG: <<Neg:j\d+>> [<<Par>>] + /// CHECK-DAG: <<Sel:j\d+>> Select [<<Par>>,<<Neg>>,<<Cnd>>] + /// CHECK-DAG: Return [<<Sel>>] + // + /// CHECK-START: long Main.abs7(long) instruction_simplifier$after_inlining (after) + /// CHECK-DAG: <<Par:j\d+>> ParameterValue + /// CHECK-DAG: <<Abs:j\d+>> Abs [<<Par>>] + /// CHECK-DAG: Return [<<Abs>>] + // + /// CHECK-START: long Main.abs7(long) instruction_simplifier$after_inlining (after) + /// CHECK-NOT: Select + public static long abs7(long a) { + return a >= 0 ? a : -a; + } + + // + // Complications. + // + + /// CHECK-START: int Main.abs0(int[]) instruction_simplifier$after_inlining (before) + /// CHECK-DAG: <<Zer:i\d+>> IntConstant 0 + /// CHECK-DAG: <<Arr:i\d+>> ArrayGet [{{l\d+}},{{i\d+}}] + /// CHECK-DAG: <<Cnd:z\d+>> LessThan [<<Arr>>,<<Zer>>] + /// CHECK-DAG: <<Neg:i\d+>> [<<Arr>>] + /// CHECK-DAG: <<Sel:i\d+>> Select [<<Arr>>,<<Neg>>,<<Cnd>>] + /// CHECK-DAG: Return [<<Sel>>] + // + /// CHECK-START: int Main.abs0(int[]) instruction_simplifier$after_inlining (after) + /// CHECK-DAG: <<Arr:i\d+>> ArrayGet [{{l\d+}},{{i\d+}}] + /// CHECK-DAG: <<Abs:i\d+>> Abs [<<Arr>>] + /// CHECK-DAG: Return [<<Abs>>] + // + /// CHECK-START: int Main.abs0(int[]) instruction_simplifier$after_inlining (after) + /// CHECK-NOT: Select + public static int abs0(int[] a) { + return a[0] >= 0 ? a[0] : -a[0]; + } + + // + // Nop zero extension. + // + + /// CHECK-START: int Main.zabs1(byte) instruction_simplifier (before) + /// CHECK-DAG: <<Par:b\d+>> ParameterValue + /// CHECK-DAG: <<Msk:i\d+>> IntConstant 255 + /// CHECK-DAG: <<And:i\d+>> [<<Par>>,<<Msk>>] + /// CHECK-DAG: <<Abs:i\d+>> InvokeStaticOrDirect [<<And>>] intrinsic:MathAbsInt + /// CHECK-DAG: Return [<<Abs>>] + // + /// CHECK-START: int Main.zabs1(byte) instruction_simplifier (after) + /// CHECK-DAG: <<Par:b\d+>> ParameterValue + /// CHECK-DAG: <<Cnv:a\d+>> TypeConversion [<<Par>>] + /// CHECK-DAG: Return [<<Cnv>>] + // + /// CHECK-START: int Main.zabs1(byte) instruction_simplifier (after) + /// CHECK-NOT: InvokeStaticOrDirect + /// CHECK-NOT: Abs + public static int zabs1(byte a) { + return Math.abs(a & 0xff); + } + + /// CHECK-START: int Main.zabs2(short) instruction_simplifier (before) + /// CHECK-DAG: <<Par:s\d+>> ParameterValue + /// CHECK-DAG: <<Msk:i\d+>> IntConstant 65535 + /// CHECK-DAG: <<And:i\d+>> [<<Msk>>,<<Par>>] + /// CHECK-DAG: <<Abs:i\d+>> InvokeStaticOrDirect [<<And>>] intrinsic:MathAbsInt + /// CHECK-DAG: Return [<<Abs>>] + // + /// CHECK-START: int Main.zabs2(short) instruction_simplifier (after) + /// CHECK-DAG: <<Par:s\d+>> ParameterValue + /// CHECK-DAG: <<Cnv:c\d+>> TypeConversion [<<Par>>] + /// CHECK-DAG: Return [<<Cnv>>] + // + /// CHECK-START: int Main.zabs2(short) instruction_simplifier (after) + /// CHECK-NOT: InvokeStaticOrDirect + /// CHECK-NOT: Abs + public static int zabs2(short a) { + return Math.abs(a & 0xffff); + } + + /// CHECK-START: int Main.zabs3(char) instruction_simplifier (before) + /// CHECK-DAG: <<Par:c\d+>> ParameterValue + /// CHECK-DAG: <<Abs:i\d+>> InvokeStaticOrDirect [<<Par>>] intrinsic:MathAbsInt + /// CHECK-DAG: Return [<<Abs>>] + // + /// CHECK-START: int Main.zabs3(char) instruction_simplifier (after) + /// CHECK-DAG: <<Par:c\d+>> ParameterValue + /// CHECK-DAG: <<Abs:i\d+>> Abs [<<Par>>] + /// CHECK-DAG: Return [<<Abs>>] + // + /// CHECK-START: int Main.zabs3(char) instruction_simplifier$after_inlining (after) + /// CHECK-DAG: <<Par:c\d+>> ParameterValue + /// CHECK-DAG: Return [<<Par>>] + // + /// CHECK-START: int Main.zabs3(char) instruction_simplifier$after_inlining (after) + /// CHECK-NOT: InvokeStaticOrDirect + /// CHECK-NOT: Abs + public static int zabs3(char a) { + return Math.abs(a); + } + + public static void main(String[] args) { + // Intrinsics. + expectEquals(10, absI(-10)); + expectEquals(20, absI(20)); + expectEquals(10L, absL(-10L)); + expectEquals(20L, absL(20L)); + // Types. + expectEquals(10, abs1(-10)); + expectEquals(20, abs1(20)); + expectEquals(10, abs2(-10)); + expectEquals(20, abs2(20)); + expectEquals(10, abs3(-10)); + expectEquals(20, abs3(20)); + expectEquals(10, abs4(-10)); + expectEquals(20, abs4(20)); + expectEquals(10, abs4((short) -10)); + expectEquals(20, abs4((short) 20)); + expectEquals(10, abs6((byte) -10)); + expectEquals(20, abs6((byte) 20)); + expectEquals(10L, abs7(-10L)); + expectEquals(20L, abs7(20L)); + // Complications. + int[] a = { 13 }; + int[] b = { -11 }; + expectEquals(13, abs0(a)); + expectEquals(11, abs0(b)); + // Nop zero extension. + expectEquals(1, zabs1((byte) 1)); + expectEquals(0xff, zabs1((byte) -1)); + expectEquals(1, zabs2((short) 1)); + expectEquals(0xffff, zabs2((short) -1)); + expectEquals(1, zabs3((char) 1)); + expectEquals(0xffff, zabs3((char) -1)); + System.out.println("passed"); + } + + private static void expectEquals(int expected, int result) { + if (expected != result) { + throw new Error("Expected: " + expected + ", found: " + result); + } + } + + private static void expectEquals(long expected, long result) { + if (expected != result) { + throw new Error("Expected: " + expected + ", found: " + result); + } + } +} diff --git a/test/704-multiply-accumulate/build b/test/704-multiply-accumulate/build new file mode 100644 index 00000000000..d85147f17bd --- /dev/null +++ b/test/704-multiply-accumulate/build @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Copyright 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See b/65168732 +export USE_D8=false + +./default-build "$@" diff --git a/test/706-checker-scheduler/build b/test/706-checker-scheduler/build new file mode 100644 index 00000000000..d85147f17bd --- /dev/null +++ b/test/706-checker-scheduler/build @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Copyright 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See b/65168732 +export USE_D8=false + +./default-build "$@" diff --git a/test/952-invoke-custom/expected.txt b/test/952-invoke-custom/expected.txt index 767cc7e7343..7e8ffa6f412 100644 --- a/test/952-invoke-custom/expected.txt +++ b/test/952-invoke-custom/expected.txt @@ -18,3 +18,67 @@ testStaticFieldAccessors testInstanceFieldAccessors testInvokeVirtual => max(77, -3) = 77 testConstructor => class TestInvocationKinds$Widget +TestDynamicArguments +bsm +0, One, 3.141592653589793 +bsm +1, Two, 2.718281828459045 +bsm +2, Three, 0.0 +0, One, 3.141592653589793 +1, Two, 2.718281828459045 +2, Three, 0.0 +TestBadBootstrapArguments +bsm(class TestBadBootstrapArguments, happy, ()void, -1, very) +happy +invokeWrongParameterTypes => class java.lang.NoSuchMethodError +invokeMissingParameterTypes => class java.lang.NoSuchMethodError +invokeExtraArguments => class java.lang.BootstrapMethodError => class java.lang.invoke.WrongMethodTypeException +invokeWrongArguments => class java.lang.BootstrapMethodError => class java.lang.ClassCastException +invokeWrongArguments => class java.lang.BootstrapMethodError => class java.lang.ClassCastException +invokeWrongArgumentsAgain => class java.lang.BootstrapMethodError => class java.lang.ClassCastException +invokeNarrowArguments => class java.lang.BootstrapMethodError => class java.lang.ClassCastException +bsmDJ(..., 1.7976931348623157E308, 2147483647) +wideningArguments +bsmDoubleLong(..., 1.7976931348623157E308, 9223372036854775807) +boxingArguments +invokeWideningBoxingArguments => class java.lang.BootstrapMethodError => class java.lang.ClassCastException +bsm returning void value. +invokeVoidReturnType() => class java.lang.BootstrapMethodError => class java.lang.ClassCastException +bsm returning Object value. +invokeObjectReturnType() => class java.lang.BootstrapMethodError => class java.lang.ClassCastException +bsm returning Integer value. +invokeIntegerReturnType() => class java.lang.BootstrapMethodError => class java.lang.ClassCastException +Hello! +bsmWithStringArray(TestVariableArityLinkerMethod, methodA, ()void, [Aachen, Aalborg, Aalto]); +methodA +bsmWithStringArray(TestVariableArityLinkerMethod, methodB, ()void, [barium]); +methodB +bsmWithStringArray(TestVariableArityLinkerMethod, methodC, ()void, []); +methodC +methodA +methodB +methodC +bsmWithIntAndStringArray(TestVariableArityLinkerMethod, methodD, ()void, 101, [zoo, zoogene, zoogenic]); +methodD +bsmWithIntAndStringArray(TestVariableArityLinkerMethod, methodE, ()void, 102, [zonic]); +methodE +bsmWithIntAndStringArray(TestVariableArityLinkerMethod, methodF, ()void, 103, []); +methodF +methodD +methodE +methodF +bsmWithLongAndIntArray(TestVariableArityLinkerMethod, methodG, ()void, 81985529216486895, [1, -1, 2, -2]); +methodG +bsmWithFloatAndLongArray(TestVariableArityLinkerMethod, methodH, ()void, -2.7182817, [999999999999, -8888888888888]); +methodH +bsmWithClassAndFloatArray(TestVariableArityLinkerMethod, methodI, ()void, class java.lang.Throwable, [3.4028235E38, 1.4E-45, 3.1415927, -3.1415927]); +methodI +bsmWithDoubleArray(TestVariableArityLinkerMethod, methodJ, ()void, [1.7976931348623157E308, 4.9E-324, 2.718281828459045, -3.141592653589793]); +methodJ +bsmWithClassArray(TestVariableArityLinkerMethod, methodK, ()void, [class java.lang.Integer, class java.lang.invoke.MethodHandles, class java.util.Arrays]); +methodK +methodO => class java.lang.BootstrapMethodError => class java.lang.ClassCastException +methodP => class java.lang.BootstrapMethodError => class java.lang.ClassCastException +methodQ => class java.lang.BootstrapMethodError => class java.lang.invoke.WrongMethodTypeException +methodR => class java.lang.BootstrapMethodError => class java.lang.invoke.WrongMethodTypeException diff --git a/test/952-invoke-custom/src/Main.java b/test/952-invoke-custom/src/Main.java index 0b1c1fffe56..d2250a9647b 100644 --- a/test/952-invoke-custom/src/Main.java +++ b/test/952-invoke-custom/src/Main.java @@ -74,18 +74,15 @@ public class Main extends TestBase { TestLinkerMethodMinimalArguments.FAILURE_TYPE_NONE, 10, 13); } - private static void TestInvokeCustomWithConcurrentThreads() throws Throwable { - // This is a concurrency test that attempts to run invoke-custom on the same - // call site. - TestInvokeCustomWithConcurrentThreads.test(); - } - public static void main(String[] args) throws Throwable { TestUninitializedCallSite(); TestLinkerMethodMinimalArguments(); TestLinkerMethodMultipleArgumentTypes(); TestLinkerUnrelatedBSM.test(); - TestInvokeCustomWithConcurrentThreads(); + TestInvokeCustomWithConcurrentThreads.test(); TestInvocationKinds.test(); + TestDynamicBootstrapArguments.test(); + TestBadBootstrapArguments.test(); + TestVariableArityLinkerMethod.test(); } } diff --git a/test/952-invoke-custom/src/TestBadBootstrapArguments.java b/test/952-invoke-custom/src/TestBadBootstrapArguments.java new file mode 100644 index 00000000000..25d8b59eaa4 --- /dev/null +++ b/test/952-invoke-custom/src/TestBadBootstrapArguments.java @@ -0,0 +1,583 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import annotations.BootstrapMethod; +import annotations.CalledByIndy; +import annotations.Constant; +import java.lang.invoke.CallSite; +import java.lang.invoke.ConstantCallSite; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.lang.invoke.WrongMethodTypeException; + +public class TestBadBootstrapArguments extends TestBase { + private static CallSite bsm( + MethodHandles.Lookup lookup, + String methodName, + MethodType methodType, + int extraInt, + String extraString) + throws Throwable { + System.out.print("bsm("); + System.out.print(lookup.lookupClass()); + System.out.print(", "); + System.out.print(methodName); + System.out.print(", "); + System.out.print(methodType); + System.out.print(", "); + System.out.print(extraInt); + System.out.print(", "); + System.out.print(extraString); + System.out.println(")"); + MethodHandle mh = lookup.findStatic(lookup.lookupClass(), methodName, methodType); + return new ConstantCallSite(mh); + } + + @CalledByIndy( + bootstrapMethod = + @BootstrapMethod( + enclosingType = TestBadBootstrapArguments.class, + name = "bsm", + parameterTypes = { + MethodHandles.Lookup.class, + String.class, + MethodType.class, + int.class, + String.class + } + ), + fieldOrMethodName = "happy", + constantArgumentsForBootstrapMethod = { + @Constant(intValue = -1), + @Constant(stringValue = "very") + } + ) + private static void invokeHappy() { + assertNotReached(); + } + + private static void happy() { + System.out.println("happy"); + } + + // BootstrapMethod.parameterTypes != parameterTypesOf(constantArgumentsForBootstrapMethod) + @CalledByIndy( + bootstrapMethod = + @BootstrapMethod( + enclosingType = TestBadBootstrapArguments.class, + name = "bsm", + parameterTypes = { + MethodHandles.Lookup.class, + String.class, + MethodType.class, + int.class, + double.class + } + ), + fieldOrMethodName = "wrongParameterTypes", + constantArgumentsForBootstrapMethod = { + @Constant(intValue = -1), + @Constant(stringValue = "very") + } + ) + private static void invokeWrongParameterTypes() throws NoSuchMethodError { + assertNotReached(); + } + + private static void wrongParameterTypes() { + System.out.println("wrongParameterTypes"); + } + + // BootstrapMethod.parameterTypes != parameterTypesOf(constantArgumentsForBootstrapMethod) + // (missing constantArgumentTypes)) + @CalledByIndy( + bootstrapMethod = + @BootstrapMethod( + enclosingType = TestBadBootstrapArguments.class, + name = "bsm", + parameterTypes = { + MethodHandles.Lookup.class, + String.class, + MethodType.class, + int.class, + double.class + } + ), + fieldOrMethodName = "missingParameterTypes", + constantArgumentsForBootstrapMethod = {} + ) + private static void invokeMissingParameterTypes() throws NoSuchMethodError { + assertNotReached(); + } + + private static void missingParameterTypes() { + System.out.println("missingParameterTypes"); + } + + // BootstrapMethod.parameterTypes != parameterTypesOf(constantArgumentsForBootstrapMethod): + // extra constant present + @CalledByIndy( + bootstrapMethod = + @BootstrapMethod( + enclosingType = TestBadBootstrapArguments.class, + name = "bsm", + parameterTypes = { + MethodHandles.Lookup.class, + String.class, + MethodType.class, + int.class, + String.class + } + ), + fieldOrMethodName = "extraArguments", + constantArgumentsForBootstrapMethod = { + @Constant(intValue = 1), + @Constant(stringValue = "2"), + @Constant(intValue = 3) + } + ) + private static void invokeExtraArguments() { + assertNotReached(); + } + + private static void extraArguments() { + System.out.println("extraArguments"); + } + + // constantArgumentTypes do not correspond to expected parameter types + @CalledByIndy( + bootstrapMethod = + @BootstrapMethod( + enclosingType = TestBadBootstrapArguments.class, + name = "bsm", + parameterTypes = { + MethodHandles.Lookup.class, + String.class, + MethodType.class, + int.class, + String.class + } + ), + fieldOrMethodName = "wrongArguments", + constantArgumentsForBootstrapMethod = { + @Constant(stringValue = "1"), + @Constant(doubleValue = Math.PI) + } + ) + private static void invokeWrongArguments() { + assertNotReached(); + } + + private static void wrongArguments() { + System.out.println("wrongArguments"); + } + + // constantArgumentTypes do not correspond to expected parameter types + @CalledByIndy( + bootstrapMethod = + @BootstrapMethod( + enclosingType = TestBadBootstrapArguments.class, + name = "bsm", + parameterTypes = { + MethodHandles.Lookup.class, + String.class, + MethodType.class, + int.class, + String.class + } + ), + fieldOrMethodName = "wrongArgumentsAgain", + constantArgumentsForBootstrapMethod = { + @Constant(doubleValue = Math.PI), + @Constant(stringValue = "pie") + } + ) + private static void invokeWrongArgumentsAgain() { + assertNotReached(); + } + + private static void wrongArgumentsAgain() { + System.out.println("wrongArgumentsAgain"); + } + + // Primitive argument types not supported {Z, B, C, S}. + private static CallSite bsmZBCS( + MethodHandles.Lookup lookup, + String methodName, + MethodType methodType, + boolean extraArg0, + byte extraArg1, + char extraArg2, + short extraArg3) + throws Throwable { + assertNotReached(); + return null; + } + + // Arguments are narrower than supported. + @CalledByIndy( + bootstrapMethod = + @BootstrapMethod( + enclosingType = TestBadBootstrapArguments.class, + name = "bsmZBCS", + parameterTypes = { + MethodHandles.Lookup.class, + String.class, + MethodType.class, + boolean.class, + byte.class, + char.class, + short.class + } + ), + fieldOrMethodName = "narrowArguments", + constantArgumentsForBootstrapMethod = { + @Constant(booleanValue = true), + @Constant(byteValue = Byte.MAX_VALUE), + @Constant(charValue = 'A'), + @Constant(shortValue = Short.MIN_VALUE) + } + ) + private static void invokeNarrowArguments() { + assertNotReached(); + } + + private static void narrowArguments() { + assertNotReached(); + } + + private static CallSite bsmDJ( + MethodHandles.Lookup lookup, + String methodName, + MethodType methodType, + double extraArg0, + long extraArg1) + throws Throwable { + System.out.print("bsmDJ(..., "); + System.out.print(extraArg0); + System.out.print(", "); + System.out.print(extraArg1); + System.out.println(")"); + MethodHandle mh = lookup.findStatic(lookup.lookupClass(), methodName, methodType); + return new ConstantCallSite(mh); + } + + // Arguments need widening to parameter types. + @CalledByIndy( + bootstrapMethod = + @BootstrapMethod( + enclosingType = TestBadBootstrapArguments.class, + name = "bsmDJ", + parameterTypes = { + MethodHandles.Lookup.class, + String.class, + MethodType.class, + double.class, + long.class + } + ), + fieldOrMethodName = "wideningArguments", + constantArgumentsForBootstrapMethod = { + @Constant(doubleValue = Double.MAX_VALUE), + @Constant(intValue = Integer.MAX_VALUE) + } + ) + private static void invokeWideningArguments() { + assertNotReached(); + } + + private static void wideningArguments() { + System.out.println("wideningArguments"); + } + + private static CallSite bsmDoubleLong( + MethodHandles.Lookup lookup, + String methodName, + MethodType methodType, + Double extraArg0, + Long extraArg1) + throws Throwable { + System.out.print("bsmDoubleLong(..., "); + System.out.print(extraArg0); + System.out.print(", "); + System.out.print(extraArg1); + System.out.println(")"); + MethodHandle mh = lookup.findStatic(lookup.lookupClass(), methodName, methodType); + return new ConstantCallSite(mh); + } + + // Arguments need boxing to parameter types + @CalledByIndy( + bootstrapMethod = + @BootstrapMethod( + enclosingType = TestBadBootstrapArguments.class, + name = "bsmDoubleLong", + parameterTypes = { + MethodHandles.Lookup.class, + String.class, + MethodType.class, + Double.class, + Long.class + } + ), + fieldOrMethodName = "boxingArguments", + constantArgumentsForBootstrapMethod = { + @Constant(doubleValue = Double.MAX_VALUE), + @Constant(longValue = Long.MAX_VALUE) + } + ) + private static void invokeBoxingArguments() { + assertNotReached(); + } + + private static void boxingArguments() { + System.out.println("boxingArguments"); + } + + // Arguments need widening and boxing to parameter types + @CalledByIndy( + bootstrapMethod = + @BootstrapMethod( + enclosingType = TestBadBootstrapArguments.class, + name = "bsmDoubleLong", + parameterTypes = { + MethodHandles.Lookup.class, + String.class, + MethodType.class, + Double.class, + Long.class + } + ), + fieldOrMethodName = "wideningBoxingArguments", + constantArgumentsForBootstrapMethod = { + @Constant(floatValue = Float.MAX_VALUE), + @Constant(longValue = Integer.MAX_VALUE) + } + ) + private static void invokeWideningBoxingArguments() { + assertNotReached(); + } + + private static void wideningBoxingArguments() { + System.out.println("wideningBoxingArguments"); + } + + static void bsmReturningVoid(MethodHandles.Lookup lookup, String name, MethodType type) { + System.out.println("bsm returning void value."); + } + + @CalledByIndy( + bootstrapMethod = + @BootstrapMethod( + enclosingType = TestBadBootstrapArguments.class, + name = "bsmReturningVoid", + parameterTypes = {MethodHandles.Lookup.class, String.class, MethodType.class}, + returnType = void.class + ), + fieldOrMethodName = "voidReturnType" + ) + private static void invokeVoidReturnType() { + assertNotReached(); + } + + private static void voidReturnType() { + assertNotReached(); + } + + static Object bsmReturningObject(MethodHandles.Lookup lookup, String name, MethodType type) { + System.out.println("bsm returning Object value."); + return new Object(); + } + + @CalledByIndy( + bootstrapMethod = + @BootstrapMethod( + enclosingType = TestBadBootstrapArguments.class, + name = "bsmReturningObject", + parameterTypes = {MethodHandles.Lookup.class, String.class, MethodType.class}, + returnType = Object.class + ), + fieldOrMethodName = "ObjectReturnType" + ) + private static void invokeObjectReturnType() { + assertNotReached(); + } + + private static void objectReturnType() { + assertNotReached(); + } + + static Integer bsmReturningInteger(MethodHandles.Lookup lookup, String name, MethodType type) { + System.out.println("bsm returning Integer value."); + return Integer.valueOf(3); + } + + @CalledByIndy( + bootstrapMethod = + @BootstrapMethod( + enclosingType = TestBadBootstrapArguments.class, + name = "bsmReturningInteger", + parameterTypes = {MethodHandles.Lookup.class, String.class, MethodType.class}, + returnType = Integer.class + ), + fieldOrMethodName = "integerReturnType" + ) + private static void invokeIntegerReturnType() { + assertNotReached(); + } + + private static void integerReturnType() { + assertNotReached(); + } + + static class TestersConstantCallSite extends ConstantCallSite { + public TestersConstantCallSite(MethodHandle mh) { + super(mh); + } + } + + static TestersConstantCallSite bsmReturningTestersConstantCallsite( + MethodHandles.Lookup lookup, String name, MethodType type) throws Throwable { + return new TestersConstantCallSite(lookup.findStatic(lookup.lookupClass(), name, type)); + } + + @CalledByIndy( + bootstrapMethod = + @BootstrapMethod( + enclosingType = TestBadBootstrapArguments.class, + name = "bsmReturningTestersConstantCallsite", + parameterTypes = {MethodHandles.Lookup.class, String.class, MethodType.class}, + returnType = TestersConstantCallSite.class + ), + fieldOrMethodName = "sayHello" + ) + private static void invokeViaCustomCallSiteClass() { + assertNotReached(); + } + + private static void sayHello() { + System.out.println("Hello!"); + } + + static void test() { + System.out.println("TestBadBootstrapArguments"); + invokeHappy(); + try { + invokeWrongParameterTypes(); + assertNotReached(); + } catch (NoSuchMethodError expected) { + System.out.print("invokeWrongParameterTypes => "); + System.out.println(expected.getClass()); + } + try { + invokeMissingParameterTypes(); + assertNotReached(); + } catch (NoSuchMethodError expected) { + System.out.print("invokeMissingParameterTypes => "); + System.out.println(expected.getClass()); + } + try { + invokeExtraArguments(); + assertNotReached(); + } catch (BootstrapMethodError expected) { + assertEquals(WrongMethodTypeException.class, expected.getCause().getClass()); + System.out.print("invokeExtraArguments => "); + System.out.print(expected.getClass()); + System.out.print(" => "); + System.out.println(expected.getCause().getClass()); + } + try { + invokeWrongArguments(); + assertNotReached(); + } catch (BootstrapMethodError expected) { + assertEquals(ClassCastException.class, expected.getCause().getClass()); + System.out.print("invokeWrongArguments => "); + System.out.print(expected.getClass()); + System.out.print(" => "); + System.out.println(expected.getCause().getClass()); + } + try { + invokeWrongArguments(); + assertNotReached(); + } catch (BootstrapMethodError expected) { + assertEquals(ClassCastException.class, expected.getCause().getClass()); + System.out.print("invokeWrongArguments => "); + System.out.print(expected.getClass()); + System.out.print(" => "); + System.out.println(expected.getCause().getClass()); + } + try { + invokeWrongArgumentsAgain(); + assertNotReached(); + } catch (BootstrapMethodError expected) { + assertEquals(ClassCastException.class, expected.getCause().getClass()); + System.out.print("invokeWrongArgumentsAgain => "); + System.out.print(expected.getClass()); + System.out.print(" => "); + System.out.println(expected.getCause().getClass()); + } + try { + invokeNarrowArguments(); + assertNotReached(); + } catch (BootstrapMethodError expected) { + assertEquals(ClassCastException.class, expected.getCause().getClass()); + System.out.print("invokeNarrowArguments => "); + System.out.print(expected.getClass()); + System.out.print(" => "); + System.out.println(expected.getCause().getClass()); + } + invokeWideningArguments(); + invokeBoxingArguments(); + try { + invokeWideningBoxingArguments(); + assertNotReached(); + } catch (BootstrapMethodError expected) { + System.out.print("invokeWideningBoxingArguments => "); + System.out.print(expected.getClass()); + System.out.print(" => "); + System.out.println(expected.getCause().getClass()); + } + try { + invokeVoidReturnType(); + assertNotReached(); + } catch (BootstrapMethodError expected) { + System.out.print("invokeVoidReturnType() => "); + System.out.print(expected.getClass()); + System.out.print(" => "); + System.out.println(expected.getCause().getClass()); + } + try { + invokeObjectReturnType(); + assertNotReached(); + } catch (BootstrapMethodError expected) { + System.out.print("invokeObjectReturnType() => "); + System.out.print(expected.getClass()); + System.out.print(" => "); + System.out.println(expected.getCause().getClass()); + } + try { + invokeIntegerReturnType(); + assertNotReached(); + } catch (BootstrapMethodError expected) { + System.out.print("invokeIntegerReturnType() => "); + System.out.print(expected.getClass()); + System.out.print(" => "); + System.out.println(expected.getCause().getClass()); + } + invokeViaCustomCallSiteClass(); + } +} diff --git a/test/952-invoke-custom/src/TestDynamicBootstrapArguments.java b/test/952-invoke-custom/src/TestDynamicBootstrapArguments.java new file mode 100644 index 00000000000..782feca6da2 --- /dev/null +++ b/test/952-invoke-custom/src/TestDynamicBootstrapArguments.java @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import annotations.BootstrapMethod; +import annotations.CalledByIndy; +import annotations.Constant; +import java.lang.invoke.CallSite; +import java.lang.invoke.ConstantCallSite; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; + +class TestDynamicBootstrapArguments extends TestBase { + private static int bsmCalls = 0; + + static CallSite bsm( + MethodHandles.Lookup lookup, + String name, + MethodType methodType, + String otherNameComponent, + long nameSuffix) + throws Throwable { + bsmCalls = bsmCalls + 1; + Class<?> definingClass = TestDynamicBootstrapArguments.class; + String methodName = name + otherNameComponent + nameSuffix; + MethodHandle mh = lookup.findStatic(definingClass, methodName, methodType); + System.out.println("bsm"); + return new ConstantCallSite(mh); + } + + @CalledByIndy( + bootstrapMethod = + @BootstrapMethod( + enclosingType = TestDynamicBootstrapArguments.class, + name = "bsm", + parameterTypes = { + MethodHandles.Lookup.class, + String.class, + MethodType.class, + String.class, + long.class + } + ), + fieldOrMethodName = "target", + returnType = int.class, + parameterTypes = {int.class, String.class, double.class}, + constantArgumentsForBootstrapMethod = { + @Constant(stringValue = "A"), + @Constant(longValue = 100000000l) + } + ) + private static int testDynamic(int i, String s, Double d) { + assertNotReached(); + return 0; + } + + private static int targetA100000000(int i, String s, Double d) { + System.out.print(i); + System.out.print(", "); + System.out.print(s); + System.out.print(", "); + System.out.println(d); + return i; + } + + static void testCallSites() { + assertEquals(0, testDynamic(0, "One", Math.PI)); + assertEquals(1, testDynamic(1, "Two", Math.E)); + assertEquals(2, testDynamic(2, "Three", 0.0)); + } + + static void test() { + System.out.println("TestDynamicArguments"); + testCallSites(); + assertEquals(3, bsmCalls); + testCallSites(); + assertEquals(3, bsmCalls); + } +} diff --git a/test/952-invoke-custom/src/TestInvocationKinds.java b/test/952-invoke-custom/src/TestInvocationKinds.java index 7b88c18c669..f743bef158f 100644 --- a/test/952-invoke-custom/src/TestInvocationKinds.java +++ b/test/952-invoke-custom/src/TestInvocationKinds.java @@ -173,6 +173,7 @@ class TestInvocationKinds extends TestBase { static class Widget { int value; + public Widget(int value) {} } diff --git a/test/952-invoke-custom/src/TestVariableArityLinkerMethod.java b/test/952-invoke-custom/src/TestVariableArityLinkerMethod.java new file mode 100644 index 00000000000..597273c8ecf --- /dev/null +++ b/test/952-invoke-custom/src/TestVariableArityLinkerMethod.java @@ -0,0 +1,569 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import annotations.BootstrapMethod; +import annotations.CalledByIndy; +import annotations.Constant; +import java.lang.invoke.CallSite; +import java.lang.invoke.ConstantCallSite; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.util.Arrays; + +public class TestVariableArityLinkerMethod extends TestBase { + private static void printBsmArgs(String method, Object... args) { + System.out.print(method); + System.out.print("("); + for (int i = 0; i < args.length; ++i) { + if (i != 0) { + System.out.print(", "); + } + if (args[i] != null && args[i].getClass().isArray()) { + Object array = args[i]; + if (array.getClass() == int[].class) { + System.out.print(Arrays.toString((int[]) array)); + } else if (array.getClass() == long[].class) { + System.out.print(Arrays.toString((long[]) array)); + } else if (array.getClass() == float[].class) { + System.out.print(Arrays.toString((float[]) array)); + } else if (array.getClass() == double[].class) { + System.out.print(Arrays.toString((double[]) array)); + } else { + System.out.print(Arrays.toString((Object[]) array)); + } + } else { + System.out.print(args[i]); + } + } + System.out.println(");"); + } + + private static CallSite bsmWithStringArray( + MethodHandles.Lookup lookup, + String methodName, + MethodType methodType, + String... arityArgs) + throws Throwable { + printBsmArgs("bsmWithStringArray", lookup, methodName, methodType, arityArgs); + MethodHandle mh = lookup.findStatic(lookup.lookupClass(), methodName, methodType); + return new ConstantCallSite(mh); + } + + @CalledByIndy( + bootstrapMethod = + @BootstrapMethod( + enclosingType = TestVariableArityLinkerMethod.class, + name = "bsmWithStringArray", + parameterTypes = { + MethodHandles.Lookup.class, + String.class, + MethodType.class, + String[].class + } + ), + fieldOrMethodName = "methodA", + constantArgumentsForBootstrapMethod = { + @Constant(stringValue = "Aachen"), + @Constant(stringValue = "Aalborg"), + @Constant(stringValue = "Aalto") + } + ) + private static void methodA() { + System.out.println("methodA"); + } + + @CalledByIndy( + bootstrapMethod = + @BootstrapMethod( + enclosingType = TestVariableArityLinkerMethod.class, + name = "bsmWithStringArray", + parameterTypes = { + MethodHandles.Lookup.class, + String.class, + MethodType.class, + String[].class + } + ), + fieldOrMethodName = "methodB", + constantArgumentsForBootstrapMethod = {@Constant(stringValue = "barium")} + ) + private static void methodB() { + System.out.println("methodB"); + } + + @CalledByIndy( + bootstrapMethod = + @BootstrapMethod( + enclosingType = TestVariableArityLinkerMethod.class, + name = "bsmWithStringArray", + parameterTypes = { + MethodHandles.Lookup.class, + String.class, + MethodType.class, + String[].class + } + ), + fieldOrMethodName = "methodC" + ) + private static void methodC() { + System.out.println("methodC"); + } + + private static CallSite bsmWithIntAndStringArray( + MethodHandles.Lookup lookup, + String methodName, + MethodType methodType, + int extraInt, + String... extraArityArgs) + throws Throwable { + printBsmArgs( + "bsmWithIntAndStringArray", + lookup, + methodName, + methodType, + extraInt, + extraArityArgs); + MethodHandle mh = lookup.findStatic(lookup.lookupClass(), methodName, methodType); + return new ConstantCallSite(mh); + } + + @CalledByIndy( + bootstrapMethod = + @BootstrapMethod( + enclosingType = TestVariableArityLinkerMethod.class, + name = "bsmWithIntAndStringArray", + parameterTypes = { + MethodHandles.Lookup.class, + String.class, + MethodType.class, + int.class, + String[].class + } + ), + fieldOrMethodName = "methodD", + constantArgumentsForBootstrapMethod = { + @Constant(intValue = 101), + @Constant(stringValue = "zoo"), + @Constant(stringValue = "zoogene"), + @Constant(stringValue = "zoogenic") + } + ) + private static void methodD() { + System.out.println("methodD"); + } + + @CalledByIndy( + bootstrapMethod = + @BootstrapMethod( + enclosingType = TestVariableArityLinkerMethod.class, + name = "bsmWithIntAndStringArray", + parameterTypes = { + MethodHandles.Lookup.class, + String.class, + MethodType.class, + int.class, + String[].class + } + ), + fieldOrMethodName = "methodE", + constantArgumentsForBootstrapMethod = { + @Constant(intValue = 102), + @Constant(stringValue = "zonic") + } + ) + private static void methodE() { + System.out.println("methodE"); + } + + @CalledByIndy( + bootstrapMethod = + @BootstrapMethod( + enclosingType = TestVariableArityLinkerMethod.class, + name = "bsmWithIntAndStringArray", + parameterTypes = { + MethodHandles.Lookup.class, + String.class, + MethodType.class, + int.class, + String[].class + } + ), + fieldOrMethodName = "methodF", + constantArgumentsForBootstrapMethod = {@Constant(intValue = 103)} + ) + private static void methodF() { + System.out.println("methodF"); + } + + private static CallSite bsmWithLongAndIntArray( + MethodHandles.Lookup lookup, + String methodName, + MethodType methodType, + long extraArg, + int... arityArgs) + throws Throwable { + printBsmArgs("bsmWithLongAndIntArray", lookup, methodName, methodType, extraArg, arityArgs); + MethodHandle mh = lookup.findStatic(lookup.lookupClass(), methodName, methodType); + return new ConstantCallSite(mh); + } + + @CalledByIndy( + bootstrapMethod = + @BootstrapMethod( + enclosingType = TestVariableArityLinkerMethod.class, + name = "bsmWithLongAndIntArray", + parameterTypes = { + MethodHandles.Lookup.class, + String.class, + MethodType.class, + long.class, + int[].class + } + ), + fieldOrMethodName = "methodG", + constantArgumentsForBootstrapMethod = { + @Constant(longValue = 0x123456789abcdefl), + @Constant(intValue = +1), + @Constant(intValue = -1), + @Constant(intValue = +2), + @Constant(intValue = -2) + } + ) + private static void methodG() { + System.out.println("methodG"); + } + + private static CallSite bsmWithFloatAndLongArray( + MethodHandles.Lookup lookup, + String methodName, + MethodType methodType, + float extraArg, + long... arityArgs) + throws Throwable { + printBsmArgs( + "bsmWithFloatAndLongArray", lookup, methodName, methodType, extraArg, arityArgs); + MethodHandle mh = lookup.findStatic(lookup.lookupClass(), methodName, methodType); + return new ConstantCallSite(mh); + } + + @CalledByIndy( + bootstrapMethod = + @BootstrapMethod( + enclosingType = TestVariableArityLinkerMethod.class, + name = "bsmWithFloatAndLongArray", + parameterTypes = { + MethodHandles.Lookup.class, + String.class, + MethodType.class, + float.class, + long[].class + } + ), + fieldOrMethodName = "methodH", + constantArgumentsForBootstrapMethod = { + @Constant(floatValue = (float) -Math.E), + @Constant(longValue = 999999999999l), + @Constant(longValue = -8888888888888l) + } + ) + private static void methodH() { + System.out.println("methodH"); + } + + private static CallSite bsmWithClassAndFloatArray( + MethodHandles.Lookup lookup, + String methodName, + MethodType methodType, + Class<?> extraArg, + float... arityArgs) + throws Throwable { + printBsmArgs( + "bsmWithClassAndFloatArray", lookup, methodName, methodType, extraArg, arityArgs); + MethodHandle mh = lookup.findStatic(lookup.lookupClass(), methodName, methodType); + return new ConstantCallSite(mh); + } + + @CalledByIndy( + bootstrapMethod = + @BootstrapMethod( + enclosingType = TestVariableArityLinkerMethod.class, + name = "bsmWithClassAndFloatArray", + parameterTypes = { + MethodHandles.Lookup.class, + String.class, + MethodType.class, + Class.class, + float[].class + } + ), + fieldOrMethodName = "methodI", + constantArgumentsForBootstrapMethod = { + @Constant(classValue = Throwable.class), + @Constant(floatValue = Float.MAX_VALUE), + @Constant(floatValue = Float.MIN_VALUE), + @Constant(floatValue = (float) Math.PI), + @Constant(floatValue = (float) -Math.PI) + } + ) + private static void methodI() { + System.out.println("methodI"); + } + + private static CallSite bsmWithDoubleArray( + MethodHandles.Lookup lookup, + String methodName, + MethodType methodType, + double... arityArgs) + throws Throwable { + printBsmArgs("bsmWithDoubleArray", lookup, methodName, methodType, arityArgs); + MethodHandle mh = lookup.findStatic(lookup.lookupClass(), methodName, methodType); + return new ConstantCallSite(mh); + } + + @CalledByIndy( + bootstrapMethod = + @BootstrapMethod( + enclosingType = TestVariableArityLinkerMethod.class, + name = "bsmWithDoubleArray", + parameterTypes = { + MethodHandles.Lookup.class, + String.class, + MethodType.class, + double[].class + } + ), + fieldOrMethodName = "methodJ", + constantArgumentsForBootstrapMethod = { + @Constant(doubleValue = Double.MAX_VALUE), + @Constant(doubleValue = Double.MIN_VALUE), + @Constant(doubleValue = Math.E), + @Constant(doubleValue = -Math.PI) + } + ) + private static void methodJ() { + System.out.println("methodJ"); + } + + private static CallSite bsmWithClassArray( + MethodHandles.Lookup lookup, + String methodName, + MethodType methodType, + Class... arityArgs) + throws Throwable { + printBsmArgs("bsmWithClassArray", lookup, methodName, methodType, arityArgs); + MethodHandle mh = lookup.findStatic(lookup.lookupClass(), methodName, methodType); + return new ConstantCallSite(mh); + } + + @CalledByIndy( + bootstrapMethod = + @BootstrapMethod( + enclosingType = TestVariableArityLinkerMethod.class, + name = "bsmWithClassArray", + parameterTypes = { + MethodHandles.Lookup.class, + String.class, + MethodType.class, + Class[].class + } + ), + fieldOrMethodName = "methodK", + constantArgumentsForBootstrapMethod = { + @Constant(classValue = Integer.class), + @Constant(classValue = MethodHandles.class), + @Constant(classValue = Arrays.class) + } + ) + private static void methodK() { + System.out.println("methodK"); + } + + @CalledByIndy( + bootstrapMethod = + @BootstrapMethod( + enclosingType = TestVariableArityLinkerMethod.class, + name = "bsmWithIntAndStringArray", + parameterTypes = { + MethodHandles.Lookup.class, + String.class, + MethodType.class, + int.class, + String[].class + } + ), + fieldOrMethodName = "methodO", + constantArgumentsForBootstrapMethod = {@Constant(intValue = 103), @Constant(intValue = 104)} + ) + private static void methodO() { + // Arguments are not compatible + assertNotReached(); + } + + @CalledByIndy( + bootstrapMethod = + @BootstrapMethod( + enclosingType = TestVariableArityLinkerMethod.class, + name = "bsmWithIntAndStringArray", + parameterTypes = { + MethodHandles.Lookup.class, + String.class, + MethodType.class, + int.class, + String[].class + } + ), + fieldOrMethodName = "methodP", + constantArgumentsForBootstrapMethod = { + @Constant(intValue = 103), + @Constant(stringValue = "A"), + @Constant(stringValue = "B"), + @Constant(intValue = 42) + } + ) + private static void methodP() { + // Arguments are not compatible - specifically, the third + // component of potential collector array is an integer + // argument (42). + assertNotReached(); + } + + private static CallSite bsmWithWiderArray( + MethodHandles.Lookup lookup, String methodName, MethodType methodType, long[] extraArgs) + throws Throwable { + printBsmArgs("bsmWithWiderArray", lookup, methodName, methodType, extraArgs); + MethodHandle mh = lookup.findStatic(lookup.lookupClass(), methodName, methodType); + return new ConstantCallSite(mh); + } + + @CalledByIndy( + bootstrapMethod = + @BootstrapMethod( + enclosingType = TestVariableArityLinkerMethod.class, + name = "bsmWithWiderArray", + parameterTypes = { + MethodHandles.Lookup.class, + String.class, + MethodType.class, + long[].class + } + ), + fieldOrMethodName = "methodQ", + constantArgumentsForBootstrapMethod = {@Constant(intValue = 103), @Constant(intValue = 42)} + ) + private static void methodQ() { + assertNotReached(); + } + + private static CallSite bsmWithBoxedArray( + MethodHandles.Lookup lookup, + String methodName, + MethodType methodType, + Integer[] extraArgs) + throws Throwable { + printBsmArgs("bsmWithBoxedArray", lookup, methodName, methodType, extraArgs); + MethodHandle mh = lookup.findStatic(lookup.lookupClass(), methodName, methodType); + return new ConstantCallSite(mh); + } + + @CalledByIndy( + bootstrapMethod = + @BootstrapMethod( + enclosingType = TestVariableArityLinkerMethod.class, + name = "bsmWithBoxedArray", + parameterTypes = { + MethodHandles.Lookup.class, + String.class, + MethodType.class, + Integer[].class + } + ), + fieldOrMethodName = "methodR", + constantArgumentsForBootstrapMethod = { + @Constant(intValue = 1030), + @Constant(intValue = 420) + } + ) + private static void methodR() { + assertNotReached(); + } + + static void test() { + // Happy cases + for (int i = 0; i < 2; ++i) { + methodA(); + methodB(); + methodC(); + } + for (int i = 0; i < 2; ++i) { + methodD(); + methodE(); + methodF(); + } + methodG(); + methodH(); + methodI(); + methodJ(); + methodK(); + + // Broken cases + try { + // bsm has incompatible static methods. Collector + // component type is String, the corresponding static + // arguments are int values. + methodO(); + assertNotReached(); + } catch (BootstrapMethodError expected) { + System.out.print("methodO => "); + System.out.print(expected.getClass()); + System.out.print(" => "); + System.out.println(expected.getCause().getClass()); + } + try { + // bsm has a trailing String array for the collector array. + // There is an int value amongst the String values. + methodP(); + assertNotReached(); + } catch (BootstrapMethodError expected) { + System.out.print("methodP => "); + System.out.print(expected.getClass()); + System.out.print(" => "); + System.out.println(expected.getCause().getClass()); + } + try { + // bsm has as trailing long[] element for the collector array. + // The corresponding static bsm arguments are of type int. + methodQ(); + assertNotReached(); + } catch (BootstrapMethodError expected) { + System.out.print("methodQ => "); + System.out.print(expected.getClass()); + System.out.print(" => "); + System.out.println(expected.getCause().getClass()); + } + try { + // bsm has as trailing Integer[] element for the collector array. + // The corresponding static bsm arguments are of type int. + methodR(); + assertNotReached(); + } catch (BootstrapMethodError expected) { + System.out.print("methodR => "); + System.out.print(expected.getClass()); + System.out.print(" => "); + System.out.println(expected.getCause().getClass()); + } + } +} diff --git a/test/979-const-method-handle/build b/test/979-const-method-handle/build index 495557e3df5..ce931a96d11 100644..100755 --- a/test/979-const-method-handle/build +++ b/test/979-const-method-handle/build @@ -1,12 +1,12 @@ #!/bin/bash # -# Copyright (C) 2017 The Android Open Source Project +# Copyright 2018 The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -14,9 +14,42 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Stop if something fails. +# make us exit on a failure set -e -${DX} --dex --min-sdk-version=28 --output=classes.dex classes +ASM_JAR="${ANDROID_BUILD_TOP}/prebuilts/misc/common/asm/asm-6.0.jar" +INTERMEDIATE_CLASSES=classes-intermediate +TRANSFORMER_CLASSES=classes-transformer +CLASSES=classes -zip $TEST_NAME.jar classes.dex +DEXER="${DX:-dx}" +if [ "${USE_D8=false}" = "true" ]; then + DEXER="${ANDROID_HOST_OUT}/bin/d8-compat-dx" +fi + +# Create directories for classes +for class_dir in "${INTERMEDIATE_CLASSES}" "${TRANSFORMER_CLASSES}" "${CLASSES}"; do + rm -rf "${class_dir}" + mkdir "${class_dir}" +done + +# Build transformer +${JAVAC:-javac} ${JAVAC_ARGS} -cp "${ASM_JAR}" -d ${TRANSFORMER_CLASSES} $(find util-src -name '*.java') + +# Generate intermediate classes that will allow transform to be applied to test classes +JAVAC_ARGS="${JAVAC_ARGS} -source 1.8 -target 1.8" +${JAVAC:-javac} ${JAVAC_ARGS} -cp ${TRANSFORMER_CLASSES} -d ${INTERMEDIATE_CLASSES} $(find src -name '*.java') + +# Run transform +for class in ${INTERMEDIATE_CLASSES}/*.class ; do + transformed_class=${CLASSES}/$(basename ${class}) + ${JAVA:-java} -cp "${ASM_JAR}:${TRANSFORMER_CLASSES}" \ + transformer.ConstantTransformer ${class} ${transformed_class} +done + +# Create DEX +DX_FLAGS="${DX_FLAGS} --min-sdk-version=28 --debug --dump-width=1000" +${DEXER} -JXmx256m --dex ${DX_FLAGS} --dump-to=${CLASSES}.lst --output=classes.dex ${CLASSES} ${TRANSFORMER_CLASSES} + +# Zip DEX to file name expected by test runner +zip ${TEST_NAME:-classes-dex}.jar classes.dex diff --git a/test/979-const-method-handle/classes/Main.class b/test/979-const-method-handle/classes/Main.class Binary files differdeleted file mode 100644 index 8d6b7d88bba..00000000000 --- a/test/979-const-method-handle/classes/Main.class +++ /dev/null diff --git a/test/979-const-method-handle/classes/constmethodhandle/ConstTest.class b/test/979-const-method-handle/classes/constmethodhandle/ConstTest.class Binary files differdeleted file mode 100644 index a21b0a336c4..00000000000 --- a/test/979-const-method-handle/classes/constmethodhandle/ConstTest.class +++ /dev/null diff --git a/test/979-const-method-handle/expected.txt b/test/979-const-method-handle/expected.txt index 573b80da995..bc943e368e4 100644 --- a/test/979-const-method-handle/expected.txt +++ b/test/979-const-method-handle/expected.txt @@ -1,2 +1,6 @@ -MethodHandle MethodHandle(Object)Class => class java.lang.Float -MethodType (char,short,int,long,float,double,Object)boolean +(int,Integer,System)String +Hello World! And Hello Zog +Hello World! And Hello Zorba +name is HoverFly +2.718281828459045 +Attempting to set Math.E raised IAE diff --git a/test/979-const-method-handle/info.txt b/test/979-const-method-handle/info.txt index e8514ce91fe..fc909db299e 100644 --- a/test/979-const-method-handle/info.txt +++ b/test/979-const-method-handle/info.txt @@ -1,7 +1 @@ This test checks const-method-handle and const-method-type bytecodes. - -The class files in this test come from: - - dalvik/dx/tests/142-const-method-handle - -and are built using ASM bytecode manipulation library. diff --git a/test/979-const-method-handle/src/Main.java b/test/979-const-method-handle/src/Main.java new file mode 100644 index 00000000000..663814f232d --- /dev/null +++ b/test/979-const-method-handle/src/Main.java @@ -0,0 +1,102 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import annotations.ConstantMethodHandle; +import annotations.ConstantMethodType; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodType; + +class Main { + private static String name = "default"; + + private static void unreachable() { + throw new Error("Unreachable"); + } + + @ConstantMethodType( + returnType = String.class, + parameterTypes = {int.class, Integer.class, System.class} + ) + private static MethodType methodType0() { + unreachable(); + return null; + } + + static void helloWorld(String who) { + System.out.print("Hello World! And Hello "); + System.out.println(who); + } + + @ConstantMethodHandle( + kind = ConstantMethodHandle.INVOKE_STATIC, + owner = "Main", + fieldOrMethodName = "helloWorld", + descriptor = "(Ljava/lang/String;)V" + ) + private static MethodHandle printHelloHandle() { + unreachable(); + return null; + } + + @ConstantMethodHandle( + kind = ConstantMethodHandle.STATIC_PUT, + owner = "Main", + fieldOrMethodName = "name", + descriptor = "Ljava/lang/String;" + ) + private static MethodHandle setNameHandle() { + unreachable(); + return null; + } + + @ConstantMethodHandle( + kind = ConstantMethodHandle.STATIC_GET, + owner = "java/lang/Math", + fieldOrMethodName = "E", + descriptor = "D" + ) + private static MethodHandle getMathE() { + unreachable(); + return null; + } + + @ConstantMethodHandle( + kind = ConstantMethodHandle.STATIC_PUT, + owner = "java/lang/Math", + fieldOrMethodName = "E", + descriptor = "D" + ) + private static MethodHandle putMathE() { + unreachable(); + return null; + } + + public static void main(String[] args) throws Throwable { + System.out.println(methodType0()); + printHelloHandle().invokeExact("Zog"); + printHelloHandle().invokeExact("Zorba"); + setNameHandle().invokeExact("HoverFly"); + System.out.print("name is "); + System.out.println(name); + System.out.println(getMathE().invoke()); + try { + putMathE().invokeExact(Math.PI); + unreachable(); + } catch (IllegalAccessError expected) { + System.out.println("Attempting to set Math.E raised IAE"); + } + } +} diff --git a/test/979-const-method-handle/util-src/annotations/ConstantMethodHandle.java b/test/979-const-method-handle/util-src/annotations/ConstantMethodHandle.java new file mode 100644 index 00000000000..40785ebc6ec --- /dev/null +++ b/test/979-const-method-handle/util-src/annotations/ConstantMethodHandle.java @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * This annotation can be set on method to specify that if this method + * is statically invoked then the invocation is replaced by a + * load-constant bytecode with the MethodHandle constant described by + * the annotation. + */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.METHOD) +public @interface ConstantMethodHandle { + /* Method handle kinds */ + public static final int STATIC_PUT = 0; + public static final int STATIC_GET = 1; + public static final int INSTANCE_PUT = 2; + public static final int INSTANCE_GET = 3; + public static final int INVOKE_STATIC = 4; + public static final int INVOKE_VIRTUAL = 5; + public static final int INVOKE_SPECIAL = 6; + public static final int NEW_INVOKE_SPECIAL = 7; + public static final int INVOKE_INTERFACE = 8; + + /** Kind of method handle. */ + int kind(); + + /** Class name owning the field or method. */ + String owner(); + + /** The field or method name addressed by the MethodHandle. */ + String fieldOrMethodName(); + + /** Descriptor for the field (type) or method (method-type) */ + String descriptor(); + + /** Whether the owner is an interface. */ + boolean ownerIsInterface() default false; +} diff --git a/test/979-const-method-handle/util-src/annotations/ConstantMethodType.java b/test/979-const-method-handle/util-src/annotations/ConstantMethodType.java new file mode 100644 index 00000000000..c89fa013fe2 --- /dev/null +++ b/test/979-const-method-handle/util-src/annotations/ConstantMethodType.java @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * This annotation can be set on method to specify that if this method + * is statically invoked then the invocation is replaced by a + * load-constant bytecode with the MethodType constant described by + * the annotation. + */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.METHOD) +public @interface ConstantMethodType { + /** Return type of method() or field getter() */ + Class<?> returnType() default void.class; + + /** Types of parameters for method or field setter() */ + Class<?>[] parameterTypes() default {}; +} diff --git a/test/979-const-method-handle/util-src/transformer/ConstantTransformer.java b/test/979-const-method-handle/util-src/transformer/ConstantTransformer.java new file mode 100644 index 00000000000..9356426492e --- /dev/null +++ b/test/979-const-method-handle/util-src/transformer/ConstantTransformer.java @@ -0,0 +1,229 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package transformer; + +import annotations.ConstantMethodHandle; +import annotations.ConstantMethodType; +import java.io.InputStream; +import java.io.OutputStream; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodType; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.ClassWriter; +import org.objectweb.asm.Handle; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; +import org.objectweb.asm.Type; + +/** + * Class for transforming invoke static bytecodes into constant method handle loads and and constant + * method type loads. + * + * <p>When a parameterless private static method returning a MethodHandle is defined and annotated + * with {@code ConstantMethodHandle}, this transformer will replace static invocations of the method + * with a load constant bytecode with a method handle in the constant pool. + * + * <p>Suppose a method is annotated as: <code> + * @ConstantMethodHandle( + * kind = ConstantMethodHandle.STATIC_GET, + * owner = "java/lang/Math", + * fieldOrMethodName = "E", + * descriptor = "D" + * ) + * private static MethodHandle getMathE() { + * unreachable(); + * return null; + * } + * </code> Then invocations of {@code getMathE} will be replaced by a load from the constant pool + * with the constant method handle described in the {@code ConstantMethodHandle} annotation. + * + * <p>Similarly, a parameterless private static method returning a {@code MethodType} and annotated + * with {@code ConstantMethodType}, will have invocations replaced by a load constant bytecode with + * a method type in the constant pool. + */ +class ConstantTransformer { + static class ConstantBuilder extends ClassVisitor { + private final Map<String, ConstantMethodHandle> constantMethodHandles; + private final Map<String, ConstantMethodType> constantMethodTypes; + + ConstantBuilder( + int api, + ClassVisitor cv, + Map<String, ConstantMethodHandle> constantMethodHandles, + Map<String, ConstantMethodType> constantMethodTypes) { + super(api, cv); + this.constantMethodHandles = constantMethodHandles; + this.constantMethodTypes = constantMethodTypes; + } + + @Override + public MethodVisitor visitMethod( + int access, String name, String desc, String signature, String[] exceptions) { + MethodVisitor mv = cv.visitMethod(access, name, desc, signature, exceptions); + return new MethodVisitor(this.api, mv) { + @Override + public void visitMethodInsn( + int opcode, String owner, String name, String desc, boolean itf) { + if (opcode == org.objectweb.asm.Opcodes.INVOKESTATIC) { + ConstantMethodHandle constantMethodHandle = constantMethodHandles.get(name); + if (constantMethodHandle != null) { + insertConstantMethodHandle(constantMethodHandle); + return; + } + ConstantMethodType constantMethodType = constantMethodTypes.get(name); + if (constantMethodType != null) { + insertConstantMethodType(constantMethodType); + return; + } + } + mv.visitMethodInsn(opcode, owner, name, desc, itf); + } + + private Type buildMethodType(Class<?> returnType, Class<?>[] parameterTypes) { + Type rType = Type.getType(returnType); + Type[] pTypes = new Type[parameterTypes.length]; + for (int i = 0; i < pTypes.length; ++i) { + pTypes[i] = Type.getType(parameterTypes[i]); + } + return Type.getMethodType(rType, pTypes); + } + + private int getHandleTag(int kind) { + switch (kind) { + case ConstantMethodHandle.STATIC_PUT: + return Opcodes.H_PUTSTATIC; + case ConstantMethodHandle.STATIC_GET: + return Opcodes.H_GETSTATIC; + case ConstantMethodHandle.INSTANCE_PUT: + return Opcodes.H_PUTFIELD; + case ConstantMethodHandle.INSTANCE_GET: + return Opcodes.H_GETFIELD; + case ConstantMethodHandle.INVOKE_STATIC: + return Opcodes.H_INVOKESTATIC; + case ConstantMethodHandle.INVOKE_VIRTUAL: + return Opcodes.H_INVOKEVIRTUAL; + case ConstantMethodHandle.INVOKE_SPECIAL: + return Opcodes.H_INVOKESPECIAL; + case ConstantMethodHandle.NEW_INVOKE_SPECIAL: + return Opcodes.H_NEWINVOKESPECIAL; + case ConstantMethodHandle.INVOKE_INTERFACE: + return Opcodes.H_INVOKEINTERFACE; + } + throw new Error("Unhandled kind " + kind); + } + + private void insertConstantMethodHandle(ConstantMethodHandle constantMethodHandle) { + Handle handle = + new Handle( + getHandleTag(constantMethodHandle.kind()), + constantMethodHandle.owner(), + constantMethodHandle.fieldOrMethodName(), + constantMethodHandle.descriptor(), + constantMethodHandle.ownerIsInterface()); + mv.visitLdcInsn(handle); + } + + private void insertConstantMethodType(ConstantMethodType constantMethodType) { + Type methodType = + buildMethodType( + constantMethodType.returnType(), + constantMethodType.parameterTypes()); + mv.visitLdcInsn(methodType); + } + }; + } + } + + private static void throwAnnotationError( + Method method, Class<?> annotationClass, String reason) { + StringBuilder sb = new StringBuilder(); + sb.append("Error in annotation ") + .append(annotationClass) + .append(" on method ") + .append(method) + .append(": ") + .append(reason); + throw new Error(sb.toString()); + } + + private static void checkMethodToBeReplaced( + Method method, Class<?> annotationClass, Class<?> returnType) { + final int PRIVATE_STATIC = Modifier.STATIC | Modifier.PRIVATE; + if ((method.getModifiers() & PRIVATE_STATIC) != PRIVATE_STATIC) { + throwAnnotationError(method, annotationClass, " method is not private and static"); + } + if (method.getTypeParameters().length != 0) { + throwAnnotationError(method, annotationClass, " method expects parameters"); + } + if (!method.getReturnType().equals(returnType)) { + throwAnnotationError(method, annotationClass, " wrong return type"); + } + } + + private static void transform(Path inputClassPath, Path outputClassPath) throws Throwable { + Path classLoadPath = inputClassPath.toAbsolutePath().getParent(); + URLClassLoader classLoader = + new URLClassLoader(new URL[] {classLoadPath.toUri().toURL()}, + ClassLoader.getSystemClassLoader()); + String inputClassName = inputClassPath.getFileName().toString().replace(".class", ""); + Class<?> inputClass = classLoader.loadClass(inputClassName); + + final Map<String, ConstantMethodHandle> constantMethodHandles = new HashMap<>(); + final Map<String, ConstantMethodType> constantMethodTypes = new HashMap<>(); + + for (Method m : inputClass.getDeclaredMethods()) { + ConstantMethodHandle constantMethodHandle = m.getAnnotation(ConstantMethodHandle.class); + if (constantMethodHandle != null) { + checkMethodToBeReplaced(m, ConstantMethodHandle.class, MethodHandle.class); + constantMethodHandles.put(m.getName(), constantMethodHandle); + continue; + } + + ConstantMethodType constantMethodType = m.getAnnotation(ConstantMethodType.class); + if (constantMethodType != null) { + checkMethodToBeReplaced(m, ConstantMethodType.class, MethodType.class); + constantMethodTypes.put(m.getName(), constantMethodType); + continue; + } + } + ClassWriter cw = new ClassWriter(ClassWriter.COMPUTE_FRAMES); + try (InputStream is = Files.newInputStream(inputClassPath)) { + ClassReader cr = new ClassReader(is); + ConstantBuilder cb = + new ConstantBuilder( + Opcodes.ASM6, cw, constantMethodHandles, constantMethodTypes); + cr.accept(cb, 0); + } + try (OutputStream os = Files.newOutputStream(outputClassPath)) { + os.write(cw.toByteArray()); + } + } + + public static void main(String[] args) throws Throwable { + transform(Paths.get(args[0]), Paths.get(args[1])); + } +} diff --git a/test/Android.run-test.mk b/test/Android.run-test.mk index cf781d7f2b6..f8bebdd35f1 100644 --- a/test/Android.run-test.mk +++ b/test/Android.run-test.mk @@ -21,18 +21,12 @@ include art/build/Android.common_test.mk TEST_ART_RUN_TEST_DEPENDENCIES := \ $(HOST_OUT_EXECUTABLES)/dx \ $(HOST_OUT_EXECUTABLES)/d8 \ + $(HOST_OUT_EXECUTABLES)/d8-compat-dx \ $(HOST_OUT_EXECUTABLES)/hiddenapi \ $(HOST_OUT_EXECUTABLES)/jasmin \ $(HOST_OUT_EXECUTABLES)/smali \ - $(HOST_OUT_EXECUTABLES)/dexmerger \ $(HOST_OUT_JAVA_LIBRARIES)/desugar.jar -# Add d8 dependency, if enabled. -ifeq ($(USE_D8),true) -TEST_ART_RUN_TEST_DEPENDENCIES += \ - $(HOST_OUT_EXECUTABLES)/d8-compat-dx -endif - # We need dex2oat and dalvikvm on the target as well as the core images (all images as we sync # only once). TEST_ART_TARGET_SYNC_DEPS += $(ART_TARGET_EXECUTABLES) $(TARGET_CORE_IMG_OUTS) @@ -103,7 +97,7 @@ endif # Host executables. host_prereq_rules := $(ART_TEST_HOST_RUN_TEST_DEPENDENCIES) -# Required for dx, jasmin, smali, dexmerger. +# Required for dx, jasmin, smali. host_prereq_rules += $(TEST_ART_RUN_TEST_DEPENDENCIES) # Sync test files to the target, depends upon all things that must be pushed diff --git a/test/etc/default-build b/test/etc/default-build index 3e6577cfda4..8bb898c7c1f 100755 --- a/test/etc/default-build +++ b/test/etc/default-build @@ -145,7 +145,7 @@ JAVAC_EXPERIMENTAL_ARGS["agents"]="-source 1.8 -target 1.8" declare -A DX_EXPERIMENTAL_ARGS DX_EXPERIMENTAL_ARGS["method-handles"]="--min-sdk-version=26" DX_EXPERIMENTAL_ARGS["parameter-annotations"]="--min-sdk-version=25" -DX_EXPERIMENTAL_ARGS["var-handles"]="--min-sdk-version=26" +DX_EXPERIMENTAL_ARGS["var-handles"]="--min-sdk-version=28" while true; do if [ "x$1" = "x--dx-option" ]; then @@ -317,7 +317,7 @@ function make_dex() { fi local dexer="${DX}" - if [ ${USE_D8} = "true" ]; then + if [[ "${USE_D8}" != "false" ]]; then dexer="${ANDROID_HOST_OUT}/bin/d8-compat-dx" fi @@ -341,8 +341,26 @@ function make_dexmerge() { shift done - # Should have at least 1 dex_files_to_merge here, otherwise dxmerger will print the help. - ${DXMERGER} "$dst_file" "${dex_files_to_merge[@]}" + # Skip merge if we are not merging anything. IE: input = output. + if [[ "${#dex_files_to_merge[@]}" -eq "1" ]]; then + local single_input=${dex_files_to_merge[0]} + if [[ "$dst_file" != "$single_input" ]]; then + mv "$single_input" "$dst_file"; + return + fi + fi + + # We assume the dexer did all the API level checks and just merge away. + mkdir d8_merge_out + ${DXMERGER} --min-api 1000 --output ./d8_merge_out "${dex_files_to_merge[@]}" + + if [[ -e "./d8_merge_out/classes2.dex" ]]; then + echo "Cannot merge all dex files into a single dex" + exit 1 + fi + + mv ./d8_merge_out/classes.dex "$dst_file"; + rmdir d8_merge_out } function make_hiddenapi() { diff --git a/test/etc/run-test-jar b/test/etc/run-test-jar index fa6ef54c935..e9127a8101a 100755 --- a/test/etc/run-test-jar +++ b/test/etc/run-test-jar @@ -792,6 +792,7 @@ dalvikvm_cmdline="$INVOKE_WITH $GDB $ANDROID_ROOT/bin/$DALVIKVM \ $DEBUGGER_OPTS \ $DALVIKVM_BOOT_OPT \ $TMP_DIR_OPTION \ + -XX:DumpNativeStackOnSigQuit:false \ -cp $DEX_LOCATION/$TEST_NAME.jar$SECONDARY_DEX $MAIN $ARGS" # Remove whitespace. @@ -811,6 +812,25 @@ if [ "x$RUN_TEST_ASAN_OPTIONS" != "x" ] ; then fi RUN_TEST_ASAN_OPTIONS="${RUN_TEST_ASAN_OPTIONS}detect_leaks=0" +# For running, we must turn off logging when dex2oat or patchoat are missing. Otherwise we use +# the same defaults as for prebuilt: everything when --dev, otherwise errors and above only. +if [ "$EXTERNAL_LOG_TAGS" = "n" ]; then + if [ "$DEV_MODE" = "y" ]; then + export ANDROID_LOG_TAGS='*:d' + elif [ "$USE_DEX2OAT_AND_PATCHOAT" = "n" ]; then + # All tests would log the error of failing dex2oat/patchoat. Be silent here and only + # log fatal events. + export ANDROID_LOG_TAGS='*:s' + elif [ "$HAVE_IMAGE" = "n" ]; then + # All tests would log the error of missing image. Be silent here and only log fatal + # events. + export ANDROID_LOG_TAGS='*:s' + else + # We are interested in LOG(ERROR) output. + export ANDROID_LOG_TAGS='*:e' + fi +fi + if [ "$HOST" = "n" ]; then adb root > /dev/null adb wait-for-device @@ -849,7 +869,12 @@ if [ "$HOST" = "n" ]; then fi # System libraries needed by libarttestd.so - PUBLIC_LIBS=libart.so:libartd.so:libc++.so:libbacktrace.so:libdexfile.so:libdexfiled.so:libbase.so:libnativehelper.so + PUBLIC_LIBS=libc++.so:libbacktrace.so:libbase.so:libnativehelper.so + if [ "$TEST_IS_NDEBUG" = "y" ]; then + PUBLIC_LIBS=$PUBLIC_LIBS:libart.so:libdexfile.so + else + PUBLIC_LIBS=$PUBLIC_LIBS:libartd.so:libdexfiled.so + fi # Create a script with the command. The command can get longer than the longest # allowed adb command and there is no way to get the exit status from a adb shell @@ -861,6 +886,7 @@ if [ "$HOST" = "n" ]; then export ANDROID_ADDITIONAL_PUBLIC_LIBRARIES=$PUBLIC_LIBS && \ export DEX_LOCATION=$DEX_LOCATION && \ export ANDROID_ROOT=$ANDROID_ROOT && \ + export ANDROID_LOG_TAGS=$ANDROID_LOG_TAGS && \ rm -rf ${DEX_LOCATION}/dalvik-cache/ && \ mkdir -p ${mkdir_locations} && \ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH && \ @@ -898,16 +924,6 @@ else # Host run. export ANDROID_PRINTF_LOG=brief - # By default, and for prebuild dex2oat, we are interested in errors being logged. In dev mode - # we want debug messages. - if [ "$EXTERNAL_LOG_TAGS" = "n" ]; then - if [ "$DEV_MODE" = "y" ]; then - export ANDROID_LOG_TAGS='*:d' - else - export ANDROID_LOG_TAGS='*:e' - fi - fi - export ANDROID_DATA="$DEX_LOCATION" export ANDROID_ROOT="${ANDROID_ROOT}" export LD_LIBRARY_PATH="${ANDROID_ROOT}/${LIBRARY_DIRECTORY}:${ANDROID_ROOT}/${TEST_DIRECTORY}" @@ -968,25 +984,6 @@ else $strip_cmdline || { echo "Strip failed." >&2 ; exit 3; } $sync_cmdline || { echo "Sync failed." >&2 ; exit 4; } - # For running, we must turn off logging when dex2oat or patchoat are missing. Otherwise we use - # the same defaults as for prebuilt: everything when --dev, otherwise errors and above only. - if [ "$EXTERNAL_LOG_TAGS" = "n" ]; then - if [ "$DEV_MODE" = "y" ]; then - export ANDROID_LOG_TAGS='*:d' - elif [ "$USE_DEX2OAT_AND_PATCHOAT" = "n" ]; then - # All tests would log the error of failing dex2oat/patchoat. Be silent here and only - # log fatal events. - export ANDROID_LOG_TAGS='*:s' - elif [ "$HAVE_IMAGE" = "n" ]; then - # All tests would log the error of missing image. Be silent here and only log fatal - # events. - export ANDROID_LOG_TAGS='*:s' - else - # We are interested in LOG(ERROR) output. - export ANDROID_LOG_TAGS='*:e' - fi - fi - if [ "$DRY_RUN" = "y" ]; then exit 0 fi diff --git a/test/knownfailures.json b/test/knownfailures.json index dd92843c528..6d8abe13b2c 100644 --- a/test/knownfailures.json +++ b/test/knownfailures.json @@ -1,11 +1,5 @@ [ { - "tests": "1934-jvmti-signal-thread", - "description": ["Disables 1934-jvmti-signal-thread in tracing configurations"], - "variant": "trace | stream", - "bug": "http://b/67384421" - }, - { "tests": "153-reference-stress", "description": ["Disable 153-reference-stress temporarily until a fix", "arrives."], @@ -282,12 +276,8 @@ }, { "tests": "596-app-images", - "variant": "npictest" - }, - { - "tests": "639-checker-code-sinking", - "variant": "pictest", - "bug": "http://b/65366606" + "description": "Code being tested has been disabled", + "bug": "b/70734839" }, { "tests": "055-enum-performance", @@ -350,8 +340,7 @@ { "tests": ["537-checker-arraycopy", "641-checker-arraycopy"], - "env_vars": {"ART_USE_READ_BARRIER": "true"}, - "variant": "interpreter | optimizing | regalloc_gc | jit" + "env_vars": {"ART_READ_BARRIER_TYPE": "TABLELOOKUP"} }, { "tests": ["476-clinit-inline-static-invoke", @@ -503,6 +492,7 @@ { "tests": [ "031-class-attributes", + "715-clinit-implicit-parameter-annotations", "911-get-stack-trace" ], "description": [ @@ -668,12 +658,6 @@ "description": ["Time out"] }, { - "tests": ["130-hprof"], - "env_vars": {"SANITIZE_HOST": "address"}, - "bug": "b/73060923", - "description": ["ASAN issue"] - }, - { "tests": ["1941-dispose-stress", "522-checker-regression-monitor-exit"], "variant": "jvm", "bug": "b/73888836", @@ -932,7 +916,6 @@ "946-obsolete-throw", "948-change-annotations", "950-redefine-intrinsic", - "952-invoke-custom", "954-invoke-polymorphic-verifier", "955-methodhandles-smali", "956-methodhandles", @@ -978,6 +961,13 @@ "description": ["Doesn't run on RI."] }, { + "tests": ["121-modifiers", + "1929-exception-catch-exception"], + "variant": "jvm", + "bug": "b/76399183", + "description": ["New failures to be investigated."] + }, + { "tests": ["616-cha-unloading"], "variant": "trace", "description": ["Trace prevents class unloading."] @@ -986,5 +976,11 @@ "tests": "677-fsi", "variant": "no-dex2oat | no-image | no-prebuild | relocate-npatchoat | jvm", "description": ["Test requires a successful dex2oat invocation"] + }, + { + "tests": ["990-field-trace", + "991-field-trace-2"], + "variant": "gcstress & debug & target", + "description": ["Test can time out on gcstress with debug"] } ] diff --git a/test/run-test b/test/run-test index 260a65a056f..5f85b0875bd 100755 --- a/test/run-test +++ b/test/run-test @@ -45,16 +45,23 @@ export JAVAC="javac -g -Xlint:-options" export RUN="${progdir}/etc/run-test-jar" export DEX_LOCATION=/data/run-test/${test_dir} export NEED_DEX="true" -export USE_D8="false" +export USE_D8="true" export USE_JACK="false" export USE_DESUGAR="true" export SMALI_ARGS="" +# If d8 was not set by the environment variable, assume it is in the path. +if [ -z "$D8" ]; then + export D8="d8" +fi + # If dx was not set by the environment variable, assume it is in the path. if [ -z "$DX" ]; then export DX="dx" fi +export DXMERGER="$D8" + # If jasmin was not set by the environment variable, assume it is in the path. if [ -z "$JASMIN" ]; then export JASMIN="jasmin" @@ -65,11 +72,6 @@ if [ -z "$SMALI" ]; then export SMALI="smali" fi -# If dexmerger was not set by the environment variable, assume it is in the path. -if [ -z "$DXMERGER" ]; then - export DXMERGER="dexmerger" -fi - # If jack was not set by the environment variable, assume it is in the path. if [ -z "$JACK" ]; then export JACK="jack" @@ -363,9 +365,6 @@ while true; do elif [ "x$1" = "x--build-only" ]; then build_only="yes" shift - elif [ "x$1" = "x--build-with-d8" ]; then - USE_D8="true" - shift elif [ "x$1" = "x--build-with-javac-dx" ]; then USE_JACK="false" shift diff --git a/test/testrunner/env.py b/test/testrunner/env.py index 55569629eac..7564f5a6b4a 100644 --- a/test/testrunner/env.py +++ b/test/testrunner/env.py @@ -17,6 +17,16 @@ import re import tempfile import subprocess +# begin import $ANDROID_BUILD_TOP/art/tools/build/var_cache.py +_THIS_DIR = os.path.dirname(os.path.realpath(__file__)) +_TOP = os.path.join(_THIS_DIR, "../../..") +_VAR_CACHE_DIR = os.path.join(_TOP, "art/tools/build/") + +import sys +sys.path.append(_VAR_CACHE_DIR) +import var_cache +# end import var_cache.py + _env = dict(os.environ) def _getEnvBoolean(var, default): @@ -28,55 +38,8 @@ def _getEnvBoolean(var, default): return False return default -_DUMP_MANY_VARS_LIST = ['HOST_2ND_ARCH_PREFIX', - 'TARGET_2ND_ARCH', - 'TARGET_ARCH', - 'HOST_PREFER_32_BIT', - 'HOST_OUT_EXECUTABLES', - 'ANDROID_JAVA_TOOLCHAIN', - 'ANDROID_COMPILE_WITH_JACK', - 'USE_D8_BY_DEFAULT'] -_DUMP_MANY_VARS = None # To be set to a dictionary with above list being the keys, - # and the build variable being the value. -def _dump_many_vars(var_name): - """ - Reach into the Android build system to dump many build vars simultaneously. - Since the make system is so slow, we want to avoid calling into build frequently. - """ - global _DUMP_MANY_VARS - global _DUMP_MANY_VARS_LIST - - # Look up var from cache. - if _DUMP_MANY_VARS: - return _DUMP_MANY_VARS[var_name] - - all_vars=" ".join(_DUMP_MANY_VARS_LIST) - - # The command is taken from build/envsetup.sh to fetch build variables. - command = ("build/soong/soong_ui.bash --dumpvars-mode --vars=\"%s\"") % (all_vars) - - config = subprocess.Popen(command, - stdout=subprocess.PIPE, - universal_newlines=True, - shell=True, - cwd=ANDROID_BUILD_TOP).communicate()[0] # read until EOF, select stdin - # Prints out something like: - # TARGET_ARCH='arm64' - # HOST_ARCH='x86_64' - _DUMP_MANY_VARS = {} - for line in config.split("\n"): - # Split out "$key='$value'" via regex. - match = re.search("([^=]+)='([^']*)", line) - if not match: - continue - key = match.group(1) - value = match.group(2) - _DUMP_MANY_VARS[key] = value - - return _DUMP_MANY_VARS[var_name] - def _get_build_var(var_name): - return _dump_many_vars(var_name) + return var_cache.get_build_var(var_name) def _get_build_var_boolean(var, default): val = _get_build_var(var) @@ -108,9 +71,6 @@ ANDROID_BUILD_TOP = _get_android_build_top() # Compiling with jack? Possible values in (True, False, 'default') ANDROID_COMPILE_WITH_JACK = _get_build_var_boolean('ANDROID_COMPILE_WITH_JACK', 'default') -# Follow the build system's D8 usage. -USE_D8_BY_DEFAULT = _get_build_var_boolean('USE_D8_BY_DEFAULT', False) - # Directory used for temporary test files on the host. ART_HOST_TEST_DIR = tempfile.mkdtemp(prefix = 'test-art-') @@ -173,9 +133,8 @@ HOST_OUT_EXECUTABLES = os.path.join(ANDROID_BUILD_TOP, _get_build_var("HOST_OUT_EXECUTABLES")) # Set up default values for $JACK, $DX, $SMALI, etc to the $HOST_OUT_EXECUTABLES/$name path. -for tool in ['jack', 'dx', 'smali', 'jasmin', 'dxmerger']: - binary = tool if tool != 'dxmerger' else 'dexmerger' - os.environ.setdefault(tool.upper(), HOST_OUT_EXECUTABLES + '/' + binary) +for tool in ['jack', 'dx', 'smali', 'jasmin', 'd8']: + os.environ.setdefault(tool.upper(), HOST_OUT_EXECUTABLES + '/' + tool) ANDROID_JAVA_TOOLCHAIN = os.path.join(ANDROID_BUILD_TOP, _get_build_var('ANDROID_JAVA_TOOLCHAIN')) diff --git a/test/testrunner/run_build_test_target.py b/test/testrunner/run_build_test_target.py index fcc5505a954..e0ccc3e14c0 100755 --- a/test/testrunner/run_build_test_target.py +++ b/test/testrunner/run_build_test_target.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Copyright 2017, The Android Open Source Project # @@ -45,9 +45,9 @@ options = parser.parse_args() ########## if options.list: - print "List of all known build_target: " - for k in sorted(target_config.iterkeys()): - print " * " + k + print("List of all known build_target: ") + for k in sorted(target_config.keys()): + print(" * " + k) # TODO: would be nice if this was the same order as the target config file. sys.exit(1) @@ -59,10 +59,10 @@ target = target_config[options.build_target] n_threads = options.n_threads custom_env = target.get('env', {}) custom_env['SOONG_ALLOW_MISSING_DEPENDENCIES'] = 'true' -print custom_env +print(custom_env) os.environ.update(custom_env) -if target.has_key('make'): +if 'make' in target: build_command = 'make' build_command += ' DX=' build_command += ' -j' + str(n_threads) @@ -75,7 +75,7 @@ if target.has_key('make'): if subprocess.call(build_command.split()): sys.exit(1) -if target.has_key('golem'): +if 'golem' in target: machine_type = target.get('golem') # use art-opt-cc by default since it mimics the default preopt config. default_golem_config = 'art-opt-cc' @@ -93,7 +93,7 @@ if target.has_key('golem'): if subprocess.call(cmd): sys.exit(1) -if target.has_key('run-test'): +if 'run-test' in target: run_test_command = [os.path.join(env.ANDROID_BUILD_TOP, 'art/test/testrunner/testrunner.py')] test_flags = target.get('run-test', []) diff --git a/test/testrunner/target_config.py b/test/testrunner/target_config.py index 9d0377510a0..e0757abbe0a 100644 --- a/test/testrunner/target_config.py +++ b/test/testrunner/target_config.py @@ -24,10 +24,7 @@ target_config = { 'art-test' : { 'make' : 'test-art-host-gtest', - 'run-test' : [], - 'env' : { - 'ART_USE_READ_BARRIER' : 'true' - } + 'run-test' : [] }, 'art-test-javac' : { @@ -38,82 +35,74 @@ target_config = { # (calls testrunner which builds and then runs the test targets) 'art-ndebug' : { - 'run-test' : ['--ndebug'], - 'env' : { - 'ART_USE_READ_BARRIER' : 'true' - } + 'run-test' : ['--ndebug'] }, 'art-interpreter' : { - 'run-test' : ['--interpreter'], - 'env' : { - 'ART_USE_READ_BARRIER' : 'true' - } + 'run-test' : ['--interpreter'] }, 'art-interpreter-access-checks' : { - 'run-test' : ['--interp-ac'], - 'env' : { - 'ART_USE_READ_BARRIER' : 'true' - } + 'run-test' : ['--interp-ac'] }, 'art-jit' : { - 'run-test' : ['--jit'], - 'env' : { - 'ART_USE_READ_BARRIER' : 'true' - } + 'run-test' : ['--jit'] + }, + 'art-jit-on-first-use' : { + 'run-test' : ['--jit', + '--runtime-option=-Xjitthreshold:0'] }, 'art-pictest' : { 'run-test' : ['--pictest', - '--optimizing'], - 'env' : { - 'ART_USE_READ_BARRIER' : 'true' - } + '--optimizing'] }, 'art-gcstress-gcverify': { - 'run-test': ['--gcstress', - '--gcverify'], - 'env' : { - 'ART_USE_READ_BARRIER' : 'false', - 'ART_DEFAULT_GC_TYPE' : 'SS' - } - }, + # Do not exercise '--interpreter', '--optimizing', nor '--jit' in this + # configuration, as they are covered by the 'art-interpreter-gcstress', + # 'art-optimizing-gcstress' and 'art-jit-gcstress' configurations below. + 'run-test': ['--interp-ac', + '--speed-profile', + '--gcstress', + '--gcverify'] + }, + # Rename this configuration as 'art-interpreter-gcstress-gcverify' (b/62611253). 'art-interpreter-gcstress' : { 'run-test' : ['--interpreter', - '--gcstress'], - 'env' : { - 'ART_USE_READ_BARRIER' : 'false', - 'ART_DEFAULT_GC_TYPE' : 'SS' - } + '--gcstress', + '--gcverify'] }, + # Rename this configuration as 'art-optimizing-gcstress-gcverify' (b/62611253). 'art-optimizing-gcstress' : { - 'run-test' : ['--gcstress', - '--optimizing'], - 'env' : { - 'ART_USE_READ_BARRIER' : 'false', - 'ART_DEFAULT_GC_TYPE' : 'SS' - } + 'run-test' : ['--optimizing', + '--gcstress', + '--gcverify'] }, + # Rename this configuration as 'art-jit-gcstress-gcverify' (b/62611253). 'art-jit-gcstress' : { 'run-test' : ['--jit', - '--gcstress'], - 'env' : { - 'ART_USE_READ_BARRIER' : 'false', - 'ART_DEFAULT_GC_TYPE' : 'SS' - } + '--gcstress', + '--gcverify'] }, + 'art-jit-on-first-use-gcstress' : { + 'run-test' : ['--jit', + '--gcstress', + '--runtime-option=-Xjitthreshold:0'] + }, + # TODO: Rename or repurpose this configuration as + # 'art-read-barrier-heap-poisoning' (b/62611253). 'art-read-barrier' : { 'run-test': ['--interpreter', '--optimizing'], 'env' : { - 'ART_USE_READ_BARRIER' : 'true', 'ART_HEAP_POISONING' : 'true' } }, + # TODO: Remove or disable this configuration, as it is now covered + # by 'art-interpreter-gcstress' and 'art-optimizing-gcstress' -- + # except for heap poisoning, but that's fine (b/62611253). 'art-read-barrier-gcstress' : { 'run-test' : ['--interpreter', '--optimizing', '--gcstress'], 'env' : { - 'ART_USE_READ_BARRIER' : 'true', 'ART_HEAP_POISONING' : 'true' } }, @@ -121,7 +110,6 @@ target_config = { 'run-test' : ['--interpreter', '--optimizing'], 'env' : { - 'ART_USE_READ_BARRIER' : 'true', 'ART_READ_BARRIER_TYPE' : 'TABLELOOKUP', 'ART_HEAP_POISONING' : 'true' } @@ -173,54 +161,30 @@ target_config = { } }, 'art-tracing' : { - 'run-test' : ['--trace'], - 'env' : { - 'ART_USE_READ_BARRIER' : 'true' - } + 'run-test' : ['--trace'] }, 'art-interpreter-tracing' : { 'run-test' : ['--interpreter', - '--trace'], - 'env' : { - 'ART_USE_READ_BARRIER' : 'true', - } + '--trace'] }, 'art-forcecopy' : { - 'run-test' : ['--forcecopy'], - 'env' : { - 'ART_USE_READ_BARRIER' : 'true', - } + 'run-test' : ['--forcecopy'] }, 'art-no-prebuild' : { - 'run-test' : ['--no-prebuild'], - 'env' : { - 'ART_USE_READ_BARRIER' : 'true', - } + 'run-test' : ['--no-prebuild'] }, 'art-no-image' : { - 'run-test' : ['--no-image'], - 'env' : { - 'ART_USE_READ_BARRIER' : 'true', - } + 'run-test' : ['--no-image'] }, 'art-interpreter-no-image' : { 'run-test' : ['--interpreter', - '--no-image'], - 'env' : { - 'ART_USE_READ_BARRIER' : 'true', - } + '--no-image'] }, 'art-relocate-no-patchoat' : { - 'run-test' : ['--relocate-npatchoat'], - 'env' : { - 'ART_USE_READ_BARRIER' : 'true', - } + 'run-test' : ['--relocate-npatchoat'] }, 'art-no-dex2oat' : { - 'run-test' : ['--no-dex2oat'], - 'env' : { - 'ART_USE_READ_BARRIER' : 'true', - } + 'run-test' : ['--no-dex2oat'] }, 'art-heap-poisoning' : { 'run-test' : ['--interpreter', @@ -240,32 +204,24 @@ target_config = { 'run-test' : ['--pictest', '--prebuild', '--relocate', - '--jit'], - 'env' : { - 'ART_USE_READ_BARRIER' : 'true' - } + '--jit'] }, # ART gtest configurations # (calls make 'target' which builds and then runs the gtests). 'art-gtest' : { - 'make' : 'test-art-host-gtest', - 'env' : { - 'ART_USE_READ_BARRIER' : 'true' - } + 'make' : 'test-art-host-gtest' }, 'art-gtest-read-barrier': { 'make' : 'test-art-host-gtest', 'env' : { - 'ART_USE_READ_BARRIER' : 'true', 'ART_HEAP_POISONING' : 'true' } }, 'art-gtest-read-barrier-table-lookup': { 'make' : 'test-art-host-gtest', 'env': { - 'ART_USE_READ_BARRIER' : 'true', 'ART_READ_BARRIER_TYPE' : 'TABLELOOKUP', 'ART_HEAP_POISONING' : 'true' } diff --git a/test/testrunner/testrunner.py b/test/testrunner/testrunner.py index 3974ccb4f91..88b509d3b70 100755 --- a/test/testrunner/testrunner.py +++ b/test/testrunner/testrunner.py @@ -114,6 +114,7 @@ ignore_skips = False build = False gdb = False gdb_arg = '' +runtime_option = '' stop_testrunner = False dex2oat_jobs = -1 # -1 corresponds to default threads for dex2oat run_all_configs = False @@ -173,56 +174,37 @@ def setup_test_env(): global _user_input_variants global run_all_configs + # These are the default variant-options we will use if nothing in the group is specified. + default_variants = { + 'target': {'host', 'target'}, + 'pictest': {'npictest'}, + 'prebuild': {'prebuild'}, + 'cdex_level': {'cdex-fast'}, + 'jvmti': { 'no-jvmti'}, + 'compiler': {'optimizing', + 'jit', + 'interpreter', + 'interp-ac', + 'speed-profile'}, + 'relocate': {'no-relocate'}, + 'trace': {'ntrace'}, + 'gc': {'cms'}, + 'jni': {'checkjni'}, + 'image': {'picimage'}, + 'pictest': {'pictest'}, + 'debuggable': {'ndebuggable'}, + 'run': {'debug'}, + # address_sizes_target depends on the target so it is dealt with below. + } + # We want to pull these early since the full VARIANT_TYPE_DICT has a few additional ones we don't + # want to pick up if we pass --all. + default_variants_keys = default_variants.keys() if run_all_configs: - target_types = _user_input_variants['target'] - _user_input_variants = VARIANT_TYPE_DICT - _user_input_variants['target'] = target_types - - if not _user_input_variants['target']: - _user_input_variants['target'].add('host') - _user_input_variants['target'].add('target') - - if not _user_input_variants['prebuild']: # Default - _user_input_variants['prebuild'].add('prebuild') - - if not _user_input_variants['cdex_level']: # Default - _user_input_variants['cdex_level'].add('cdex-fast') - - # By default only run without jvmti - if not _user_input_variants['jvmti']: - _user_input_variants['jvmti'].add('no-jvmti') - - # By default we run all 'compiler' variants. - if not _user_input_variants['compiler'] and _user_input_variants['target'] != 'jvm': - _user_input_variants['compiler'].add('optimizing') - _user_input_variants['compiler'].add('jit') - _user_input_variants['compiler'].add('interpreter') - _user_input_variants['compiler'].add('interp-ac') - _user_input_variants['compiler'].add('speed-profile') - - if not _user_input_variants['relocate']: # Default - _user_input_variants['relocate'].add('no-relocate') - - if not _user_input_variants['trace']: # Default - _user_input_variants['trace'].add('ntrace') + default_variants = VARIANT_TYPE_DICT - if not _user_input_variants['gc']: # Default - _user_input_variants['gc'].add('cms') - - if not _user_input_variants['jni']: # Default - _user_input_variants['jni'].add('checkjni') - - if not _user_input_variants['image']: # Default - _user_input_variants['image'].add('picimage') - - if not _user_input_variants['pictest']: # Default - _user_input_variants['pictest'].add('npictest') - - if not _user_input_variants['debuggable']: # Default - _user_input_variants['debuggable'].add('ndebuggable') - - if not _user_input_variants['run']: # Default - _user_input_variants['run'].add('debug') + for key in default_variants_keys: + if not _user_input_variants[key]: + _user_input_variants[key] = default_variants[key] _user_input_variants['address_sizes_target'] = collections.defaultdict(set) if not _user_input_variants['address_sizes']: @@ -346,6 +328,10 @@ def run_tests(tests): if gdb_arg: options_all += ' --gdb-arg ' + gdb_arg + if runtime_option: + for opt in runtime_option: + options_all += ' --runtime-option ' + opt + if dex2oat_jobs != -1: options_all += ' --dex2oat-jobs ' + str(dex2oat_jobs) @@ -499,9 +485,6 @@ def run_tests(tests): elif env.ANDROID_COMPILE_WITH_JACK == False: options_test += ' --build-with-javac-dx' - if env.USE_D8_BY_DEFAULT == True: - options_test += ' --build-with-d8' - # TODO(http://36039166): This is a temporary solution to # fix build breakages. options_test = (' --output-path %s') % ( @@ -921,39 +904,51 @@ def parse_option(): global build global gdb global gdb_arg + global runtime_option global timeout global dex2oat_jobs global run_all_configs parser = argparse.ArgumentParser(description="Runs all or a subset of the ART test suite.") parser.add_argument('-t', '--test', action='append', dest='tests', help='name(s) of the test(s)') - parser.add_argument('-j', type=int, dest='n_thread') - parser.add_argument('--timeout', default=timeout, type=int, dest='timeout') - for variant in TOTAL_VARIANTS_SET: - flag = '--' + variant - parser.add_argument(flag, action='store_true', dest=variant) - parser.add_argument('--verbose', '-v', action='store_true', dest='verbose') - parser.add_argument('--dry-run', action='store_true', dest='dry_run') - parser.add_argument("--skip", action="append", dest="skips", default=[], - help="Skip the given test in all circumstances.") - parser.add_argument("--no-skips", dest="ignore_skips", action="store_true", default=False, - help="Don't skip any run-test configurations listed in knownfailures.json.") - parser.add_argument('--no-build-dependencies', - action='store_false', dest='build', - help="Don't build dependencies under any circumstances. This is the " + - "behavior if ART_TEST_RUN_TEST_ALWAYS_BUILD is not set to 'true'.") - parser.add_argument('-b', '--build-dependencies', - action='store_true', dest='build', - help="Build dependencies under all circumstances. By default we will " + - "not build dependencies unless ART_TEST_RUN_TEST_BUILD=true.") - parser.add_argument('--build-target', dest='build_target', help='master-art-host targets') - parser.set_defaults(build = env.ART_TEST_RUN_TEST_BUILD) - parser.add_argument('--gdb', action='store_true', dest='gdb') - parser.add_argument('--gdb-arg', dest='gdb_arg') - parser.add_argument('--dex2oat-jobs', type=int, dest='dex2oat_jobs', - help='Number of dex2oat jobs') - parser.add_argument('-a', '--all', action='store_true', dest='run_all', - help="Run all the possible configurations for the input test set") + global_group = parser.add_argument_group('Global options', + 'Options that affect all tests being run') + global_group.add_argument('-j', type=int, dest='n_thread') + global_group.add_argument('--timeout', default=timeout, type=int, dest='timeout') + global_group.add_argument('--verbose', '-v', action='store_true', dest='verbose') + global_group.add_argument('--dry-run', action='store_true', dest='dry_run') + global_group.add_argument("--skip", action='append', dest="skips", default=[], + help="Skip the given test in all circumstances.") + global_group.add_argument("--no-skips", dest="ignore_skips", action='store_true', default=False, + help="""Don't skip any run-test configurations listed in + knownfailures.json.""") + global_group.add_argument('--no-build-dependencies', + action='store_false', dest='build', + help="""Don't build dependencies under any circumstances. This is the + behavior if ART_TEST_RUN_TEST_ALWAYS_BUILD is not set to 'true'.""") + global_group.add_argument('-b', '--build-dependencies', + action='store_true', dest='build', + help="""Build dependencies under all circumstances. By default we will + not build dependencies unless ART_TEST_RUN_TEST_BUILD=true.""") + global_group.add_argument('--build-target', dest='build_target', help='master-art-host targets') + global_group.set_defaults(build = env.ART_TEST_RUN_TEST_BUILD) + global_group.add_argument('--gdb', action='store_true', dest='gdb') + global_group.add_argument('--gdb-arg', dest='gdb_arg') + global_group.add_argument('--runtime-option', action='append', dest='runtime_option', + help="""Pass an option to the runtime. Runtime options + starting with a '-' must be separated by a '=', for + example '--runtime-option=-Xjitthreshold:0'.""") + global_group.add_argument('--dex2oat-jobs', type=int, dest='dex2oat_jobs', + help='Number of dex2oat jobs') + global_group.add_argument('-a', '--all', action='store_true', dest='run_all', + help="Run all the possible configurations for the input test set") + for variant_type, variant_set in VARIANT_TYPE_DICT.items(): + var_group = parser.add_argument_group( + '{}-type Options'.format(variant_type), + "Options that control the '{}' variants.".format(variant_type)) + for variant in variant_set: + flag = '--' + variant + var_group.add_argument(flag, action='store_true', dest=variant) options = vars(parser.parse_args()) if options['build_target']: @@ -986,6 +981,7 @@ def parse_option(): gdb = True if options['gdb_arg']: gdb_arg = options['gdb_arg'] + runtime_option = options['runtime_option']; timeout = options['timeout'] if options['dex2oat_jobs']: dex2oat_jobs = options['dex2oat_jobs'] diff --git a/tools/ahat/Android.bp b/tools/ahat/Android.bp new file mode 100644 index 00000000000..dc9f098ff34 --- /dev/null +++ b/tools/ahat/Android.bp @@ -0,0 +1,25 @@ +// Copyright 2018 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +droiddoc_host { + name: "ahat-docs", + srcs: [ + "src/main/**/*.java", + ], + custom_template: "droiddoc-templates-sdk", + args: "-stubpackages com.android.ahat:com.android.ahat.*", + api_tag_name: "AHAT", + api_filename: "ahat_api.txt", + removed_api_filename: "ahat_removed_api.txt", +} diff --git a/tools/ahat/Android.mk b/tools/ahat/Android.mk index bf79751659a..ad33233159f 100644 --- a/tools/ahat/Android.mk +++ b/tools/ahat/Android.mk @@ -37,23 +37,10 @@ LOCAL_COMPATIBILITY_SUITE := general-tests include $(BUILD_HOST_JAVA_LIBRARY) AHAT_JAR := $(LOCAL_BUILT_MODULE) -AHAT_API := $(intermediates.COMMON)/ahat_api.txt -AHAT_REMOVED_API := $(intermediates.COMMON)/ahat_removed_api.txt # --- api check for ahat.jar ---------- -include $(CLEAR_VARS) -LOCAL_SRC_FILES := $(call all-java-files-under, src/main) -LOCAL_IS_HOST_MODULE := true -LOCAL_MODULE_TAGS := optional -LOCAL_MODULE_CLASS := JAVA_LIBRARIES -LOCAL_MODULE := ahat -LOCAL_DROIDDOC_OPTIONS := \ - -stubpackages com.android.ahat:com.android.ahat.* \ - -api $(AHAT_API) \ - -removedApi $(AHAT_REMOVED_API) -LOCAL_DROIDDOC_CUSTOM_TEMPLATE_DIR := external/doclava/res/assets/templates-sdk -include $(BUILD_DROIDDOC) -$(AHAT_API): $(full_target) +AHAT_API := $(INTERNAL_PLATFORM_AHAT_API_FILE) +AHAT_REMOVED_API := $(INTERNAL_PLATFORM_AHAT_REMOVED_API_FILE) $(eval $(call check-api, \ ahat-check-api, \ diff --git a/tools/bisection_search/bisection_search.py b/tools/bisection_search/bisection_search.py index 27bd599aaaf..250b5d178ac 100755 --- a/tools/bisection_search/bisection_search.py +++ b/tools/bisection_search/bisection_search.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.4 +#!/usr/bin/env python3 # # Copyright (C) 2016 The Android Open Source Project # @@ -166,10 +166,6 @@ class Dex2OatWrapperTestable(object): """Prepare command to run.""" cmd = self._base_cmd[0:self._arguments_position] # insert additional arguments before the first argument - if compiled_methods is not None: - self._test_env.WriteLines(self._compiled_methods_path, compiled_methods) - cmd += ['-Xcompiler-option', '--compiled-methods={0}'.format( - self._compiled_methods_path)] if passes_to_run is not None: self._test_env.WriteLines(self._passes_to_run_path, passes_to_run) cmd += ['-Xcompiler-option', '--run-passes={0}'.format( diff --git a/tools/bisection_search/bisection_test.py b/tools/bisection_search/bisection_test.py index 9aa08fb5b12..b6a73c02679 100755 --- a/tools/bisection_search/bisection_test.py +++ b/tools/bisection_search/bisection_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.4 +#!/usr/bin/env python3 # # Copyright (C) 2016 The Android Open Source Project # diff --git a/tools/bootjars.sh b/tools/bootjars.sh index f710de99bba..dca209d5803 100755 --- a/tools/bootjars.sh +++ b/tools/bootjars.sh @@ -21,7 +21,7 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" TOP="$DIR/../.." -source "${TOP}/build/envsetup.sh" >&/dev/null # import get_build_var +source "${TOP}/art/tools/build/var_cache.sh" >&/dev/null # import get_build_var selected_env_var= core_jars_only=n diff --git a/tools/build/var_cache.py b/tools/build/var_cache.py new file mode 100644 index 00000000000..9e616faafc9 --- /dev/null +++ b/tools/build/var_cache.py @@ -0,0 +1,148 @@ +# Copyright 2018, The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# !!! Keep up-to-date with var_cache.sh +# + +# +# Provide a soong-build variable query mechanism that is cached +# in the current process and any other subchild process that knows +# how to parse the exported variable: +# +# export ART_TOOLS_BUILD_VAR_CACHE="..." +# +# Of the format: +# +# <key1>='<value1>'\n +# <key2>='<value2>'\n +# ... +# <keyN>='<valueN>' +# +# Note: This is intentionally the same output format as +# build/soong/soong_ui.bash --dumpvars-mode --vars "key1 key2 ... keyN" +# +# For example, this would be a valid var-cache: +# +# export ART_TOOLS_BUILD_VAR_CACHE="TARGET_CORE_JARS='core-oj core-libart' +# HOST_CORE_JARS='core-oj-hostdex core-libart-hostdex'" +# +# Calling into soong repeatedly is very slow; whenever it needs to be done +# more than once, the var_cache.py or var_cache.sh script should be used instead. +# + +import os +import subprocess +import sys + +def get_build_var(name): + """ + Query soong build for a variable value and return it as a string. + + Var lookup is cached, subsequent var lookups in any child process + (that includes a var-cache is free). The var must be in 'var_list' + to participate in the cache. + + Example: + host_out = var_cache.get_build_var('HOST_OUT') + + Note that build vars can often have spaces in them, + so the caller must take care to ensure space-correctness. + + Raises KeyError if the variable name is not in 'var_list'. + """ + _populate() + _build_dict() + + value = _var_cache_dict.get(name) + if value is None: + _debug(_var_cache_dict) + raise KeyError("The variable '%s' is not in 'var_list', can't lookup" %(name)) + + return value + +_var_cache_dict = None +_THIS_DIR = os.path.dirname(os.path.realpath(__file__)) +_TOP = os.path.join(_THIS_DIR, "../../..") +_VAR_LIST_PATH = os.path.join(_THIS_DIR, "var_list") +_SOONG_UI_SCRIPT = os.path.join(_TOP, "build/soong/soong_ui.bash") +_DEBUG = False + +def _populate(): + if os.environ.get('ART_TOOLS_BUILD_VAR_CACHE'): + return + + _debug("Varcache missing (PY)... repopulate") + + interesting_vars=[] + with open(_VAR_LIST_PATH) as f: + for line in f.readlines(): + line = line.strip() + if not line or line.startswith('#'): + continue + + _debug(line) + + interesting_vars.append(line) + + _debug("Interesting vars: ", interesting_vars) + + # Invoke soong exactly once for optimal performance. + var_values = subprocess.check_output([ + _SOONG_UI_SCRIPT, '--dumpvars-mode', '-vars', " ".join(interesting_vars)], + cwd=_TOP) + + # Export the ART_TOOLS_BUILD_VAR_CACHE in the same format as soong_ui.bash --dumpvars-mode. + os.environb[b'ART_TOOLS_BUILD_VAR_CACHE'] = var_values + + _debug("Soong output: ", var_values) + +def _build_dict(): + global _var_cache_dict + + if _var_cache_dict: + return + + _debug("_var_cache_build_dict()") + + _var_cache_dict = {} + + # Parse $ART_TOOLS_BUILD_VAR_CACHE, e.g. + # TARGET_CORE_JARS='core-oj core-libart conscrypt okhttp bouncycastle apache-xml' + # HOST_CORE_JARS='core-oj-hostdex core-libart-hostdex ...' + + for line in os.environ['ART_TOOLS_BUILD_VAR_CACHE'].splitlines(): + _debug(line) + var_name, var_value = line.split("=") + var_value = var_value.strip("'") + + _debug("Var name =", var_name) + _debug("Var value =", var_value) + + _var_cache_dict[var_name] = var_value + + _debug("Num entries in dict: ", len(_var_cache_dict)) + +def _debug(*args): + if _DEBUG: + print(*args, file=sys.stderr) + +# Below definitions are for interactive use only, e.g. +# python -c 'import var_cache; var_cache._dump_cache()' + +def _dump_cache(): + _populate() + print(os.environ['ART_TOOLS_BUILD_VAR_CACHE']) + +#get_build_var("xyz") diff --git a/tools/build/var_cache.sh b/tools/build/var_cache.sh new file mode 100755 index 00000000000..26e9770f956 --- /dev/null +++ b/tools/build/var_cache.sh @@ -0,0 +1,195 @@ +#!/bin/bash +# +# Copyright (C) 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# !!! Keep up-to-date with var_cache.py +# + +# +# Provide a soong-build variable query mechanism that is cached +# in the current process and any other subchild process that knows +# how to parse the exported variable: +# +# export ART_TOOLS_BUILD_VAR_CACHE="..." +# +# Of the format: +# +# <key1>='<value1>'\n +# <key2>='<value2>'\n +# ... +# <keyN>='<valueN>' +# +# Note: This is intentionally the same output format as +# build/soong/soong_ui.bash --dumpvars-mode --vars "key1 key2 ... keyN" +# +# For example, this would be a valid var-cache: +# +# export ART_TOOLS_BUILD_VAR_CACHE="TARGET_CORE_JARS='core-oj core-libart' +# HOST_CORE_JARS='core-oj-hostdex core-libart-hostdex'" +# +# Calling into soong repeatedly is very slow; whenever it needs to be done +# more than once, the var_cache.py or var_cache.sh script should be used instead. +# + +# ------------------------------------------------------- + +# Echoes the result of get_build_var <var_name>. +# Var lookup is cached, subsequent var lookups in any child process +# (that includes a var-cache is free). The var must be in 'var_list' +# to participate in the cache. +# +# Example: +# local host_out="$(get_build_var HOST_OUT)" +# +# Note that build vars can often have spaces in them, +# so the caller must take care to ensure space-correctness. +get_build_var() { + local var_name="$1" + + _var_cache_populate + _var_cache_build_dict + + if [[ ${_VAR_CACHE_DICT[$var_name]+exists} ]]; then + echo "${_VAR_CACHE_DICT[$var_name]}" + return 0 + else + echo "[ERROR] get_build_var: The variable '$var_name' is not in 'var_list', can't lookup." >&2 + return 1 + fi +} + +# The above functions are "public" and are intentionally not exported. +# User scripts must have "source var_cache.sh" to take advantage of caching. + +# ------------------------------------------------------- +# Below functions are "private"; +# do not call them outside of this file. + +_var_cache_populate() { + if [[ -n $ART_TOOLS_BUILD_VAR_CACHE ]]; then + _var_cache_debug "ART_TOOLS_BUILD_VAR_CACHE preset to (quotes added)..." + _var_cache_debug \""$ART_TOOLS_BUILD_VAR_CACHE"\" + return 0 + fi + + _var_cache_debug "Varcache missing... repopulate" + + local this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + local top="$this_dir/../../.." + + local interesting_vars=() + while read -r line; do + if [[ -z $line ]] || [[ $line == '#'* ]]; then + continue; + fi + interesting_vars+=($line) + done < "$this_dir"/var_list + + _var_cache_debug "Interesting vars: " ${interesting_vars[@]} + + local flat_vars="${interesting_vars[*]}" + + local var_values + _var_cache_show_command "$top"/build/soong/soong_ui.bash --dumpvars-mode -vars=\"${interesting_vars[*]}\" + + # Invoke soong exactly once for optimal performance. + # soong_ui.bash must be invoked from $ANDROID_BUILD_TOP or it gets confused and breaks. + var_values="$(cd "$top" && "$top"/build/soong/soong_ui.bash --dumpvars-mode -vars="$flat_vars")" + + # Export the ART_TOOLS_BUILD_VAR_CACHE in the same format as soong_ui.bash --dumpvars-mode. + export ART_TOOLS_BUILD_VAR_CACHE="$var_values" + + _var_cache_debug ART_TOOLS_BUILD_VAR_CACHE=\"$var_values\" +} + +_var_cache_build_dict() { + if [[ ${#_VAR_CACHE_DICT[@]} -ne 0 ]]; then + # Associative arrays cannot be exported, have + # a separate step to reconstruct the associative + # array from a flat variable. + return 0 + fi + + # Parse $ART_TOOLS_BUILD_VAR_CACHE, e.g. + # TARGET_CORE_JARS='core-oj core-libart conscrypt okhttp bouncycastle apache-xml' + # HOST_CORE_JARS='core-oj-hostdex core-libart-hostdex ...' + + local var_name + local var_value + local strip_quotes + + _var_cache_debug "_var_cache_build_dict()" + + declare -g -A _VAR_CACHE_DICT # global associative array. + while IFS='=' read -r var_name var_value; do + if [[ -z $var_name ]]; then + # skip empty lines, e.g. blank newline at the end + continue + fi + _var_cache_debug "Var_name was $var_name" + _var_cache_debug "Var_value was $var_value" + strip_quotes=${var_value//\'/} + _VAR_CACHE_DICT["$var_name"]="$strip_quotes" + done < <(echo "$ART_TOOLS_BUILD_VAR_CACHE") + + _var_cache_debug ${#_VAR_CACHE_DICT[@]} -eq 0 +} + +_var_cache_debug() { + if ((_var_cache_debug_enabled)); then + echo "[DBG]: " "$@" >&2 + fi +} + +_var_cache_show_command() { + if (( _var_cache_show_commands || _var_cache_debug_enabled)); then + echo "$@" >&2 + fi +} + +while true; do + case $1 in + --help) + echo "Usage: $0 [--debug] [--show-commands] [--dump-cache] [--var <name>] [--var <name2>...]" + echo "" + echo "Exposes a function 'get_build_var' which returns the result of" + echo "a soong build variable." + echo "" + echo "Primarily intended to be used as 'source var_cache.sh'," + echo "but also allows interactive command line usage for simplifying development." + exit 0 + ;; + --var) + echo -ne "$2=" + get_build_var "$2" + shift + ;; + --debug) + _var_cache_debug_enabled=1 + ;; + --show-commands) + _var_cache_show_commands=1 + ;; + --dump-cache) + _var_cache_populate + echo "ART_TOOLS_BUILD_VAR_CACHE=\"$ART_TOOLS_BUILD_VAR_CACHE\"" + ;; + *) + break + ;; + esac + shift +done diff --git a/tools/build/var_list b/tools/build/var_list new file mode 100644 index 00000000000..adcb066f7c6 --- /dev/null +++ b/tools/build/var_list @@ -0,0 +1,37 @@ +# +# Copyright (C) 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# This file contains a list of all the build vars that need to be eagerly cached +# by the var_cache.sh and var_cache.py scripts. +# Lines starting with '#' or blank lines are ignored. +# + +# javac-helper.sh +TARGET_CORE_JARS +PRODUCT_BOOT_JARS +TARGET_OUT_COMMON_INTERMEDIATES +HOST_CORE_JARS +HOST_OUT_COMMON_INTERMEDIATES + +# testrunner/env.py +HOST_2ND_ARCH_PREFIX +TARGET_2ND_ARCH +TARGET_ARCH +HOST_PREFER_32_BIT +HOST_OUT_EXECUTABLES +ANDROID_JAVA_TOOLCHAIN +ANDROID_COMPILE_WITH_JACK + diff --git a/tools/buildbot-build.sh b/tools/buildbot-build.sh index a20175531d9..e447ab4cf4b 100755 --- a/tools/buildbot-build.sh +++ b/tools/buildbot-build.sh @@ -74,6 +74,10 @@ if [[ $mode == "host" ]]; then make_command+=" dx-tests" mode_suffix="-host" elif [[ $mode == "target" ]]; then + if [[ -z "$TARGET_PRODUCT" ]]; then + echo 'TARGET_PRODUCT environment variable is empty; did you forget to run `lunch`?' + exit 1 + fi make_command="make $j_arg $extra_args $showcommands build-art-target-tests $common_targets" make_command+=" libjavacrypto-target libnetd_client-target linker toybox toolbox sh" make_command+=" ${out_dir}/host/linux-x86/bin/adb libstdc++ " diff --git a/tools/common/common.py b/tools/common/common.py index b822dcadb7a..735bbaa4a41 100755 --- a/tools/common/common.py +++ b/tools/common/common.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.4 +#!/usr/bin/env python3 # # Copyright (C) 2016 The Android Open Source Project # diff --git a/tools/dexanalyze/Android.bp b/tools/dexanalyze/Android.bp new file mode 100644 index 00000000000..2754e6445e2 --- /dev/null +++ b/tools/dexanalyze/Android.bp @@ -0,0 +1,53 @@ +// +// Copyright (C) 2018 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +cc_defaults { + name: "dexanalyze-defaults", + defaults: ["art_defaults"], + host_supported: true, + srcs: [ + "dexanalyze.cc", + "dexanalyze_experiments.cc", + ], + target: { + android: { + shared_libs: ["libcutils"], + }, + }, + header_libs: [ + "art_cmdlineparser_headers", + ], +} + +art_cc_binary { + name: "dexanalyze", + defaults: ["dexanalyze-defaults"], + shared_libs: [ + "libdexfile", + "libbase", + ], +} + +art_cc_test { + name: "art_dexanalyze_tests", + required: ["dexanalyze"], + defaults: [ + "art_gtest_defaults", + ], + srcs: [ + "dexanalyze_test.cc", + ], +} diff --git a/tools/dexanalyze/dexanalyze.cc b/tools/dexanalyze/dexanalyze.cc new file mode 100644 index 00000000000..a5f647cc562 --- /dev/null +++ b/tools/dexanalyze/dexanalyze.cc @@ -0,0 +1,160 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <cstdint> +#include <set> +#include <sstream> + +#include <android-base/file.h> + +#include "dexanalyze_experiments.h" +#include "dex/code_item_accessors-inl.h" +#include "dex/dex_file.h" +#include "dex/dex_file_loader.h" +#include "dex/dex_instruction-inl.h" + +namespace art { + +class DexAnalyze { + static const int kExitCodeUsageError = 1; + + static int Usage(char** argv) { + LOG(ERROR) + << "Usage " << argv[0] << " [options] <dex files>\n" + << " [options] is a combination of the following\n" + << " -count_indices (Count dex indices accessed from code items)\n" + << " -i (Ignore DEX checksum failure)\n" + << " -a (Run all experiments)\n" + << " -d (Dump on per DEX basis)\n"; + return kExitCodeUsageError; + } + + struct Options { + int Parse(int argc, char** argv) { + int i; + for (i = 1; i < argc; ++i) { + const std::string arg = argv[i]; + if (arg == "-i") { + verify_checksum_ = false; + } else if (arg == "-a") { + run_all_experiments_ = true; + } else if (arg == "-count-indices") { + exp_count_indices_ = true; + } else if (arg == "-d") { + dump_per_input_dex_ = true; + } else if (!arg.empty() && arg[0] == '-') { + return Usage(argv); + } else { + break; + } + } + filenames_.insert(filenames_.end(), argv + i, argv + argc); + if (filenames_.empty()) { + return Usage(argv); + } + return 0; + } + + bool verify_checksum_ = true; + bool run_dex_file_verifier_ = true; + bool dump_per_input_dex_ = false; + bool exp_count_indices_ = false; + bool run_all_experiments_ = false; + std::vector<std::string> filenames_; + }; + + class Analysis { + public: + explicit Analysis(const Options* options) : options_(options) { + if (options->run_all_experiments_ || options->exp_count_indices_) { + experiments_.emplace_back(new CountDexIndices); + } + } + + bool ProcessDexFile(const DexFile& dex_file) { + for (std::unique_ptr<Experiment>& experiment : experiments_) { + experiment->ProcessDexFile(dex_file); + } + ++dex_count_; + return true; + } + + void Dump(std::ostream& os) { + for (std::unique_ptr<Experiment>& experiment : experiments_) { + experiment->Dump(os); + } + } + + const Options* const options_; + std::vector<std::unique_ptr<Experiment>> experiments_; + size_t dex_count_ = 0; + }; + + public: + static int Run(int argc, char** argv) { + Options options; + int result = options.Parse(argc, argv); + if (result != 0) { + return result; + } + + std::string error_msg; + Analysis cumulative(&options); + for (const std::string& filename : options.filenames_) { + std::string content; + // TODO: once added, use an api to android::base to read a std::vector<uint8_t>. + if (!android::base::ReadFileToString(filename.c_str(), &content)) { + LOG(ERROR) << "ReadFileToString failed for " + filename << std::endl; + continue; + } + std::vector<std::unique_ptr<const DexFile>> dex_files; + const DexFileLoader dex_file_loader; + if (!dex_file_loader.OpenAll(reinterpret_cast<const uint8_t*>(content.data()), + content.size(), + filename.c_str(), + options.run_dex_file_verifier_, + options.verify_checksum_, + &error_msg, + &dex_files)) { + LOG(ERROR) << "OpenAll failed for " + filename << " with " << error_msg << std::endl; + continue; + } + for (std::unique_ptr<const DexFile>& dex_file : dex_files) { + if (options.dump_per_input_dex_) { + Analysis current(&options); + if (!current.ProcessDexFile(*dex_file)) { + LOG(ERROR) << "Failed to process " << filename << " with error " << error_msg; + continue; + } + LOG(INFO) << "Analysis for " << dex_file->GetLocation() << std::endl; + current.Dump(LOG_STREAM(INFO)); + } + cumulative.ProcessDexFile(*dex_file); + } + } + LOG(INFO) << "Cumulative analysis for " << cumulative.dex_count_ << " DEX files" << std::endl; + cumulative.Dump(LOG_STREAM(INFO)); + return 0; + } +}; + +} // namespace art + +int main(int argc, char** argv) { + android::base::SetLogger(android::base::StderrLogger); + return art::DexAnalyze::Run(argc, argv); +} + diff --git a/tools/dexanalyze/dexanalyze_experiments.cc b/tools/dexanalyze/dexanalyze_experiments.cc new file mode 100644 index 00000000000..e1f119df59a --- /dev/null +++ b/tools/dexanalyze/dexanalyze_experiments.cc @@ -0,0 +1,133 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "dexanalyze_experiments.h" +#include "dex/code_item_accessors-inl.h" +#include "dex/dex_instruction-inl.h" +#include "dex/standard_dex_file.h" + +namespace art { + +void CountDexIndices::ProcessDexFile(const DexFile& dex_file) { + num_string_ids_ += dex_file.NumStringIds(); + num_method_ids_ += dex_file.NumMethodIds(); + num_field_ids_ += dex_file.NumFieldIds(); + num_type_ids_ += dex_file.NumTypeIds(); + num_class_defs_ += dex_file.NumClassDefs(); + for (size_t class_def_index = 0; class_def_index < dex_file.NumClassDefs(); ++class_def_index) { + const DexFile::ClassDef& class_def = dex_file.GetClassDef(class_def_index); + const uint8_t* class_data = dex_file.GetClassData(class_def); + if (class_data == nullptr) { + continue; + } + ClassDataItemIterator it(dex_file, class_data); + it.SkipAllFields(); + std::set<size_t> unique_method_ids; + std::set<size_t> unique_string_ids; + while (it.HasNextMethod()) { + const DexFile::CodeItem* code_item = it.GetMethodCodeItem(); + if (code_item != nullptr) { + CodeItemInstructionAccessor instructions(dex_file, code_item); + const uint16_t* code_ptr = instructions.Insns(); + dex_code_bytes_ += instructions.InsnsSizeInCodeUnits() * sizeof(code_ptr[0]); + for (const DexInstructionPcPair& inst : instructions) { + switch (inst->Opcode()) { + case Instruction::CONST_STRING: { + const dex::StringIndex string_index(inst->VRegB_21c()); + unique_string_ids.insert(string_index.index_); + ++num_string_ids_from_code_; + break; + } + case Instruction::CONST_STRING_JUMBO: { + const dex::StringIndex string_index(inst->VRegB_31c()); + unique_string_ids.insert(string_index.index_); + ++num_string_ids_from_code_; + break; + } + // Invoke cases. + case Instruction::INVOKE_VIRTUAL: + case Instruction::INVOKE_VIRTUAL_RANGE: { + bool is_range = (inst->Opcode() == Instruction::INVOKE_VIRTUAL_RANGE); + uint32_t method_idx = is_range ? inst->VRegB_3rc() : inst->VRegB_35c(); + if (dex_file.GetMethodId(method_idx).class_idx_ == class_def.class_idx_) { + ++same_class_virtual_; + } else { + ++other_class_virtual_; + unique_method_ids.insert(method_idx); + } + break; + } + case Instruction::INVOKE_DIRECT: + case Instruction::INVOKE_DIRECT_RANGE: { + bool is_range = (inst->Opcode() == Instruction::INVOKE_DIRECT_RANGE); + uint32_t method_idx = (is_range) ? inst->VRegB_3rc() : inst->VRegB_35c(); + if (dex_file.GetMethodId(method_idx).class_idx_ == class_def.class_idx_) { + ++same_class_direct_; + } else { + ++other_class_direct_; + unique_method_ids.insert(method_idx); + } + break; + } + case Instruction::INVOKE_STATIC: + case Instruction::INVOKE_STATIC_RANGE: { + bool is_range = (inst->Opcode() == Instruction::INVOKE_STATIC_RANGE); + uint32_t method_idx = (is_range) ? inst->VRegB_3rc() : inst->VRegB_35c(); + if (dex_file.GetMethodId(method_idx).class_idx_ == class_def.class_idx_) { + ++same_class_static_; + } else { + ++other_class_static_; + unique_method_ids.insert(method_idx); + } + break; + } + default: + break; + } + } + } + it.Next(); + } + DCHECK(!it.HasNext()); + total_unique_method_idx_ += unique_method_ids.size(); + total_unique_string_ids_ += unique_string_ids.size(); + } +} + +void CountDexIndices::Dump(std::ostream& os) const { + os << "Num string ids: " << num_string_ids_ << "\n"; + os << "Num method ids: " << num_method_ids_ << "\n"; + os << "Num field ids: " << num_field_ids_ << "\n"; + os << "Num type ids: " << num_type_ids_ << "\n"; + os << "Num class defs: " << num_class_defs_ << "\n"; + os << "Same class direct: " << same_class_direct_ << "\n"; + os << "Other class direct: " << other_class_direct_ << "\n"; + os << "Same class virtual: " << same_class_virtual_ << "\n"; + os << "Other class virtual: " << other_class_virtual_ << "\n"; + os << "Same class static: " << same_class_static_ << "\n"; + os << "Other class static: " << other_class_static_ << "\n"; + os << "Num strings accessed from code: " << num_string_ids_from_code_ << "\n"; + os << "Unique(per class) method ids accessed from code: " << total_unique_method_idx_ << "\n"; + os << "Unique(per class) string ids accessed from code: " << total_unique_string_ids_ << "\n"; + size_t same_class_total = same_class_direct_ + same_class_virtual_ + same_class_static_; + size_t other_class_total = other_class_direct_ + other_class_virtual_ + other_class_static_; + os << "Same class invoke: " << same_class_total << "\n"; + os << "Other class invoke: " << other_class_total << "\n"; + os << "Invokes from code: " << (same_class_total + other_class_total) << "\n"; +} + +} // namespace art + diff --git a/tools/dexanalyze/dexanalyze_experiments.h b/tools/dexanalyze/dexanalyze_experiments.h new file mode 100644 index 00000000000..5d0f51b8210 --- /dev/null +++ b/tools/dexanalyze/dexanalyze_experiments.h @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_TOOLS_DEXANALYZE_DEXANALYZE_EXPERIMENTS_H_ +#define ART_TOOLS_DEXANALYZE_DEXANALYZE_EXPERIMENTS_H_ + +#include <iosfwd> +#include <set> + +namespace art { + +class DexFile; + +// An experiment a stateful visitor that runs on dex files. Results are cumulative. +class Experiment { + public: + virtual ~Experiment() {} + virtual void ProcessDexFile(const DexFile& dex_file) = 0; + virtual void Dump(std::ostream& os) const = 0; +}; + +// Count numbers of dex indices. +class CountDexIndices : public Experiment { + public: + void ProcessDexFile(const DexFile& dex_file); + + void Dump(std::ostream& os) const; + + private: + // Total string ids loaded from dex code. + size_t num_string_ids_from_code_ = 0; + size_t total_unique_method_idx_ = 0; + size_t total_unique_string_ids_ = 0; + + // Other dex ids. + size_t dex_code_bytes_ = 0; + size_t num_string_ids_ = 0; + size_t num_method_ids_ = 0; + size_t num_field_ids_ = 0; + size_t num_type_ids_ = 0; + size_t num_class_defs_ = 0; + + // Invokes + size_t same_class_direct_ = 0; + size_t other_class_direct_ = 0; + size_t same_class_virtual_ = 0; + size_t other_class_virtual_ = 0; + size_t same_class_static_ = 0; + size_t other_class_static_ = 0; +}; + +} // namespace art + +#endif // ART_TOOLS_DEXANALYZE_DEXANALYZE_EXPERIMENTS_H_ + diff --git a/tools/dexanalyze/dexanalyze_test.cc b/tools/dexanalyze/dexanalyze_test.cc new file mode 100644 index 00000000000..c9b8f53d240 --- /dev/null +++ b/tools/dexanalyze/dexanalyze_test.cc @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common_runtime_test.h" +#include "exec_utils.h" + +namespace art { + +class DexAnalyzeTest : public CommonRuntimeTest { + public: + std::string GetDexAnalyzePath() { + return GetTestAndroidRoot() + "/bin/dexanalyze"; + } + + void DexAnalyzeExec(const std::vector<std::string>& args, bool expect_success) { + std::string binary = GetDexAnalyzePath(); + CHECK(OS::FileExists(binary.c_str())) << binary << " should be a valid file path"; + std::vector<std::string> argv; + argv.push_back(binary); + argv.insert(argv.end(), args.begin(), args.end()); + std::string error_msg; + ASSERT_EQ(::art::Exec(argv, &error_msg), expect_success) << error_msg; + } +}; + +TEST_F(DexAnalyzeTest, TestAnalyzeMultidex) { + DexAnalyzeExec({ "-a", GetTestDexFileName("MultiDex") }, /*expect_success*/ true); +} + +TEST_F(DexAnalyzeTest, TestInvalidArg) { + DexAnalyzeExec({ "-invalid-option" }, /*expect_success*/ false); +} + +} // namespace art diff --git a/tools/external_oj_libjdwp_art_failures.txt b/tools/external_oj_libjdwp_art_failures.txt index 6c2206fe46b..9b6ff984f89 100644 --- a/tools/external_oj_libjdwp_art_failures.txt +++ b/tools/external_oj_libjdwp_art_failures.txt @@ -13,12 +13,6 @@ name: "org.apache.harmony.jpda.tests.jdwp.ThreadReference.ThreadGroup002Test#testThreadGroup002" }, { - description: "Test fails due to modifiers not including ACC_SUPER", - result: EXEC_FAILED, - bug: 66906055, - name: "org.apache.harmony.jpda.tests.jdwp.ReferenceType.ModifiersTest#testModifiers001" -}, -{ description: "Test fails due to static values not being set correctly.", result: EXEC_FAILED, bug: 66905894, diff --git a/tools/hiddenapi/find_api_violations.pl b/tools/hiddenapi/find_api_violations.pl new file mode 100755 index 00000000000..caa4f543a18 --- /dev/null +++ b/tools/hiddenapi/find_api_violations.pl @@ -0,0 +1,133 @@ +#!/usr/bin/perl +# Copyright (C) 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +use strict; +use Getopt::Long; +use Pod::Usage; + +=pod + +=head1 DESCRIPTION + +This script parses API violations from C<adb logcat>. Output is in CSV format +with columns C<package>, C<symbol>, C<list>. + +The package name is mapped from a PID, parsed from the same log. To ensure you +get all packages names, you should process the logcat from device boot time. + +=head1 SYNOPSIS + + adb logcat | perl find_api_violations.pl > violations.csv + cat bugreport.txt | perl find_api_violations.pl --bugreport > violations.csv + +=head1 OPTIONS + +=over + +=item --[no]lightgrey + +(Don't) show light grey list accesses (default true) + +=item --[no]darkgrey + +(Don't) show dark grey list accesses (default true) + +=item --[no]black + +(Don't) show black list accesses (default true) + +=item --bugreport|-b + +Process a bugreport, rather than raw logcat + +=item --short|-s + +Output API signatures only, don't include CSV header/package names/list name. + +=item --help + +=back + +=cut + +my $lightgrey = 1; +my $darkgrey = 1; +my $black = 1; +my $bugreport = 0; +my $short = 0; +my $help = 0; + +GetOptions("lightgrey!" => \$lightgrey, + "darkgrey!" => \$darkgrey, + "black!" => \$black, + "bugreport|b" => \$bugreport, + "short|s" => \$short, + "help" => \$help) + or pod2usage(q(-verbose) => 1); + +pod2usage(q(-verbose) => 2) if ($help); + +my $in; + +if ($bugreport) { + my $found_main = 0; + while (my $line = <>) { + chomp $line; + if ($line =~ m/^------ SYSTEM LOG /) { + $found_main = 1; + last; + } + } + if (!$found_main) { + die "Couldn't find main log in bugreport\n"; + } +} + +my $procmap = {}; +print "package,symbol,list\n" unless $short; +while (my $line = <>) { + chomp $line; + last if $bugreport and $line =~ m/^------ \d+\.\d+s was the duration of 'SYSTEM LOG' ------/; + next if $line =~ m/^--------- beginning of/; + unless ($line =~ m/^\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}\s+(?:\w+\s+)?(\d+)\s+(\d+)\s+([VWIDE])\s+(.*?): (.*)$/) { + die "Cannot match line: $line\n"; + next; + } + my ($pid, $tid, $class, $tag, $msg) = ($1, $2, $3, $4, $5); + if ($tag eq "ActivityManager" && $msg =~ m/^Start proc (\d+):(.*?) for /) { + my ($new_pid, $proc_name) = ($1, $2); + my $package; + if ($proc_name =~ m/^(.*?)(:.*?)?\/(.*)$/) { + $package = $1; + } else { + $package = $proc_name; + } + $procmap->{$new_pid} = $package; + } + if ($msg =~ m/Accessing hidden (\w+) (L.*?) \((.*list), (.*?)\)/) { + my ($member_type, $symbol, $list, $access_type) = ($1, $2, $3, $4); + my $package = $procmap->{$pid} || "unknown($pid)"; + if (($list =~ m/light/ && $lightgrey) + || ($list =~ m/dark/ && $darkgrey) + || ($list =~ m/black/ && $black)) { + if ($short) { + print "$symbol\n"; + } else { + print "$package,$symbol,$list\n" + } + } + } +} + diff --git a/tools/hiddenapi/hiddenapi.cc b/tools/hiddenapi/hiddenapi.cc index d22998ae1b4..97e7f4cf3c2 100644 --- a/tools/hiddenapi/hiddenapi.cc +++ b/tools/hiddenapi/hiddenapi.cc @@ -21,12 +21,12 @@ #include "android-base/stringprintf.h" #include "android-base/strings.h" +#include "base/mem_map.h" #include "base/os.h" #include "base/unix_file/fd_file.h" #include "dex/art_dex_file_loader.h" #include "dex/dex_file-inl.h" #include "dex/hidden_api_access_flags.h" -#include "mem_map.h" namespace art { diff --git a/tools/hiddenapi/hiddenapi_test.cc b/tools/hiddenapi/hiddenapi_test.cc index af1439520f7..ed880e0e922 100644 --- a/tools/hiddenapi/hiddenapi_test.cc +++ b/tools/hiddenapi/hiddenapi_test.cc @@ -17,11 +17,11 @@ #include <fstream> #include "base/unix_file/fd_file.h" +#include "base/zip_archive.h" #include "common_runtime_test.h" #include "dex/art_dex_file_loader.h" #include "dex/dex_file-inl.h" #include "exec_utils.h" -#include "zip_archive.h" namespace art { diff --git a/tools/jfuzz/run_dex_fuzz_test.py b/tools/jfuzz/run_dex_fuzz_test.py index 203e03d678d..47fe072b6df 100755 --- a/tools/jfuzz/run_dex_fuzz_test.py +++ b/tools/jfuzz/run_dex_fuzz_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.4 +#!/usr/bin/env python3 # # Copyright (C) 2016 The Android Open Source Project # diff --git a/tools/jfuzz/run_jfuzz_test.py b/tools/jfuzz/run_jfuzz_test.py index 4a54a3a4f2e..3ff9f450a1b 100755 --- a/tools/jfuzz/run_jfuzz_test.py +++ b/tools/jfuzz/run_jfuzz_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.4 +#!/usr/bin/env python3 # # Copyright (C) 2016 The Android Open Source Project # diff --git a/tools/jfuzz/run_jfuzz_test_nightly.py b/tools/jfuzz/run_jfuzz_test_nightly.py index e6c216d1f7c..fecf116d8e0 100755 --- a/tools/jfuzz/run_jfuzz_test_nightly.py +++ b/tools/jfuzz/run_jfuzz_test_nightly.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.4 +#!/usr/bin/env python3 # # Copyright (C) 2016 The Android Open Source Project # diff --git a/tools/libcore_network_failures.txt b/tools/libcore_network_failures.txt new file mode 100644 index 00000000000..e7e31dbe671 --- /dev/null +++ b/tools/libcore_network_failures.txt @@ -0,0 +1,92 @@ +/* + * This file contains extra expectations for ART's buildbot regarding network tests. + * The script that uses this file is art/tools/run-libcore-tests.sh. + */ + +[ +{ + description: "Ignore failure of network-related tests on new devices running Android O", + result: EXEC_FAILED, + bug: 74725685, + modes: [device], + names: ["libcore.libcore.io.OsTest#test_byteBufferPositions_sendto_recvfrom_af_inet", + "libcore.libcore.net.NetworkSecurityPolicyTest#testCleartextTrafficPolicyWithFtpURLConnection", + "libcore.libcore.net.NetworkSecurityPolicyTest#testCleartextTrafficPolicyWithHttpURLConnection", + "libcore.libcore.net.NetworkSecurityPolicyTest#testCleartextTrafficPolicyWithJarFtpURLConnection", + "libcore.libcore.net.NetworkSecurityPolicyTest#testCleartextTrafficPolicyWithJarHttpURLConnection", + "libcore.libcore.net.NetworkSecurityPolicyTest#testCleartextTrafficPolicyWithLoggingSocketHandler", + "org.apache.harmony.tests.java.util.ScannerTest#test_40555", + "org.apache.harmony.tests.java.util.ScannerTest#test_ConstructorLjava_io_File", + "org.apache.harmony.tests.java.util.ScannerTest#test_ConstructorLjava_io_FileLjava_lang_String", + "org.apache.harmony.tests.java.util.ScannerTest#test_ConstructorLjava_io_InputStream", + "org.apache.harmony.tests.java.util.ScannerTest#test_ConstructorLjava_io_InputStreamLjava_lang_String", + "org.apache.harmony.tests.java.util.ScannerTest#test_ConstructorLjava_lang_Readable", + "org.apache.harmony.tests.java.util.ScannerTest#test_ConstructorLjava_lang_String", + "org.apache.harmony.tests.java.util.ScannerTest#test_ConstructorLjava_nio_channels_ReadableByteChannel", + "org.apache.harmony.tests.java.util.ScannerTest#test_ConstructorLjava_nio_channels_ReadableByteChannelLjava_lang_String", + "org.apache.harmony.tests.java.util.ScannerTest#test_ConstructorLjava_nio_file_Path", + "org.apache.harmony.tests.java.util.ScannerTest#test_ConstructorLjava_nio_file_PathLjava_lang_String", + "org.apache.harmony.tests.java.util.ScannerTest#test_ConstructorLjava_nio_file_PathLjava_lang_String_Exception", + "org.apache.harmony.tests.java.util.ScannerTest#test_ConstructorLjava_nio_file_Path_Exception", + "org.apache.harmony.tests.java.util.ScannerTest#test_close", + "org.apache.harmony.tests.java.util.ScannerTest#test_delimiter", + "org.apache.harmony.tests.java.util.ScannerTest#test_findInLine_LPattern", + "org.apache.harmony.tests.java.util.ScannerTest#test_findInLine_LString", + "org.apache.harmony.tests.java.util.ScannerTest#test_findInLine_LString_NPEs", + "org.apache.harmony.tests.java.util.ScannerTest#test_findWithinHorizon_LPatternI", + "org.apache.harmony.tests.java.util.ScannerTest#test_hasNext", + "org.apache.harmony.tests.java.util.ScannerTest#test_hasNextBigDecimal", + "org.apache.harmony.tests.java.util.ScannerTest#test_hasNextBigInteger", + "org.apache.harmony.tests.java.util.ScannerTest#test_hasNextBigIntegerI", + "org.apache.harmony.tests.java.util.ScannerTest#test_hasNextBigIntegerI_cache", + "org.apache.harmony.tests.java.util.ScannerTest#test_hasNextBoolean", + "org.apache.harmony.tests.java.util.ScannerTest#test_hasNextByte", + "org.apache.harmony.tests.java.util.ScannerTest#test_hasNextByteI", + "org.apache.harmony.tests.java.util.ScannerTest#test_hasNextByteI_cache", + "org.apache.harmony.tests.java.util.ScannerTest#test_hasNextDouble", + "org.apache.harmony.tests.java.util.ScannerTest#test_hasNextFloat", + "org.apache.harmony.tests.java.util.ScannerTest#test_hasNextInt", + "org.apache.harmony.tests.java.util.ScannerTest#test_hasNextIntI", + "org.apache.harmony.tests.java.util.ScannerTest#test_hasNextIntI_cache", + "org.apache.harmony.tests.java.util.ScannerTest#test_hasNextLPattern", + "org.apache.harmony.tests.java.util.ScannerTest#test_hasNextLString", + "org.apache.harmony.tests.java.util.ScannerTest#test_hasNextLine", + "org.apache.harmony.tests.java.util.ScannerTest#test_hasNextLine_sequence", + "org.apache.harmony.tests.java.util.ScannerTest#test_hasNextLong", + "org.apache.harmony.tests.java.util.ScannerTest#test_hasNextLongI", + "org.apache.harmony.tests.java.util.ScannerTest#test_hasNextLongI_cache", + "org.apache.harmony.tests.java.util.ScannerTest#test_hasNextShort", + "org.apache.harmony.tests.java.util.ScannerTest#test_hasNextShortI", + "org.apache.harmony.tests.java.util.ScannerTest#test_hasNextShortI_cache", + "org.apache.harmony.tests.java.util.ScannerTest#test_ioException", + "org.apache.harmony.tests.java.util.ScannerTest#test_locale", + "org.apache.harmony.tests.java.util.ScannerTest#test_match", + "org.apache.harmony.tests.java.util.ScannerTest#test_next", + "org.apache.harmony.tests.java.util.ScannerTest#test_nextBigDecimal", + "org.apache.harmony.tests.java.util.ScannerTest#test_nextBigInteger", + "org.apache.harmony.tests.java.util.ScannerTest#test_nextBigIntegerI", + "org.apache.harmony.tests.java.util.ScannerTest#test_nextBoolean", + "org.apache.harmony.tests.java.util.ScannerTest#test_nextByte", + "org.apache.harmony.tests.java.util.ScannerTest#test_nextByteI", + "org.apache.harmony.tests.java.util.ScannerTest#test_nextDouble", + "org.apache.harmony.tests.java.util.ScannerTest#test_nextFloat", + "org.apache.harmony.tests.java.util.ScannerTest#test_nextInt", + "org.apache.harmony.tests.java.util.ScannerTest#test_nextIntI", + "org.apache.harmony.tests.java.util.ScannerTest#test_nextLPattern", + "org.apache.harmony.tests.java.util.ScannerTest#test_nextLString", + "org.apache.harmony.tests.java.util.ScannerTest#test_nextLine", + "org.apache.harmony.tests.java.util.ScannerTest#test_nextLong", + "org.apache.harmony.tests.java.util.ScannerTest#test_nextLongI", + "org.apache.harmony.tests.java.util.ScannerTest#test_nextShort", + "org.apache.harmony.tests.java.util.ScannerTest#test_nextShortI", + "org.apache.harmony.tests.java.util.ScannerTest#test_radix", + "org.apache.harmony.tests.java.util.ScannerTest#test_remove", + "org.apache.harmony.tests.java.util.ScannerTest#test_skip_LPattern", + "org.apache.harmony.tests.java.util.ScannerTest#test_skip_LString", + "org.apache.harmony.tests.java.util.ScannerTest#test_toString", + "org.apache.harmony.tests.java.util.ScannerTest#test_useDelimiter_LPattern", + "org.apache.harmony.tests.java.util.ScannerTest#test_useDelimiter_String", + "org.apache.harmony.tests.java.util.ScannerTest#test_useLocale_LLocale", + "org.apache.harmony.tests.java.util.ScannerTest#test_useRadix_I"] +} +] diff --git a/tools/run-jdwp-tests.sh b/tools/run-jdwp-tests.sh index c58411c60b6..21ddcbc0629 100755 --- a/tools/run-jdwp-tests.sh +++ b/tools/run-jdwp-tests.sh @@ -19,6 +19,16 @@ if [ ! -d libcore ]; then exit 1 fi +# Prevent JDWP tests from running on the following devices running +# Android O (they are failing because of a network-related issue), as +# a workaround for b/74725685: +# - FA7BN1A04406 (walleye device testing configuration aosp-poison/volantis-armv7-poison-debug) +# - FA7BN1A04412 (walleye device testing configuration aosp-poison/volantis-armv8-poison-ndebug) +# - FA7BN1A04433 (walleye device testing configuration aosp-poison/volantis-armv8-poison-debug) +case "$ANDROID_SERIAL" in + (FA7BN1A04406|FA7BN1A04412|FA7BN1A04433) exit 0;; +esac + source build/envsetup.sh >&/dev/null # for get_build_var, setpaths setpaths # include platform prebuilt java, javac, etc in $PATH. @@ -26,12 +36,17 @@ if [ -z "$ANDROID_HOST_OUT" ] ; then ANDROID_HOST_OUT=${OUT_DIR-$ANDROID_BUILD_TOP/out}/host/linux-x86 fi +android_root="/system" +if [ -n "$ART_TEST_ANDROID_ROOT" ]; then + android_root="$ART_TEST_ANDROID_ROOT" +fi + java_lib_location="${ANDROID_HOST_OUT}/../common/obj/JAVA_LIBRARIES" make_target_name="apache-harmony-jdwp-tests-hostdex" vm_args="" -art="/data/local/tmp/system/bin/art" -art_debugee="sh /data/local/tmp/system/bin/art" +art="$android_root/bin/art" +art_debugee="sh $android_root/bin/art" args=$@ debuggee_args="-Xcompiler-option --debuggable" device_dir="--device-dir=/data/local/tmp" @@ -201,10 +216,7 @@ else if [[ "$mode" == "host" ]]; then dump_command="/bin/kill -3" else - # TODO It would be great to be able to use this on target too but we need to - # be able to walk /proc to figure out what the actual child pid is. - dump_command="/system/bin/true" - # dump_command="/system/xbin/su root /data/local/tmp/system/bin/debuggerd -b" + dump_command="/system/xbin/su root /data/local/tmp/system/bin/debuggerd" fi if [[ $has_gdb = "yes" ]]; then if [[ $mode == "target" ]]; then diff --git a/tools/run-libcore-tests.sh b/tools/run-libcore-tests.sh index 739646a7542..7f0383d55d0 100755 --- a/tools/run-libcore-tests.sh +++ b/tools/run-libcore-tests.sh @@ -28,6 +28,11 @@ else JAVA_LIBRARIES=${ANDROID_PRODUCT_OUT}/../../common/obj/JAVA_LIBRARIES fi +android_root="/system" +if [ -n "$ART_TEST_ANDROID_ROOT" ]; then + android_root="$ART_TEST_ANDROID_ROOT" +fi + function classes_jar_path { local var="$1" local suffix="jar" @@ -103,7 +108,7 @@ debug=false while true; do if [[ "$1" == "--mode=device" ]]; then vogar_args="$vogar_args --device-dir=/data/local/tmp" - vogar_args="$vogar_args --vm-command=/data/local/tmp/system/bin/art" + vogar_args="$vogar_args --vm-command=$android_root/bin/art" vogar_args="$vogar_args --vm-arg -Ximage:/data/art-test/core.art" shift elif [[ "$1" == "--mode=host" ]]; then @@ -156,6 +161,16 @@ if [[ $gcstress ]]; then fi fi +# Disable network-related libcore tests that are failing on the following +# devices running Android O, as a workaround for b/74725685: +# - FA7BN1A04406 (walleye device testing configuration aosp-poison/volantis-armv7-poison-debug) +# - FA7BN1A04412 (walleye device testing configuration aosp-poison/volantis-armv8-poison-ndebug) +# - FA7BN1A04433 (walleye device testing configuration aosp-poison/volantis-armv8-poison-debug) +case "$ANDROID_SERIAL" in + (FA7BN1A04406|FA7BN1A04412|FA7BN1A04433) + expectations="$expectations --expectations art/tools/libcore_network_failures.txt";; +esac + # Run the tests using vogar. echo "Running tests for the following test packages:" echo ${working_packages[@]} | tr " " "\n" diff --git a/tools/setup-buildbot-device.sh b/tools/setup-buildbot-device.sh index 546a6bf55b6..5ce7f5244eb 100755 --- a/tools/setup-buildbot-device.sh +++ b/tools/setup-buildbot-device.sh @@ -57,6 +57,21 @@ echo -e "${green}Setting local loopback${nc}" adb shell ifconfig lo up adb shell ifconfig +# Ensure netd is running, as otherwise the logcat would be spammed +# with the following messages on devices running Android O: +# +# E NetdConnector: Communications error: java.io.IOException: No such file or directory +# E mDnsConnector: Communications error: java.io.IOException: No such file or directory +# +# Netd was initially disabled as an attempt to solve issues with +# network-related libcore and JDWP tests failing on devices running +# Android O (MR1) (see b/74725685). These tests are currently +# disabled. When a better solution has been found, we should remove +# the following lines. +echo -e "${green}Turning on netd${nc}" +adb shell start netd +adb shell getprop init.svc.netd + echo -e "${green}List properties${nc}" adb shell getprop diff --git a/tools/symbolize-buildbot-crashes.sh b/tools/symbolize-buildbot-crashes.sh index 8dc4e278855..0200346fa05 100755 --- a/tools/symbolize-buildbot-crashes.sh +++ b/tools/symbolize-buildbot-crashes.sh @@ -17,7 +17,11 @@ # We push art and its dependencies to '/data/local/tmp', but the 'stack' # script expect things to be in '/'. So we just remove the # '/data/local/tmp' prefix. -adb logcat -d | sed 's,/data/local/tmp,,g' | development/scripts/stack +if [[ -n "$1" ]]; then + cat $1 +else + adb logcat -d +fi | sed 's,/data/local/tmp,,g' | development/scripts/stack # Always return 0 to avoid having the buildbot complain about wrong stacks. exit 0 diff --git a/tools/wrap-logcat.py b/tools/wrap-logcat.py new file mode 100755 index 00000000000..067890e8702 --- /dev/null +++ b/tools/wrap-logcat.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# +# Copyright 2018, The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import subprocess +import sys +import shlex + +def main(): + parser = argparse.ArgumentParser( + description='Runs a command while collecting logcat in the background') + parser.add_argument('--output', '-o', + type=lambda f: open(f,'w'), + required=True, + action='store', + help='File to store the logcat to. Will be created if not already existing') + parser.add_argument('--logcat-invoke', + action='store', + default='adb logcat', + help="""Command to run to retrieve logcat data. Defaults to 'adb logcat'. + It will be run in the background and killed by SIGTERM when the 'command' + finishes.""") + parser.add_argument('command', + action='store', + nargs=argparse.REMAINDER, + help='The command to run with logcat in the background.') + args = parser.parse_args() + if len(args.command) == 0: + print("Must have some command to run.", file=sys.stderr) + parser.print_help(file=sys.stderr) + return 1 + # Send all output from logcat to the file. + with subprocess.Popen(shlex.split(args.logcat_invoke), + stdout=args.output, + stderr=subprocess.STDOUT, + shell=False, + universal_newlines=True) as logcat_proc: + # Let the run-test-proc inherit our stdout FDs + with subprocess.Popen(shlex.split(args.command[0]) if len(args.command) == 1 else args.command, + stdout=None, + stderr=None, + shell=False) as run_test_proc: + # Don't actually do anything. Just let the run-test proc finish. + pass + # Send SIGTERM to the logcat process. + logcat_proc.kill() + return 0 + +if __name__ == '__main__': + sys.exit(main()) + |