From 7f5c7d293c2dad462dc9c0f1f1a160fb2c2c9a9b Mon Sep 17 00:00:00 2001 From: Xusong Wang Date: Tue, 5 Oct 2021 14:10:41 -0700 Subject: Reusable execution at HAL level -- HAL. This CL modifies the canonical/AIDL adapter to use IExecution object if available. Bug: 202405342 Bug: 202431255 Test: NNT_static Test: CtsNNAPITestCases Test: VtsHalNeuralnetworksTargetTest Change-Id: I6aac3c57f97ac87a5ba3f78cfd843fcc403decff --- neuralnetworks/aidl/utils/src/PreparedModel.cpp | 158 ++++++++++++++++-------- 1 file changed, 106 insertions(+), 52 deletions(-) (limited to 'neuralnetworks/aidl/utils/src/PreparedModel.cpp') diff --git a/neuralnetworks/aidl/utils/src/PreparedModel.cpp b/neuralnetworks/aidl/utils/src/PreparedModel.cpp index f25c2c8825..6d1de569d0 100644 --- a/neuralnetworks/aidl/utils/src/PreparedModel.cpp +++ b/neuralnetworks/aidl/utils/src/PreparedModel.cpp @@ -54,21 +54,77 @@ nn::GeneralResult> convertFencedExecutionResul return std::make_pair(NN_TRY(nn::convert(timingLaunched)), NN_TRY(nn::convert(timingFenced))); } +nn::ExecutionResult, nn::Timing>> handleExecutionResult( + const ExecutionResult& result, const hal::utils::RequestRelocation& relocation) { + if (!result.outputSufficientSize) { + auto canonicalOutputShapes = + nn::convert(result.outputShapes).value_or(std::vector{}); + return NN_ERROR(nn::ErrorStatus::OUTPUT_INSUFFICIENT_SIZE, std::move(canonicalOutputShapes)) + << "execution failed with " << nn::ErrorStatus::OUTPUT_INSUFFICIENT_SIZE; + } + auto [outputShapes, timing] = + NN_TRY(convertExecutionResults(result.outputShapes, result.timing)); + + if (relocation.output) { + relocation.output->flush(); + } + return std::make_pair(std::move(outputShapes), timing); +} + +nn::GeneralResult> +handleFencedExecutionResult(const FencedExecutionResult& result, + const hal::utils::RequestRelocation& relocation) { + auto resultSyncFence = nn::SyncFence::createAsSignaled(); + if (result.syncFence.get() != -1) { + resultSyncFence = nn::SyncFence::create(NN_TRY(nn::convert(result.syncFence))).value(); + } + + auto callback = result.callback; + if (callback == nullptr) { + return NN_ERROR(nn::ErrorStatus::GENERAL_FAILURE) << "callback is null"; + } + + // If computeFenced required the request memory to be moved into shared memory, block here until + // the fenced execution has completed and flush the memory back. + if (relocation.output) { + const auto state = resultSyncFence.syncWait({}); + if (state != nn::SyncFence::FenceState::SIGNALED) { + return NN_ERROR() << "syncWait failed with " << state; + } + relocation.output->flush(); + } + + // Create callback which can be used to retrieve the execution error status and timings. + nn::ExecuteFencedInfoCallback resultCallback = + [callback]() -> nn::GeneralResult> { + ErrorStatus errorStatus; + Timing timingLaunched; + Timing timingFenced; + const auto ret = callback->getExecutionInfo(&timingLaunched, &timingFenced, &errorStatus); + HANDLE_ASTATUS(ret) << "fenced execution callback getExecutionInfo failed"; + return convertFencedExecutionResults(errorStatus, timingLaunched, timingFenced); + }; + + return std::make_pair(std::move(resultSyncFence), std::move(resultCallback)); +} + } // namespace nn::GeneralResult> PreparedModel::create( - std::shared_ptr preparedModel) { + std::shared_ptr preparedModel, nn::Version featureLevel) { if (preparedModel == nullptr) { return NN_ERROR() << "aidl_hal::utils::PreparedModel::create must have non-null preparedModel"; } - return std::make_shared(PrivateConstructorTag{}, std::move(preparedModel)); + return std::make_shared(PrivateConstructorTag{}, std::move(preparedModel), + featureLevel); } PreparedModel::PreparedModel(PrivateConstructorTag /*tag*/, - std::shared_ptr preparedModel) - : kPreparedModel(std::move(preparedModel)) {} + std::shared_ptr preparedModel, + nn::Version featureLevel) + : kPreparedModel(std::move(preparedModel)), kFeatureLevel(featureLevel) {} nn::ExecutionResult, nn::Timing>> PreparedModel::execute( const nn::Request& request, nn::MeasureTiming measure, @@ -101,19 +157,7 @@ PreparedModel::executeInternal(const Request& request, bool measure, int64_t dea const auto ret = kPreparedModel->executeSynchronously(request, measure, deadline, loopTimeoutDuration, &executionResult); HANDLE_ASTATUS(ret) << "executeSynchronously failed"; - if (!executionResult.outputSufficientSize) { - auto canonicalOutputShapes = - nn::convert(executionResult.outputShapes).value_or(std::vector{}); - return NN_ERROR(nn::ErrorStatus::OUTPUT_INSUFFICIENT_SIZE, std::move(canonicalOutputShapes)) - << "execution failed with " << nn::ErrorStatus::OUTPUT_INSUFFICIENT_SIZE; - } - auto [outputShapes, timing] = - NN_TRY(convertExecutionResults(executionResult.outputShapes, executionResult.timing)); - - if (relocation.output) { - relocation.output->flush(); - } - return std::make_pair(std::move(outputShapes), timing); + return handleExecutionResult(executionResult, relocation); } nn::GeneralResult> @@ -154,39 +198,7 @@ PreparedModel::executeFencedInternal(const Request& request, kPreparedModel->executeFenced(request, waitFor, measure, deadline, loopTimeoutDuration, timeoutDurationAfterFence, &result); HANDLE_ASTATUS(ret) << "executeFenced failed"; - - auto resultSyncFence = nn::SyncFence::createAsSignaled(); - if (result.syncFence.get() != -1) { - resultSyncFence = nn::SyncFence::create(NN_TRY(nn::convert(result.syncFence))).value(); - } - - auto callback = result.callback; - if (callback == nullptr) { - return NN_ERROR(nn::ErrorStatus::GENERAL_FAILURE) << "callback is null"; - } - - // If executeFenced required the request memory to be moved into shared memory, block here until - // the fenced execution has completed and flush the memory back. - if (relocation.output) { - const auto state = resultSyncFence.syncWait({}); - if (state != nn::SyncFence::FenceState::SIGNALED) { - return NN_ERROR() << "syncWait failed with " << state; - } - relocation.output->flush(); - } - - // Create callback which can be used to retrieve the execution error status and timings. - nn::ExecuteFencedInfoCallback resultCallback = - [callback]() -> nn::GeneralResult> { - ErrorStatus errorStatus; - Timing timingLaunched; - Timing timingFenced; - const auto ret = callback->getExecutionInfo(&timingLaunched, &timingFenced, &errorStatus); - HANDLE_ASTATUS(ret) << "fenced execution callback getExecutionInfo failed"; - return convertFencedExecutionResults(errorStatus, timingLaunched, timingFenced); - }; - - return std::make_pair(std::move(resultSyncFence), std::move(resultCallback)); + return handleFencedExecutionResult(result, relocation); } nn::GeneralResult PreparedModel::createReusableExecution( @@ -202,8 +214,18 @@ nn::GeneralResult PreparedModel::createReusableExecution( auto aidlRequest = NN_TRY(convert(requestInShared)); auto aidlMeasure = NN_TRY(convert(measure)); auto aidlLoopTimeoutDuration = NN_TRY(convert(loopTimeoutDuration)); - return Execution::create(shared_from_this(), std::move(aidlRequest), std::move(relocation), - aidlMeasure, aidlLoopTimeoutDuration); + + if (kFeatureLevel.level >= nn::Version::Level::FEATURE_LEVEL_8) { + std::shared_ptr execution; + const auto ret = kPreparedModel->createReusableExecution( + aidlRequest, aidlMeasure, aidlLoopTimeoutDuration, &execution); + HANDLE_ASTATUS(ret) << "createReusableExecution failed"; + return Execution::create(std::move(execution), std::move(relocation)); + } + + return ExecutionWithCachedRequest::create(shared_from_this(), std::move(aidlRequest), + std::move(relocation), aidlMeasure, + aidlLoopTimeoutDuration); } nn::GeneralResult PreparedModel::configureExecutionBurst() const { @@ -218,4 +240,36 @@ std::any PreparedModel::getUnderlyingResource() const { return resource; } +nn::ExecutionResult, nn::Timing>> Execution::compute( + const nn::OptionalTimePoint& deadline) const { + const auto aidlDeadline = NN_TRY(convert(deadline)); + + if (kRelocation.input) { + kRelocation.input->flush(); + } + + ExecutionResult executionResult; + auto ret = kExecution->executeSynchronously(aidlDeadline, &executionResult); + HANDLE_ASTATUS(ret) << "executeSynchronously failed"; + return handleExecutionResult(executionResult, kRelocation); +} + +nn::GeneralResult> Execution::computeFenced( + const std::vector& waitFor, const nn::OptionalTimePoint& deadline, + const nn::OptionalDuration& timeoutDurationAfterFence) const { + const auto aidlWaitFor = NN_TRY(convert(waitFor)); + const auto aidlDeadline = NN_TRY(convert(deadline)); + const auto aidlTimeoutDurationAfterFence = NN_TRY(convert(timeoutDurationAfterFence)); + + if (kRelocation.input) { + kRelocation.input->flush(); + } + + FencedExecutionResult result; + const auto ret = kExecution->executeFenced(aidlWaitFor, aidlDeadline, + aidlTimeoutDurationAfterFence, &result); + HANDLE_ASTATUS(ret) << "executeFenced failed"; + return handleFencedExecutionResult(result, kRelocation); +} + } // namespace aidl::android::hardware::neuralnetworks::utils -- cgit v1.2.3