diff options
153 files changed, 4006 insertions, 936 deletions
diff --git a/apex/ld.config.txt b/apex/ld.config.txt index a5937fd9d8..af8ec066c0 100644 --- a/apex/ld.config.txt +++ b/apex/ld.config.txt @@ -37,9 +37,11 @@ namespace.default.link.platform.shared_libs = libEGL.so:libGLESv1_CM.so:libGLESv namespace.platform.isolated = true -namespace.platform.search.paths = /system/${LIB} +namespace.platform.search.paths = /system/${LIB} +namespace.platform.search.paths += /apex/com.android.runtime/${LIB} namespace.platform.asan.search.paths = /data/asan/system/${LIB} namespace.platform.asan.search.paths += /system/${LIB} +namespace.platform.asan.search.paths += /apex/com.android.runtime/${LIB} # /system/lib/libc.so, etc are symlinks to /apex/com.android.lib/lib/bionic/libc.so, etc. # Add /apex/... pat to the permitted paths because linker uses realpath(3) diff --git a/apex/manifest.json b/apex/manifest.json index 3011ee832e..b11187de6a 100644 --- a/apex/manifest.json +++ b/apex/manifest.json @@ -1,4 +1,4 @@ { "name": "com.android.media", - "version": 290000000 + "version": 299900000 } diff --git a/apex/manifest_codec.json b/apex/manifest_codec.json index 83a517859b..09c436d1a7 100644 --- a/apex/manifest_codec.json +++ b/apex/manifest_codec.json @@ -1,4 +1,4 @@ { "name": "com.android.media.swcodec", - "version": 290000000 + "version": 299900000 } diff --git a/camera/aidl/android/hardware/ICameraServiceListener.aidl b/camera/aidl/android/hardware/ICameraServiceListener.aidl index e9dcbdb6f6..47580f8e68 100644 --- a/camera/aidl/android/hardware/ICameraServiceListener.aidl +++ b/camera/aidl/android/hardware/ICameraServiceListener.aidl @@ -83,4 +83,12 @@ interface ICameraServiceListener * can retry after receiving this callback. */ oneway void onCameraAccessPrioritiesChanged(); + + /** + * Notify registered clients about cameras being opened/closed. + * Only clients with android.permission.CAMERA_OPEN_CLOSE_LISTENER permission + * will receive such callbacks. + */ + oneway void onCameraOpened(String cameraId, String clientPackageId); + oneway void onCameraClosed(String cameraId); } diff --git a/camera/cameraserver/main_cameraserver.cpp b/camera/cameraserver/main_cameraserver.cpp index 53b3d84894..cef8ef5456 100644 --- a/camera/cameraserver/main_cameraserver.cpp +++ b/camera/cameraserver/main_cameraserver.cpp @@ -34,6 +34,7 @@ int main(int argc __unused, char** argv __unused) sp<IServiceManager> sm = defaultServiceManager(); ALOGI("ServiceManager: %p", sm.get()); CameraService::instantiate(); + ALOGI("ServiceManager: %p done instantiate", sm.get()); ProcessState::self()->startThreadPool(); IPCThreadState::self()->joinThreadPool(); } diff --git a/camera/ndk/impl/ACameraCaptureSession.cpp b/camera/ndk/impl/ACameraCaptureSession.cpp index d6f14122ea..68db233d29 100644 --- a/camera/ndk/impl/ACameraCaptureSession.cpp +++ b/camera/ndk/impl/ACameraCaptureSession.cpp @@ -33,7 +33,9 @@ ACameraCaptureSession::~ACameraCaptureSession() { dev->unlockDevice(); } // Fire onClosed callback - (*mUserSessionCallback.onClosed)(mUserSessionCallback.context, this); + if (mUserSessionCallback.onClosed != nullptr) { + (*mUserSessionCallback.onClosed)(mUserSessionCallback.context, this); + } ALOGV("~ACameraCaptureSession: %p is deleted", this); } diff --git a/camera/ndk/impl/ACameraDevice.cpp b/camera/ndk/impl/ACameraDevice.cpp index d24cb814a3..46a8dae317 100644 --- a/camera/ndk/impl/ACameraDevice.cpp +++ b/camera/ndk/impl/ACameraDevice.cpp @@ -29,7 +29,7 @@ #include "ACameraCaptureSession.inc" ACameraDevice::~ACameraDevice() { - mDevice->stopLooper(); + mDevice->stopLooperAndDisconnect(); } namespace android { @@ -112,19 +112,7 @@ CameraDevice::CameraDevice( } } -// Device close implementaiton -CameraDevice::~CameraDevice() { - sp<ACameraCaptureSession> session = mCurrentSession.promote(); - { - Mutex::Autolock _l(mDeviceLock); - if (!isClosed()) { - disconnectLocked(session); - } - LOG_ALWAYS_FATAL_IF(mCbLooper != nullptr, - "CameraDevice looper should've been stopped before ~CameraDevice"); - mCurrentSession = nullptr; - } -} +CameraDevice::~CameraDevice() { } void CameraDevice::postSessionMsgAndCleanup(sp<AMessage>& msg) { @@ -892,8 +880,14 @@ CameraDevice::onCaptureErrorLocked( return; } -void CameraDevice::stopLooper() { +void CameraDevice::stopLooperAndDisconnect() { Mutex::Autolock _l(mDeviceLock); + sp<ACameraCaptureSession> session = mCurrentSession.promote(); + if (!isClosed()) { + disconnectLocked(session); + } + mCurrentSession = nullptr; + if (mCbLooper != nullptr) { mCbLooper->unregisterHandler(mHandler->id()); mCbLooper->stop(); diff --git a/camera/ndk/impl/ACameraDevice.h b/camera/ndk/impl/ACameraDevice.h index 7a35bf0e81..6c2ceb300e 100644 --- a/camera/ndk/impl/ACameraDevice.h +++ b/camera/ndk/impl/ACameraDevice.h @@ -40,6 +40,7 @@ #include <camera/NdkCameraManager.h> #include <camera/NdkCameraCaptureSession.h> + #include "ACameraMetadata.h" namespace android { @@ -110,7 +111,7 @@ class CameraDevice final : public RefBase { inline ACameraDevice* getWrapper() const { return mWrapper; }; // Stop the looper thread and unregister the handler - void stopLooper(); + void stopLooperAndDisconnect(); private: friend ACameraCaptureSession; diff --git a/camera/ndk/impl/ACameraManager.h b/camera/ndk/impl/ACameraManager.h index 8c1da36418..28fec826a1 100644 --- a/camera/ndk/impl/ACameraManager.h +++ b/camera/ndk/impl/ACameraManager.h @@ -92,6 +92,12 @@ class CameraManagerGlobal final : public RefBase { } virtual binder::Status onCameraAccessPrioritiesChanged(); + virtual binder::Status onCameraOpened(const String16&, const String16&) { + return binder::Status::ok(); + } + virtual binder::Status onCameraClosed(const String16&) { + return binder::Status::ok(); + } private: const wp<CameraManagerGlobal> mCameraManager; diff --git a/camera/ndk/ndk_vendor/impl/ACameraDevice.cpp b/camera/ndk/ndk_vendor/impl/ACameraDevice.cpp index 35c83551c2..e511a3f81d 100644 --- a/camera/ndk/ndk_vendor/impl/ACameraDevice.cpp +++ b/camera/ndk/ndk_vendor/impl/ACameraDevice.cpp @@ -45,7 +45,7 @@ using namespace android; ACameraDevice::~ACameraDevice() { - mDevice->stopLooper(); + mDevice->stopLooperAndDisconnect(); } namespace android { @@ -125,19 +125,7 @@ CameraDevice::CameraDevice( } } -// Device close implementaiton -CameraDevice::~CameraDevice() { - sp<ACameraCaptureSession> session = mCurrentSession.promote(); - { - Mutex::Autolock _l(mDeviceLock); - if (!isClosed()) { - disconnectLocked(session); - } - mCurrentSession = nullptr; - LOG_ALWAYS_FATAL_IF(mCbLooper != nullptr, - "CameraDevice looper should've been stopped before ~CameraDevice"); - } -} +CameraDevice::~CameraDevice() { } void CameraDevice::postSessionMsgAndCleanup(sp<AMessage>& msg) { @@ -1388,6 +1376,7 @@ CameraDevice::checkAndFireSequenceCompleteLocked() { // before cbh goes out of scope and causing we call the session // destructor while holding device lock cbh.mSession.clear(); + postSessionMsgAndCleanup(msg); } @@ -1400,8 +1389,13 @@ CameraDevice::checkAndFireSequenceCompleteLocked() { } } -void CameraDevice::stopLooper() { +void CameraDevice::stopLooperAndDisconnect() { Mutex::Autolock _l(mDeviceLock); + sp<ACameraCaptureSession> session = mCurrentSession.promote(); + if (!isClosed()) { + disconnectLocked(session); + } + mCurrentSession = nullptr; if (mCbLooper != nullptr) { mCbLooper->unregisterHandler(mHandler->id()); mCbLooper->stop(); diff --git a/camera/ndk/ndk_vendor/impl/ACameraDevice.h b/camera/ndk/ndk_vendor/impl/ACameraDevice.h index 3328a85686..7fc699e16e 100644 --- a/camera/ndk/ndk_vendor/impl/ACameraDevice.h +++ b/camera/ndk/ndk_vendor/impl/ACameraDevice.h @@ -36,6 +36,7 @@ #include <camera/NdkCameraManager.h> #include <camera/NdkCameraCaptureSession.h> + #include "ACameraMetadata.h" #include "utils.h" @@ -134,10 +135,11 @@ class CameraDevice final : public RefBase { inline ACameraDevice* getWrapper() const { return mWrapper; }; // Stop the looper thread and unregister the handler - void stopLooper(); + void stopLooperAndDisconnect(); private: friend ACameraCaptureSession; + friend ACameraDevice; camera_status_t checkCameraClosedOrErrorLocked() const; @@ -387,7 +389,6 @@ struct ACameraDevice { mDevice(new android::acam::CameraDevice(id, cb, std::move(chars), this)) {} ~ACameraDevice(); - /******************* * NDK public APIs * *******************/ diff --git a/camera/ndk/ndk_vendor/tests/AImageReaderVendorTest.cpp b/camera/ndk/ndk_vendor/tests/AImageReaderVendorTest.cpp index 37de30ab01..938b5f55b8 100644 --- a/camera/ndk/ndk_vendor/tests/AImageReaderVendorTest.cpp +++ b/camera/ndk/ndk_vendor/tests/AImageReaderVendorTest.cpp @@ -24,6 +24,7 @@ #include <algorithm> #include <mutex> #include <string> +#include <variant> #include <vector> #include <stdio.h> #include <stdio.h> @@ -49,6 +50,7 @@ static constexpr int kTestImageHeight = 480; static constexpr int kTestImageFormat = AIMAGE_FORMAT_YUV_420_888; using android::hardware::camera::common::V1_0::helper::VendorTagDescriptorCache; +using ConfiguredWindows = std::set<native_handle_t *>; class CameraHelper { public: @@ -60,9 +62,12 @@ class CameraHelper { const char* physicalCameraId; native_handle_t* anw; }; - int initCamera(native_handle_t* imgReaderAnw, + + // Retaining the error code in case the caller needs to analyze it. + std::variant<int, ConfiguredWindows> initCamera(native_handle_t* imgReaderAnw, const std::vector<PhysicalImgReaderInfo>& physicalImgReaders, bool usePhysicalSettings) { + ConfiguredWindows configuredWindows; if (imgReaderAnw == nullptr) { ALOGE("Cannot initialize camera before image reader get initialized."); return -1; @@ -78,7 +83,7 @@ class CameraHelper { ret = ACameraManager_openCamera(mCameraManager, mCameraId, &mDeviceCb, &mDevice); if (ret != AMEDIA_OK || mDevice == nullptr) { ALOGE("Failed to open camera, ret=%d, mDevice=%p.", ret, mDevice); - return -1; + return ret; } // Create capture session @@ -97,8 +102,9 @@ class CameraHelper { ALOGE("ACaptureSessionOutputContainer_add failed, ret=%d", ret); return ret; } - + configuredWindows.insert(mImgReaderAnw); std::vector<const char*> idPointerList; + std::set<const native_handle_t*> physicalStreamMap; for (auto& physicalStream : physicalImgReaders) { ACaptureSessionOutput* sessionOutput = nullptr; ret = ACaptureSessionPhysicalOutput_create(physicalStream.anw, @@ -112,21 +118,25 @@ class CameraHelper { ALOGE("ACaptureSessionOutputContainer_add failed, ret=%d", ret); return ret; } - mExtraOutputs.push_back(sessionOutput); + ret = ACameraDevice_isSessionConfigurationSupported(mDevice, mOutputs); + if (ret != ACAMERA_OK && ret != ACAMERA_ERROR_UNSUPPORTED_OPERATION) { + ALOGW("ACameraDevice_isSessionConfigurationSupported failed, ret=%d camera id %s", + ret, mCameraId); + ACaptureSessionOutputContainer_remove(mOutputs, sessionOutput); + ACaptureSessionOutput_free(sessionOutput); + continue; + } + configuredWindows.insert(physicalStream.anw); // Assume that at most one physical stream per physical camera. mPhysicalCameraIds.push_back(physicalStream.physicalCameraId); idPointerList.push_back(physicalStream.physicalCameraId); + physicalStreamMap.insert(physicalStream.anw); + mSessionPhysicalOutputs.push_back(sessionOutput); } ACameraIdList cameraIdList; cameraIdList.numCameras = idPointerList.size(); cameraIdList.cameraIds = idPointerList.data(); - ret = ACameraDevice_isSessionConfigurationSupported(mDevice, mOutputs); - if (ret != ACAMERA_OK && ret != ACAMERA_ERROR_UNSUPPORTED_OPERATION) { - ALOGE("ACameraDevice_isSessionConfigurationSupported failed, ret=%d", ret); - return ret; - } - ret = ACameraDevice_createCaptureSession(mDevice, mOutputs, &mSessionCb, &mSession); if (ret != AMEDIA_OK) { ALOGE("ACameraDevice_createCaptureSession failed, ret=%d", ret); @@ -157,6 +167,10 @@ class CameraHelper { } for (auto& physicalStream : physicalImgReaders) { + if (physicalStreamMap.find(physicalStream.anw) == physicalStreamMap.end()) { + ALOGI("Skipping physicalStream anw=%p", physicalStream.anw); + continue; + } ACameraOutputTarget* outputTarget = nullptr; ret = ACameraOutputTarget_create(physicalStream.anw, &outputTarget); if (ret != AMEDIA_OK) { @@ -168,11 +182,11 @@ class CameraHelper { ALOGE("ACaptureRequest_addTarget failed, ret=%d", ret); return ret; } - mReqExtraOutputs.push_back(outputTarget); + mReqPhysicalOutputs.push_back(outputTarget); } mIsCameraReady = true; - return 0; + return configuredWindows; } @@ -184,10 +198,10 @@ class CameraHelper { ACameraOutputTarget_free(mReqImgReaderOutput); mReqImgReaderOutput = nullptr; } - for (auto& outputTarget : mReqExtraOutputs) { + for (auto& outputTarget : mReqPhysicalOutputs) { ACameraOutputTarget_free(outputTarget); } - mReqExtraOutputs.clear(); + mReqPhysicalOutputs.clear(); if (mStillRequest) { ACaptureRequest_free(mStillRequest); mStillRequest = nullptr; @@ -201,10 +215,10 @@ class CameraHelper { ACaptureSessionOutput_free(mImgReaderOutput); mImgReaderOutput = nullptr; } - for (auto& extraOutput : mExtraOutputs) { + for (auto& extraOutput : mSessionPhysicalOutputs) { ACaptureSessionOutput_free(extraOutput); } - mExtraOutputs.clear(); + mSessionPhysicalOutputs.clear(); if (mOutputs) { ACaptureSessionOutputContainer_free(mOutputs); mOutputs = nullptr; @@ -239,21 +253,9 @@ class CameraHelper { return true; } - static void onDeviceDisconnected(void* /*obj*/, ACameraDevice* /*device*/) {} - - static void onDeviceError(void* /*obj*/, ACameraDevice* /*device*/, int /*errorCode*/) {} - - static void onSessionClosed(void* /*obj*/, ACameraCaptureSession* /*session*/) {} - - static void onSessionReady(void* /*obj*/, ACameraCaptureSession* /*session*/) {} - - static void onSessionActive(void* /*obj*/, ACameraCaptureSession* /*session*/) {} - private: - ACameraDevice_StateCallbacks mDeviceCb{this, onDeviceDisconnected, - onDeviceError}; - ACameraCaptureSession_stateCallbacks mSessionCb{ - this, onSessionClosed, onSessionReady, onSessionActive}; + ACameraDevice_StateCallbacks mDeviceCb{this, nullptr, nullptr}; + ACameraCaptureSession_stateCallbacks mSessionCb{ this, nullptr, nullptr, nullptr}; native_handle_t* mImgReaderAnw = nullptr; // not owned by us. @@ -262,13 +264,13 @@ class CameraHelper { // Capture session ACaptureSessionOutputContainer* mOutputs = nullptr; ACaptureSessionOutput* mImgReaderOutput = nullptr; - std::vector<ACaptureSessionOutput*> mExtraOutputs; + std::vector<ACaptureSessionOutput*> mSessionPhysicalOutputs; ACameraCaptureSession* mSession = nullptr; // Capture request ACaptureRequest* mStillRequest = nullptr; ACameraOutputTarget* mReqImgReaderOutput = nullptr; - std::vector<ACameraOutputTarget*> mReqExtraOutputs; + std::vector<ACameraOutputTarget*> mReqPhysicalOutputs; bool mIsCameraReady = false; const char* mCameraId; @@ -581,9 +583,11 @@ class AImageReaderVendorTest : public ::testing::Test { } CameraHelper cameraHelper(id, mCameraManager); - ret = cameraHelper.initCamera(testCase.getNativeWindow(), - {}/*physicalImageReaders*/, false/*usePhysicalSettings*/); - if (ret < 0) { + std::variant<int, ConfiguredWindows> retInit = + cameraHelper.initCamera(testCase.getNativeWindow(), {}/*physicalImageReaders*/, + false/*usePhysicalSettings*/); + int *retp = std::get_if<int>(&retInit); + if (retp) { ALOGE("Unable to initialize camera helper"); return false; } @@ -751,10 +755,15 @@ class AImageReaderVendorTest : public ::testing::Test { physicalImgReaderInfo.push_back({physicalCameraIds[0], testCases[1]->getNativeWindow()}); physicalImgReaderInfo.push_back({physicalCameraIds[1], testCases[2]->getNativeWindow()}); - int ret = cameraHelper.initCamera(testCases[0]->getNativeWindow(), - physicalImgReaderInfo, usePhysicalSettings); - ASSERT_EQ(ret, 0); - + std::variant<int, ConfiguredWindows> retInit = + cameraHelper.initCamera(testCases[0]->getNativeWindow(), physicalImgReaderInfo, + usePhysicalSettings); + int *retp = std::get_if<int>(&retInit); + ASSERT_EQ(retp, nullptr); + ConfiguredWindows *configuredWindowsp = std::get_if<ConfiguredWindows>(&retInit); + ASSERT_NE(configuredWindowsp, nullptr); + ASSERT_LE(configuredWindowsp->size(), testCases.size()); + int ret = 0; if (!cameraHelper.isCameraReady()) { ALOGW("Camera is not ready after successful initialization. It's either due to camera " "on board lacks BACKWARDS_COMPATIBLE capability or the device does not have " @@ -776,9 +785,15 @@ class AImageReaderVendorTest : public ::testing::Test { break; } } - ASSERT_EQ(testCases[0]->getAcquiredImageCount(), pictureCount); - ASSERT_EQ(testCases[1]->getAcquiredImageCount(), pictureCount); - ASSERT_EQ(testCases[2]->getAcquiredImageCount(), pictureCount); + for(auto &testCase : testCases) { + auto it = configuredWindowsp->find(testCase->getNativeWindow()); + if (it == configuredWindowsp->end()) { + continue; + } + ALOGI("Testing window %p", testCase->getNativeWindow()); + ASSERT_EQ(testCase->getAcquiredImageCount(), pictureCount); + } + ASSERT_TRUE(cameraHelper.checkCallbacks(pictureCount)); ACameraMetadata_free(staticMetadata); diff --git a/camera/tests/CameraBinderTests.cpp b/camera/tests/CameraBinderTests.cpp index 8fe029aea7..dc5afc557b 100644 --- a/camera/tests/CameraBinderTests.cpp +++ b/camera/tests/CameraBinderTests.cpp @@ -95,6 +95,17 @@ public: return binder::Status::ok(); } + virtual binder::Status onCameraOpened(const String16& /*cameraId*/, + const String16& /*clientPackageName*/) { + // No op + return binder::Status::ok(); + } + + virtual binder::Status onCameraClosed(const String16& /*cameraId*/) { + // No op + return binder::Status::ok(); + } + bool waitForNumCameras(size_t num) const { Mutex::Autolock l(mLock); diff --git a/drm/libmediadrm/IDrm.cpp b/drm/libmediadrm/IDrm.cpp index 51274d16c1..c2cc429859 100644 --- a/drm/libmediadrm/IDrm.cpp +++ b/drm/libmediadrm/IDrm.cpp @@ -1071,6 +1071,7 @@ status_t BnDrm::onTransact( Vector<uint8_t> keySetId; readVector(data, keySetId); DrmPlugin::OfflineLicenseState state; + state = DrmPlugin::OfflineLicenseState::kOfflineLicenseStateUnknown; status_t result = getOfflineLicenseState(keySetId, &state); reply->writeInt32(static_cast<DrmPlugin::OfflineLicenseState>(state)); reply->writeInt32(result); diff --git a/drm/mediadrm/plugins/clearkey/default/InitDataParser.cpp b/drm/mediadrm/plugins/clearkey/default/InitDataParser.cpp index caff3939be..121a4e2952 100644 --- a/drm/mediadrm/plugins/clearkey/default/InitDataParser.cpp +++ b/drm/mediadrm/plugins/clearkey/default/InitDataParser.cpp @@ -76,10 +76,21 @@ android::status_t InitDataParser::parse(const Vector<uint8_t>& initData, android::status_t InitDataParser::parsePssh(const Vector<uint8_t>& initData, Vector<const uint8_t*>* keyIds) { + // Description of PSSH format: + // https://w3c.github.io/encrypted-media/format-registry/initdata/cenc.html size_t readPosition = 0; - // Validate size field uint32_t expectedSize = initData.size(); + const char psshIdentifier[4] = {'p', 's', 's', 'h'}; + const uint8_t psshVersion1[4] = {1, 0, 0, 0}; + uint32_t keyIdCount = 0; + size_t headerSize = sizeof(expectedSize) + sizeof(psshIdentifier) + + sizeof(psshVersion1) + kSystemIdSize + sizeof(keyIdCount); + if (initData.size() < headerSize) { + return android::ERROR_DRM_CANNOT_HANDLE; + } + + // Validate size field expectedSize = htonl(expectedSize); if (memcmp(&initData[readPosition], &expectedSize, sizeof(expectedSize)) != 0) { @@ -88,7 +99,6 @@ android::status_t InitDataParser::parsePssh(const Vector<uint8_t>& initData, readPosition += sizeof(expectedSize); // Validate PSSH box identifier - const char psshIdentifier[4] = {'p', 's', 's', 'h'}; if (memcmp(&initData[readPosition], psshIdentifier, sizeof(psshIdentifier)) != 0) { return android::ERROR_DRM_CANNOT_HANDLE; @@ -96,7 +106,6 @@ android::status_t InitDataParser::parsePssh(const Vector<uint8_t>& initData, readPosition += sizeof(psshIdentifier); // Validate EME version number - const uint8_t psshVersion1[4] = {1, 0, 0, 0}; if (memcmp(&initData[readPosition], psshVersion1, sizeof(psshVersion1)) != 0) { return android::ERROR_DRM_CANNOT_HANDLE; @@ -110,12 +119,14 @@ android::status_t InitDataParser::parsePssh(const Vector<uint8_t>& initData, readPosition += kSystemIdSize; // Read key ID count - uint32_t keyIdCount; memcpy(&keyIdCount, &initData[readPosition], sizeof(keyIdCount)); keyIdCount = ntohl(keyIdCount); readPosition += sizeof(keyIdCount); - if (readPosition + ((uint64_t)keyIdCount * kKeyIdSize) != - initData.size() - sizeof(uint32_t)) { + + uint64_t psshSize = 0; + if (__builtin_mul_overflow(keyIdCount, kKeyIdSize, &psshSize) || + __builtin_add_overflow(readPosition, psshSize, &psshSize) || + psshSize != initData.size() - sizeof(uint32_t) /* DataSize(0) */) { return android::ERROR_DRM_CANNOT_HANDLE; } diff --git a/drm/mediadrm/plugins/clearkey/hidl/Android.bp b/drm/mediadrm/plugins/clearkey/hidl/Android.bp index fdb18d07e1..e91e918b1d 100644 --- a/drm/mediadrm/plugins/clearkey/hidl/Android.bp +++ b/drm/mediadrm/plugins/clearkey/hidl/Android.bp @@ -37,7 +37,7 @@ cc_defaults { relative_install_path: "hw", - cflags: ["-Wall", "-Werror", "-Wthread-safety"], + cflags: ["-Wall", "-Werror"], shared_libs: [ "android.hardware.drm@1.0", diff --git a/drm/mediadrm/plugins/clearkey/hidl/DrmPlugin.cpp b/drm/mediadrm/plugins/clearkey/hidl/DrmPlugin.cpp index f33b6480df..4274c3613f 100644 --- a/drm/mediadrm/plugins/clearkey/hidl/DrmPlugin.cpp +++ b/drm/mediadrm/plugins/clearkey/hidl/DrmPlugin.cpp @@ -111,6 +111,8 @@ void DrmPlugin::initProperties() { // The content in this secure stop is implementation dependent, the clearkey // secureStop does not serve as a reference implementation. void DrmPlugin::installSecureStop(const hidl_vec<uint8_t>& sessionId) { + Mutex::Autolock lock(mSecureStopLock); + ClearkeySecureStop clearkeySecureStop; clearkeySecureStop.id = uint32ToVector(++mNextSecureStopId); clearkeySecureStop.data.assign(sessionId.begin(), sessionId.end()); @@ -213,7 +215,6 @@ Status_V1_2 DrmPlugin::getKeyRequestCommon(const hidl_vec<uint8_t>& scope, if (requestString.find(kOfflineLicense) != std::string::npos) { std::string emptyResponse; std::string keySetIdString(keySetId.begin(), keySetId.end()); - Mutex::Autolock lock(mFileHandleLock); if (!mFileHandle.StoreLicense(keySetIdString, DeviceFiles::kLicenseStateReleasing, emptyResponse)) { @@ -329,7 +330,6 @@ bool DrmPlugin::makeKeySetId(std::string* keySetId) { } *keySetId = kKeySetIdPrefix + ByteArrayToHexString( reinterpret_cast<const uint8_t*>(randomData.data()), randomData.size()); - Mutex::Autolock lock(mFileHandleLock); if (mFileHandle.LicenseExists(*keySetId)) { // collision, regenerate ALOGV("Retry generating KeySetId"); @@ -382,7 +382,6 @@ Return<void> DrmPlugin::provideKeyResponse( if (status == Status::OK) { if (isOfflineLicense) { if (isRelease) { - Mutex::Autolock lock(mFileHandleLock); mFileHandle.DeleteLicense(keySetId); } else { if (!makeKeySetId(&keySetId)) { @@ -390,7 +389,6 @@ Return<void> DrmPlugin::provideKeyResponse( return Void(); } - Mutex::Autolock lock(mFileHandleLock); bool ok = mFileHandle.StoreLicense( keySetId, DeviceFiles::kLicenseStateActive, @@ -445,7 +443,6 @@ Return<Status> DrmPlugin::restoreKeys( DeviceFiles::LicenseState licenseState; std::string offlineLicense; Status status = Status::OK; - Mutex::Autolock lock(mFileHandleLock); if (!mFileHandle.RetrieveLicense(std::string(keySetId.begin(), keySetId.end()), &licenseState, &offlineLicense)) { ALOGE("Failed to restore offline license"); @@ -697,7 +694,6 @@ Return<void> DrmPlugin::getMetrics(getMetrics_cb _hidl_cb) { } Return<void> DrmPlugin::getOfflineLicenseKeySetIds(getOfflineLicenseKeySetIds_cb _hidl_cb) { - Mutex::Autolock lock(mFileHandleLock); std::vector<std::string> licenseNames = mFileHandle.ListLicenses(); std::vector<KeySetId> keySetIds; if (mMockError != Status_V1_2::OK) { @@ -718,7 +714,6 @@ Return<Status> DrmPlugin::removeOfflineLicense(const KeySetId& keySetId) { return toStatus_1_0(mMockError); } std::string licenseName(keySetId.begin(), keySetId.end()); - Mutex::Autolock lock(mFileHandleLock); if (mFileHandle.DeleteLicense(licenseName)) { return Status::OK; } @@ -727,8 +722,6 @@ Return<Status> DrmPlugin::removeOfflineLicense(const KeySetId& keySetId) { Return<void> DrmPlugin::getOfflineLicenseState(const KeySetId& keySetId, getOfflineLicenseState_cb _hidl_cb) { - Mutex::Autolock lock(mFileHandleLock); - std::string licenseName(keySetId.begin(), keySetId.end()); DeviceFiles::LicenseState state; std::string license; @@ -755,6 +748,7 @@ Return<void> DrmPlugin::getOfflineLicenseState(const KeySetId& keySetId, } Return<void> DrmPlugin::getSecureStops(getSecureStops_cb _hidl_cb) { + mSecureStopLock.lock(); std::vector<SecureStop> stops; for (auto itr = mSecureStops.begin(); itr != mSecureStops.end(); ++itr) { ClearkeySecureStop clearkeyStop = itr->second; @@ -766,26 +760,32 @@ Return<void> DrmPlugin::getSecureStops(getSecureStops_cb _hidl_cb) { stop.opaqueData = toHidlVec(stopVec); stops.push_back(stop); } + mSecureStopLock.unlock(); + _hidl_cb(Status::OK, stops); return Void(); } Return<void> DrmPlugin::getSecureStop(const hidl_vec<uint8_t>& secureStopId, getSecureStop_cb _hidl_cb) { - SecureStop stop; + std::vector<uint8_t> stopVec; + + mSecureStopLock.lock(); auto itr = mSecureStops.find(toVector(secureStopId)); if (itr != mSecureStops.end()) { ClearkeySecureStop clearkeyStop = itr->second; - std::vector<uint8_t> stopVec; stopVec.insert(stopVec.end(), clearkeyStop.id.begin(), clearkeyStop.id.end()); stopVec.insert(stopVec.end(), clearkeyStop.data.begin(), clearkeyStop.data.end()); + } + mSecureStopLock.unlock(); + SecureStop stop; + if (!stopVec.empty()) { stop.opaqueData = toHidlVec(stopVec); _hidl_cb(Status::OK, stop); } else { _hidl_cb(Status::BAD_VALUE, stop); } - return Void(); } @@ -798,23 +798,35 @@ Return<Status> DrmPlugin::releaseAllSecureStops() { } Return<void> DrmPlugin::getSecureStopIds(getSecureStopIds_cb _hidl_cb) { + mSecureStopLock.lock(); std::vector<SecureStopId> ids; for (auto itr = mSecureStops.begin(); itr != mSecureStops.end(); ++itr) { ids.push_back(itr->first); } + mSecureStopLock.unlock(); _hidl_cb(Status::OK, toHidlVec(ids)); return Void(); } Return<Status> DrmPlugin::releaseSecureStops(const SecureStopRelease& ssRelease) { - if (ssRelease.opaqueData.size() == 0) { + // OpaqueData starts with 4 byte decimal integer string + const size_t kFourBytesOffset = 4; + if (ssRelease.opaqueData.size() < kFourBytesOffset) { + ALOGE("Invalid secureStopRelease length"); return Status::BAD_VALUE; } Status status = Status::OK; std::vector<uint8_t> input = toVector(ssRelease.opaqueData); + if (input.size() < kSecureStopIdSize + kFourBytesOffset) { + // The minimum size of SecureStopRelease has to contain + // a 4 bytes count and one secureStop id + ALOGE("Total size of secureStops is too short"); + return Status::BAD_VALUE; + } + // The format of opaqueData is shared between the server // and the drm service. The clearkey implementation consists of: // count - number of secure stops @@ -830,24 +842,35 @@ Return<Status> DrmPlugin::releaseSecureStops(const SecureStopRelease& ssRelease) // Avoid divide by 0 below. if (count == 0) { + ALOGE("Invalid 0 secureStop count"); return Status::BAD_VALUE; } - size_t secureStopSize = (input.size() - countBufferSize) / count; - uint8_t buffer[secureStopSize]; - size_t offset = countBufferSize; // skip the count + // Computes the fixed length secureStop size + size_t secureStopSize = (input.size() - kFourBytesOffset) / count; + if (secureStopSize < kSecureStopIdSize) { + // A valid secureStop contains the id plus data + ALOGE("Invalid secureStop size"); + return Status::BAD_VALUE; + } + uint8_t* buffer = new uint8_t[secureStopSize]; + size_t offset = kFourBytesOffset; // skip the count for (size_t i = 0; i < count; ++i, offset += secureStopSize) { memcpy(buffer, input.data() + offset, secureStopSize); - std::vector<uint8_t> id(buffer, buffer + kSecureStopIdSize); + // A secureStop contains id+data, we only use the id for removal + std::vector<uint8_t> id(buffer, buffer + kSecureStopIdSize); status = removeSecureStop(toHidlVec(id)); if (Status::OK != status) break; } + delete[] buffer; return status; } Return<Status> DrmPlugin::removeSecureStop(const hidl_vec<uint8_t>& secureStopId) { + Mutex::Autolock lock(mSecureStopLock); + if (1 != mSecureStops.erase(toVector(secureStopId))) { return Status::BAD_VALUE; } @@ -855,6 +878,8 @@ Return<Status> DrmPlugin::removeSecureStop(const hidl_vec<uint8_t>& secureStopId } Return<Status> DrmPlugin::removeAllSecureStops() { + Mutex::Autolock lock(mSecureStopLock); + mSecureStops.clear(); mNextSecureStopId = kSecureStopIdStart; return Status::OK; diff --git a/drm/mediadrm/plugins/clearkey/hidl/InitDataParser.cpp b/drm/mediadrm/plugins/clearkey/hidl/InitDataParser.cpp index b988ce03ca..ebb6fb8fc2 100644 --- a/drm/mediadrm/plugins/clearkey/hidl/InitDataParser.cpp +++ b/drm/mediadrm/plugins/clearkey/hidl/InitDataParser.cpp @@ -85,10 +85,22 @@ Status InitDataParser::parse(const std::vector<uint8_t>& initData, Status InitDataParser::parsePssh(const std::vector<uint8_t>& initData, std::vector<const uint8_t*>* keyIds) { + // Description of PSSH format: + // https://w3c.github.io/encrypted-media/format-registry/initdata/cenc.html size_t readPosition = 0; - // Validate size field uint32_t expectedSize = initData.size(); + const char psshIdentifier[4] = {'p', 's', 's', 'h'}; + const uint8_t psshVersion1[4] = {1, 0, 0, 0}; + uint32_t keyIdCount = 0; + size_t headerSize = sizeof(expectedSize) + sizeof(psshIdentifier) + + sizeof(psshVersion1) + kSystemIdSize + sizeof(keyIdCount); + if (initData.size() < headerSize) { + android_errorWriteLog(0x534e4554, "137282168"); + return Status::ERROR_DRM_CANNOT_HANDLE; + } + + // Validate size field expectedSize = htonl(expectedSize); if (memcmp(&initData[readPosition], &expectedSize, sizeof(expectedSize)) != 0) { @@ -97,7 +109,6 @@ Status InitDataParser::parsePssh(const std::vector<uint8_t>& initData, readPosition += sizeof(expectedSize); // Validate PSSH box identifier - const char psshIdentifier[4] = {'p', 's', 's', 'h'}; if (memcmp(&initData[readPosition], psshIdentifier, sizeof(psshIdentifier)) != 0) { return Status::ERROR_DRM_CANNOT_HANDLE; @@ -105,7 +116,6 @@ Status InitDataParser::parsePssh(const std::vector<uint8_t>& initData, readPosition += sizeof(psshIdentifier); // Validate EME version number - const uint8_t psshVersion1[4] = {1, 0, 0, 0}; if (memcmp(&initData[readPosition], psshVersion1, sizeof(psshVersion1)) != 0) { return Status::ERROR_DRM_CANNOT_HANDLE; @@ -119,12 +129,14 @@ Status InitDataParser::parsePssh(const std::vector<uint8_t>& initData, readPosition += kSystemIdSize; // Read key ID count - uint32_t keyIdCount; memcpy(&keyIdCount, &initData[readPosition], sizeof(keyIdCount)); keyIdCount = ntohl(keyIdCount); readPosition += sizeof(keyIdCount); - if (readPosition + ((uint64_t)keyIdCount * kKeyIdSize) != - initData.size() - sizeof(uint32_t)) { + + uint64_t psshSize = 0; + if (__builtin_mul_overflow(keyIdCount, kKeyIdSize, &psshSize) || + __builtin_add_overflow(readPosition, psshSize, &psshSize) || + psshSize != initData.size() - sizeof(uint32_t) /* DataSize(0) */) { return Status::ERROR_DRM_CANNOT_HANDLE; } diff --git a/drm/mediadrm/plugins/clearkey/hidl/MemoryFileSystem.cpp b/drm/mediadrm/plugins/clearkey/hidl/MemoryFileSystem.cpp index d29acac30a..2dcd00fb27 100644 --- a/drm/mediadrm/plugins/clearkey/hidl/MemoryFileSystem.cpp +++ b/drm/mediadrm/plugins/clearkey/hidl/MemoryFileSystem.cpp @@ -24,13 +24,11 @@ std::string MemoryFileSystem::GetFileName(const std::string& path) { } bool MemoryFileSystem::FileExists(const std::string& fileName) const { - std::lock_guard<std::mutex> lock(mMemoryFileSystemLock); auto result = mMemoryFileSystem.find(fileName); return result != mMemoryFileSystem.end(); } ssize_t MemoryFileSystem::GetFileSize(const std::string& fileName) const { - std::lock_guard<std::mutex> lock(mMemoryFileSystemLock); auto result = mMemoryFileSystem.find(fileName); if (result != mMemoryFileSystem.end()) { return static_cast<ssize_t>(result->second.getFileSize()); @@ -42,7 +40,6 @@ ssize_t MemoryFileSystem::GetFileSize(const std::string& fileName) const { std::vector<std::string> MemoryFileSystem::ListFiles() const { std::vector<std::string> list; - std::lock_guard<std::mutex> lock(mMemoryFileSystemLock); for (const auto& filename : mMemoryFileSystem) { list.push_back(filename.first); } @@ -51,7 +48,6 @@ std::vector<std::string> MemoryFileSystem::ListFiles() const { size_t MemoryFileSystem::Read(const std::string& path, std::string* buffer) { std::string key = GetFileName(path); - std::lock_guard<std::mutex> lock(mMemoryFileSystemLock); auto result = mMemoryFileSystem.find(key); if (result != mMemoryFileSystem.end()) { std::string serializedHashFile = result->second.getContent(); @@ -65,7 +61,6 @@ size_t MemoryFileSystem::Read(const std::string& path, std::string* buffer) { size_t MemoryFileSystem::Write(const std::string& path, const MemoryFile& memoryFile) { std::string key = GetFileName(path); - std::lock_guard<std::mutex> lock(mMemoryFileSystemLock); auto result = mMemoryFileSystem.find(key); if (result != mMemoryFileSystem.end()) { mMemoryFileSystem.erase(key); @@ -75,7 +70,6 @@ size_t MemoryFileSystem::Write(const std::string& path, const MemoryFile& memory } bool MemoryFileSystem::RemoveFile(const std::string& fileName) { - std::lock_guard<std::mutex> lock(mMemoryFileSystemLock); auto result = mMemoryFileSystem.find(fileName); if (result != mMemoryFileSystem.end()) { mMemoryFileSystem.erase(result); @@ -87,7 +81,6 @@ bool MemoryFileSystem::RemoveFile(const std::string& fileName) { } bool MemoryFileSystem::RemoveAllFiles() { - std::lock_guard<std::mutex> lock(mMemoryFileSystemLock); mMemoryFileSystem.clear(); return mMemoryFileSystem.empty(); } diff --git a/drm/mediadrm/plugins/clearkey/hidl/include/DrmPlugin.h b/drm/mediadrm/plugins/clearkey/hidl/include/DrmPlugin.h index 7ea58e07a9..076beb8a0d 100644 --- a/drm/mediadrm/plugins/clearkey/hidl/include/DrmPlugin.h +++ b/drm/mediadrm/plugins/clearkey/hidl/include/DrmPlugin.h @@ -416,8 +416,8 @@ private: mMockError = Status_V1_2::OK; } - Mutex mFileHandleLock; - DeviceFiles mFileHandle GUARDED_BY(mFileHandleLock); + DeviceFiles mFileHandle; + Mutex mSecureStopLock; CLEARKEY_DISALLOW_COPY_AND_ASSIGN_AND_NEW(DrmPlugin); }; diff --git a/drm/mediadrm/plugins/clearkey/hidl/include/MemoryFileSystem.h b/drm/mediadrm/plugins/clearkey/hidl/include/MemoryFileSystem.h index 6ac0e2c4bb..bcd9fd631a 100644 --- a/drm/mediadrm/plugins/clearkey/hidl/include/MemoryFileSystem.h +++ b/drm/mediadrm/plugins/clearkey/hidl/include/MemoryFileSystem.h @@ -5,9 +5,7 @@ #ifndef CLEARKEY_MEMORY_FILE_SYSTEM_H_ #define CLEARKEY_MEMORY_FILE_SYSTEM_H_ -#include <android-base/thread_annotations.h> #include <map> -#include <mutex> #include <string> #include "ClearKeyTypes.h" @@ -51,12 +49,10 @@ class MemoryFileSystem { size_t Write(const std::string& pathName, const MemoryFile& memoryFile); private: - mutable std::mutex mMemoryFileSystemLock; - // License file name is made up of a unique keySetId, therefore, // the filename can be used as the key to locate licenses in the // memory file system. - std::map<std::string, MemoryFile> mMemoryFileSystem GUARDED_BY(mMemoryFileSystemLock); + std::map<std::string, MemoryFile> mMemoryFileSystem; std::string GetFileName(const std::string& path); diff --git a/media/bufferpool/1.0/AccessorImpl.cpp b/media/bufferpool/1.0/AccessorImpl.cpp index fa17f15f8c..a5366f62fc 100644 --- a/media/bufferpool/1.0/AccessorImpl.cpp +++ b/media/bufferpool/1.0/AccessorImpl.cpp @@ -151,6 +151,7 @@ ResultStatus Accessor::Impl::connect( newConnection->initialize(accessor, id); *connection = newConnection; *pConnectionId = id; + mBufferPool.mConnectionIds.insert(id); ++sSeqId; } } @@ -305,7 +306,12 @@ bool Accessor::Impl::BufferPool::handleTransferTo(const BufferStatusMessage &mes found->second->mSenderValidated = true; return true; } - // TODO: verify there is target connection Id + if (mConnectionIds.find(message.targetConnectionId) == mConnectionIds.end()) { + // N.B: it could be fake or receive connection already closed. + ALOGD("bufferpool %p receiver connection %lld is no longer valid", + this, (long long)message.targetConnectionId); + return false; + } mStats.onBufferSent(); mTransactions.insert(std::make_pair( message.transactionId, @@ -450,6 +456,7 @@ bool Accessor::Impl::BufferPool::handleClose(ConnectionId connectionId) { } } } + mConnectionIds.erase(connectionId); return true; } diff --git a/media/bufferpool/1.0/AccessorImpl.h b/media/bufferpool/1.0/AccessorImpl.h index c04dbf3687..84cb6854b0 100644 --- a/media/bufferpool/1.0/AccessorImpl.h +++ b/media/bufferpool/1.0/AccessorImpl.h @@ -94,6 +94,7 @@ private: std::map<BufferId, std::unique_ptr<InternalBuffer>> mBuffers; std::set<BufferId> mFreeBuffers; + std::set<ConnectionId> mConnectionIds; /// Buffer pool statistics which tracks allocation and transfer statistics. struct Stats { diff --git a/media/bufferpool/2.0/AccessorImpl.cpp b/media/bufferpool/2.0/AccessorImpl.cpp index 94cf006952..0d591d78a9 100644 --- a/media/bufferpool/2.0/AccessorImpl.cpp +++ b/media/bufferpool/2.0/AccessorImpl.cpp @@ -37,7 +37,7 @@ namespace { static constexpr int64_t kLogDurationUs = 5000000; // 5 secs static constexpr size_t kMinAllocBytesForEviction = 1024*1024*15; - static constexpr size_t kMinBufferCountForEviction = 40; + static constexpr size_t kMinBufferCountForEviction = 25; } // Buffer structure in bufferpool process @@ -163,6 +163,7 @@ ResultStatus Accessor::Impl::connect( *connection = newConnection; *pConnectionId = id; *pMsgId = mBufferPool.mInvalidation.mInvalidationId; + mBufferPool.mConnectionIds.insert(id); mBufferPool.mInvalidationChannel.getDesc(invDescPtr); mBufferPool.mInvalidation.onConnect(id, observer); ++sSeqId; @@ -474,7 +475,12 @@ bool Accessor::Impl::BufferPool::handleTransferTo(const BufferStatusMessage &mes found->second->mSenderValidated = true; return true; } - // TODO: verify there is target connection Id + if (mConnectionIds.find(message.targetConnectionId) == mConnectionIds.end()) { + // N.B: it could be fake or receive connection already closed. + ALOGD("bufferpool2 %p receiver connection %lld is no longer valid", + this, (long long)message.targetConnectionId); + return false; + } mStats.onBufferSent(); mTransactions.insert(std::make_pair( message.transactionId, @@ -644,6 +650,7 @@ bool Accessor::Impl::BufferPool::handleClose(ConnectionId connectionId) { } } } + mConnectionIds.erase(connectionId); return true; } @@ -711,8 +718,8 @@ void Accessor::Impl::BufferPool::cleanUp(bool clearCache) { mStats.mTotalFetches, mStats.mTotalTransfers); } for (auto freeIt = mFreeBuffers.begin(); freeIt != mFreeBuffers.end();) { - if (!clearCache && mStats.mSizeCached < kMinAllocBytesForEviction - && mBuffers.size() < kMinBufferCountForEviction) { + if (!clearCache && (mStats.mSizeCached < kMinAllocBytesForEviction + || mBuffers.size() < kMinBufferCountForEviction)) { break; } auto it = mBuffers.find(*freeIt); @@ -774,11 +781,19 @@ void Accessor::Impl::invalidatorThread( std::mutex &mutex, std::condition_variable &cv, bool &ready) { + constexpr uint32_t NUM_SPIN_TO_INCREASE_SLEEP = 1024; + constexpr uint32_t NUM_SPIN_TO_LOG = 1024*8; + constexpr useconds_t MAX_SLEEP_US = 10000; + uint32_t numSpin = 0; + useconds_t sleepUs = 1; + while(true) { std::map<uint32_t, const std::weak_ptr<Accessor::Impl>> copied; { std::unique_lock<std::mutex> lock(mutex); if (!ready) { + numSpin = 0; + sleepUs = 1; cv.wait(lock); } copied.insert(accessors.begin(), accessors.end()); @@ -800,9 +815,20 @@ void Accessor::Impl::invalidatorThread( if (accessors.size() == 0) { ready = false; } else { - // prevent draining cpu. + // TODO Use an efficient way to wait over FMQ. + // N.B. Since there is not a efficient way to wait over FMQ, + // polling over the FMQ is the current way to prevent draining + // CPU. lock.unlock(); - std::this_thread::yield(); + ++numSpin; + if (numSpin % NUM_SPIN_TO_INCREASE_SLEEP == 0 && + sleepUs < MAX_SLEEP_US) { + sleepUs *= 10; + } + if (numSpin % NUM_SPIN_TO_LOG == 0) { + ALOGW("invalidator thread spinning"); + } + ::usleep(sleepUs); } } } diff --git a/media/bufferpool/2.0/AccessorImpl.h b/media/bufferpool/2.0/AccessorImpl.h index eea72b98fb..807e0f1444 100644 --- a/media/bufferpool/2.0/AccessorImpl.h +++ b/media/bufferpool/2.0/AccessorImpl.h @@ -111,6 +111,7 @@ private: std::map<BufferId, std::unique_ptr<InternalBuffer>> mBuffers; std::set<BufferId> mFreeBuffers; + std::set<ConnectionId> mConnectionIds; struct Invalidation { static std::atomic<std::uint32_t> sInvSeqId; diff --git a/media/bufferpool/2.0/Android.bp b/media/bufferpool/2.0/Android.bp index c71ac17d32..e8a69c9ce1 100644 --- a/media/bufferpool/2.0/Android.bp +++ b/media/bufferpool/2.0/Android.bp @@ -1,9 +1,5 @@ -cc_library { - name: "libstagefright_bufferpool@2.0", - vendor_available: true, - vndk: { - enabled: true, - }, +cc_defaults { + name: "libstagefright_bufferpool@2.0-default", srcs: [ "Accessor.cpp", "AccessorImpl.cpp", @@ -31,3 +27,23 @@ cc_library { "android.hardware.media.bufferpool@2.0", ], } + +cc_library { + name: "libstagefright_bufferpool@2.0.1", + defaults: ["libstagefright_bufferpool@2.0-default"], + vendor_available: true, + cflags: [ + "-DBUFFERPOOL_CLONE_HANDLES", + ], +} + +// Deprecated. Do not use. Use libstagefright_bufferpool@2.0.1 instead. +cc_library { + name: "libstagefright_bufferpool@2.0", + defaults: ["libstagefright_bufferpool@2.0-default"], + vendor_available: true, + vndk: { + enabled: true, + }, +} + diff --git a/media/bufferpool/2.0/ClientManager.cpp b/media/bufferpool/2.0/ClientManager.cpp index c31d313ab1..87ee4e848e 100644 --- a/media/bufferpool/2.0/ClientManager.cpp +++ b/media/bufferpool/2.0/ClientManager.cpp @@ -351,7 +351,21 @@ ResultStatus ClientManager::Impl::allocate( } client = it->second; } +#ifdef BUFFERPOOL_CLONE_HANDLES + native_handle_t *origHandle; + ResultStatus res = client->allocate(params, &origHandle, buffer); + if (res != ResultStatus::OK) { + return res; + } + *handle = native_handle_clone(origHandle); + if (handle == NULL) { + buffer->reset(); + return ResultStatus::NO_MEMORY; + } + return ResultStatus::OK; +#else return client->allocate(params, handle, buffer); +#endif } ResultStatus ClientManager::Impl::receive( @@ -367,7 +381,22 @@ ResultStatus ClientManager::Impl::receive( } client = it->second; } +#ifdef BUFFERPOOL_CLONE_HANDLES + native_handle_t *origHandle; + ResultStatus res = client->receive( + transactionId, bufferId, timestampUs, &origHandle, buffer); + if (res != ResultStatus::OK) { + return res; + } + *handle = native_handle_clone(origHandle); + if (handle == NULL) { + buffer->reset(); + return ResultStatus::NO_MEMORY; + } + return ResultStatus::OK; +#else return client->receive(transactionId, bufferId, timestampUs, handle, buffer); +#endif } ResultStatus ClientManager::Impl::postSend( diff --git a/media/bufferpool/2.0/include/bufferpool/ClientManager.h b/media/bufferpool/2.0/include/bufferpool/ClientManager.h index 953c3049a4..24b61f4fb5 100644 --- a/media/bufferpool/2.0/include/bufferpool/ClientManager.h +++ b/media/bufferpool/2.0/include/bufferpool/ClientManager.h @@ -104,7 +104,9 @@ struct ClientManager : public IClientManager { ResultStatus flush(ConnectionId connectionId); /** - * Allocates a buffer from the specified connection. + * Allocates a buffer from the specified connection. The output parameter + * handle is cloned from the internal handle. So it is safe to use directly, + * and it should be deleted and destroyed after use. * * @param connectionId The id of the connection. * @param params The allocation parameters. @@ -123,7 +125,9 @@ struct ClientManager : public IClientManager { std::shared_ptr<BufferPoolData> *buffer); /** - * Receives a buffer for the transaction. + * Receives a buffer for the transaction. The output parameter handle is + * cloned from the internal handle. So it is safe to use directly, and it + * should be deleted and destoyed after use. * * @param connectionId The id of the receiving connection. * @param transactionId The id for the transaction. diff --git a/media/codec2/components/aac/C2SoftAacEnc.cpp b/media/codec2/components/aac/C2SoftAacEnc.cpp index 8e3852c110..be52a1db34 100644 --- a/media/codec2/components/aac/C2SoftAacEnc.cpp +++ b/media/codec2/components/aac/C2SoftAacEnc.cpp @@ -157,9 +157,10 @@ C2SoftAacEnc::C2SoftAacEnc( mSentCodecSpecificData(false), mInputTimeSet(false), mInputSize(0), - mInputTimeUs(0), + mNextFrameTimestampUs(0), mSignalledError(false), - mOutIndex(0u) { + mOutIndex(0u), + mRemainderLen(0u) { } C2SoftAacEnc::~C2SoftAacEnc() { @@ -183,8 +184,9 @@ c2_status_t C2SoftAacEnc::onStop() { mSentCodecSpecificData = false; mInputTimeSet = false; mInputSize = 0u; - mInputTimeUs = 0; + mNextFrameTimestampUs = 0; mSignalledError = false; + mRemainderLen = 0; return C2_OK; } @@ -201,7 +203,7 @@ c2_status_t C2SoftAacEnc::onFlush_sm() { mSentCodecSpecificData = false; mInputTimeSet = false; mInputSize = 0u; - mInputTimeUs = 0; + mNextFrameTimestampUs = 0; return C2_OK; } @@ -365,21 +367,25 @@ void C2SoftAacEnc::process( capacity = view.capacity(); } if (!mInputTimeSet && capacity > 0) { - mInputTimeUs = work->input.ordinal.timestamp; + mNextFrameTimestampUs = work->input.ordinal.timestamp; mInputTimeSet = true; } - size_t numFrames = (capacity + mInputSize + (eos ? mNumBytesPerInputFrame - 1 : 0)) - / mNumBytesPerInputFrame; - ALOGV("capacity = %zu; mInputSize = %zu; numFrames = %zu mNumBytesPerInputFrame = %u", - capacity, mInputSize, numFrames, mNumBytesPerInputFrame); + size_t numFrames = + (mRemainderLen + capacity + mInputSize + (eos ? mNumBytesPerInputFrame - 1 : 0)) + / mNumBytesPerInputFrame; + ALOGV("capacity = %zu; mInputSize = %zu; numFrames = %zu " + "mNumBytesPerInputFrame = %u inputTS = %lld remaining = %zu", + capacity, mInputSize, numFrames, + mNumBytesPerInputFrame, work->input.ordinal.timestamp.peekll(), + mRemainderLen); std::shared_ptr<C2LinearBlock> block; - std::shared_ptr<C2Buffer> buffer; std::unique_ptr<C2WriteView> wView; uint8_t *outPtr = temp; size_t outAvailable = 0u; uint64_t inputIndex = work->input.ordinal.frameIndex.peeku(); + size_t bytesPerSample = channelCount * sizeof(int16_t); AACENC_InArgs inargs; AACENC_OutArgs outargs; @@ -442,9 +448,31 @@ void C2SoftAacEnc::process( const std::shared_ptr<C2Buffer> mBuffer; }; - C2WorkOrdinalStruct outOrdinal = work->input.ordinal; + struct OutputBuffer { + std::shared_ptr<C2Buffer> buffer; + c2_cntr64_t timestampUs; + }; + std::list<OutputBuffer> outputBuffers; - while (encoderErr == AACENC_OK && inargs.numInSamples > 0) { + if (mRemainderLen > 0) { + size_t offset = 0; + for (; mRemainderLen < bytesPerSample && offset < capacity; ++offset) { + mRemainder[mRemainderLen++] = data[offset]; + } + data += offset; + capacity -= offset; + if (mRemainderLen == bytesPerSample) { + inBuffer[0] = mRemainder; + inBufferSize[0] = bytesPerSample; + inargs.numInSamples = channelCount; + mRemainderLen = 0; + ALOGV("Processing remainder"); + } else { + // We have exhausted the input already + inargs.numInSamples = 0; + } + } + while (encoderErr == AACENC_OK && inargs.numInSamples >= channelCount) { if (numFrames && !block) { C2MemoryUsage usage = { C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE }; // TODO: error handling, proper usage, etc. @@ -473,43 +501,40 @@ void C2SoftAacEnc::process( &outargs); if (encoderErr == AACENC_OK) { - if (buffer) { - outOrdinal.frameIndex = mOutIndex++; - outOrdinal.timestamp = mInputTimeUs; - cloneAndSend( - inputIndex, - work, - FillWork(C2FrameData::FLAG_INCOMPLETE, outOrdinal, buffer)); - buffer.reset(); - } - if (outargs.numOutBytes > 0) { mInputSize = 0; int consumed = (capacity / sizeof(int16_t)) - inargs.numInSamples + outargs.numInSamples; - mInputTimeUs = work->input.ordinal.timestamp + c2_cntr64_t currentFrameTimestampUs = mNextFrameTimestampUs; + mNextFrameTimestampUs = work->input.ordinal.timestamp + (consumed * 1000000ll / channelCount / sampleRate); - buffer = createLinearBuffer(block, 0, outargs.numOutBytes); -#if defined(LOG_NDEBUG) && !LOG_NDEBUG + std::shared_ptr<C2Buffer> buffer = createLinearBuffer(block, 0, outargs.numOutBytes); +#if 0 hexdump(outPtr, std::min(outargs.numOutBytes, 256)); #endif outPtr = temp; outAvailable = 0; block.reset(); + + outputBuffers.push_back({buffer, currentFrameTimestampUs}); } else { mInputSize += outargs.numInSamples * sizeof(int16_t); } - if (outargs.numInSamples > 0) { + if (inBuffer[0] == mRemainder) { + inBuffer[0] = const_cast<uint8_t *>(data); + inBufferSize[0] = capacity; + inargs.numInSamples = capacity / sizeof(int16_t); + } else if (outargs.numInSamples > 0) { inBuffer[0] = (int16_t *)inBuffer[0] + outargs.numInSamples; inBufferSize[0] -= outargs.numInSamples * sizeof(int16_t); inargs.numInSamples -= outargs.numInSamples; } } - ALOGV("encoderErr = %d mInputSize = %zu inargs.numInSamples = %d, mInputTimeUs = %lld", - encoderErr, mInputSize, inargs.numInSamples, mInputTimeUs.peekll()); + ALOGV("encoderErr = %d mInputSize = %zu " + "inargs.numInSamples = %d, mNextFrameTimestampUs = %lld", + encoderErr, mInputSize, inargs.numInSamples, mNextFrameTimestampUs.peekll()); } - if (eos && inBufferSize[0] > 0) { if (numFrames && !block) { C2MemoryUsage usage = { C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE }; @@ -540,12 +565,37 @@ void C2SoftAacEnc::process( &outBufDesc, &inargs, &outargs); + inBufferSize[0] = 0; + } + + if (inBufferSize[0] > 0) { + for (size_t i = 0; i < inBufferSize[0]; ++i) { + mRemainder[i] = static_cast<uint8_t *>(inBuffer[0])[i]; + } + mRemainderLen = inBufferSize[0]; } - outOrdinal.frameIndex = mOutIndex++; - outOrdinal.timestamp = mInputTimeUs; + while (outputBuffers.size() > 1) { + const OutputBuffer& front = outputBuffers.front(); + C2WorkOrdinalStruct ordinal = work->input.ordinal; + ordinal.frameIndex = mOutIndex++; + ordinal.timestamp = front.timestampUs; + cloneAndSend( + inputIndex, + work, + FillWork(C2FrameData::FLAG_INCOMPLETE, ordinal, front.buffer)); + outputBuffers.pop_front(); + } + std::shared_ptr<C2Buffer> buffer; + C2WorkOrdinalStruct ordinal = work->input.ordinal; + ordinal.frameIndex = mOutIndex++; + if (!outputBuffers.empty()) { + ordinal.timestamp = outputBuffers.front().timestampUs; + buffer = outputBuffers.front().buffer; + } + // Mark the end of frame FillWork((C2FrameData::flags_t)(eos ? C2FrameData::FLAG_END_OF_STREAM : 0), - outOrdinal, buffer)(work); + ordinal, buffer)(work); } c2_status_t C2SoftAacEnc::drain( @@ -569,7 +619,7 @@ c2_status_t C2SoftAacEnc::drain( mSentCodecSpecificData = false; mInputTimeSet = false; mInputSize = 0u; - mInputTimeUs = 0; + mNextFrameTimestampUs = 0; // TODO: we don't have any pending work at this time to drain. return C2_OK; diff --git a/media/codec2/components/aac/C2SoftAacEnc.h b/media/codec2/components/aac/C2SoftAacEnc.h index a38be190a1..6ecfbdd1fe 100644 --- a/media/codec2/components/aac/C2SoftAacEnc.h +++ b/media/codec2/components/aac/C2SoftAacEnc.h @@ -56,11 +56,15 @@ private: bool mSentCodecSpecificData; bool mInputTimeSet; size_t mInputSize; - c2_cntr64_t mInputTimeUs; + c2_cntr64_t mNextFrameTimestampUs; bool mSignalledError; std::atomic_uint64_t mOutIndex; + // We support max 6 channels + uint8_t mRemainder[6 * sizeof(int16_t)]; + size_t mRemainderLen; + status_t initEncoder(); status_t setAudioParams(); diff --git a/media/codec2/components/aom/Android.bp b/media/codec2/components/aom/Android.bp index 0fabf5cf87..61dbd4ccc2 100644 --- a/media/codec2/components/aom/Android.bp +++ b/media/codec2/components/aom/Android.bp @@ -1,10 +1,16 @@ cc_library_shared { - name: "libcodec2_soft_av1dec", + name: "libcodec2_soft_av1dec_aom", defaults: [ "libcodec2_soft-defaults", "libcodec2_soft_sanitize_all-defaults", ], + // coordinated with frameworks/av/media/codec2/components/gav1/Android.bp + // so only 1 of them has the official c2.android.av1.decoder name + cflags: [ + "-DCODECNAME=\"c2.android.av1-aom.decoder\"", + ], + srcs: ["C2SoftAomDec.cpp"], static_libs: ["libaom"], diff --git a/media/codec2/components/aom/C2SoftAomDec.cpp b/media/codec2/components/aom/C2SoftAomDec.cpp index 769895cec9..36137e6089 100644 --- a/media/codec2/components/aom/C2SoftAomDec.cpp +++ b/media/codec2/components/aom/C2SoftAomDec.cpp @@ -29,7 +29,8 @@ namespace android { -constexpr char COMPONENT_NAME[] = "c2.android.av1.decoder"; +// codecname set and passed in as a compile flag from Android.bp +constexpr char COMPONENT_NAME[] = CODECNAME; class C2SoftAomDec::IntfImpl : public SimpleInterface<void>::BaseParams { public: @@ -340,6 +341,7 @@ status_t C2SoftAomDec::initDecoder() { aom_codec_flags_t flags; memset(&flags, 0, sizeof(aom_codec_flags_t)); + ALOGV("Using libaom AV1 software decoder."); aom_codec_err_t err; if ((err = aom_codec_dec_init(mCodecCtx, aom_codec_av1_dx(), &cfg, 0))) { ALOGE("av1 decoder failed to initialize. (%d)", err); diff --git a/media/codec2/components/avc/C2SoftAvcDec.cpp b/media/codec2/components/avc/C2SoftAvcDec.cpp index 3f015b464c..fa98178187 100644 --- a/media/codec2/components/avc/C2SoftAvcDec.cpp +++ b/media/codec2/components/avc/C2SoftAvcDec.cpp @@ -33,7 +33,8 @@ namespace android { namespace { constexpr char COMPONENT_NAME[] = "c2.android.avc.decoder"; - +constexpr uint32_t kDefaultOutputDelay = 8; +constexpr uint32_t kMaxOutputDelay = 16; } // namespace class C2SoftAvcDec::IntfImpl : public SimpleInterface<void>::BaseParams { @@ -54,7 +55,9 @@ public: // TODO: Proper support for reorder depth. addParameter( DefineParam(mActualOutputDelay, C2_PARAMKEY_OUTPUT_DELAY) - .withConstValue(new C2PortActualDelayTuning::output(8u)) + .withDefault(new C2PortActualDelayTuning::output(kDefaultOutputDelay)) + .withFields({C2F(mActualOutputDelay, value).inRange(0, kMaxOutputDelay)}) + .withSetter(Setter<decltype(*mActualOutputDelay)>::StrictValueWithNoDeps) .build()); // TODO: output latency and reordering @@ -196,7 +199,6 @@ public: 0u, HAL_PIXEL_FORMAT_YCBCR_420_888)) .build()); } - static C2R SizeSetter(bool mayBlock, const C2P<C2StreamPictureSizeInfo::output> &oldMe, C2P<C2StreamPictureSizeInfo::output> &me) { (void)mayBlock; @@ -333,6 +335,7 @@ C2SoftAvcDec::C2SoftAvcDec( mDecHandle(nullptr), mOutBufferFlush(nullptr), mIvColorFormat(IV_YUV_420P), + mOutputDelay(kDefaultOutputDelay), mWidth(320), mHeight(240), mHeaderDecoded(false), @@ -882,6 +885,26 @@ void C2SoftAvcDec::process( work->result = C2_CORRUPTED; return; } + if (s_decode_op.i4_reorder_depth >= 0 && mOutputDelay != s_decode_op.i4_reorder_depth) { + mOutputDelay = s_decode_op.i4_reorder_depth; + ALOGV("New Output delay %d ", mOutputDelay); + + C2PortActualDelayTuning::output outputDelay(mOutputDelay); + std::vector<std::unique_ptr<C2SettingResult>> failures; + c2_status_t err = + mIntf->config({&outputDelay}, C2_MAY_BLOCK, &failures); + if (err == OK) { + work->worklets.front()->output.configUpdate.push_back( + C2Param::Copy(outputDelay)); + } else { + ALOGE("Cannot set output delay"); + mSignalledError = true; + work->workletsProcessed = 1u; + work->result = C2_CORRUPTED; + return; + } + continue; + } if (0 < s_decode_op.u4_pic_wd && 0 < s_decode_op.u4_pic_ht) { if (mHeaderDecoded == false) { mHeaderDecoded = true; diff --git a/media/codec2/components/avc/C2SoftAvcDec.h b/media/codec2/components/avc/C2SoftAvcDec.h index 72ee583a33..4414a26c96 100644 --- a/media/codec2/components/avc/C2SoftAvcDec.h +++ b/media/codec2/components/avc/C2SoftAvcDec.h @@ -157,7 +157,7 @@ private: size_t mNumCores; IV_COLOR_FORMAT_T mIvColorFormat; - + uint32_t mOutputDelay; uint32_t mWidth; uint32_t mHeight; uint32_t mStride; diff --git a/media/codec2/components/g711/C2SoftG711Dec.cpp b/media/codec2/components/g711/C2SoftG711Dec.cpp index 43b843ae88..4ff079361a 100644 --- a/media/codec2/components/g711/C2SoftG711Dec.cpp +++ b/media/codec2/components/g711/C2SoftG711Dec.cpp @@ -74,7 +74,7 @@ public: addParameter( DefineParam(mChannelCount, C2_PARAMKEY_CHANNEL_COUNT) .withDefault(new C2StreamChannelCountInfo::output(0u, 1)) - .withFields({C2F(mChannelCount, value).equalTo(1)}) + .withFields({C2F(mChannelCount, value).inRange(1, 6)}) .withSetter(Setter<decltype(*mChannelCount)>::StrictValueWithNoDeps) .build()); diff --git a/media/codec2/components/gav1/Android.bp b/media/codec2/components/gav1/Android.bp new file mode 100644 index 0000000000..5c4abb7e30 --- /dev/null +++ b/media/codec2/components/gav1/Android.bp @@ -0,0 +1,20 @@ +cc_library_shared { + name: "libcodec2_soft_av1dec_gav1", + defaults: [ + "libcodec2_soft-defaults", + "libcodec2_soft_sanitize_all-defaults", + ], + + // coordinated with frameworks/av/media/codec2/components/aom/Android.bp + // so only 1 of them has the official c2.android.av1.decoder name + cflags: [ + "-DCODECNAME=\"c2.android.av1.decoder\"", + ], + + srcs: ["C2SoftGav1Dec.cpp"], + static_libs: ["libgav1"], + + include_dirs: [ + "external/libgav1/libgav1/", + ], +} diff --git a/media/codec2/components/gav1/C2SoftGav1Dec.cpp b/media/codec2/components/gav1/C2SoftGav1Dec.cpp new file mode 100644 index 0000000000..ec5f549049 --- /dev/null +++ b/media/codec2/components/gav1/C2SoftGav1Dec.cpp @@ -0,0 +1,792 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "C2SoftGav1Dec" +#include "C2SoftGav1Dec.h" + +#include <C2Debug.h> +#include <C2PlatformSupport.h> +#include <SimpleC2Interface.h> +#include <log/log.h> +#include <media/stagefright/foundation/AUtils.h> +#include <media/stagefright/foundation/MediaDefs.h> + +namespace android { + +// codecname set and passed in as a compile flag from Android.bp +constexpr char COMPONENT_NAME[] = CODECNAME; + +class C2SoftGav1Dec::IntfImpl : public SimpleInterface<void>::BaseParams { + public: + explicit IntfImpl(const std::shared_ptr<C2ReflectorHelper> &helper) + : SimpleInterface<void>::BaseParams( + helper, COMPONENT_NAME, C2Component::KIND_DECODER, + C2Component::DOMAIN_VIDEO, MEDIA_MIMETYPE_VIDEO_AV1) { + noPrivateBuffers(); // TODO: account for our buffers here. + noInputReferences(); + noOutputReferences(); + noInputLatency(); + noTimeStretch(); + + addParameter(DefineParam(mAttrib, C2_PARAMKEY_COMPONENT_ATTRIBUTES) + .withConstValue(new C2ComponentAttributesSetting( + C2Component::ATTRIB_IS_TEMPORAL)) + .build()); + + addParameter( + DefineParam(mSize, C2_PARAMKEY_PICTURE_SIZE) + .withDefault(new C2StreamPictureSizeInfo::output(0u, 320, 240)) + .withFields({ + C2F(mSize, width).inRange(2, 2048, 2), + C2F(mSize, height).inRange(2, 2048, 2), + }) + .withSetter(SizeSetter) + .build()); + + addParameter(DefineParam(mProfileLevel, C2_PARAMKEY_PROFILE_LEVEL) + .withDefault(new C2StreamProfileLevelInfo::input( + 0u, C2Config::PROFILE_AV1_0, C2Config::LEVEL_AV1_2_1)) + .withFields({C2F(mProfileLevel, profile) + .oneOf({C2Config::PROFILE_AV1_0, + C2Config::PROFILE_AV1_1}), + C2F(mProfileLevel, level) + .oneOf({ + C2Config::LEVEL_AV1_2, + C2Config::LEVEL_AV1_2_1, + C2Config::LEVEL_AV1_2_2, + C2Config::LEVEL_AV1_3, + C2Config::LEVEL_AV1_3_1, + C2Config::LEVEL_AV1_3_2, + })}) + .withSetter(ProfileLevelSetter, mSize) + .build()); + + mHdr10PlusInfoInput = C2StreamHdr10PlusInfo::input::AllocShared(0); + addParameter( + DefineParam(mHdr10PlusInfoInput, C2_PARAMKEY_INPUT_HDR10_PLUS_INFO) + .withDefault(mHdr10PlusInfoInput) + .withFields({ + C2F(mHdr10PlusInfoInput, m.value).any(), + }) + .withSetter(Hdr10PlusInfoInputSetter) + .build()); + + mHdr10PlusInfoOutput = C2StreamHdr10PlusInfo::output::AllocShared(0); + addParameter( + DefineParam(mHdr10PlusInfoOutput, C2_PARAMKEY_OUTPUT_HDR10_PLUS_INFO) + .withDefault(mHdr10PlusInfoOutput) + .withFields({ + C2F(mHdr10PlusInfoOutput, m.value).any(), + }) + .withSetter(Hdr10PlusInfoOutputSetter) + .build()); + + addParameter( + DefineParam(mMaxSize, C2_PARAMKEY_MAX_PICTURE_SIZE) + .withDefault(new C2StreamMaxPictureSizeTuning::output(0u, 320, 240)) + .withFields({ + C2F(mSize, width).inRange(2, 2048, 2), + C2F(mSize, height).inRange(2, 2048, 2), + }) + .withSetter(MaxPictureSizeSetter, mSize) + .build()); + + addParameter(DefineParam(mMaxInputSize, C2_PARAMKEY_INPUT_MAX_BUFFER_SIZE) + .withDefault(new C2StreamMaxBufferSizeInfo::input( + 0u, 320 * 240 * 3 / 4)) + .withFields({ + C2F(mMaxInputSize, value).any(), + }) + .calculatedAs(MaxInputSizeSetter, mMaxSize) + .build()); + + C2ChromaOffsetStruct locations[1] = {C2ChromaOffsetStruct::ITU_YUV_420_0()}; + std::shared_ptr<C2StreamColorInfo::output> defaultColorInfo = + C2StreamColorInfo::output::AllocShared(1u, 0u, 8u /* bitDepth */, + C2Color::YUV_420); + memcpy(defaultColorInfo->m.locations, locations, sizeof(locations)); + + defaultColorInfo = C2StreamColorInfo::output::AllocShared( + {C2ChromaOffsetStruct::ITU_YUV_420_0()}, 0u, 8u /* bitDepth */, + C2Color::YUV_420); + helper->addStructDescriptors<C2ChromaOffsetStruct>(); + + addParameter(DefineParam(mColorInfo, C2_PARAMKEY_CODED_COLOR_INFO) + .withConstValue(defaultColorInfo) + .build()); + + addParameter( + DefineParam(mDefaultColorAspects, C2_PARAMKEY_DEFAULT_COLOR_ASPECTS) + .withDefault(new C2StreamColorAspectsTuning::output( + 0u, C2Color::RANGE_UNSPECIFIED, C2Color::PRIMARIES_UNSPECIFIED, + C2Color::TRANSFER_UNSPECIFIED, C2Color::MATRIX_UNSPECIFIED)) + .withFields( + {C2F(mDefaultColorAspects, range) + .inRange(C2Color::RANGE_UNSPECIFIED, C2Color::RANGE_OTHER), + C2F(mDefaultColorAspects, primaries) + .inRange(C2Color::PRIMARIES_UNSPECIFIED, + C2Color::PRIMARIES_OTHER), + C2F(mDefaultColorAspects, transfer) + .inRange(C2Color::TRANSFER_UNSPECIFIED, + C2Color::TRANSFER_OTHER), + C2F(mDefaultColorAspects, matrix) + .inRange(C2Color::MATRIX_UNSPECIFIED, + C2Color::MATRIX_OTHER)}) + .withSetter(DefaultColorAspectsSetter) + .build()); + + // TODO: support more formats? + addParameter(DefineParam(mPixelFormat, C2_PARAMKEY_PIXEL_FORMAT) + .withConstValue(new C2StreamPixelFormatInfo::output( + 0u, HAL_PIXEL_FORMAT_YCBCR_420_888)) + .build()); + } + + static C2R SizeSetter(bool mayBlock, + const C2P<C2StreamPictureSizeInfo::output> &oldMe, + C2P<C2StreamPictureSizeInfo::output> &me) { + (void)mayBlock; + C2R res = C2R::Ok(); + if (!me.F(me.v.width).supportsAtAll(me.v.width)) { + res = res.plus(C2SettingResultBuilder::BadValue(me.F(me.v.width))); + me.set().width = oldMe.v.width; + } + if (!me.F(me.v.height).supportsAtAll(me.v.height)) { + res = res.plus(C2SettingResultBuilder::BadValue(me.F(me.v.height))); + me.set().height = oldMe.v.height; + } + return res; + } + + static C2R MaxPictureSizeSetter( + bool mayBlock, C2P<C2StreamMaxPictureSizeTuning::output> &me, + const C2P<C2StreamPictureSizeInfo::output> &size) { + (void)mayBlock; + // TODO: get max width/height from the size's field helpers vs. + // hardcoding + me.set().width = c2_min(c2_max(me.v.width, size.v.width), 4096u); + me.set().height = c2_min(c2_max(me.v.height, size.v.height), 4096u); + return C2R::Ok(); + } + + static C2R MaxInputSizeSetter( + bool mayBlock, C2P<C2StreamMaxBufferSizeInfo::input> &me, + const C2P<C2StreamMaxPictureSizeTuning::output> &maxSize) { + (void)mayBlock; + // assume compression ratio of 2 + me.set().value = + (((maxSize.v.width + 63) / 64) * ((maxSize.v.height + 63) / 64) * 3072); + return C2R::Ok(); + } + + static C2R DefaultColorAspectsSetter( + bool mayBlock, C2P<C2StreamColorAspectsTuning::output> &me) { + (void)mayBlock; + if (me.v.range > C2Color::RANGE_OTHER) { + me.set().range = C2Color::RANGE_OTHER; + } + if (me.v.primaries > C2Color::PRIMARIES_OTHER) { + me.set().primaries = C2Color::PRIMARIES_OTHER; + } + if (me.v.transfer > C2Color::TRANSFER_OTHER) { + me.set().transfer = C2Color::TRANSFER_OTHER; + } + if (me.v.matrix > C2Color::MATRIX_OTHER) { + me.set().matrix = C2Color::MATRIX_OTHER; + } + return C2R::Ok(); + } + + static C2R ProfileLevelSetter( + bool mayBlock, C2P<C2StreamProfileLevelInfo::input> &me, + const C2P<C2StreamPictureSizeInfo::output> &size) { + (void)mayBlock; + (void)size; + (void)me; // TODO: validate + return C2R::Ok(); + } + + std::shared_ptr<C2StreamColorAspectsTuning::output> + getDefaultColorAspects_l() { + return mDefaultColorAspects; + } + + static C2R Hdr10PlusInfoInputSetter(bool mayBlock, + C2P<C2StreamHdr10PlusInfo::input> &me) { + (void)mayBlock; + (void)me; // TODO: validate + return C2R::Ok(); + } + + static C2R Hdr10PlusInfoOutputSetter(bool mayBlock, + C2P<C2StreamHdr10PlusInfo::output> &me) { + (void)mayBlock; + (void)me; // TODO: validate + return C2R::Ok(); + } + + private: + std::shared_ptr<C2StreamProfileLevelInfo::input> mProfileLevel; + std::shared_ptr<C2StreamPictureSizeInfo::output> mSize; + std::shared_ptr<C2StreamMaxPictureSizeTuning::output> mMaxSize; + std::shared_ptr<C2StreamMaxBufferSizeInfo::input> mMaxInputSize; + std::shared_ptr<C2StreamColorInfo::output> mColorInfo; + std::shared_ptr<C2StreamPixelFormatInfo::output> mPixelFormat; + std::shared_ptr<C2StreamColorAspectsTuning::output> mDefaultColorAspects; + std::shared_ptr<C2StreamHdr10PlusInfo::input> mHdr10PlusInfoInput; + std::shared_ptr<C2StreamHdr10PlusInfo::output> mHdr10PlusInfoOutput; +}; + +C2SoftGav1Dec::C2SoftGav1Dec(const char *name, c2_node_id_t id, + const std::shared_ptr<IntfImpl> &intfImpl) + : SimpleC2Component( + std::make_shared<SimpleInterface<IntfImpl>>(name, id, intfImpl)), + mIntf(intfImpl), + mCodecCtx(nullptr) { + gettimeofday(&mTimeStart, nullptr); + gettimeofday(&mTimeEnd, nullptr); +} + +C2SoftGav1Dec::~C2SoftGav1Dec() { onRelease(); } + +c2_status_t C2SoftGav1Dec::onInit() { + return initDecoder() ? C2_OK : C2_CORRUPTED; +} + +c2_status_t C2SoftGav1Dec::onStop() { + mSignalledError = false; + mSignalledOutputEos = false; + return C2_OK; +} + +void C2SoftGav1Dec::onReset() { + (void)onStop(); + c2_status_t err = onFlush_sm(); + if (err != C2_OK) { + ALOGW("Failed to flush the av1 decoder. Trying to hard reset."); + destroyDecoder(); + if (!initDecoder()) { + ALOGE("Hard reset failed."); + } + } +} + +void C2SoftGav1Dec::onRelease() { destroyDecoder(); } + +c2_status_t C2SoftGav1Dec::onFlush_sm() { + Libgav1StatusCode status = + mCodecCtx->EnqueueFrame(/*data=*/nullptr, /*size=*/0, + /*user_private_data=*/0); + if (status != kLibgav1StatusOk) { + ALOGE("Failed to flush av1 decoder. status: %d.", status); + return C2_CORRUPTED; + } + + // Dequeue frame (if any) that was enqueued previously. + const libgav1::DecoderBuffer *buffer; + status = mCodecCtx->DequeueFrame(&buffer); + if (status != kLibgav1StatusOk) { + ALOGE("Failed to dequeue frame after flushing the av1 decoder. status: %d", + status); + return C2_CORRUPTED; + } + + mSignalledError = false; + mSignalledOutputEos = false; + + return C2_OK; +} + +static int GetCPUCoreCount() { + int cpuCoreCount = 1; +#if defined(_SC_NPROCESSORS_ONLN) + cpuCoreCount = sysconf(_SC_NPROCESSORS_ONLN); +#else + // _SC_NPROC_ONLN must be defined... + cpuCoreCount = sysconf(_SC_NPROC_ONLN); +#endif + CHECK(cpuCoreCount >= 1); + ALOGV("Number of CPU cores: %d", cpuCoreCount); + return cpuCoreCount; +} + +bool C2SoftGav1Dec::initDecoder() { + mSignalledError = false; + mSignalledOutputEos = false; + mCodecCtx.reset(new libgav1::Decoder()); + + if (mCodecCtx == nullptr) { + ALOGE("mCodecCtx is null"); + return false; + } + + libgav1::DecoderSettings settings = {}; + settings.threads = GetCPUCoreCount(); + + ALOGV("Using libgav1 AV1 software decoder."); + Libgav1StatusCode status = mCodecCtx->Init(&settings); + if (status != kLibgav1StatusOk) { + ALOGE("av1 decoder failed to initialize. status: %d.", status); + return false; + } + + return true; +} + +void C2SoftGav1Dec::destroyDecoder() { mCodecCtx = nullptr; } + +void fillEmptyWork(const std::unique_ptr<C2Work> &work) { + uint32_t flags = 0; + if (work->input.flags & C2FrameData::FLAG_END_OF_STREAM) { + flags |= C2FrameData::FLAG_END_OF_STREAM; + ALOGV("signalling eos"); + } + work->worklets.front()->output.flags = (C2FrameData::flags_t)flags; + work->worklets.front()->output.buffers.clear(); + work->worklets.front()->output.ordinal = work->input.ordinal; + work->workletsProcessed = 1u; +} + +void C2SoftGav1Dec::finishWork(uint64_t index, + const std::unique_ptr<C2Work> &work, + const std::shared_ptr<C2GraphicBlock> &block) { + std::shared_ptr<C2Buffer> buffer = + createGraphicBuffer(block, C2Rect(mWidth, mHeight)); + auto fillWork = [buffer, index](const std::unique_ptr<C2Work> &work) { + uint32_t flags = 0; + if ((work->input.flags & C2FrameData::FLAG_END_OF_STREAM) && + (c2_cntr64_t(index) == work->input.ordinal.frameIndex)) { + flags |= C2FrameData::FLAG_END_OF_STREAM; + ALOGV("signalling eos"); + } + work->worklets.front()->output.flags = (C2FrameData::flags_t)flags; + work->worklets.front()->output.buffers.clear(); + work->worklets.front()->output.buffers.push_back(buffer); + work->worklets.front()->output.ordinal = work->input.ordinal; + work->workletsProcessed = 1u; + }; + if (work && c2_cntr64_t(index) == work->input.ordinal.frameIndex) { + fillWork(work); + } else { + finish(index, fillWork); + } +} + +void C2SoftGav1Dec::process(const std::unique_ptr<C2Work> &work, + const std::shared_ptr<C2BlockPool> &pool) { + work->result = C2_OK; + work->workletsProcessed = 0u; + work->worklets.front()->output.configUpdate.clear(); + work->worklets.front()->output.flags = work->input.flags; + if (mSignalledError || mSignalledOutputEos) { + work->result = C2_BAD_VALUE; + return; + } + + size_t inOffset = 0u; + size_t inSize = 0u; + C2ReadView rView = mDummyReadView; + if (!work->input.buffers.empty()) { + rView = work->input.buffers[0]->data().linearBlocks().front().map().get(); + inSize = rView.capacity(); + if (inSize && rView.error()) { + ALOGE("read view map failed %d", rView.error()); + work->result = C2_CORRUPTED; + return; + } + } + + bool codecConfig = + ((work->input.flags & C2FrameData::FLAG_CODEC_CONFIG) != 0); + bool eos = ((work->input.flags & C2FrameData::FLAG_END_OF_STREAM) != 0); + + ALOGV("in buffer attr. size %zu timestamp %d frameindex %d, flags %x", inSize, + (int)work->input.ordinal.timestamp.peeku(), + (int)work->input.ordinal.frameIndex.peeku(), work->input.flags); + + if (codecConfig) { + fillEmptyWork(work); + return; + } + + int64_t frameIndex = work->input.ordinal.frameIndex.peekll(); + if (inSize) { + uint8_t *bitstream = const_cast<uint8_t *>(rView.data() + inOffset); + int32_t decodeTime = 0; + int32_t delay = 0; + + GETTIME(&mTimeStart, nullptr); + TIME_DIFF(mTimeEnd, mTimeStart, delay); + + const Libgav1StatusCode status = + mCodecCtx->EnqueueFrame(bitstream, inSize, frameIndex); + + GETTIME(&mTimeEnd, nullptr); + TIME_DIFF(mTimeStart, mTimeEnd, decodeTime); + ALOGV("decodeTime=%4d delay=%4d\n", decodeTime, delay); + + if (status != kLibgav1StatusOk) { + ALOGE("av1 decoder failed to decode frame. status: %d.", status); + work->result = C2_CORRUPTED; + work->workletsProcessed = 1u; + mSignalledError = true; + return; + } + + } else { + const Libgav1StatusCode status = + mCodecCtx->EnqueueFrame(/*data=*/nullptr, /*size=*/0, + /*user_private_data=*/0); + if (status != kLibgav1StatusOk) { + ALOGE("Failed to flush av1 decoder. status: %d.", status); + work->result = C2_CORRUPTED; + work->workletsProcessed = 1u; + mSignalledError = true; + return; + } + } + + (void)outputBuffer(pool, work); + + if (eos) { + drainInternal(DRAIN_COMPONENT_WITH_EOS, pool, work); + mSignalledOutputEos = true; + } else if (!inSize) { + fillEmptyWork(work); + } +} + +static void copyOutputBufferToYV12Frame(uint8_t *dst, const uint8_t *srcY, + const uint8_t *srcU, + const uint8_t *srcV, size_t srcYStride, + size_t srcUStride, size_t srcVStride, + uint32_t width, uint32_t height) { + const size_t dstYStride = align(width, 16); + const size_t dstUVStride = align(dstYStride / 2, 16); + uint8_t *const dstStart = dst; + + for (size_t i = 0; i < height; ++i) { + memcpy(dst, srcY, width); + srcY += srcYStride; + dst += dstYStride; + } + + dst = dstStart + dstYStride * height; + for (size_t i = 0; i < height / 2; ++i) { + memcpy(dst, srcV, width / 2); + srcV += srcVStride; + dst += dstUVStride; + } + + dst = dstStart + (dstYStride * height) + (dstUVStride * height / 2); + for (size_t i = 0; i < height / 2; ++i) { + memcpy(dst, srcU, width / 2); + srcU += srcUStride; + dst += dstUVStride; + } +} + +static void convertYUV420Planar16ToY410(uint32_t *dst, const uint16_t *srcY, + const uint16_t *srcU, + const uint16_t *srcV, size_t srcYStride, + size_t srcUStride, size_t srcVStride, + size_t dstStride, size_t width, + size_t height) { + // Converting two lines at a time, slightly faster + for (size_t y = 0; y < height; y += 2) { + uint32_t *dstTop = (uint32_t *)dst; + uint32_t *dstBot = (uint32_t *)(dst + dstStride); + uint16_t *ySrcTop = (uint16_t *)srcY; + uint16_t *ySrcBot = (uint16_t *)(srcY + srcYStride); + uint16_t *uSrc = (uint16_t *)srcU; + uint16_t *vSrc = (uint16_t *)srcV; + + uint32_t u01, v01, y01, y23, y45, y67, uv0, uv1; + size_t x = 0; + for (; x < width - 3; x += 4) { + u01 = *((uint32_t *)uSrc); + uSrc += 2; + v01 = *((uint32_t *)vSrc); + vSrc += 2; + + y01 = *((uint32_t *)ySrcTop); + ySrcTop += 2; + y23 = *((uint32_t *)ySrcTop); + ySrcTop += 2; + y45 = *((uint32_t *)ySrcBot); + ySrcBot += 2; + y67 = *((uint32_t *)ySrcBot); + ySrcBot += 2; + + uv0 = (u01 & 0x3FF) | ((v01 & 0x3FF) << 20); + uv1 = (u01 >> 16) | ((v01 >> 16) << 20); + + *dstTop++ = 3 << 30 | ((y01 & 0x3FF) << 10) | uv0; + *dstTop++ = 3 << 30 | ((y01 >> 16) << 10) | uv0; + *dstTop++ = 3 << 30 | ((y23 & 0x3FF) << 10) | uv1; + *dstTop++ = 3 << 30 | ((y23 >> 16) << 10) | uv1; + + *dstBot++ = 3 << 30 | ((y45 & 0x3FF) << 10) | uv0; + *dstBot++ = 3 << 30 | ((y45 >> 16) << 10) | uv0; + *dstBot++ = 3 << 30 | ((y67 & 0x3FF) << 10) | uv1; + *dstBot++ = 3 << 30 | ((y67 >> 16) << 10) | uv1; + } + + // There should be at most 2 more pixels to process. Note that we don't + // need to consider odd case as the buffer is always aligned to even. + if (x < width) { + u01 = *uSrc; + v01 = *vSrc; + y01 = *((uint32_t *)ySrcTop); + y45 = *((uint32_t *)ySrcBot); + uv0 = (u01 & 0x3FF) | ((v01 & 0x3FF) << 20); + *dstTop++ = ((y01 & 0x3FF) << 10) | uv0; + *dstTop++ = ((y01 >> 16) << 10) | uv0; + *dstBot++ = ((y45 & 0x3FF) << 10) | uv0; + *dstBot++ = ((y45 >> 16) << 10) | uv0; + } + + srcY += srcYStride * 2; + srcU += srcUStride; + srcV += srcVStride; + dst += dstStride * 2; + } +} + +static void convertYUV420Planar16ToYUV420Planar( + uint8_t *dst, const uint16_t *srcY, const uint16_t *srcU, + const uint16_t *srcV, size_t srcYStride, size_t srcUStride, + size_t srcVStride, size_t dstStride, size_t width, size_t height) { + uint8_t *dstY = (uint8_t *)dst; + size_t dstYSize = dstStride * height; + size_t dstUVStride = align(dstStride / 2, 16); + size_t dstUVSize = dstUVStride * height / 2; + uint8_t *dstV = dstY + dstYSize; + uint8_t *dstU = dstV + dstUVSize; + + for (size_t y = 0; y < height; ++y) { + for (size_t x = 0; x < width; ++x) { + dstY[x] = (uint8_t)(srcY[x] >> 2); + } + + srcY += srcYStride; + dstY += dstStride; + } + + for (size_t y = 0; y < (height + 1) / 2; ++y) { + for (size_t x = 0; x < (width + 1) / 2; ++x) { + dstU[x] = (uint8_t)(srcU[x] >> 2); + dstV[x] = (uint8_t)(srcV[x] >> 2); + } + + srcU += srcUStride; + srcV += srcVStride; + dstU += dstUVStride; + dstV += dstUVStride; + } +} + +bool C2SoftGav1Dec::outputBuffer(const std::shared_ptr<C2BlockPool> &pool, + const std::unique_ptr<C2Work> &work) { + if (!(work && pool)) return false; + + const libgav1::DecoderBuffer *buffer; + const Libgav1StatusCode status = mCodecCtx->DequeueFrame(&buffer); + + if (status != kLibgav1StatusOk) { + ALOGE("av1 decoder DequeueFrame failed. status: %d.", status); + return false; + } + + // |buffer| can be NULL if status was equal to kLibgav1StatusOk. This is not + // an error. This could mean one of two things: + // - The EnqueueFrame() call was either a flush (called with nullptr). + // - The enqueued frame did not have any displayable frames. + if (!buffer) { + return false; + } + + const int width = buffer->displayed_width[0]; + const int height = buffer->displayed_height[0]; + if (width != mWidth || height != mHeight) { + mWidth = width; + mHeight = height; + + C2StreamPictureSizeInfo::output size(0u, mWidth, mHeight); + std::vector<std::unique_ptr<C2SettingResult>> failures; + c2_status_t err = mIntf->config({&size}, C2_MAY_BLOCK, &failures); + if (err == C2_OK) { + work->worklets.front()->output.configUpdate.push_back( + C2Param::Copy(size)); + } else { + ALOGE("Config update size failed"); + mSignalledError = true; + work->result = C2_CORRUPTED; + work->workletsProcessed = 1u; + return false; + } + } + + // TODO(vigneshv): Add support for monochrome videos since AV1 supports it. + CHECK(buffer->image_format == libgav1::kImageFormatYuv420); + + std::shared_ptr<C2GraphicBlock> block; + uint32_t format = HAL_PIXEL_FORMAT_YV12; + if (buffer->bitdepth == 10) { + IntfImpl::Lock lock = mIntf->lock(); + std::shared_ptr<C2StreamColorAspectsTuning::output> defaultColorAspects = + mIntf->getDefaultColorAspects_l(); + + if (defaultColorAspects->primaries == C2Color::PRIMARIES_BT2020 && + defaultColorAspects->matrix == C2Color::MATRIX_BT2020 && + defaultColorAspects->transfer == C2Color::TRANSFER_ST2084) { + format = HAL_PIXEL_FORMAT_RGBA_1010102; + } + } + C2MemoryUsage usage = {C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE}; + + c2_status_t err = pool->fetchGraphicBlock(align(mWidth, 16), mHeight, format, + usage, &block); + + if (err != C2_OK) { + ALOGE("fetchGraphicBlock for Output failed with status %d", err); + work->result = err; + return false; + } + + C2GraphicView wView = block->map().get(); + + if (wView.error()) { + ALOGE("graphic view map failed %d", wView.error()); + work->result = C2_CORRUPTED; + return false; + } + + ALOGV("provided (%dx%d) required (%dx%d), out frameindex %d", block->width(), + block->height(), mWidth, mHeight, (int)buffer->user_private_data); + + uint8_t *dst = const_cast<uint8_t *>(wView.data()[C2PlanarLayout::PLANE_Y]); + size_t srcYStride = buffer->stride[0]; + size_t srcUStride = buffer->stride[1]; + size_t srcVStride = buffer->stride[2]; + + if (buffer->bitdepth == 10) { + const uint16_t *srcY = (const uint16_t *)buffer->plane[0]; + const uint16_t *srcU = (const uint16_t *)buffer->plane[1]; + const uint16_t *srcV = (const uint16_t *)buffer->plane[2]; + + if (format == HAL_PIXEL_FORMAT_RGBA_1010102) { + convertYUV420Planar16ToY410( + (uint32_t *)dst, srcY, srcU, srcV, srcYStride / 2, srcUStride / 2, + srcVStride / 2, align(mWidth, 16), mWidth, mHeight); + } else { + convertYUV420Planar16ToYUV420Planar(dst, srcY, srcU, srcV, srcYStride / 2, + srcUStride / 2, srcVStride / 2, + align(mWidth, 16), mWidth, mHeight); + } + } else { + const uint8_t *srcY = (const uint8_t *)buffer->plane[0]; + const uint8_t *srcU = (const uint8_t *)buffer->plane[1]; + const uint8_t *srcV = (const uint8_t *)buffer->plane[2]; + copyOutputBufferToYV12Frame(dst, srcY, srcU, srcV, srcYStride, srcUStride, + srcVStride, mWidth, mHeight); + } + finishWork(buffer->user_private_data, work, std::move(block)); + block = nullptr; + return true; +} + +c2_status_t C2SoftGav1Dec::drainInternal( + uint32_t drainMode, const std::shared_ptr<C2BlockPool> &pool, + const std::unique_ptr<C2Work> &work) { + if (drainMode == NO_DRAIN) { + ALOGW("drain with NO_DRAIN: no-op"); + return C2_OK; + } + if (drainMode == DRAIN_CHAIN) { + ALOGW("DRAIN_CHAIN not supported"); + return C2_OMITTED; + } + + Libgav1StatusCode status = + mCodecCtx->EnqueueFrame(/*data=*/nullptr, /*size=*/0, + /*user_private_data=*/0); + if (status != kLibgav1StatusOk) { + ALOGE("Failed to flush av1 decoder. status: %d.", status); + return C2_CORRUPTED; + } + + while (outputBuffer(pool, work)) { + } + + if (drainMode == DRAIN_COMPONENT_WITH_EOS && work && + work->workletsProcessed == 0u) { + fillEmptyWork(work); + } + + return C2_OK; +} + +c2_status_t C2SoftGav1Dec::drain(uint32_t drainMode, + const std::shared_ptr<C2BlockPool> &pool) { + return drainInternal(drainMode, pool, nullptr); +} + +class C2SoftGav1Factory : public C2ComponentFactory { + public: + C2SoftGav1Factory() + : mHelper(std::static_pointer_cast<C2ReflectorHelper>( + GetCodec2PlatformComponentStore()->getParamReflector())) {} + + virtual c2_status_t createComponent( + c2_node_id_t id, std::shared_ptr<C2Component> *const component, + std::function<void(C2Component *)> deleter) override { + *component = std::shared_ptr<C2Component>( + new C2SoftGav1Dec(COMPONENT_NAME, id, + std::make_shared<C2SoftGav1Dec::IntfImpl>(mHelper)), + deleter); + return C2_OK; + } + + virtual c2_status_t createInterface( + c2_node_id_t id, std::shared_ptr<C2ComponentInterface> *const interface, + std::function<void(C2ComponentInterface *)> deleter) override { + *interface = std::shared_ptr<C2ComponentInterface>( + new SimpleInterface<C2SoftGav1Dec::IntfImpl>( + COMPONENT_NAME, id, + std::make_shared<C2SoftGav1Dec::IntfImpl>(mHelper)), + deleter); + return C2_OK; + } + + virtual ~C2SoftGav1Factory() override = default; + + private: + std::shared_ptr<C2ReflectorHelper> mHelper; +}; + +} // namespace android + +extern "C" ::C2ComponentFactory *CreateCodec2Factory() { + ALOGV("in %s", __func__); + return new ::android::C2SoftGav1Factory(); +} + +extern "C" void DestroyCodec2Factory(::C2ComponentFactory *factory) { + ALOGV("in %s", __func__); + delete factory; +} diff --git a/media/codec2/components/gav1/C2SoftGav1Dec.h b/media/codec2/components/gav1/C2SoftGav1Dec.h new file mode 100644 index 0000000000..a7c08bbc39 --- /dev/null +++ b/media/codec2/components/gav1/C2SoftGav1Dec.h @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ANDROID_C2_SOFT_GAV1_DEC_H_ +#define ANDROID_C2_SOFT_GAV1_DEC_H_ + +#include <SimpleC2Component.h> +#include "libgav1/src/decoder.h" +#include "libgav1/src/decoder_settings.h" + +#define GETTIME(a, b) gettimeofday(a, b); +#define TIME_DIFF(start, end, diff) \ + diff = (((end).tv_sec - (start).tv_sec) * 1000000) + \ + ((end).tv_usec - (start).tv_usec); + +namespace android { + +struct C2SoftGav1Dec : public SimpleC2Component { + class IntfImpl; + + C2SoftGav1Dec(const char* name, c2_node_id_t id, + const std::shared_ptr<IntfImpl>& intfImpl); + ~C2SoftGav1Dec(); + + // Begin SimpleC2Component overrides. + c2_status_t onInit() override; + c2_status_t onStop() override; + void onReset() override; + void onRelease() override; + c2_status_t onFlush_sm() override; + void process(const std::unique_ptr<C2Work>& work, + const std::shared_ptr<C2BlockPool>& pool) override; + c2_status_t drain(uint32_t drainMode, + const std::shared_ptr<C2BlockPool>& pool) override; + // End SimpleC2Component overrides. + + private: + std::shared_ptr<IntfImpl> mIntf; + std::unique_ptr<libgav1::Decoder> mCodecCtx; + + uint32_t mWidth; + uint32_t mHeight; + bool mSignalledOutputEos; + bool mSignalledError; + + struct timeval mTimeStart; // Time at the start of decode() + struct timeval mTimeEnd; // Time at the end of decode() + + bool initDecoder(); + void destroyDecoder(); + void finishWork(uint64_t index, const std::unique_ptr<C2Work>& work, + const std::shared_ptr<C2GraphicBlock>& block); + bool outputBuffer(const std::shared_ptr<C2BlockPool>& pool, + const std::unique_ptr<C2Work>& work); + c2_status_t drainInternal(uint32_t drainMode, + const std::shared_ptr<C2BlockPool>& pool, + const std::unique_ptr<C2Work>& work); + + C2_DO_NOT_COPY(C2SoftGav1Dec); +}; + +} // namespace android + +#endif // ANDROID_C2_SOFT_GAV1_DEC_H_ diff --git a/media/codec2/components/hevc/C2SoftHevcDec.cpp b/media/codec2/components/hevc/C2SoftHevcDec.cpp index 7232572789..df677c284e 100644 --- a/media/codec2/components/hevc/C2SoftHevcDec.cpp +++ b/media/codec2/components/hevc/C2SoftHevcDec.cpp @@ -33,7 +33,8 @@ namespace android { namespace { constexpr char COMPONENT_NAME[] = "c2.android.hevc.decoder"; - +constexpr uint32_t kDefaultOutputDelay = 8; +constexpr uint32_t kMaxOutputDelay = 16; } // namespace class C2SoftHevcDec::IntfImpl : public SimpleInterface<void>::BaseParams { @@ -54,7 +55,9 @@ public: // TODO: Proper support for reorder depth. addParameter( DefineParam(mActualOutputDelay, C2_PARAMKEY_OUTPUT_DELAY) - .withConstValue(new C2PortActualDelayTuning::output(8u)) + .withDefault(new C2PortActualDelayTuning::output(kDefaultOutputDelay)) + .withFields({C2F(mActualOutputDelay, value).inRange(0, kMaxOutputDelay)}) + .withSetter(Setter<decltype(*mActualOutputDelay)>::StrictValueWithNoDeps) .build()); addParameter( @@ -327,6 +330,7 @@ C2SoftHevcDec::C2SoftHevcDec( mDecHandle(nullptr), mOutBufferFlush(nullptr), mIvColorformat(IV_YUV_420P), + mOutputDelay(kDefaultOutputDelay), mWidth(320), mHeight(240), mHeaderDecoded(false), @@ -877,6 +881,26 @@ void C2SoftHevcDec::process( work->result = C2_CORRUPTED; return; } + if (s_decode_op.i4_reorder_depth >= 0 && mOutputDelay != s_decode_op.i4_reorder_depth) { + mOutputDelay = s_decode_op.i4_reorder_depth; + ALOGV("New Output delay %d ", mOutputDelay); + + C2PortActualDelayTuning::output outputDelay(mOutputDelay); + std::vector<std::unique_ptr<C2SettingResult>> failures; + c2_status_t err = + mIntf->config({&outputDelay}, C2_MAY_BLOCK, &failures); + if (err == OK) { + work->worklets.front()->output.configUpdate.push_back( + C2Param::Copy(outputDelay)); + } else { + ALOGE("Cannot set output delay"); + mSignalledError = true; + work->workletsProcessed = 1u; + work->result = C2_CORRUPTED; + return; + } + continue; + } if (0 < s_decode_op.u4_pic_wd && 0 < s_decode_op.u4_pic_ht) { if (mHeaderDecoded == false) { mHeaderDecoded = true; diff --git a/media/codec2/components/hevc/C2SoftHevcDec.h b/media/codec2/components/hevc/C2SoftHevcDec.h index b7664e65e5..ce63a6c4c3 100644 --- a/media/codec2/components/hevc/C2SoftHevcDec.h +++ b/media/codec2/components/hevc/C2SoftHevcDec.h @@ -115,7 +115,7 @@ struct C2SoftHevcDec : public SimpleC2Component { size_t mNumCores; IV_COLOR_FORMAT_T mIvColorformat; - + uint32_t mOutputDelay; uint32_t mWidth; uint32_t mHeight; uint32_t mStride; diff --git a/media/codec2/components/mpeg4_h263/C2SoftMpeg4Enc.cpp b/media/codec2/components/mpeg4_h263/C2SoftMpeg4Enc.cpp index 36053f6885..54c8c47e96 100644 --- a/media/codec2/components/mpeg4_h263/C2SoftMpeg4Enc.cpp +++ b/media/codec2/components/mpeg4_h263/C2SoftMpeg4Enc.cpp @@ -517,9 +517,11 @@ void C2SoftMpeg4Enc::process( if (layout.planes[layout.PLANE_Y].colInc == 1 && layout.planes[layout.PLANE_U].colInc == 1 && layout.planes[layout.PLANE_V].colInc == 1 + && yStride == align(width, 16) && uStride == vStride && yStride == 2 * vStride) { - // I420 compatible - planes are already set up above + // I420 compatible with yStride being equal to aligned width + // planes are already set up above break; } diff --git a/media/codec2/components/raw/C2SoftRawDec.cpp b/media/codec2/components/raw/C2SoftRawDec.cpp index 95eb909206..7b6f21aee4 100644 --- a/media/codec2/components/raw/C2SoftRawDec.cpp +++ b/media/codec2/components/raw/C2SoftRawDec.cpp @@ -58,7 +58,7 @@ public: addParameter( DefineParam(mSampleRate, C2_PARAMKEY_SAMPLE_RATE) .withDefault(new C2StreamSampleRateInfo::output(0u, 44100)) - .withFields({C2F(mSampleRate, value).inRange(8000, 384000)}) + .withFields({C2F(mSampleRate, value).greaterThan(0)}) .withSetter((Setter<decltype(*mSampleRate)>::StrictValueWithNoDeps)) .build()); diff --git a/media/codec2/components/vpx/C2SoftVpxDec.cpp b/media/codec2/components/vpx/C2SoftVpxDec.cpp index a52ca15777..a759e8f3f6 100644 --- a/media/codec2/components/vpx/C2SoftVpxDec.cpp +++ b/media/codec2/components/vpx/C2SoftVpxDec.cpp @@ -593,12 +593,10 @@ void C2SoftVpxDec::process( } } - int64_t frameIndex = work->input.ordinal.frameIndex.peekll(); - if (inSize) { uint8_t *bitstream = const_cast<uint8_t *>(rView.data() + inOffset); vpx_codec_err_t err = vpx_codec_decode( - mCodecCtx, bitstream, inSize, &frameIndex, 0); + mCodecCtx, bitstream, inSize, &work->input.ordinal.frameIndex, 0); if (err != VPX_CODEC_OK) { ALOGE("on2 decoder failed to decode frame. err: %d", err); mSignalledError = true; @@ -608,7 +606,20 @@ void C2SoftVpxDec::process( } } - (void)outputBuffer(pool, work); + status_t err = outputBuffer(pool, work); + if (err == NOT_ENOUGH_DATA) { + if (inSize > 0) { + ALOGV("Maybe non-display frame at %lld.", + work->input.ordinal.frameIndex.peekll()); + // send the work back with empty buffer. + inSize = 0; + } + } else if (err != OK) { + ALOGD("Error while getting the output frame out"); + // work->result would be already filled; do fillEmptyWork() below to + // send the work back. + inSize = 0; + } if (eos) { drainInternal(DRAIN_COMPONENT_WITH_EOS, pool, work); @@ -742,16 +753,16 @@ static void convertYUV420Planar16ToYUV420Planar(uint8_t *dst, } return; } -bool C2SoftVpxDec::outputBuffer( +status_t C2SoftVpxDec::outputBuffer( const std::shared_ptr<C2BlockPool> &pool, const std::unique_ptr<C2Work> &work) { - if (!(work && pool)) return false; + if (!(work && pool)) return BAD_VALUE; vpx_codec_iter_t iter = nullptr; vpx_image_t *img = vpx_codec_get_frame(mCodecCtx, &iter); - if (!img) return false; + if (!img) return NOT_ENOUGH_DATA; if (img->d_w != mWidth || img->d_h != mHeight) { mWidth = img->d_w; @@ -768,7 +779,7 @@ bool C2SoftVpxDec::outputBuffer( mSignalledError = true; work->workletsProcessed = 1u; work->result = C2_CORRUPTED; - return false; + return UNKNOWN_ERROR; } } @@ -791,18 +802,19 @@ bool C2SoftVpxDec::outputBuffer( if (err != C2_OK) { ALOGE("fetchGraphicBlock for Output failed with status %d", err); work->result = err; - return false; + return UNKNOWN_ERROR; } C2GraphicView wView = block->map().get(); if (wView.error()) { ALOGE("graphic view map failed %d", wView.error()); work->result = C2_CORRUPTED; - return false; + return UNKNOWN_ERROR; } - ALOGV("provided (%dx%d) required (%dx%d), out frameindex %d", - block->width(), block->height(), mWidth, mHeight, (int)*(int64_t *)img->user_priv); + ALOGV("provided (%dx%d) required (%dx%d), out frameindex %lld", + block->width(), block->height(), mWidth, mHeight, + ((c2_cntr64_t *)img->user_priv)->peekll()); uint8_t *dst = const_cast<uint8_t *>(wView.data()[C2PlanarLayout::PLANE_Y]); size_t srcYStride = img->stride[VPX_PLANE_Y]; @@ -858,8 +870,8 @@ bool C2SoftVpxDec::outputBuffer( dstYStride, dstUVStride, mWidth, mHeight); } - finishWork(*(int64_t *)img->user_priv, work, std::move(block)); - return true; + finishWork(((c2_cntr64_t *)img->user_priv)->peekull(), work, std::move(block)); + return OK; } c2_status_t C2SoftVpxDec::drainInternal( @@ -875,7 +887,7 @@ c2_status_t C2SoftVpxDec::drainInternal( return C2_OMITTED; } - while ((outputBuffer(pool, work))) { + while (outputBuffer(pool, work) == OK) { } if (drainMode == DRAIN_COMPONENT_WITH_EOS && diff --git a/media/codec2/components/vpx/C2SoftVpxDec.h b/media/codec2/components/vpx/C2SoftVpxDec.h index e51bcee0de..206516557a 100644 --- a/media/codec2/components/vpx/C2SoftVpxDec.h +++ b/media/codec2/components/vpx/C2SoftVpxDec.h @@ -85,7 +85,7 @@ struct C2SoftVpxDec : public SimpleC2Component { status_t destroyDecoder(); void finishWork(uint64_t index, const std::unique_ptr<C2Work> &work, const std::shared_ptr<C2GraphicBlock> &block); - bool outputBuffer( + status_t outputBuffer( const std::shared_ptr<C2BlockPool> &pool, const std::unique_ptr<C2Work> &work); c2_status_t drainInternal( diff --git a/media/codec2/hidl/1.0/utils/Android.bp b/media/codec2/hidl/1.0/utils/Android.bp index 63fe36bfe5..f1f1536617 100644 --- a/media/codec2/hidl/1.0/utils/Android.bp +++ b/media/codec2/hidl/1.0/utils/Android.bp @@ -24,7 +24,7 @@ cc_library { "libgui", "libhidlbase", "liblog", - "libstagefright_bufferpool@2.0", + "libstagefright_bufferpool@2.0.1", "libui", "libutils", ], @@ -37,7 +37,7 @@ cc_library { "android.hardware.media.c2@1.0", "libcodec2", "libgui", - "libstagefright_bufferpool@2.0", + "libstagefright_bufferpool@2.0.1", "libui", ], } @@ -83,7 +83,7 @@ cc_library { "libhidltransport", "libhwbinder", "liblog", - "libstagefright_bufferpool@2.0", + "libstagefright_bufferpool@2.0.1", "libstagefright_bufferqueue_helper", "libui", "libutils", @@ -98,7 +98,7 @@ cc_library { "libcodec2", "libcodec2_vndk", "libhidlbase", - "libstagefright_bufferpool@2.0", + "libstagefright_bufferpool@2.0.1", "libui", ], } diff --git a/media/codec2/hidl/1.0/utils/InputBufferManager.cpp b/media/codec2/hidl/1.0/utils/InputBufferManager.cpp index a023a05dc8..8c0d0a4ba0 100644 --- a/media/codec2/hidl/1.0/utils/InputBufferManager.cpp +++ b/media/codec2/hidl/1.0/utils/InputBufferManager.cpp @@ -70,7 +70,7 @@ void InputBufferManager::_registerFrameData( << "."; std::lock_guard<std::mutex> lock(mMutex); - std::set<TrackedBuffer> &bufferIds = + std::set<TrackedBuffer*> &bufferIds = mTrackedBuffersMap[listener][frameIndex]; for (size_t i = 0; i < input.buffers.size(); ++i) { @@ -79,13 +79,14 @@ void InputBufferManager::_registerFrameData( << "Input buffer at index " << i << " is null."; continue; } - const TrackedBuffer &bufferId = - *bufferIds.emplace(listener, frameIndex, i, input.buffers[i]). - first; + TrackedBuffer *bufferId = + new TrackedBuffer(listener, frameIndex, i, input.buffers[i]); + mTrackedBufferCache.emplace(bufferId); + bufferIds.emplace(bufferId); c2_status_t status = input.buffers[i]->registerOnDestroyNotify( onBufferDestroyed, - const_cast<void*>(reinterpret_cast<const void*>(&bufferId))); + reinterpret_cast<void*>(bufferId)); if (status != C2_OK) { LOG(DEBUG) << "InputBufferManager::_registerFrameData -- " << "registerOnDestroyNotify() failed " @@ -119,31 +120,32 @@ void InputBufferManager::_unregisterFrameData( auto findListener = mTrackedBuffersMap.find(listener); if (findListener != mTrackedBuffersMap.end()) { - std::map<uint64_t, std::set<TrackedBuffer>> &frameIndex2BufferIds + std::map<uint64_t, std::set<TrackedBuffer*>> &frameIndex2BufferIds = findListener->second; auto findFrameIndex = frameIndex2BufferIds.find(frameIndex); if (findFrameIndex != frameIndex2BufferIds.end()) { - std::set<TrackedBuffer> &bufferIds = findFrameIndex->second; - for (const TrackedBuffer& bufferId : bufferIds) { - std::shared_ptr<C2Buffer> buffer = bufferId.buffer.lock(); + std::set<TrackedBuffer*> &bufferIds = findFrameIndex->second; + for (TrackedBuffer* bufferId : bufferIds) { + std::shared_ptr<C2Buffer> buffer = bufferId->buffer.lock(); if (buffer) { c2_status_t status = buffer->unregisterOnDestroyNotify( onBufferDestroyed, - const_cast<void*>( - reinterpret_cast<const void*>(&bufferId))); + reinterpret_cast<void*>(bufferId)); if (status != C2_OK) { LOG(DEBUG) << "InputBufferManager::_unregisterFrameData " << "-- unregisterOnDestroyNotify() failed " << "(listener @ 0x" << std::hex - << bufferId.listener.unsafe_get() + << bufferId->listener.unsafe_get() << ", frameIndex = " - << std::dec << bufferId.frameIndex - << ", bufferIndex = " << bufferId.bufferIndex + << std::dec << bufferId->frameIndex + << ", bufferIndex = " << bufferId->bufferIndex << ") => status = " << status << "."; } } + mTrackedBufferCache.erase(bufferId); + delete bufferId; } frameIndex2BufferIds.erase(findFrameIndex); @@ -179,31 +181,32 @@ void InputBufferManager::_unregisterFrameData( auto findListener = mTrackedBuffersMap.find(listener); if (findListener != mTrackedBuffersMap.end()) { - std::map<uint64_t, std::set<TrackedBuffer>> &frameIndex2BufferIds = + std::map<uint64_t, std::set<TrackedBuffer*>> &frameIndex2BufferIds = findListener->second; for (auto findFrameIndex = frameIndex2BufferIds.begin(); findFrameIndex != frameIndex2BufferIds.end(); ++findFrameIndex) { - std::set<TrackedBuffer> &bufferIds = findFrameIndex->second; - for (const TrackedBuffer& bufferId : bufferIds) { - std::shared_ptr<C2Buffer> buffer = bufferId.buffer.lock(); + std::set<TrackedBuffer*> &bufferIds = findFrameIndex->second; + for (TrackedBuffer* bufferId : bufferIds) { + std::shared_ptr<C2Buffer> buffer = bufferId->buffer.lock(); if (buffer) { c2_status_t status = buffer->unregisterOnDestroyNotify( onBufferDestroyed, - const_cast<void*>( - reinterpret_cast<const void*>(&bufferId))); + reinterpret_cast<void*>(bufferId)); if (status != C2_OK) { LOG(DEBUG) << "InputBufferManager::_unregisterFrameData " << "-- unregisterOnDestroyNotify() failed " << "(listener @ 0x" << std::hex - << bufferId.listener.unsafe_get() + << bufferId->listener.unsafe_get() << ", frameIndex = " - << std::dec << bufferId.frameIndex - << ", bufferIndex = " << bufferId.bufferIndex + << std::dec << bufferId->frameIndex + << ", bufferIndex = " << bufferId->bufferIndex << ") => status = " << status << "."; } + mTrackedBufferCache.erase(bufferId); + delete bufferId; } } } @@ -236,50 +239,59 @@ void InputBufferManager::_onBufferDestroyed(const C2Buffer* buf, void* arg) { << std::dec << "."; return; } - TrackedBuffer id(*reinterpret_cast<TrackedBuffer*>(arg)); + + std::lock_guard<std::mutex> lock(mMutex); + TrackedBuffer *bufferId = reinterpret_cast<TrackedBuffer*>(arg); + + if (mTrackedBufferCache.find(bufferId) == mTrackedBufferCache.end()) { + LOG(VERBOSE) << "InputBufferManager::_onBufferDestroyed -- called with " + << "unregistered buffer: " + << "buf @ 0x" << std::hex << buf + << ", arg @ 0x" << std::hex << arg + << std::dec << "."; + return; + } + LOG(VERBOSE) << "InputBufferManager::_onBufferDestroyed -- called with " << "buf @ 0x" << std::hex << buf << ", arg @ 0x" << std::hex << arg << std::dec << " -- " - << "listener @ 0x" << std::hex << id.listener.unsafe_get() - << ", frameIndex = " << std::dec << id.frameIndex - << ", bufferIndex = " << id.bufferIndex + << "listener @ 0x" << std::hex << bufferId->listener.unsafe_get() + << ", frameIndex = " << std::dec << bufferId->frameIndex + << ", bufferIndex = " << bufferId->bufferIndex << "."; - - std::lock_guard<std::mutex> lock(mMutex); - - auto findListener = mTrackedBuffersMap.find(id.listener); + auto findListener = mTrackedBuffersMap.find(bufferId->listener); if (findListener == mTrackedBuffersMap.end()) { - LOG(DEBUG) << "InputBufferManager::_onBufferDestroyed -- " - << "received invalid listener: " - << "listener @ 0x" << std::hex << id.listener.unsafe_get() - << " (frameIndex = " << std::dec << id.frameIndex - << ", bufferIndex = " << id.bufferIndex - << ")."; + LOG(VERBOSE) << "InputBufferManager::_onBufferDestroyed -- " + << "received invalid listener: " + << "listener @ 0x" << std::hex << bufferId->listener.unsafe_get() + << " (frameIndex = " << std::dec << bufferId->frameIndex + << ", bufferIndex = " << bufferId->bufferIndex + << ")."; return; } - std::map<uint64_t, std::set<TrackedBuffer>> &frameIndex2BufferIds + std::map<uint64_t, std::set<TrackedBuffer*>> &frameIndex2BufferIds = findListener->second; - auto findFrameIndex = frameIndex2BufferIds.find(id.frameIndex); + auto findFrameIndex = frameIndex2BufferIds.find(bufferId->frameIndex); if (findFrameIndex == frameIndex2BufferIds.end()) { LOG(DEBUG) << "InputBufferManager::_onBufferDestroyed -- " << "received invalid frame index: " - << "frameIndex = " << id.frameIndex - << " (listener @ 0x" << std::hex << id.listener.unsafe_get() - << ", bufferIndex = " << std::dec << id.bufferIndex + << "frameIndex = " << bufferId->frameIndex + << " (listener @ 0x" << std::hex << bufferId->listener.unsafe_get() + << ", bufferIndex = " << std::dec << bufferId->bufferIndex << ")."; return; } - std::set<TrackedBuffer> &bufferIds = findFrameIndex->second; - auto findBufferId = bufferIds.find(id); + std::set<TrackedBuffer*> &bufferIds = findFrameIndex->second; + auto findBufferId = bufferIds.find(bufferId); if (findBufferId == bufferIds.end()) { LOG(DEBUG) << "InputBufferManager::_onBufferDestroyed -- " << "received invalid buffer index: " - << "bufferIndex = " << id.bufferIndex - << " (frameIndex = " << id.frameIndex - << ", listener @ 0x" << std::hex << id.listener.unsafe_get() + << "bufferIndex = " << bufferId->bufferIndex + << " (frameIndex = " << bufferId->frameIndex + << ", listener @ 0x" << std::hex << bufferId->listener.unsafe_get() << std::dec << ")."; return; } @@ -292,10 +304,13 @@ void InputBufferManager::_onBufferDestroyed(const C2Buffer* buf, void* arg) { } } - DeathNotifications &deathNotifications = mDeathNotifications[id.listener]; - deathNotifications.indices[id.frameIndex].emplace_back(id.bufferIndex); + DeathNotifications &deathNotifications = mDeathNotifications[bufferId->listener]; + deathNotifications.indices[bufferId->frameIndex].emplace_back(bufferId->bufferIndex); ++deathNotifications.count; mOnBufferDestroyed.notify_one(); + + mTrackedBufferCache.erase(bufferId); + delete bufferId; } // Notify the clients about buffer destructions. diff --git a/media/codec2/hidl/1.0/utils/include/codec2/hidl/1.0/InputBufferManager.h b/media/codec2/hidl/1.0/utils/include/codec2/hidl/1.0/InputBufferManager.h index b6857d5f71..42fa557a6b 100644 --- a/media/codec2/hidl/1.0/utils/include/codec2/hidl/1.0/InputBufferManager.h +++ b/media/codec2/hidl/1.0/utils/include/codec2/hidl/1.0/InputBufferManager.h @@ -196,13 +196,9 @@ private: frameIndex(frameIndex), bufferIndex(bufferIndex), buffer(buffer) {} - TrackedBuffer(const TrackedBuffer&) = default; - bool operator<(const TrackedBuffer& other) const { - return bufferIndex < other.bufferIndex; - } }; - // Map: listener -> frameIndex -> set<TrackedBuffer>. + // Map: listener -> frameIndex -> set<TrackedBuffer*>. // Essentially, this is used to store triples (listener, frameIndex, // bufferIndex) that's searchable by listener and (listener, frameIndex). // However, the value of the innermost map is TrackedBuffer, which also @@ -210,7 +206,7 @@ private: // because onBufferDestroyed() needs to know listener and frameIndex too. typedef std::map<wp<IComponentListener>, std::map<uint64_t, - std::set<TrackedBuffer>>> TrackedBuffersMap; + std::set<TrackedBuffer*>>> TrackedBuffersMap; // Storage for pending (unsent) death notifications for one listener. // Each pair in member named "indices" are (frameIndex, bufferIndex) from @@ -247,6 +243,16 @@ private: // Mutex for the management of all input buffers. std::mutex mMutex; + // Cache for all TrackedBuffers. + // + // Whenever registerOnDestroyNotify() is called, an argument of type + // TrackedBuffer is created and stored into this cache. + // Whenever unregisterOnDestroyNotify() or onBufferDestroyed() is called, + // the TrackedBuffer is removed from this cache. + // + // mTrackedBuffersMap stores references to TrackedBuffers inside this cache. + std::set<TrackedBuffer*> mTrackedBufferCache; + // Tracked input buffers. TrackedBuffersMap mTrackedBuffersMap; diff --git a/media/codec2/hidl/1.0/utils/types.cpp b/media/codec2/hidl/1.0/utils/types.cpp index 07dbf6777f..04fa59ccd0 100644 --- a/media/codec2/hidl/1.0/utils/types.cpp +++ b/media/codec2/hidl/1.0/utils/types.cpp @@ -1434,6 +1434,11 @@ bool objcpy(C2BaseBlock* d, const BaseBlock& s) { d->type = C2BaseBlock::GRAPHIC; return true; } + if (cHandle) { + // Though we got cloned handle, creating block failed. + native_handle_close(cHandle); + native_handle_delete(cHandle); + } LOG(ERROR) << "Unknown handle type in BaseBlock::pooledBlock."; return false; diff --git a/media/codec2/hidl/client/Android.bp b/media/codec2/hidl/client/Android.bp index 6038a4043f..e18422337b 100644 --- a/media/codec2/hidl/client/Android.bp +++ b/media/codec2/hidl/client/Android.bp @@ -19,7 +19,7 @@ cc_library { "libhidlbase", "libhidltransport", "liblog", - "libstagefright_bufferpool@2.0", + "libstagefright_bufferpool@2.0.1", "libui", "libutils", ], diff --git a/media/codec2/hidl/client/client.cpp b/media/codec2/hidl/client/client.cpp index 5ed54f1a24..c747190397 100644 --- a/media/codec2/hidl/client/client.cpp +++ b/media/codec2/hidl/client/client.cpp @@ -125,6 +125,9 @@ public: if (!mClient) { mClient = Codec2Client::_CreateFromIndex(mIndex); } + CHECK(mClient) << "Failed to create Codec2Client to service \"" + << GetServiceNames()[mIndex] << "\". (Index = " + << mIndex << ")."; return mClient; } @@ -832,6 +835,7 @@ std::shared_ptr<Codec2Client> Codec2Client::_CreateFromIndex(size_t index) { c2_status_t Codec2Client::ForAllServices( const std::string &key, + size_t numberOfAttempts, std::function<c2_status_t(const std::shared_ptr<Codec2Client>&)> predicate) { c2_status_t status = C2_NO_INIT; // no IComponentStores present @@ -860,33 +864,45 @@ c2_status_t Codec2Client::ForAllServices( for (size_t index : indices) { Cache& cache = Cache::List()[index]; - std::shared_ptr<Codec2Client> client{cache.getClient()}; - if (client) { + for (size_t tries = numberOfAttempts; tries > 0; --tries) { + std::shared_ptr<Codec2Client> client{cache.getClient()}; status = predicate(client); if (status == C2_OK) { std::scoped_lock lock{key2IndexMutex}; key2Index[key] = index; // update last known client index return C2_OK; + } else if (status == C2_TRANSACTION_FAILED) { + LOG(WARNING) << "\"" << key << "\" failed for service \"" + << client->getName() + << "\" due to transaction failure. " + << "(Service may have crashed.)" + << (tries > 1 ? " Retrying..." : ""); + cache.invalidate(); + continue; } - } - if (wasMapped) { - LOG(INFO) << "Could not find \"" << key << "\"" - " in the last instance. Retrying..."; - wasMapped = false; - cache.invalidate(); + if (wasMapped) { + LOG(INFO) << "\"" << key << "\" became invalid in service \"" + << client->getName() << "\". Retrying..."; + wasMapped = false; + } + break; } } - return status; // return the last status from a valid client + return status; // return the last status from a valid client } std::shared_ptr<Codec2Client::Component> Codec2Client::CreateComponentByName( const char* componentName, const std::shared_ptr<Listener>& listener, - std::shared_ptr<Codec2Client>* owner) { + std::shared_ptr<Codec2Client>* owner, + size_t numberOfAttempts) { + std::string key{"create:"}; + key.append(componentName); std::shared_ptr<Component> component; c2_status_t status = ForAllServices( - componentName, + key, + numberOfAttempts, [owner, &component, componentName, &listener]( const std::shared_ptr<Codec2Client> &client) -> c2_status_t { @@ -907,8 +923,9 @@ std::shared_ptr<Codec2Client::Component> return status; }); if (status != C2_OK) { - LOG(DEBUG) << "Could not create component \"" << componentName << "\". " - "Status = " << status << "."; + LOG(DEBUG) << "Failed to create component \"" << componentName + << "\" from all known services. " + "Last returned status = " << status << "."; } return component; } @@ -916,10 +933,14 @@ std::shared_ptr<Codec2Client::Component> std::shared_ptr<Codec2Client::Interface> Codec2Client::CreateInterfaceByName( const char* interfaceName, - std::shared_ptr<Codec2Client>* owner) { + std::shared_ptr<Codec2Client>* owner, + size_t numberOfAttempts) { + std::string key{"create:"}; + key.append(interfaceName); std::shared_ptr<Interface> interface; c2_status_t status = ForAllServices( - interfaceName, + key, + numberOfAttempts, [owner, &interface, interfaceName]( const std::shared_ptr<Codec2Client> &client) -> c2_status_t { @@ -939,8 +960,9 @@ std::shared_ptr<Codec2Client::Interface> return status; }); if (status != C2_OK) { - LOG(DEBUG) << "Could not create interface \"" << interfaceName << "\". " - "Status = " << status << "."; + LOG(DEBUG) << "Failed to create interface \"" << interfaceName + << "\" from all known services. " + "Last returned status = " << status << "."; } return interface; } diff --git a/media/codec2/hidl/client/include/codec2/hidl/client.h b/media/codec2/hidl/client/include/codec2/hidl/client.h index b8a7fb5d5b..c37407f5a4 100644 --- a/media/codec2/hidl/client/include/codec2/hidl/client.h +++ b/media/codec2/hidl/client/include/codec2/hidl/client.h @@ -179,17 +179,21 @@ struct Codec2Client : public Codec2ConfigurableClient { static std::vector<std::shared_ptr<Codec2Client>> CreateFromAllServices(); // Try to create a component with a given name from all known - // IComponentStore services. + // IComponentStore services. numberOfAttempts determines the number of times + // to retry the HIDL call if the transaction fails. static std::shared_ptr<Component> CreateComponentByName( char const* componentName, std::shared_ptr<Listener> const& listener, - std::shared_ptr<Codec2Client>* owner = nullptr); + std::shared_ptr<Codec2Client>* owner = nullptr, + size_t numberOfAttempts = 10); // Try to create a component interface with a given name from all known - // IComponentStore services. + // IComponentStore services. numberOfAttempts determines the number of times + // to retry the HIDL call if the transaction fails. static std::shared_ptr<Interface> CreateInterfaceByName( char const* interfaceName, - std::shared_ptr<Codec2Client>* owner = nullptr); + std::shared_ptr<Codec2Client>* owner = nullptr, + size_t numberOfAttempts = 10); // List traits from all known IComponentStore services. static std::vector<C2Component::Traits> const& ListComponents(); @@ -204,11 +208,25 @@ struct Codec2Client : public Codec2ConfigurableClient { protected: sp<Base> mBase; - // Finds the first store where the predicate returns OK, and returns the last - // predicate result. Uses key to remember the last store found, and if cached, - // it tries that store before trying all stores (one retry). + // Finds the first store where the predicate returns C2_OK and returns the + // last predicate result. The predicate will be tried on all stores. The + // function will return C2_OK the first time the predicate returns C2_OK, + // or it will return the value from the last time that predicate is tried. + // (The latter case corresponds to a failure on every store.) The order of + // the stores to try is the same as the return value of GetServiceNames(). + // + // key is used to remember the last store with which the predicate last + // succeeded. If the last successful store is cached, it will be tried + // first before all the stores are tried. Note that the last successful + // store will be tried twice---first before all the stores, and another time + // with all the stores. + // + // If an attempt to evaluate the predicate results in a transaction failure, + // repeated attempts will be made until the predicate returns without a + // transaction failure or numberOfAttempts attempts have been made. static c2_status_t ForAllServices( const std::string& key, + size_t numberOfAttempts, std::function<c2_status_t(std::shared_ptr<Codec2Client> const&)> predicate); diff --git a/media/codec2/sfplugin/C2OMXNode.cpp b/media/codec2/sfplugin/C2OMXNode.cpp index 78d221ebb3..c7588e9a51 100644 --- a/media/codec2/sfplugin/C2OMXNode.cpp +++ b/media/codec2/sfplugin/C2OMXNode.cpp @@ -62,7 +62,7 @@ public: android::base::unique_fd &&fd0, android::base::unique_fd &&fd1) { Mutexed<Jobs>::Locked jobs(mJobs); - auto it = jobs->queues.try_emplace(comp, comp, systemTime()).first; + auto it = jobs->queues.try_emplace(comp, comp).first; it->second.workList.emplace_back( std::move(work), fenceFd, std::move(fd0), std::move(fd1)); jobs->cond.broadcast(); @@ -79,7 +79,8 @@ protected: for (auto it = jobs->queues.begin(); it != jobs->queues.end(); ) { Queue &queue = it->second; if (queue.workList.empty() - || nowNs - queue.lastQueuedTimestampNs < kIntervalNs) { + || (queue.lastQueuedTimestampNs != 0 && + nowNs - queue.lastQueuedTimestampNs < kIntervalNs)) { ++it; continue; } @@ -104,6 +105,7 @@ protected: sp<Fence> fence(new Fence(fenceFd)); fence->waitForever(LOG_TAG); } + queue.lastQueuedTimestampNs = nowNs; comp->queue(&items); for (android::base::unique_fd &ufd : uniqueFds) { (void)ufd.release(); @@ -143,8 +145,8 @@ private: android::base::unique_fd fd1; }; struct Queue { - Queue(const std::shared_ptr<Codec2Client::Component> &comp, nsecs_t timestamp) - : component(comp), lastQueuedTimestampNs(timestamp) {} + Queue(const std::shared_ptr<Codec2Client::Component> &comp) + : component(comp), lastQueuedTimestampNs(0) {} Queue(const Queue &) = delete; Queue &operator =(const Queue &) = delete; diff --git a/media/codec2/sfplugin/CCodec.cpp b/media/codec2/sfplugin/CCodec.cpp index 9d1cc60419..78ddd6d569 100644 --- a/media/codec2/sfplugin/CCodec.cpp +++ b/media/codec2/sfplugin/CCodec.cpp @@ -375,7 +375,11 @@ public: // consumer usage is queried earlier. - ALOGD("ISConfig%s", status.str().c_str()); + if (status.str().empty()) { + ALOGD("ISConfig not changed"); + } else { + ALOGD("ISConfig%s", status.str().c_str()); + } return err; } @@ -810,9 +814,17 @@ void CCodec::configure(const sp<AMessage> &msg) { } { - double value; - if (msg->findDouble("time-lapse-fps", &value)) { - config->mISConfig->mCaptureFps = value; + bool captureFpsFound = false; + double timeLapseFps; + float captureRate; + if (msg->findDouble("time-lapse-fps", &timeLapseFps)) { + config->mISConfig->mCaptureFps = timeLapseFps; + captureFpsFound = true; + } else if (msg->findAsFloat(KEY_CAPTURE_RATE, &captureRate)) { + config->mISConfig->mCaptureFps = captureRate; + captureFpsFound = true; + } + if (captureFpsFound) { (void)msg->findAsFloat(KEY_FRAME_RATE, &config->mISConfig->mCodedFps); } } @@ -1274,7 +1286,8 @@ void CCodec::start() { { Mutexed<Config>::Locked config(mConfig); inputFormat = config->mInputFormat; - outputFormat = config->mOutputFormat; + // start triggers format dup + outputFormat = config->mOutputFormat = config->mOutputFormat->dup(); if (config->mInputSurface) { err2 = config->mInputSurface->start(); } @@ -1283,6 +1296,8 @@ void CCodec::start() { mCallback->onError(err2, ACTION_CODE_FATAL); return; } + // We're not starting after flush. + (void)mSentConfigAfterResume.test_and_set(); err2 = mChannel->start(inputFormat, outputFormat); if (err2 != OK) { mCallback->onError(err2, ACTION_CODE_FATAL); @@ -1511,18 +1526,26 @@ void CCodec::flush() { } void CCodec::signalResume() { - auto setResuming = [this] { + std::shared_ptr<Codec2Client::Component> comp; + auto setResuming = [this, &comp] { Mutexed<State>::Locked state(mState); if (state->get() != FLUSHED) { return UNKNOWN_ERROR; } state->set(RESUMING); + comp = state->comp; return OK; }; if (tryAndReportOnError(setResuming) != OK) { return; } + mSentConfigAfterResume.clear(); + { + Mutexed<Config>::Locked config(mConfig); + config->queryConfiguration(comp); + } + (void)mChannel->start(nullptr, nullptr); { @@ -1718,7 +1741,7 @@ void CCodec::onMessageReceived(const sp<AMessage> &msg) { // handle configuration changes in work done Mutexed<Config>::Locked config(mConfig); - bool changed = false; + bool changed = !mSentConfigAfterResume.test_and_set(); Config::Watcher<C2StreamInitDataInfo::output> initData = config->watch<C2StreamInitDataInfo::output>(); if (!work->worklets.empty() @@ -1750,7 +1773,9 @@ void CCodec::onMessageReceived(const sp<AMessage> &msg) { ++stream; } - changed = config->updateConfiguration(updates, config->mOutputDomain); + if (config->updateConfiguration(updates, config->mOutputDomain)) { + changed = true; + } // copy standard infos to graphic buffers if not already present (otherwise, we // may overwrite the actual intermediate value with a final value) diff --git a/media/codec2/sfplugin/CCodec.h b/media/codec2/sfplugin/CCodec.h index b0b3c4f175..a580d1d20c 100644 --- a/media/codec2/sfplugin/CCodec.h +++ b/media/codec2/sfplugin/CCodec.h @@ -17,6 +17,7 @@ #ifndef C_CODEC_H_ #define C_CODEC_H_ +#include <atomic> #include <chrono> #include <list> #include <memory> @@ -175,6 +176,7 @@ private: typedef CCodecConfig Config; Mutexed<Config> mConfig; Mutexed<std::list<std::unique_ptr<C2Work>>> mWorkDoneQueue; + std::atomic_flag mSentConfigAfterResume; friend class CCodecCallbackImpl; diff --git a/media/codec2/sfplugin/CCodecBufferChannel.cpp b/media/codec2/sfplugin/CCodecBufferChannel.cpp index 8308292512..4ee6c1cd4f 100644 --- a/media/codec2/sfplugin/CCodecBufferChannel.cpp +++ b/media/codec2/sfplugin/CCodecBufferChannel.cpp @@ -948,7 +948,11 @@ status_t CCodecBufferChannel::start( uint32_t outputGeneration; { Mutexed<OutputSurface>::Locked output(mOutputSurface); - output->maxDequeueBuffers = numOutputSlots + reorderDepth.value + kRenderingDepth; + output->maxDequeueBuffers = numOutputSlots + + reorderDepth.value + kRenderingDepth; + if (!secure) { + output->maxDequeueBuffers += numInputSlots; + } outputSurface = output->surface ? output->surface->getIGraphicBufferProducer() : nullptr; if (outputSurface) { @@ -1068,7 +1072,7 @@ status_t CCodecBufferChannel::start( } else { output->buffers.reset(new LinearOutputBuffers(mName)); } - output->buffers->setFormat(outputFormat->dup()); + output->buffers->setFormat(outputFormat); // Try to set output surface to created block pool if given. @@ -1272,6 +1276,24 @@ bool CCodecBufferChannel::handleWork( std::unique_ptr<C2Work> work, const sp<AMessage> &outputFormat, const C2StreamInitDataInfo::output *initData) { + if (outputFormat != nullptr) { + Mutexed<Output>::Locked output(mOutput); + ALOGD("[%s] onWorkDone: output format changed to %s", + mName, outputFormat->debugString().c_str()); + output->buffers->setFormat(outputFormat); + + AString mediaType; + if (outputFormat->findString(KEY_MIME, &mediaType) + && mediaType == MIMETYPE_AUDIO_RAW) { + int32_t channelCount; + int32_t sampleRate; + if (outputFormat->findInt32(KEY_CHANNEL_COUNT, &channelCount) + && outputFormat->findInt32(KEY_SAMPLE_RATE, &sampleRate)) { + output->buffers->updateSkipCutBuffer(sampleRate, channelCount); + } + } + } + if ((work->input.ordinal.frameIndex - mFirstValidFrameIndex.load()).peek() < 0) { // Discard frames from previous generation. ALOGD("[%s] Discard frames from previous generation.", mName); @@ -1328,13 +1350,18 @@ bool CCodecBufferChannel::handleWork( case C2PortReorderBufferDepthTuning::CORE_INDEX: { C2PortReorderBufferDepthTuning::output reorderDepth; if (reorderDepth.updateFrom(*param)) { + bool secure = mComponent->getName().find(".secure") != std::string::npos; mReorderStash.lock()->setDepth(reorderDepth.value); ALOGV("[%s] onWorkDone: updated reorder depth to %u", mName, reorderDepth.value); size_t numOutputSlots = mOutput.lock()->numSlots; + size_t numInputSlots = mInput.lock()->numSlots; Mutexed<OutputSurface>::Locked output(mOutputSurface); - output->maxDequeueBuffers = - numOutputSlots + reorderDepth.value + kRenderingDepth; + output->maxDequeueBuffers = numOutputSlots + + reorderDepth.value + kRenderingDepth; + if (!secure) { + output->maxDequeueBuffers += numInputSlots; + } if (output->surface) { output->surface->setMaxDequeuedBufferCount(output->maxDequeueBuffers); } @@ -1378,10 +1405,12 @@ bool CCodecBufferChannel::handleWork( if (outputDelay.updateFrom(*param)) { ALOGV("[%s] onWorkDone: updating output delay %u", mName, outputDelay.value); + bool secure = mComponent->getName().find(".secure") != std::string::npos; (void)mPipelineWatcher.lock()->outputDelay(outputDelay.value); bool outputBuffersChanged = false; size_t numOutputSlots = 0; + size_t numInputSlots = mInput.lock()->numSlots; { Mutexed<Output>::Locked output(mOutput); output->outputDelay = outputDelay.value; @@ -1407,6 +1436,9 @@ bool CCodecBufferChannel::handleWork( uint32_t depth = mReorderStash.lock()->depth(); Mutexed<OutputSurface>::Locked output(mOutputSurface); output->maxDequeueBuffers = numOutputSlots + depth + kRenderingDepth; + if (!secure) { + output->maxDequeueBuffers += numInputSlots; + } if (output->surface) { output->surface->setMaxDequeuedBufferCount(output->maxDequeueBuffers); } @@ -1439,24 +1471,6 @@ bool CCodecBufferChannel::handleWork( } } - if (outputFormat != nullptr) { - Mutexed<Output>::Locked output(mOutput); - ALOGD("[%s] onWorkDone: output format changed to %s", - mName, outputFormat->debugString().c_str()); - output->buffers->setFormat(outputFormat); - - AString mediaType; - if (outputFormat->findString(KEY_MIME, &mediaType) - && mediaType == MIMETYPE_AUDIO_RAW) { - int32_t channelCount; - int32_t sampleRate; - if (outputFormat->findInt32(KEY_CHANNEL_COUNT, &channelCount) - && outputFormat->findInt32(KEY_SAMPLE_RATE, &sampleRate)) { - output->buffers->updateSkipCutBuffer(sampleRate, channelCount); - } - } - } - int32_t flags = 0; if (worklet->output.flags & C2FrameData::FLAG_END_OF_STREAM) { flags |= MediaCodec::BUFFER_FLAG_EOS; diff --git a/media/codec2/sfplugin/CCodecBuffers.cpp b/media/codec2/sfplugin/CCodecBuffers.cpp index 26c702ddc0..ed8b832478 100644 --- a/media/codec2/sfplugin/CCodecBuffers.cpp +++ b/media/codec2/sfplugin/CCodecBuffers.cpp @@ -878,9 +878,10 @@ void OutputBuffersArray::realloc(const std::shared_ptr<C2Buffer> &c2buffer) { switch (c2buffer->data().type()) { case C2BufferData::LINEAR: { uint32_t size = kLinearBufferSize; - const C2ConstLinearBlock &block = c2buffer->data().linearBlocks().front(); - if (block.size() < kMaxLinearBufferSize / 2) { - size = block.size() * 2; + const std::vector<C2ConstLinearBlock> &linear_blocks = c2buffer->data().linearBlocks(); + const uint32_t block_size = linear_blocks.front().size(); + if (block_size < kMaxLinearBufferSize / 2) { + size = block_size * 2; } else { size = kMaxLinearBufferSize; } diff --git a/media/codec2/sfplugin/CCodecConfig.cpp b/media/codec2/sfplugin/CCodecConfig.cpp index 1cfdc19dad..5adcd94abc 100644 --- a/media/codec2/sfplugin/CCodecConfig.cpp +++ b/media/codec2/sfplugin/CCodecConfig.cpp @@ -235,7 +235,10 @@ struct StandardParams { const std::vector<ConfigMapper> &getConfigMappersForSdkKey(std::string key) const { auto it = mConfigMappers.find(key); if (it == mConfigMappers.end()) { - ALOGD("no c2 equivalents for %s", key.c_str()); + if (mComplained.count(key) == 0) { + ALOGD("no c2 equivalents for %s", key.c_str()); + mComplained.insert(key); + } return NO_MAPPERS; } ALOGV("found %zu eqs for %s", it->second.size(), key.c_str()); @@ -304,6 +307,7 @@ struct StandardParams { private: std::map<SdkKey, std::vector<ConfigMapper>> mConfigMappers; + mutable std::set<std::string> mComplained; }; const std::vector<ConfigMapper> StandardParams::NO_MAPPERS; @@ -508,7 +512,8 @@ void CCodecConfig::initializeStandardParams() { .limitTo(D::ENCODER & D::VIDEO)); // convert to timestamp base add(ConfigMapper(KEY_I_FRAME_INTERVAL, C2_PARAMKEY_SYNC_FRAME_INTERVAL, "value") - .withMappers([](C2Value v) -> C2Value { + .limitTo(D::VIDEO & D::ENCODER & D::CONFIG) + .withMapper([](C2Value v) -> C2Value { // convert from i32 to float int32_t i32Value; float fpValue; @@ -518,12 +523,6 @@ void CCodecConfig::initializeStandardParams() { return int64_t(c2_min(1000000 * fpValue + 0.5, (double)INT64_MAX)); } return C2Value(); - }, [](C2Value v) -> C2Value { - int64_t i64; - if (v.get(&i64)) { - return float(i64) / 1000000; - } - return C2Value(); })); // remove when codecs switch to proper coding.gop (add support for calculating gop) deprecated(ConfigMapper("i-frame-period", "coding.gop", "intra-period") @@ -1033,7 +1032,25 @@ bool CCodecConfig::updateFormats(Domain domain) { } ReflectedParamUpdater::Dict reflected = mParamUpdater->getParams(paramPointers); - ALOGD("c2 config is %s", reflected.debugString().c_str()); + std::string config = reflected.debugString(); + std::set<std::string> configLines; + std::string diff; + for (size_t start = 0; start != std::string::npos; ) { + size_t end = config.find('\n', start); + size_t count = (end == std::string::npos) + ? std::string::npos + : end - start + 1; + std::string line = config.substr(start, count); + configLines.insert(line); + if (mLastConfig.count(line) == 0) { + diff.append(line); + } + start = (end == std::string::npos) ? std::string::npos : end + 1; + } + if (!diff.empty()) { + ALOGD("c2 config diff is %s", diff.c_str()); + } + mLastConfig.swap(configLines); bool changed = false; if (domain & mInputDomain) { diff --git a/media/codec2/sfplugin/CCodecConfig.h b/media/codec2/sfplugin/CCodecConfig.h index 3bafe3faf0..a61c8b7ddc 100644 --- a/media/codec2/sfplugin/CCodecConfig.h +++ b/media/codec2/sfplugin/CCodecConfig.h @@ -134,6 +134,8 @@ struct CCodecConfig { /// For now support a validation function. std::map<C2Param::Index, LocalParamValidator> mLocalParams; + std::set<std::string> mLastConfig; + CCodecConfig(); /// initializes the members required to manage the format: descriptors, reflector, diff --git a/media/codec2/sfplugin/PipelineWatcher.cpp b/media/codec2/sfplugin/PipelineWatcher.cpp index 74d14e865e..0ee90569f5 100644 --- a/media/codec2/sfplugin/PipelineWatcher.cpp +++ b/media/codec2/sfplugin/PipelineWatcher.cpp @@ -146,7 +146,7 @@ PipelineWatcher::Clock::duration PipelineWatcher::elapsed( std::chrono::duration_cast<std::chrono::milliseconds>(elapsed).count()); durations.push_back(elapsed); } - std::nth_element(durations.begin(), durations.end(), durations.begin() + n, + std::nth_element(durations.begin(), durations.begin() + n, durations.end(), std::greater<Clock::duration>()); return durations[n]; } diff --git a/media/codec2/vndk/Android.bp b/media/codec2/vndk/Android.bp index b6ddfabf2d..52cc7adf23 100644 --- a/media/codec2/vndk/Android.bp +++ b/media/codec2/vndk/Android.bp @@ -66,7 +66,7 @@ cc_library_shared { "liblog", "libnativewindow", "libstagefright_foundation", - "libstagefright_bufferpool@2.0", + "libstagefright_bufferpool@2.0.1", "libui", "libutils", ], diff --git a/media/codec2/vndk/C2Buffer.cpp b/media/codec2/vndk/C2Buffer.cpp index 710b536ef7..2d99b53936 100644 --- a/media/codec2/vndk/C2Buffer.cpp +++ b/media/codec2/vndk/C2Buffer.cpp @@ -413,17 +413,14 @@ std::shared_ptr<C2LinearBlock> _C2BlockFactory::CreateLinearBlock( std::shared_ptr<C2LinearAllocation> alloc; if (C2AllocatorIon::isValid(cHandle)) { - native_handle_t *handle = native_handle_clone(cHandle); - if (handle) { - c2_status_t err = sAllocator->priorLinearAllocation(handle, &alloc); - const std::shared_ptr<C2PooledBlockPoolData> poolData = - std::make_shared<C2PooledBlockPoolData>(data); - if (err == C2_OK && poolData) { - // TODO: config params? - std::shared_ptr<C2LinearBlock> block = - _C2BlockFactory::CreateLinearBlock(alloc, poolData); - return block; - } + c2_status_t err = sAllocator->priorLinearAllocation(cHandle, &alloc); + const std::shared_ptr<C2PooledBlockPoolData> poolData = + std::make_shared<C2PooledBlockPoolData>(data); + if (err == C2_OK && poolData) { + // TODO: config params? + std::shared_ptr<C2LinearBlock> block = + _C2BlockFactory::CreateLinearBlock(alloc, poolData); + return block; } } return nullptr; @@ -674,17 +671,14 @@ public: ResultStatus status = mBufferPoolManager->allocate( mConnectionId, params, &cHandle, &bufferPoolData); if (status == ResultStatus::OK) { - native_handle_t *handle = native_handle_clone(cHandle); - if (handle) { - std::shared_ptr<C2LinearAllocation> alloc; - std::shared_ptr<C2PooledBlockPoolData> poolData = - std::make_shared<C2PooledBlockPoolData>(bufferPoolData); - c2_status_t err = mAllocator->priorLinearAllocation(handle, &alloc); - if (err == C2_OK && poolData && alloc) { - *block = _C2BlockFactory::CreateLinearBlock(alloc, poolData, 0, capacity); - if (*block) { - return C2_OK; - } + std::shared_ptr<C2LinearAllocation> alloc; + std::shared_ptr<C2PooledBlockPoolData> poolData = + std::make_shared<C2PooledBlockPoolData>(bufferPoolData); + c2_status_t err = mAllocator->priorLinearAllocation(cHandle, &alloc); + if (err == C2_OK && poolData && alloc) { + *block = _C2BlockFactory::CreateLinearBlock(alloc, poolData, 0, capacity); + if (*block) { + return C2_OK; } } return C2_NO_MEMORY; @@ -710,19 +704,16 @@ public: ResultStatus status = mBufferPoolManager->allocate( mConnectionId, params, &cHandle, &bufferPoolData); if (status == ResultStatus::OK) { - native_handle_t *handle = native_handle_clone(cHandle); - if (handle) { - std::shared_ptr<C2GraphicAllocation> alloc; - std::shared_ptr<C2PooledBlockPoolData> poolData = - std::make_shared<C2PooledBlockPoolData>(bufferPoolData); - c2_status_t err = mAllocator->priorGraphicAllocation( - handle, &alloc); - if (err == C2_OK && poolData && alloc) { - *block = _C2BlockFactory::CreateGraphicBlock( - alloc, poolData, C2Rect(width, height)); - if (*block) { - return C2_OK; - } + std::shared_ptr<C2GraphicAllocation> alloc; + std::shared_ptr<C2PooledBlockPoolData> poolData = + std::make_shared<C2PooledBlockPoolData>(bufferPoolData); + c2_status_t err = mAllocator->priorGraphicAllocation( + cHandle, &alloc); + if (err == C2_OK && poolData && alloc) { + *block = _C2BlockFactory::CreateGraphicBlock( + alloc, poolData, C2Rect(width, height)); + if (*block) { + return C2_OK; } } return C2_NO_MEMORY; @@ -1117,17 +1108,14 @@ std::shared_ptr<C2GraphicBlock> _C2BlockFactory::CreateGraphicBlock( std::shared_ptr<C2GraphicAllocation> alloc; if (C2AllocatorGralloc::isValid(cHandle)) { - native_handle_t *handle = native_handle_clone(cHandle); - if (handle) { - c2_status_t err = sAllocator->priorGraphicAllocation(handle, &alloc); - const std::shared_ptr<C2PooledBlockPoolData> poolData = - std::make_shared<C2PooledBlockPoolData>(data); - if (err == C2_OK && poolData) { - // TODO: config setup? - std::shared_ptr<C2GraphicBlock> block = - _C2BlockFactory::CreateGraphicBlock(alloc, poolData); - return block; - } + c2_status_t err = sAllocator->priorGraphicAllocation(cHandle, &alloc); + const std::shared_ptr<C2PooledBlockPoolData> poolData = + std::make_shared<C2PooledBlockPoolData>(data); + if (err == C2_OK && poolData) { + // TODO: config setup? + std::shared_ptr<C2GraphicBlock> block = + _C2BlockFactory::CreateGraphicBlock(alloc, poolData); + return block; } } return nullptr; diff --git a/media/codec2/vndk/C2Store.cpp b/media/codec2/vndk/C2Store.cpp index f8afa7c539..5b2bd7b62f 100644 --- a/media/codec2/vndk/C2Store.cpp +++ b/media/codec2/vndk/C2Store.cpp @@ -848,7 +848,8 @@ C2PlatformComponentStore::C2PlatformComponentStore() emplace("libcodec2_soft_amrnbenc.so"); emplace("libcodec2_soft_amrwbdec.so"); emplace("libcodec2_soft_amrwbenc.so"); - emplace("libcodec2_soft_av1dec.so"); + //emplace("libcodec2_soft_av1dec_aom.so"); // deprecated for the gav1 implementation + emplace("libcodec2_soft_av1dec_gav1.so"); emplace("libcodec2_soft_avcdec.so"); emplace("libcodec2_soft_avcenc.so"); emplace("libcodec2_soft_flacdec.so"); diff --git a/media/extractors/midi/Android.bp b/media/extractors/midi/Android.bp index 7d42e703f3..91ce78eb9d 100644 --- a/media/extractors/midi/Android.bp +++ b/media/extractors/midi/Android.bp @@ -14,7 +14,9 @@ cc_library_shared { static_libs: [ "libmedia_midiiowrapper", "libsonivox", - "libstagefright_foundation" + "libstagefright_foundation", + "libwatchdog", + "libbase", ], name: "libmidiextractor", relative_install_path: "extractors", @@ -35,5 +37,4 @@ cc_library_shared { "signed-integer-overflow", ], }, - } diff --git a/media/extractors/midi/MidiExtractor.cpp b/media/extractors/midi/MidiExtractor.cpp index 9f4f9e69b4..d0efb2f652 100644 --- a/media/extractors/midi/MidiExtractor.cpp +++ b/media/extractors/midi/MidiExtractor.cpp @@ -26,6 +26,7 @@ #include <media/stagefright/MediaDefs.h> #include <media/stagefright/MediaErrors.h> #include <libsonivox/eas_reverb.h> +#include <watchdog/Watchdog.h> namespace android { @@ -116,6 +117,7 @@ media_status_t MidiSource::read( MediaBufferHelper **outBuffer, const ReadOptions *options) { ALOGV("MidiSource::read"); + MediaBufferHelper *buffer; // process an optional seek request int64_t seekTimeUs; @@ -139,6 +141,8 @@ status_t MidiSource::init() } // MidiEngine +using namespace std::chrono_literals; +static constexpr auto kTimeout = 10s; MidiEngine::MidiEngine(CDataSource *dataSource, AMediaFormat *fileMetadata, @@ -147,6 +151,8 @@ MidiEngine::MidiEngine(CDataSource *dataSource, mEasHandle(NULL), mEasConfig(NULL), mIsInitialized(false) { + Watchdog watchdog(kTimeout); + mIoWrapper = new MidiIoWrapper(dataSource); // spin up a new EAS engine EAS_I32 temp; @@ -186,6 +192,8 @@ MidiEngine::MidiEngine(CDataSource *dataSource, } MidiEngine::~MidiEngine() { + Watchdog watchdog(kTimeout); + if (mEasHandle) { EAS_CloseFile(mEasData, mEasHandle); } @@ -217,12 +225,16 @@ status_t MidiEngine::releaseBuffers() { } status_t MidiEngine::seekTo(int64_t positionUs) { + Watchdog watchdog(kTimeout); + ALOGV("seekTo %lld", (long long)positionUs); EAS_RESULT result = EAS_Locate(mEasData, mEasHandle, positionUs / 1000, false); return result == EAS_SUCCESS ? OK : UNKNOWN_ERROR; } MediaBufferHelper* MidiEngine::readBuffer() { + Watchdog watchdog(kTimeout); + EAS_STATE state; EAS_State(mEasData, mEasHandle, &state); if ((state == EAS_STATE_STOPPED) || (state == EAS_STATE_ERROR)) { diff --git a/media/extractors/mp4/MPEG4Extractor.cpp b/media/extractors/mp4/MPEG4Extractor.cpp index 9d5890c42c..485c0cc9ef 100755 --- a/media/extractors/mp4/MPEG4Extractor.cpp +++ b/media/extractors/mp4/MPEG4Extractor.cpp @@ -1060,6 +1060,8 @@ status_t MPEG4Extractor::parseChunk(off64_t *offset, int depth) { // drop it now to reduce our footprint free(mLastTrack->mTx3gBuffer); mLastTrack->mTx3gBuffer = NULL; + mLastTrack->mTx3gFilled = 0; + mLastTrack->mTx3gSize = 0; } } else if (chunk_type == FOURCC("moov")) { @@ -2621,6 +2623,10 @@ status_t MPEG4Extractor::parseChunk(off64_t *offset, int depth) { // if those apps are compensating for it, we'd break them with such a change // + if (mLastTrack->mTx3gBuffer == NULL) { + mLastTrack->mTx3gSize = 0; + mLastTrack->mTx3gFilled = 0; + } if (mLastTrack->mTx3gSize - mLastTrack->mTx3gFilled < chunk_size) { size_t growth = kTx3gGrowth; if (growth < chunk_size) { @@ -4993,8 +4999,11 @@ status_t MPEG4Source::parseChunk(off64_t *offset) { } status_t MPEG4Source::parseSampleAuxiliaryInformationSizes( - off64_t offset, off64_t /* size */) { + off64_t offset, off64_t size) { ALOGV("parseSampleAuxiliaryInformationSizes"); + if (size < 9) { + return -EINVAL; + } // 14496-12 8.7.12 uint8_t version; if (mDataSource->readAt( @@ -5007,25 +5016,32 @@ status_t MPEG4Source::parseSampleAuxiliaryInformationSizes( return ERROR_UNSUPPORTED; } offset++; + size--; uint32_t flags; if (!mDataSource->getUInt24(offset, &flags)) { return ERROR_IO; } offset += 3; + size -= 3; if (flags & 1) { + if (size < 13) { + return -EINVAL; + } uint32_t tmp; if (!mDataSource->getUInt32(offset, &tmp)) { return ERROR_MALFORMED; } mCurrentAuxInfoType = tmp; offset += 4; + size -= 4; if (!mDataSource->getUInt32(offset, &tmp)) { return ERROR_MALFORMED; } mCurrentAuxInfoTypeParameter = tmp; offset += 4; + size -= 4; } uint8_t defsize; @@ -5034,6 +5050,7 @@ status_t MPEG4Source::parseSampleAuxiliaryInformationSizes( } mCurrentDefaultSampleInfoSize = defsize; offset++; + size--; uint32_t smplcnt; if (!mDataSource->getUInt32(offset, &smplcnt)) { @@ -5041,11 +5058,16 @@ status_t MPEG4Source::parseSampleAuxiliaryInformationSizes( } mCurrentSampleInfoCount = smplcnt; offset += 4; - + size -= 4; if (mCurrentDefaultSampleInfoSize != 0) { ALOGV("@@@@ using default sample info size of %d", mCurrentDefaultSampleInfoSize); return OK; } + if(smplcnt > size) { + ALOGW("b/124525515 - smplcnt(%u) > size(%ld)", (unsigned int)smplcnt, (unsigned long)size); + android_errorWriteLog(0x534e4554, "124525515"); + return -EINVAL; + } if (smplcnt > mCurrentSampleInfoAllocSize) { uint8_t * newPtr = (uint8_t*) realloc(mCurrentSampleInfoSizes, smplcnt); if (newPtr == NULL) { @@ -5061,26 +5083,32 @@ status_t MPEG4Source::parseSampleAuxiliaryInformationSizes( } status_t MPEG4Source::parseSampleAuxiliaryInformationOffsets( - off64_t offset, off64_t /* size */) { + off64_t offset, off64_t size) { ALOGV("parseSampleAuxiliaryInformationOffsets"); + if (size < 8) { + return -EINVAL; + } // 14496-12 8.7.13 uint8_t version; if (mDataSource->readAt(offset, &version, sizeof(version)) != 1) { return ERROR_IO; } offset++; + size--; uint32_t flags; if (!mDataSource->getUInt24(offset, &flags)) { return ERROR_IO; } offset += 3; + size -= 3; uint32_t entrycount; if (!mDataSource->getUInt32(offset, &entrycount)) { return ERROR_IO; } offset += 4; + size -= 4; if (entrycount == 0) { return OK; } @@ -5106,19 +5134,31 @@ status_t MPEG4Source::parseSampleAuxiliaryInformationOffsets( for (size_t i = 0; i < entrycount; i++) { if (version == 0) { + if (size < 4) { + ALOGW("b/124526959"); + android_errorWriteLog(0x534e4554, "124526959"); + return -EINVAL; + } uint32_t tmp; if (!mDataSource->getUInt32(offset, &tmp)) { return ERROR_IO; } mCurrentSampleInfoOffsets[i] = tmp; offset += 4; + size -= 4; } else { + if (size < 8) { + ALOGW("b/124526959"); + android_errorWriteLog(0x534e4554, "124526959"); + return -EINVAL; + } uint64_t tmp; if (!mDataSource->getUInt64(offset, &tmp)) { return ERROR_IO; } mCurrentSampleInfoOffsets[i] = tmp; offset += 8; + size -= 8; } } @@ -5440,16 +5480,12 @@ status_t MPEG4Source::parseTrackFragmentRun(off64_t offset, off64_t size) { // apply some sanity (vs strict legality) checks // - // clamp the count of entries in the trun box, to avoid spending forever parsing - // this box. Clamping (vs error) lets us play *something*. - // 1 million is about 400 msecs on a Pixel3, should be no more than a couple seconds - // on the slowest devices. - static constexpr uint32_t kMaxTrunSampleCount = 1000000; + static constexpr uint32_t kMaxTrunSampleCount = 10000; if (sampleCount > kMaxTrunSampleCount) { - ALOGW("b/123389881 clamp sampleCount(%u) @ kMaxTrunSampleCount(%u)", + ALOGW("b/123389881 sampleCount(%u) > kMaxTrunSampleCount(%u)", sampleCount, kMaxTrunSampleCount); android_errorWriteLog(0x534e4554, "124389881 count"); - + return -EINVAL; } } @@ -5493,7 +5529,12 @@ status_t MPEG4Source::parseTrackFragmentRun(off64_t offset, off64_t size) { tmp.duration = sampleDuration; tmp.compositionOffset = sampleCtsOffset; memset(tmp.iv, 0, sizeof(tmp.iv)); - mCurrentSamples.add(tmp); + if (mCurrentSamples.add(tmp) < 0) { + ALOGW("b/123389881 failed saving sample(n=%zu)", mCurrentSamples.size()); + android_errorWriteLog(0x534e4554, "124389881 allocation"); + mCurrentSamples.clear(); + return NO_MEMORY; + } dataOffset += sampleSize; } diff --git a/media/extractors/mp4/SampleTable.cpp b/media/extractors/mp4/SampleTable.cpp index bf29bf1eac..e7e8901bf1 100644 --- a/media/extractors/mp4/SampleTable.cpp +++ b/media/extractors/mp4/SampleTable.cpp @@ -391,20 +391,11 @@ status_t SampleTable::setTimeToSampleParams( } mTimeToSampleCount = U32_AT(&header[4]); - if (mTimeToSampleCount > UINT32_MAX / (2 * sizeof(uint32_t))) { - // Choose this bound because - // 1) 2 * sizeof(uint32_t) is the amount of memory needed for one - // time-to-sample entry in the time-to-sample table. - // 2) mTimeToSampleCount is the number of entries of the time-to-sample - // table. - // 3) We hope that the table size does not exceed UINT32_MAX. + if (mTimeToSampleCount > (data_size - 8) / (2 * sizeof(uint32_t))) { ALOGE("Time-to-sample table size too large."); return ERROR_OUT_OF_RANGE; } - // Note: At this point, we know that mTimeToSampleCount * 2 will not - // overflow because of the above condition. - uint64_t allocSize = (uint64_t)mTimeToSampleCount * 2 * sizeof(uint32_t); mTotalSize += allocSize; if (mTotalSize > kMaxTotalSize) { @@ -540,6 +531,12 @@ status_t SampleTable::setSyncSampleParams(off64_t data_offset, size_t data_size) } uint64_t allocSize = (uint64_t)numSyncSamples * sizeof(uint32_t); + if (allocSize > data_size - 8) { + ALOGW("b/124771364 - allocSize(%lu) > size(%lu)", + (unsigned long)allocSize, (unsigned long)(data_size - 8)); + android_errorWriteLog(0x534e4554, "124771364"); + return ERROR_MALFORMED; + } if (allocSize > kMaxTotalSize) { ALOGE("Sync sample table size too large."); return ERROR_OUT_OF_RANGE; diff --git a/media/libaaudio/src/binding/IAAudioService.cpp b/media/libaaudio/src/binding/IAAudioService.cpp index 97ad2b02a0..e017b3a122 100644 --- a/media/libaaudio/src/binding/IAAudioService.cpp +++ b/media/libaaudio/src/binding/IAAudioService.cpp @@ -237,12 +237,12 @@ IMPLEMENT_META_INTERFACE(AAudioService, "IAAudioService"); status_t BnAAudioService::onTransact(uint32_t code, const Parcel& data, Parcel* reply, uint32_t flags) { - aaudio_handle_t streamHandle; + aaudio_handle_t streamHandle = 0; aaudio::AAudioStreamRequest request; aaudio::AAudioStreamConfiguration configuration; - pid_t tid; - int64_t nanoseconds; - aaudio_result_t result; + pid_t tid = 0; + int64_t nanoseconds = 0; + aaudio_result_t result = AAUDIO_OK; status_t status = NO_ERROR; ALOGV("BnAAudioService::onTransact(%i) %i", code, flags); @@ -285,7 +285,11 @@ status_t BnAAudioService::onTransact(uint32_t code, const Parcel& data, case CLOSE_STREAM: { CHECK_INTERFACE(IAAudioService, data, reply); - data.readInt32(&streamHandle); + status = data.readInt32(&streamHandle); + if (status != NO_ERROR) { + ALOGE("BnAAudioService::%s(CLOSE_STREAM) streamHandle failed!", __func__); + return status; + } result = closeStream(streamHandle); //ALOGD("BnAAudioService::onTransact CLOSE_STREAM 0x%08X, result = %d", // streamHandle, result); @@ -297,6 +301,7 @@ status_t BnAAudioService::onTransact(uint32_t code, const Parcel& data, CHECK_INTERFACE(IAAudioService, data, reply); status = data.readInt32(&streamHandle); if (status != NO_ERROR) { + ALOGE("BnAAudioService::%s(GET_STREAM_DESCRIPTION) streamHandle failed!", __func__); return status; } aaudio::AudioEndpointParcelable parcelable; @@ -313,7 +318,11 @@ status_t BnAAudioService::onTransact(uint32_t code, const Parcel& data, case START_STREAM: { CHECK_INTERFACE(IAAudioService, data, reply); - data.readInt32(&streamHandle); + status = data.readInt32(&streamHandle); + if (status != NO_ERROR) { + ALOGE("BnAAudioService::%s(START_STREAM) streamHandle failed!", __func__); + return status; + } result = startStream(streamHandle); ALOGV("BnAAudioService::onTransact START_STREAM 0x%08X, result = %d", streamHandle, result); @@ -323,7 +332,11 @@ status_t BnAAudioService::onTransact(uint32_t code, const Parcel& data, case PAUSE_STREAM: { CHECK_INTERFACE(IAAudioService, data, reply); - data.readInt32(&streamHandle); + status = data.readInt32(&streamHandle); + if (status != NO_ERROR) { + ALOGE("BnAAudioService::%s(PAUSE_STREAM) streamHandle failed!", __func__); + return status; + } result = pauseStream(streamHandle); ALOGV("BnAAudioService::onTransact PAUSE_STREAM 0x%08X, result = %d", streamHandle, result); @@ -333,7 +346,11 @@ status_t BnAAudioService::onTransact(uint32_t code, const Parcel& data, case STOP_STREAM: { CHECK_INTERFACE(IAAudioService, data, reply); - data.readInt32(&streamHandle); + status = data.readInt32(&streamHandle); + if (status != NO_ERROR) { + ALOGE("BnAAudioService::%s(STOP_STREAM) streamHandle failed!", __func__); + return status; + } result = stopStream(streamHandle); ALOGV("BnAAudioService::onTransact STOP_STREAM 0x%08X, result = %d", streamHandle, result); @@ -343,7 +360,11 @@ status_t BnAAudioService::onTransact(uint32_t code, const Parcel& data, case FLUSH_STREAM: { CHECK_INTERFACE(IAAudioService, data, reply); - data.readInt32(&streamHandle); + status = data.readInt32(&streamHandle); + if (status != NO_ERROR) { + ALOGE("BnAAudioService::%s(FLUSH_STREAM) streamHandle failed!", __func__); + return status; + } result = flushStream(streamHandle); ALOGV("BnAAudioService::onTransact FLUSH_STREAM 0x%08X, result = %d", streamHandle, result); @@ -353,20 +374,40 @@ status_t BnAAudioService::onTransact(uint32_t code, const Parcel& data, case REGISTER_AUDIO_THREAD: { CHECK_INTERFACE(IAAudioService, data, reply); - data.readInt32(&streamHandle); - data.readInt32(&tid); - data.readInt64(&nanoseconds); + status = data.readInt32(&streamHandle); + if (status != NO_ERROR) { + ALOGE("BnAAudioService::%s(REGISTER_AUDIO_THREAD) streamHandle failed!", __func__); + return status; + } + status = data.readInt32(&tid); + if (status != NO_ERROR) { + ALOGE("BnAAudioService::%s(REGISTER_AUDIO_THREAD) tid failed!", __func__); + return status; + } + status = data.readInt64(&nanoseconds); + if (status != NO_ERROR) { + ALOGE("BnAAudioService::%s(REGISTER_AUDIO_THREAD) nanoseconds failed!", __func__); + return status; + } result = registerAudioThread(streamHandle, tid, nanoseconds); - ALOGV("BnAAudioService::onTransact REGISTER_AUDIO_THREAD 0x%08X, result = %d", - streamHandle, result); + ALOGV("BnAAudioService::%s(REGISTER_AUDIO_THREAD) 0x%08X, result = %d", + __func__, streamHandle, result); reply->writeInt32(result); return NO_ERROR; } break; case UNREGISTER_AUDIO_THREAD: { CHECK_INTERFACE(IAAudioService, data, reply); - data.readInt32(&streamHandle); - data.readInt32(&tid); + status = data.readInt32(&streamHandle); + if (status != NO_ERROR) { + ALOGE("BnAAudioService::%s(UNREGISTER_AUDIO_THREAD) streamHandle failed!", __func__); + return status; + } + status = data.readInt32(&tid); + if (status != NO_ERROR) { + ALOGE("BnAAudioService::%s(UNREGISTER_AUDIO_THREAD) tid failed!", __func__); + return status; + } result = unregisterAudioThread(streamHandle, tid); ALOGV("BnAAudioService::onTransact UNREGISTER_AUDIO_THREAD 0x%08X, result = %d", streamHandle, result); diff --git a/media/libaaudio/src/client/AudioStreamInternalCapture.cpp b/media/libaaudio/src/client/AudioStreamInternalCapture.cpp index a6cc45b464..366cc87209 100644 --- a/media/libaaudio/src/client/AudioStreamInternalCapture.cpp +++ b/media/libaaudio/src/client/AudioStreamInternalCapture.cpp @@ -89,7 +89,11 @@ aaudio_result_t AudioStreamInternalCapture::processDataNow(void *buffer, int32_t if (mAudioEndpoint.isFreeRunning()) { //ALOGD("AudioStreamInternalCapture::processDataNow() - update remote counter"); // Update data queue based on the timing model. - int64_t estimatedRemoteCounter = mClockModel.convertTimeToPosition(currentNanoTime); + // Jitter in the DSP can cause late writes to the FIFO. + // This might be caused by resampling. + // We want to read the FIFO after the latest possible time + // that the DSP could have written the data. + int64_t estimatedRemoteCounter = mClockModel.convertLatestTimeToPosition(currentNanoTime); // TODO refactor, maybe use setRemoteCounter() mAudioEndpoint.setDataWriteCounter(estimatedRemoteCounter); } @@ -139,7 +143,7 @@ aaudio_result_t AudioStreamInternalCapture::processDataNow(void *buffer, int32_t // the writeCounter might have just advanced in the background, // causing us to sleep until a later burst. int64_t nextPosition = mAudioEndpoint.getDataReadCounter() + mFramesPerBurst; - wakeTime = mClockModel.convertPositionToTime(nextPosition); + wakeTime = mClockModel.convertPositionToLatestTime(nextPosition); } break; default: diff --git a/media/libaaudio/src/client/IsochronousClockModel.cpp b/media/libaaudio/src/client/IsochronousClockModel.cpp index 747d0e14bc..9abdf53020 100644 --- a/media/libaaudio/src/client/IsochronousClockModel.cpp +++ b/media/libaaudio/src/client/IsochronousClockModel.cpp @@ -19,12 +19,11 @@ #include <log/log.h> #include <stdint.h> +#include <algorithm> #include "utility/AudioClock.h" #include "IsochronousClockModel.h" -#define MIN_LATENESS_NANOS (10 * AAUDIO_NANOS_PER_MICROSECOND) - using namespace aaudio; IsochronousClockModel::IsochronousClockModel() @@ -32,7 +31,7 @@ IsochronousClockModel::IsochronousClockModel() , mMarkerNanoTime(0) , mSampleRate(48000) , mFramesPerBurst(64) - , mMaxLatenessInNanos(0) + , mMaxMeasuredLatenessNanos(0) , mState(STATE_STOPPED) { } @@ -41,8 +40,7 @@ IsochronousClockModel::~IsochronousClockModel() { } void IsochronousClockModel::setPositionAndTime(int64_t framePosition, int64_t nanoTime) { - ALOGV("setPositionAndTime(%lld, %lld)", - (long long) framePosition, (long long) nanoTime); + ALOGV("setPositionAndTime, %lld, %lld", (long long) framePosition, (long long) nanoTime); mMarkerFramePosition = framePosition; mMarkerNanoTime = nanoTime; } @@ -54,7 +52,9 @@ void IsochronousClockModel::start(int64_t nanoTime) { } void IsochronousClockModel::stop(int64_t nanoTime) { - ALOGV("stop(nanos = %lld)\n", (long long) nanoTime); + ALOGD("stop(nanos = %lld) max lateness = %d micros\n", + (long long) nanoTime, + (int) (mMaxMeasuredLatenessNanos / 1000)); setPositionAndTime(convertTimeToPosition(nanoTime), nanoTime); // TODO should we set position? mState = STATE_STOPPED; @@ -69,9 +69,10 @@ bool IsochronousClockModel::isRunning() const { } void IsochronousClockModel::processTimestamp(int64_t framePosition, int64_t nanoTime) { -// ALOGD("processTimestamp() - framePosition = %lld at nanoTime %llu", -// (long long)framePosition, -// (long long)nanoTime); + mTimestampCount++; +// Log position and time in CSV format so we can import it easily into spreadsheets. + //ALOGD("%s() CSV, %d, %lld, %lld", __func__, + //mTimestampCount, (long long)framePosition, (long long)nanoTime); int64_t framesDelta = framePosition - mMarkerFramePosition; int64_t nanosDelta = nanoTime - mMarkerNanoTime; if (nanosDelta < 1000) { @@ -108,17 +109,56 @@ void IsochronousClockModel::processTimestamp(int64_t framePosition, int64_t nano case STATE_RUNNING: if (nanosDelta < expectedNanosDelta) { // Earlier than expected timestamp. - // This data is probably more accurate so use it. - // or we may be drifting due to a slow HW clock. -// ALOGD("processTimestamp() - STATE_RUNNING - %d < %d micros - EARLY", -// (int) (nanosDelta / 1000), (int)(expectedNanosDelta / 1000)); + // This data is probably more accurate, so use it. + // Or we may be drifting due to a fast HW clock. + //int microsDelta = (int) (nanosDelta / 1000); + //int expectedMicrosDelta = (int) (expectedNanosDelta / 1000); + //ALOGD("%s() - STATE_RUNNING - #%d, %4d micros EARLY", + //__func__, mTimestampCount, expectedMicrosDelta - microsDelta); + setPositionAndTime(framePosition, nanoTime); - } else if (nanosDelta > (expectedNanosDelta + mMaxLatenessInNanos)) { - // Later than expected timestamp. -// ALOGD("processTimestamp() - STATE_RUNNING - %d > %d + %d micros - LATE", -// (int) (nanosDelta / 1000), (int)(expectedNanosDelta / 1000), -// (int) (mMaxLatenessInNanos / 1000)); - setPositionAndTime(framePosition - mFramesPerBurst, nanoTime - mMaxLatenessInNanos); + } else if (nanosDelta > (expectedNanosDelta + (2 * mBurstPeriodNanos))) { + // In this case we do not update mMaxMeasuredLatenessNanos because it + // would force it too high. + // mMaxMeasuredLatenessNanos should range from 1 to 2 * mBurstPeriodNanos + //int32_t measuredLatenessNanos = (int32_t)(nanosDelta - expectedNanosDelta); + //ALOGD("%s() - STATE_RUNNING - #%d, lateness %d - max %d = %4d micros VERY LATE", + //__func__, + //mTimestampCount, + //measuredLatenessNanos / 1000, + //mMaxMeasuredLatenessNanos / 1000, + //(measuredLatenessNanos - mMaxMeasuredLatenessNanos) / 1000 + //); + + // This typically happens when we are modelling a service instead of a DSP. + setPositionAndTime(framePosition, nanoTime - (2 * mBurstPeriodNanos)); + } else if (nanosDelta > (expectedNanosDelta + mMaxMeasuredLatenessNanos)) { + //int32_t previousLatenessNanos = mMaxMeasuredLatenessNanos; + mMaxMeasuredLatenessNanos = (int32_t)(nanosDelta - expectedNanosDelta); + + //ALOGD("%s() - STATE_RUNNING - #%d, newmax %d - oldmax %d = %4d micros LATE", + //__func__, + //mTimestampCount, + //mMaxMeasuredLatenessNanos / 1000, + //previousLatenessNanos / 1000, + //(mMaxMeasuredLatenessNanos - previousLatenessNanos) / 1000 + //); + + // When we are late, it may be because of preemption in the kernel, + // or timing jitter caused by resampling in the DSP, + // or we may be drifting due to a slow HW clock. + // We add slight drift value just in case there is actual long term drift + // forward caused by a slower clock. + // If the clock is faster than the model will get pushed earlier + // by the code in the preceding branch. + // The two opposing forces should allow the model to track the real clock + // over a long time. + int64_t driftingTime = mMarkerNanoTime + expectedNanosDelta + kDriftNanos; + setPositionAndTime(framePosition, driftingTime); + //ALOGD("%s() - #%d, max lateness = %d micros", + //__func__, + //mTimestampCount, + //(int) (mMaxMeasuredLatenessNanos / 1000)); } break; default: @@ -138,9 +178,12 @@ void IsochronousClockModel::setFramesPerBurst(int32_t framesPerBurst) { update(); } +// Update expected lateness based on sampleRate and framesPerBurst void IsochronousClockModel::update() { - int64_t nanosLate = convertDeltaPositionToTime(mFramesPerBurst); // uses mSampleRate - mMaxLatenessInNanos = (nanosLate > MIN_LATENESS_NANOS) ? nanosLate : MIN_LATENESS_NANOS; + mBurstPeriodNanos = convertDeltaPositionToTime(mFramesPerBurst); // uses mSampleRate + // Timestamps may be late by up to a burst because we are randomly sampling the time period + // after the DSP position is actually updated. + mMaxMeasuredLatenessNanos = mBurstPeriodNanos; } int64_t IsochronousClockModel::convertDeltaPositionToTime(int64_t framesDelta) const { @@ -183,11 +226,25 @@ int64_t IsochronousClockModel::convertTimeToPosition(int64_t nanoTime) const { return position; } +int32_t IsochronousClockModel::getLateTimeOffsetNanos() const { + // This will never be < 0 because mMaxLatenessNanos starts at + // mBurstPeriodNanos and only gets bigger. + return (mMaxMeasuredLatenessNanos - mBurstPeriodNanos) + kExtraLatenessNanos; +} + +int64_t IsochronousClockModel::convertPositionToLatestTime(int64_t framePosition) const { + return convertPositionToTime(framePosition) + getLateTimeOffsetNanos(); +} + +int64_t IsochronousClockModel::convertLatestTimeToPosition(int64_t nanoTime) const { + return convertTimeToPosition(nanoTime - getLateTimeOffsetNanos()); +} + void IsochronousClockModel::dump() const { ALOGD("mMarkerFramePosition = %lld", (long long) mMarkerFramePosition); ALOGD("mMarkerNanoTime = %lld", (long long) mMarkerNanoTime); ALOGD("mSampleRate = %6d", mSampleRate); ALOGD("mFramesPerBurst = %6d", mFramesPerBurst); - ALOGD("mMaxLatenessInNanos = %6d", mMaxLatenessInNanos); + ALOGD("mMaxMeasuredLatenessNanos = %6d", mMaxMeasuredLatenessNanos); ALOGD("mState = %6d", mState); } diff --git a/media/libaaudio/src/client/IsochronousClockModel.h b/media/libaaudio/src/client/IsochronousClockModel.h index 46ca48e7af..582bf4e34a 100644 --- a/media/libaaudio/src/client/IsochronousClockModel.h +++ b/media/libaaudio/src/client/IsochronousClockModel.h @@ -18,6 +18,7 @@ #define ANDROID_AAUDIO_ISOCHRONOUS_CLOCK_MODEL_H #include <stdint.h> +#include "utility/AudioClock.h" namespace aaudio { @@ -79,6 +80,15 @@ public: int64_t convertPositionToTime(int64_t framePosition) const; /** + * Calculate the latest estimated time that the stream will be at that position. + * The more jittery the clock is then the later this will be. + * + * @param framePosition + * @return time in nanoseconds + */ + int64_t convertPositionToLatestTime(int64_t framePosition) const; + + /** * Calculate an estimated position where the stream will be at the specified time. * * @param nanoTime time of interest @@ -87,6 +97,18 @@ public: int64_t convertTimeToPosition(int64_t nanoTime) const; /** + * Calculate the corresponding estimated position based on the specified time being + * the latest possible time. + * + * For the same nanoTime, this may return an earlier position than + * convertTimeToPosition(). + * + * @param nanoTime + * @return position in frames + */ + int64_t convertLatestTimeToPosition(int64_t nanoTime) const; + + /** * @param framesDelta difference in frames * @return duration in nanoseconds */ @@ -101,6 +123,9 @@ public: void dump() const; private: + + int32_t getLateTimeOffsetNanos() const; + enum clock_model_state_t { STATE_STOPPED, STATE_STARTING, @@ -108,13 +133,23 @@ private: STATE_RUNNING }; + // Amount of time to drift forward when we get a late timestamp. + // This value was calculated to allow tracking of a clock with 50 ppm error. + static constexpr int32_t kDriftNanos = 10 * 1000; + // TODO review value of kExtraLatenessNanos + static constexpr int32_t kExtraLatenessNanos = 100 * 1000; + int64_t mMarkerFramePosition; int64_t mMarkerNanoTime; int32_t mSampleRate; int32_t mFramesPerBurst; - int32_t mMaxLatenessInNanos; + int32_t mBurstPeriodNanos; + // Includes mBurstPeriodNanos because we sample randomly over time. + int32_t mMaxMeasuredLatenessNanos; clock_model_state_t mState; + int32_t mTimestampCount = 0; + void update(); }; diff --git a/media/libaaudio/src/core/AudioStream.cpp b/media/libaaudio/src/core/AudioStream.cpp index 9b772237e0..f5c97d844e 100644 --- a/media/libaaudio/src/core/AudioStream.cpp +++ b/media/libaaudio/src/core/AudioStream.cpp @@ -473,7 +473,7 @@ AudioStream::MyPlayerBase::~MyPlayerBase() { void AudioStream::MyPlayerBase::registerWithAudioManager() { if (!mRegistered) { - init(android::PLAYER_TYPE_AAUDIO, AUDIO_USAGE_MEDIA); + init(android::PLAYER_TYPE_AAUDIO, AAudioConvert_usageToInternal(mParent->getUsage())); mRegistered = true; } } diff --git a/media/libaudioclient/AudioRecord.cpp b/media/libaudioclient/AudioRecord.cpp index a1b04caa92..271e18655e 100644 --- a/media/libaudioclient/AudioRecord.cpp +++ b/media/libaudioclient/AudioRecord.cpp @@ -884,7 +884,6 @@ status_t AudioRecord::obtainBuffer(Buffer* audioBuffer, const struct timespec *r { // previous and new IAudioRecord sequence numbers are used to detect track re-creation uint32_t oldSequence = 0; - uint32_t newSequence; Proxy::Buffer buffer; status_t status = NO_ERROR; @@ -902,7 +901,7 @@ status_t AudioRecord::obtainBuffer(Buffer* audioBuffer, const struct timespec *r // start of lock scope AutoMutex lock(mLock); - newSequence = mSequence; + uint32_t newSequence = mSequence; // did previous obtainBuffer() fail due to media server death or voluntary invalidation? if (status == DEAD_OBJECT) { // re-create track, unless someone else has already done so @@ -939,6 +938,7 @@ status_t AudioRecord::obtainBuffer(Buffer* audioBuffer, const struct timespec *r audioBuffer->frameCount = buffer.mFrameCount; audioBuffer->size = buffer.mFrameCount * mFrameSize; audioBuffer->raw = buffer.mRaw; + audioBuffer->sequence = oldSequence; if (nonContig != NULL) { *nonContig = buffer.mNonContig; } @@ -959,6 +959,12 @@ void AudioRecord::releaseBuffer(const Buffer* audioBuffer) buffer.mRaw = audioBuffer->raw; AutoMutex lock(mLock); + if (audioBuffer->sequence != mSequence) { + // This Buffer came from a different IAudioRecord instance, so ignore the releaseBuffer + ALOGD("%s is no-op due to IAudioRecord sequence mismatch %u != %u", + __func__, audioBuffer->sequence, mSequence); + return; + } mInOverrun = false; mProxy->releaseBuffer(&buffer); diff --git a/media/libaudioclient/AudioTrack.cpp b/media/libaudioclient/AudioTrack.cpp index 4a80cd3baa..9a66d48a90 100644 --- a/media/libaudioclient/AudioTrack.cpp +++ b/media/libaudioclient/AudioTrack.cpp @@ -1665,7 +1665,6 @@ status_t AudioTrack::obtainBuffer(Buffer* audioBuffer, const struct timespec *re { // previous and new IAudioTrack sequence numbers are used to detect track re-creation uint32_t oldSequence = 0; - uint32_t newSequence; Proxy::Buffer buffer; status_t status = NO_ERROR; @@ -1682,7 +1681,7 @@ status_t AudioTrack::obtainBuffer(Buffer* audioBuffer, const struct timespec *re { // start of lock scope AutoMutex lock(mLock); - newSequence = mSequence; + uint32_t newSequence = mSequence; // did previous obtainBuffer() fail due to media server death or voluntary invalidation? if (status == DEAD_OBJECT) { // re-create track, unless someone else has already done so @@ -1729,6 +1728,7 @@ status_t AudioTrack::obtainBuffer(Buffer* audioBuffer, const struct timespec *re audioBuffer->frameCount = buffer.mFrameCount; audioBuffer->size = buffer.mFrameCount * mFrameSize; audioBuffer->raw = buffer.mRaw; + audioBuffer->sequence = oldSequence; if (nonContig != NULL) { *nonContig = buffer.mNonContig; } @@ -1752,6 +1752,12 @@ void AudioTrack::releaseBuffer(const Buffer* audioBuffer) buffer.mRaw = audioBuffer->raw; AutoMutex lock(mLock); + if (audioBuffer->sequence != mSequence) { + // This Buffer came from a different IAudioTrack instance, so ignore the releaseBuffer + ALOGD("%s is no-op due to IAudioTrack sequence mismatch %u != %u", + __func__, audioBuffer->sequence, mSequence); + return; + } mReleased += stepCount; mInUnderrun = false; mProxy->releaseBuffer(&buffer); diff --git a/media/libaudioclient/IAudioFlinger.cpp b/media/libaudioclient/IAudioFlinger.cpp index dd95e347dc..efa05124e1 100644 --- a/media/libaudioclient/IAudioFlinger.cpp +++ b/media/libaudioclient/IAudioFlinger.cpp @@ -1339,10 +1339,14 @@ status_t BnAudioFlinger::onTransact( } case GET_EFFECT_DESCRIPTOR: { CHECK_INTERFACE(IAudioFlinger, data, reply); - effect_uuid_t uuid; - data.read(&uuid, sizeof(effect_uuid_t)); - effect_uuid_t type; - data.read(&type, sizeof(effect_uuid_t)); + effect_uuid_t uuid = {}; + if (data.read(&uuid, sizeof(effect_uuid_t)) != NO_ERROR) { + android_errorWriteLog(0x534e4554, "139417189"); + } + effect_uuid_t type = {}; + if (data.read(&type, sizeof(effect_uuid_t)) != NO_ERROR) { + android_errorWriteLog(0x534e4554, "139417189"); + } uint32_t preferredTypeFlag = data.readUint32(); effect_descriptor_t desc = {}; status_t status = getEffectDescriptor(&uuid, &type, preferredTypeFlag, &desc); diff --git a/media/libaudioclient/IAudioPolicyService.cpp b/media/libaudioclient/IAudioPolicyService.cpp index 64f0aca6cd..2fb9491a06 100644 --- a/media/libaudioclient/IAudioPolicyService.cpp +++ b/media/libaudioclient/IAudioPolicyService.cpp @@ -1346,7 +1346,8 @@ status_t BnAudioPolicyService::onTransact( case GET_OFFLOAD_FORMATS_A2DP: case LIST_AUDIO_VOLUME_GROUPS: case GET_VOLUME_GROUP_FOR_ATTRIBUTES: - case SET_RTT_ENABLED: { + case SET_RTT_ENABLED: + case SET_ALLOWED_CAPTURE_POLICY: { if (!isServiceUid(IPCThreadState::self()->getCallingUid())) { ALOGW("%s: transaction %d received from PID %d unauthorized UID %d", __func__, code, IPCThreadState::self()->getCallingPid(), diff --git a/media/libaudioclient/include/media/AudioRecord.h b/media/libaudioclient/include/media/AudioRecord.h index a3c0fe4ac5..574302b869 100644 --- a/media/libaudioclient/include/media/AudioRecord.h +++ b/media/libaudioclient/include/media/AudioRecord.h @@ -92,6 +92,11 @@ public: int8_t* i8; // unsigned 8-bit, offset by 0x80 // input to obtainBuffer(): unused, output: pointer to buffer }; + + uint32_t sequence; // IAudioRecord instance sequence number, as of obtainBuffer(). + // It is set by obtainBuffer() and confirmed by releaseBuffer(). + // Not "user-serviceable". + // TODO Consider sp<IMemory> instead, or in addition to this. }; /* As a convenience, if a callback is supplied, a handler thread @@ -420,14 +425,17 @@ public: * frameCount number of frames requested * size ignored * raw ignored + * sequence ignored * After error return: * frameCount 0 * size 0 * raw undefined + * sequence undefined * After successful return: * frameCount actual number of frames available, <= number requested * size actual number of bytes available * raw pointer to the buffer + * sequence IAudioRecord instance sequence number, as of obtainBuffer() */ status_t obtainBuffer(Buffer* audioBuffer, int32_t waitCount, diff --git a/media/libaudioclient/include/media/AudioTrack.h b/media/libaudioclient/include/media/AudioTrack.h index df5eabc5cc..c607918b96 100644 --- a/media/libaudioclient/include/media/AudioTrack.h +++ b/media/libaudioclient/include/media/AudioTrack.h @@ -107,6 +107,11 @@ public: int16_t* i16; // signed 16-bit int8_t* i8; // unsigned 8-bit, offset by 0x80 }; // input to obtainBuffer(): unused, output: pointer to buffer + + uint32_t sequence; // IAudioTrack instance sequence number, as of obtainBuffer(). + // It is set by obtainBuffer() and confirmed by releaseBuffer(). + // Not "user-serviceable". + // TODO Consider sp<IMemory> instead, or in addition to this. }; /* As a convenience, if a callback is supplied, a handler thread @@ -692,14 +697,17 @@ public: * frameCount number of [empty slots for] frames requested * size ignored * raw ignored + * sequence ignored * After error return: * frameCount 0 * size 0 * raw undefined + * sequence undefined * After successful return: * frameCount actual number of [empty slots for] frames available, <= number requested * size actual number of bytes available * raw pointer to the buffer + * sequence IAudioTrack instance sequence number, as of obtainBuffer() */ status_t obtainBuffer(Buffer* audioBuffer, int32_t waitCount, size_t *nonContig = NULL); diff --git a/media/libeffects/lvm/wrapper/Bundle/EffectBundle.cpp b/media/libeffects/lvm/wrapper/Bundle/EffectBundle.cpp index 3fbbc096f5..10dda19181 100644 --- a/media/libeffects/lvm/wrapper/Bundle/EffectBundle.cpp +++ b/media/libeffects/lvm/wrapper/Bundle/EffectBundle.cpp @@ -302,6 +302,8 @@ extern "C" int EffectCreate(const effect_uuid_t *uuid, for (int i = 0; i < FIVEBAND_NUMBANDS; i++) { pContext->pBundledContext->bandGaindB[i] = EQNB_5BandSoftPresets[i]; } + pContext->pBundledContext->effectProcessCalled = 0; + pContext->pBundledContext->effectInDrain = 0; ALOGV("\tEffectCreate - Calling LvmBundle_init"); ret = LvmBundle_init(pContext); @@ -394,6 +396,8 @@ extern "C" int EffectRelease(effect_handle_t handle){ // Clear the instantiated flag for the effect // protect agains the case where an effect is un-instantiated without being disabled + + int &effectInDrain = pContext->pBundledContext->effectInDrain; if(pContext->EffectType == LVM_BASS_BOOST) { ALOGV("\tEffectRelease LVM_BASS_BOOST Clearing global intstantiated flag"); pSessionContext->bBassInstantiated = LVM_FALSE; @@ -418,12 +422,16 @@ extern "C" int EffectRelease(effect_handle_t handle){ } else if(pContext->EffectType == LVM_VOLUME) { ALOGV("\tEffectRelease LVM_VOLUME Clearing global intstantiated flag"); pSessionContext->bVolumeInstantiated = LVM_FALSE; - if (pContext->pBundledContext->bVolumeEnabled == LVM_TRUE){ + // There is no samplesToExitCount for volume so we also use the drain flag to check + // if we should decrement the effects enabled. + if (pContext->pBundledContext->bVolumeEnabled == LVM_TRUE + || (effectInDrain & 1 << LVM_VOLUME) != 0) { pContext->pBundledContext->NumberEffectsEnabled--; } } else { ALOGV("\tLVM_ERROR : EffectRelease : Unsupported effect\n\n\n\n\n\n\n"); } + effectInDrain &= ~(1 << pContext->EffectType); // no need to drain if released // Disable effect, in this case ignore errors (return codes) // if an effect has already been disabled @@ -3124,8 +3132,9 @@ LVM_INT16 LVC_ToDB_s32Tos16(LVM_INT32 Lin_fix) int Effect_setEnabled(EffectContext *pContext, bool enabled) { - ALOGV("\tEffect_setEnabled() type %d, enabled %d", pContext->EffectType, enabled); - + ALOGV("%s effectType %d, enabled %d, currently enabled %d", __func__, + pContext->EffectType, enabled, pContext->pBundledContext->NumberEffectsEnabled); + int &effectInDrain = pContext->pBundledContext->effectInDrain; if (enabled) { // Bass boost or Virtualizer can be temporarily disabled if playing over device speaker due // to their nature. @@ -3139,6 +3148,7 @@ int Effect_setEnabled(EffectContext *pContext, bool enabled) if(pContext->pBundledContext->SamplesToExitCountBb <= 0){ pContext->pBundledContext->NumberEffectsEnabled++; } + effectInDrain &= ~(1 << LVM_BASS_BOOST); pContext->pBundledContext->SamplesToExitCountBb = (LVM_INT32)(pContext->pBundledContext->SamplesPerSecond*0.1); pContext->pBundledContext->bBassEnabled = LVM_TRUE; @@ -3152,6 +3162,7 @@ int Effect_setEnabled(EffectContext *pContext, bool enabled) if(pContext->pBundledContext->SamplesToExitCountEq <= 0){ pContext->pBundledContext->NumberEffectsEnabled++; } + effectInDrain &= ~(1 << LVM_EQUALIZER); pContext->pBundledContext->SamplesToExitCountEq = (LVM_INT32)(pContext->pBundledContext->SamplesPerSecond*0.1); pContext->pBundledContext->bEqualizerEnabled = LVM_TRUE; @@ -3164,6 +3175,7 @@ int Effect_setEnabled(EffectContext *pContext, bool enabled) if(pContext->pBundledContext->SamplesToExitCountVirt <= 0){ pContext->pBundledContext->NumberEffectsEnabled++; } + effectInDrain &= ~(1 << LVM_VIRTUALIZER); pContext->pBundledContext->SamplesToExitCountVirt = (LVM_INT32)(pContext->pBundledContext->SamplesPerSecond*0.1); pContext->pBundledContext->bVirtualizerEnabled = LVM_TRUE; @@ -3174,7 +3186,10 @@ int Effect_setEnabled(EffectContext *pContext, bool enabled) ALOGV("\tEffect_setEnabled() LVM_VOLUME is already enabled"); return -EINVAL; } - pContext->pBundledContext->NumberEffectsEnabled++; + if ((effectInDrain & 1 << LVM_VOLUME) == 0) { + pContext->pBundledContext->NumberEffectsEnabled++; + } + effectInDrain &= ~(1 << LVM_VOLUME); pContext->pBundledContext->bVolumeEnabled = LVM_TRUE; break; default: @@ -3192,6 +3207,7 @@ int Effect_setEnabled(EffectContext *pContext, bool enabled) return -EINVAL; } pContext->pBundledContext->bBassEnabled = LVM_FALSE; + effectInDrain |= 1 << LVM_BASS_BOOST; break; case LVM_EQUALIZER: if (pContext->pBundledContext->bEqualizerEnabled == LVM_FALSE) { @@ -3199,6 +3215,7 @@ int Effect_setEnabled(EffectContext *pContext, bool enabled) return -EINVAL; } pContext->pBundledContext->bEqualizerEnabled = LVM_FALSE; + effectInDrain |= 1 << LVM_EQUALIZER; break; case LVM_VIRTUALIZER: if (pContext->pBundledContext->bVirtualizerEnabled == LVM_FALSE) { @@ -3206,6 +3223,7 @@ int Effect_setEnabled(EffectContext *pContext, bool enabled) return -EINVAL; } pContext->pBundledContext->bVirtualizerEnabled = LVM_FALSE; + effectInDrain |= 1 << LVM_VIRTUALIZER; break; case LVM_VOLUME: if (pContext->pBundledContext->bVolumeEnabled == LVM_FALSE) { @@ -3213,6 +3231,7 @@ int Effect_setEnabled(EffectContext *pContext, bool enabled) return -EINVAL; } pContext->pBundledContext->bVolumeEnabled = LVM_FALSE; + effectInDrain |= 1 << LVM_VOLUME; break; default: ALOGV("\tEffect_setEnabled() invalid effect type"); @@ -3283,6 +3302,38 @@ int Effect_process(effect_handle_t self, ALOGV("\tLVM_ERROR : Effect_process() ERROR NULL INPUT POINTER OR FRAME COUNT IS WRONG"); return -EINVAL; } + + int &effectProcessCalled = pContext->pBundledContext->effectProcessCalled; + int &effectInDrain = pContext->pBundledContext->effectInDrain; + if ((effectProcessCalled & 1 << pContext->EffectType) != 0) { + ALOGW("Effect %d already called", pContext->EffectType); + const int undrainedEffects = effectInDrain & ~effectProcessCalled; + if ((undrainedEffects & 1 << LVM_BASS_BOOST) != 0) { + ALOGW("Draining BASS_BOOST"); + pContext->pBundledContext->SamplesToExitCountBb = 0; + --pContext->pBundledContext->NumberEffectsEnabled; + effectInDrain &= ~(1 << LVM_BASS_BOOST); + } + if ((undrainedEffects & 1 << LVM_EQUALIZER) != 0) { + ALOGW("Draining EQUALIZER"); + pContext->pBundledContext->SamplesToExitCountEq = 0; + --pContext->pBundledContext->NumberEffectsEnabled; + effectInDrain &= ~(1 << LVM_EQUALIZER); + } + if ((undrainedEffects & 1 << LVM_VIRTUALIZER) != 0) { + ALOGW("Draining VIRTUALIZER"); + pContext->pBundledContext->SamplesToExitCountVirt = 0; + --pContext->pBundledContext->NumberEffectsEnabled; + effectInDrain &= ~(1 << LVM_VIRTUALIZER); + } + if ((undrainedEffects & 1 << LVM_VOLUME) != 0) { + ALOGW("Draining VOLUME"); + --pContext->pBundledContext->NumberEffectsEnabled; + effectInDrain &= ~(1 << LVM_VOLUME); + } + } + effectProcessCalled |= 1 << pContext->EffectType; + if ((pContext->pBundledContext->bBassEnabled == LVM_FALSE)&& (pContext->EffectType == LVM_BASS_BOOST)){ //ALOGV("\tEffect_process() LVM_BASS_BOOST Effect is not enabled"); @@ -3291,9 +3342,12 @@ int Effect_process(effect_handle_t self, //ALOGV("\tEffect_process: Waiting to turn off BASS_BOOST, %d samples left", // pContext->pBundledContext->SamplesToExitCountBb); } - if(pContext->pBundledContext->SamplesToExitCountBb <= 0) { + if (pContext->pBundledContext->SamplesToExitCountBb <= 0) { status = -ENODATA; - pContext->pBundledContext->NumberEffectsEnabled--; + if ((effectInDrain & 1 << LVM_BASS_BOOST) != 0) { + pContext->pBundledContext->NumberEffectsEnabled--; + effectInDrain &= ~(1 << LVM_BASS_BOOST); + } ALOGV("\tEffect_process() this is the last frame for LVM_BASS_BOOST"); } } @@ -3301,7 +3355,10 @@ int Effect_process(effect_handle_t self, (pContext->EffectType == LVM_VOLUME)){ //ALOGV("\tEffect_process() LVM_VOLUME Effect is not enabled"); status = -ENODATA; - pContext->pBundledContext->NumberEffectsEnabled--; + if ((effectInDrain & 1 << LVM_VOLUME) != 0) { + pContext->pBundledContext->NumberEffectsEnabled--; + effectInDrain &= ~(1 << LVM_VOLUME); + } } if ((pContext->pBundledContext->bEqualizerEnabled == LVM_FALSE)&& (pContext->EffectType == LVM_EQUALIZER)){ @@ -3311,9 +3368,12 @@ int Effect_process(effect_handle_t self, //ALOGV("\tEffect_process: Waiting to turn off EQUALIZER, %d samples left", // pContext->pBundledContext->SamplesToExitCountEq); } - if(pContext->pBundledContext->SamplesToExitCountEq <= 0) { + if (pContext->pBundledContext->SamplesToExitCountEq <= 0) { status = -ENODATA; - pContext->pBundledContext->NumberEffectsEnabled--; + if ((effectInDrain & 1 << LVM_EQUALIZER) != 0) { + pContext->pBundledContext->NumberEffectsEnabled--; + effectInDrain &= ~(1 << LVM_EQUALIZER); + } ALOGV("\tEffect_process() this is the last frame for LVM_EQUALIZER"); } } @@ -3326,9 +3386,12 @@ int Effect_process(effect_handle_t self, //ALOGV("\tEffect_process: Waiting for to turn off VIRTUALIZER, %d samples left", // pContext->pBundledContext->SamplesToExitCountVirt); } - if(pContext->pBundledContext->SamplesToExitCountVirt <= 0) { + if (pContext->pBundledContext->SamplesToExitCountVirt <= 0) { status = -ENODATA; - pContext->pBundledContext->NumberEffectsEnabled--; + if ((effectInDrain & 1 << LVM_VIRTUALIZER) != 0) { + pContext->pBundledContext->NumberEffectsEnabled--; + effectInDrain &= ~(1 << LVM_VIRTUALIZER); + } ALOGV("\tEffect_process() this is the last frame for LVM_VIRTUALIZER"); } } @@ -3337,8 +3400,18 @@ int Effect_process(effect_handle_t self, pContext->pBundledContext->NumberEffectsCalled++; } - if(pContext->pBundledContext->NumberEffectsCalled == - pContext->pBundledContext->NumberEffectsEnabled){ + if (pContext->pBundledContext->NumberEffectsCalled >= + pContext->pBundledContext->NumberEffectsEnabled) { + + // We expect the # effects called to be equal to # effects enabled in sequence (including + // draining effects). Warn if this is not the case due to inconsistent calls. + ALOGW_IF(pContext->pBundledContext->NumberEffectsCalled > + pContext->pBundledContext->NumberEffectsEnabled, + "%s Number of effects called %d is greater than number of effects enabled %d", + __func__, pContext->pBundledContext->NumberEffectsCalled, + pContext->pBundledContext->NumberEffectsEnabled); + effectProcessCalled = 0; // reset our consistency check. + //ALOGV("\tEffect_process Calling process with %d effects enabled, %d called: Effect %d", //pContext->pBundledContext->NumberEffectsEnabled, //pContext->pBundledContext->NumberEffectsCalled, pContext->EffectType); diff --git a/media/libeffects/lvm/wrapper/Bundle/EffectBundle.h b/media/libeffects/lvm/wrapper/Bundle/EffectBundle.h index 6af4554fc2..e4aacd0782 100644 --- a/media/libeffects/lvm/wrapper/Bundle/EffectBundle.h +++ b/media/libeffects/lvm/wrapper/Bundle/EffectBundle.h @@ -110,6 +110,14 @@ struct BundledEffectContext{ #ifdef SUPPORT_MC LVM_INT32 ChMask; #endif + + /* Bitmask whether drain is in progress due to disabling the effect. + The corresponding bit to an effect is set by 1 << lvm_effect_en. */ + int effectInDrain; + + /* Bitmask whether process() was called for a particular effect. + The corresponding bit to an effect is set by 1 << lvm_effect_en. */ + int effectProcessCalled; }; /* SessionContext : One session */ diff --git a/media/libmedia/IResourceManagerService.cpp b/media/libmedia/IResourceManagerService.cpp index 9724fc1a12..f8a0a143e1 100644 --- a/media/libmedia/IResourceManagerService.cpp +++ b/media/libmedia/IResourceManagerService.cpp @@ -32,6 +32,7 @@ enum { CONFIG = IBinder::FIRST_CALL_TRANSACTION, ADD_RESOURCE, REMOVE_RESOURCE, + REMOVE_CLIENT, RECLAIM_RESOURCE, }; @@ -72,12 +73,14 @@ public: virtual void addResource( int pid, + int uid, int64_t clientId, const sp<IResourceManagerClient> client, const Vector<MediaResource> &resources) { Parcel data, reply; data.writeInterfaceToken(IResourceManagerService::getInterfaceDescriptor()); data.writeInt32(pid); + data.writeInt32(uid); data.writeInt64(clientId); data.writeStrongBinder(IInterface::asBinder(client)); writeToParcel(&data, resources); @@ -85,15 +88,25 @@ public: remote()->transact(ADD_RESOURCE, data, &reply); } - virtual void removeResource(int pid, int64_t clientId) { + virtual void removeResource(int pid, int64_t clientId, const Vector<MediaResource> &resources) { Parcel data, reply; data.writeInterfaceToken(IResourceManagerService::getInterfaceDescriptor()); data.writeInt32(pid); data.writeInt64(clientId); + writeToParcel(&data, resources); remote()->transact(REMOVE_RESOURCE, data, &reply); } + virtual void removeClient(int pid, int64_t clientId) { + Parcel data, reply; + data.writeInterfaceToken(IResourceManagerService::getInterfaceDescriptor()); + data.writeInt32(pid); + data.writeInt64(clientId); + + remote()->transact(REMOVE_CLIENT, data, &reply); + } + virtual bool reclaimResource(int callingPid, const Vector<MediaResource> &resources) { Parcel data, reply; data.writeInterfaceToken(IResourceManagerService::getInterfaceDescriptor()); @@ -129,6 +142,7 @@ status_t BnResourceManagerService::onTransact( case ADD_RESOURCE: { CHECK_INTERFACE(IResourceManagerService, data, reply); int pid = data.readInt32(); + int uid = data.readInt32(); int64_t clientId = data.readInt64(); sp<IResourceManagerClient> client( interface_cast<IResourceManagerClient>(data.readStrongBinder())); @@ -137,7 +151,7 @@ status_t BnResourceManagerService::onTransact( } Vector<MediaResource> resources; readFromParcel(data, &resources); - addResource(pid, clientId, client, resources); + addResource(pid, uid, clientId, client, resources); return NO_ERROR; } break; @@ -145,7 +159,17 @@ status_t BnResourceManagerService::onTransact( CHECK_INTERFACE(IResourceManagerService, data, reply); int pid = data.readInt32(); int64_t clientId = data.readInt64(); - removeResource(pid, clientId); + Vector<MediaResource> resources; + readFromParcel(data, &resources); + removeResource(pid, clientId, resources); + return NO_ERROR; + } break; + + case REMOVE_CLIENT: { + CHECK_INTERFACE(IResourceManagerService, data, reply); + int pid = data.readInt32(); + int64_t clientId = data.readInt64(); + removeClient(pid, clientId); return NO_ERROR; } break; diff --git a/media/libmedia/Visualizer.cpp b/media/libmedia/Visualizer.cpp index cb8d3750b7..2bf08020b3 100644 --- a/media/libmedia/Visualizer.cpp +++ b/media/libmedia/Visualizer.cpp @@ -77,10 +77,13 @@ status_t Visualizer::setEnabled(bool enabled) if (t != 0) { if (enabled) { if (t->exitPending()) { + mCaptureLock.unlock(); if (t->requestExitAndWait() == WOULD_BLOCK) { + mCaptureLock.lock(); ALOGE("Visualizer::enable() called from thread"); return INVALID_OPERATION; } + mCaptureLock.lock(); } } t->mLock.lock(); diff --git a/media/libmedia/include/media/IResourceManagerService.h b/media/libmedia/include/media/IResourceManagerService.h index 1e4f6dea74..8992f8bc15 100644 --- a/media/libmedia/include/media/IResourceManagerService.h +++ b/media/libmedia/include/media/IResourceManagerService.h @@ -39,11 +39,15 @@ public: virtual void addResource( int pid, + int uid, int64_t clientId, const sp<IResourceManagerClient> client, const Vector<MediaResource> &resources) = 0; - virtual void removeResource(int pid, int64_t clientId) = 0; + virtual void removeResource(int pid, int64_t clientId, + const Vector<MediaResource> &resources) = 0; + + virtual void removeClient(int pid, int64_t clientId) = 0; virtual bool reclaimResource( int callingPid, diff --git a/media/libmedia/include/media/MediaResource.h b/media/libmedia/include/media/MediaResource.h index e1fdb9be3f..10a07bb639 100644 --- a/media/libmedia/include/media/MediaResource.h +++ b/media/libmedia/include/media/MediaResource.h @@ -31,12 +31,13 @@ public: kNonSecureCodec, kGraphicMemory, kCpuBoost, + kBattery, }; enum SubType { kUnspecifiedSubType = 0, kAudioCodec, - kVideoCodec + kVideoCodec, }; MediaResource(); @@ -62,6 +63,8 @@ inline static const char *asString(MediaResource::Type i, const char *def = "??" case MediaResource::kSecureCodec: return "secure-codec"; case MediaResource::kNonSecureCodec: return "non-secure-codec"; case MediaResource::kGraphicMemory: return "graphic-memory"; + case MediaResource::kCpuBoost: return "cpu-boost"; + case MediaResource::kBattery: return "battery"; default: return def; } } diff --git a/media/libmediaplayerservice/nuplayer/StreamingSource.cpp b/media/libmediaplayerservice/nuplayer/StreamingSource.cpp index ea64f98d37..3250a4857a 100644 --- a/media/libmediaplayerservice/nuplayer/StreamingSource.cpp +++ b/media/libmediaplayerservice/nuplayer/StreamingSource.cpp @@ -131,29 +131,32 @@ void NuPlayer::StreamingSource::onReadBuffer() { } else if (n < 0) { break; } else { - if (buffer[0] == 0x00) { + if (buffer[0] == 0x00) { // OK to access buffer[0] since n must be > 0 here // XXX legacy if (extra == NULL) { extra = new AMessage; } - uint8_t type = buffer[1]; + uint8_t type = 0; + if (n > 1) { + type = buffer[1]; - if (type & 2) { - int64_t mediaTimeUs; - memcpy(&mediaTimeUs, &buffer[2], sizeof(mediaTimeUs)); + if ((type & 2) && (n >= 2 + sizeof(int64_t))) { + int64_t mediaTimeUs; + memcpy(&mediaTimeUs, &buffer[2], sizeof(mediaTimeUs)); - extra->setInt64(kATSParserKeyMediaTimeUs, mediaTimeUs); + extra->setInt64(kATSParserKeyMediaTimeUs, mediaTimeUs); + } } mTSParser->signalDiscontinuity( ((type & 1) == 0) - ? ATSParser::DISCONTINUITY_TIME - : ATSParser::DISCONTINUITY_FORMATCHANGE, + ? ATSParser::DISCONTINUITY_TIME + : ATSParser::DISCONTINUITY_FORMATCHANGE, extra); } else { - status_t err = mTSParser->feedTSPacket(buffer, sizeof(buffer)); + status_t err = mTSParser->feedTSPacket(buffer, n); if (err != OK) { ALOGE("TS Parser returned error %d", err); diff --git a/media/libstagefright/ACodec.cpp b/media/libstagefright/ACodec.cpp index 3d67c911a2..44f246d849 100644 --- a/media/libstagefright/ACodec.cpp +++ b/media/libstagefright/ACodec.cpp @@ -1826,6 +1826,23 @@ status_t ACodec::configureCodec( mRepeatFrameDelayUs = -1LL; } + if (!msg->findDouble("time-lapse-fps", &mCaptureFps)) { + float captureRate; + if (msg->findAsFloat(KEY_CAPTURE_RATE, &captureRate)) { + mCaptureFps = captureRate; + } else { + mCaptureFps = -1.0; + } + } + + if (!msg->findInt32( + KEY_CREATE_INPUT_SURFACE_SUSPENDED, + (int32_t*)&mCreateInputBuffersSuspended)) { + mCreateInputBuffersSuspended = false; + } + } + + if (encoder && (mIsVideo || mIsImage)) { // only allow 32-bit value, since we pass it as U32 to OMX. if (!msg->findInt64(KEY_MAX_PTS_GAP_TO_ENCODER, &mMaxPtsGapUs)) { mMaxPtsGapUs = 0LL; @@ -1842,16 +1859,6 @@ status_t ACodec::configureCodec( if (mMaxPtsGapUs < 0LL) { mMaxFps = -1; } - - if (!msg->findDouble("time-lapse-fps", &mCaptureFps)) { - mCaptureFps = -1.0; - } - - if (!msg->findInt32( - KEY_CREATE_INPUT_SURFACE_SUSPENDED, - (int32_t*)&mCreateInputBuffersSuspended)) { - mCreateInputBuffersSuspended = false; - } } // NOTE: we only use native window for video decoders @@ -4505,22 +4512,38 @@ status_t ACodec::setupAVCEncoderParameters(const sp<AMessage> &msg) { status_t ACodec::configureImageGrid( const sp<AMessage> &msg, sp<AMessage> &outputFormat) { int32_t tileWidth, tileHeight, gridRows, gridCols; - if (!msg->findInt32("tile-width", &tileWidth) || - !msg->findInt32("tile-height", &tileHeight) || - !msg->findInt32("grid-rows", &gridRows) || - !msg->findInt32("grid-cols", &gridCols)) { + OMX_BOOL useGrid = OMX_FALSE; + if (msg->findInt32("tile-width", &tileWidth) && + msg->findInt32("tile-height", &tileHeight) && + msg->findInt32("grid-rows", &gridRows) && + msg->findInt32("grid-cols", &gridCols)) { + useGrid = OMX_TRUE; + } else { + // when bEnabled is false, the tile info is not used, + // but clear out these too. + tileWidth = tileHeight = gridRows = gridCols = 0; + } + + if (!mIsImage && !useGrid) { return OK; } OMX_VIDEO_PARAM_ANDROID_IMAGEGRIDTYPE gridType; InitOMXParams(&gridType); gridType.nPortIndex = kPortIndexOutput; - gridType.bEnabled = OMX_TRUE; + gridType.bEnabled = useGrid; gridType.nTileWidth = tileWidth; gridType.nTileHeight = tileHeight; gridType.nGridRows = gridRows; gridType.nGridCols = gridCols; + ALOGV("sending image grid info to component: bEnabled %d, tile %dx%d, grid %dx%d", + gridType.bEnabled, + gridType.nTileWidth, + gridType.nTileHeight, + gridType.nGridRows, + gridType.nGridCols); + status_t err = mOMXNode->setParameter( (OMX_INDEXTYPE)OMX_IndexParamVideoAndroidImageGrid, &gridType, sizeof(gridType)); @@ -4541,6 +4564,13 @@ status_t ACodec::configureImageGrid( (OMX_INDEXTYPE)OMX_IndexParamVideoAndroidImageGrid, &gridType, sizeof(gridType)); + ALOGV("received image grid info from component: bEnabled %d, tile %dx%d, grid %dx%d", + gridType.bEnabled, + gridType.nTileWidth, + gridType.nTileHeight, + gridType.nGridRows, + gridType.nGridCols); + if (err == OK && gridType.bEnabled) { outputFormat->setInt32("tile-width", gridType.nTileWidth); outputFormat->setInt32("tile-height", gridType.nTileHeight); @@ -6871,7 +6901,7 @@ status_t ACodec::LoadedState::setupInputSurface() { } } - if (mCodec->mMaxPtsGapUs != 0LL) { + if (mCodec->mIsVideo && mCodec->mMaxPtsGapUs != 0LL) { OMX_PARAM_U32TYPE maxPtsGapParams; InitOMXParams(&maxPtsGapParams); maxPtsGapParams.nPortIndex = kPortIndexInput; diff --git a/media/libstagefright/MPEG4Writer.cpp b/media/libstagefright/MPEG4Writer.cpp index 2f13dc9d57..f130c9b082 100644 --- a/media/libstagefright/MPEG4Writer.cpp +++ b/media/libstagefright/MPEG4Writer.cpp @@ -1635,8 +1635,13 @@ status_t MPEG4Writer::setCaptureRate(float captureFps) { return BAD_VALUE; } + // Increase moovExtraSize once only irrespective of how many times + // setCaptureRate is called. + bool containsCaptureFps = mMetaKeys->contains(kMetaKey_CaptureFps); mMetaKeys->setFloat(kMetaKey_CaptureFps, captureFps); - mMoovExtraSize += sizeof(kMetaKey_CaptureFps) + 4 + 32; + if (!containsCaptureFps) { + mMoovExtraSize += sizeof(kMetaKey_CaptureFps) + 4 + 32; + } return OK; } diff --git a/media/libstagefright/MediaCodec.cpp b/media/libstagefright/MediaCodec.cpp index f579e9de6b..eceb84efb6 100644 --- a/media/libstagefright/MediaCodec.cpp +++ b/media/libstagefright/MediaCodec.cpp @@ -48,6 +48,7 @@ #include <media/stagefright/foundation/avc_utils.h> #include <media/stagefright/foundation/hexdump.h> #include <media/stagefright/ACodec.h> +#include <media/stagefright/BatteryChecker.h> #include <media/stagefright/BufferProducerWrapper.h> #include <media/stagefright/MediaCodec.h> #include <media/stagefright/MediaCodecList.h> @@ -57,7 +58,6 @@ #include <media/stagefright/OMXClient.h> #include <media/stagefright/PersistentSurface.h> #include <media/stagefright/SurfaceUtils.h> -#include <mediautils/BatteryNotifier.h> #include <private/android_filesystem_config.h> #include <utils/Singleton.h> @@ -166,8 +166,9 @@ private: DISALLOW_EVIL_CONSTRUCTORS(ResourceManagerClient); }; -MediaCodec::ResourceManagerServiceProxy::ResourceManagerServiceProxy(pid_t pid) - : mPid(pid) { +MediaCodec::ResourceManagerServiceProxy::ResourceManagerServiceProxy( + pid_t pid, uid_t uid) + : mPid(pid), mUid(uid) { if (mPid == MediaCodec::kNoPid) { mPid = IPCThreadState::self()->getCallingPid(); } @@ -204,15 +205,25 @@ void MediaCodec::ResourceManagerServiceProxy::addResource( if (mService == NULL) { return; } - mService->addResource(mPid, clientId, client, resources); + mService->addResource(mPid, mUid, clientId, client, resources); } -void MediaCodec::ResourceManagerServiceProxy::removeResource(int64_t clientId) { +void MediaCodec::ResourceManagerServiceProxy::removeResource( + int64_t clientId, + const Vector<MediaResource> &resources) { + Mutex::Autolock _l(mLock); + if (mService == NULL) { + return; + } + mService->removeResource(mPid, clientId, resources); +} + +void MediaCodec::ResourceManagerServiceProxy::removeClient(int64_t clientId) { Mutex::Autolock _l(mLock); if (mService == NULL) { return; } - mService->removeResource(mPid, clientId); + mService->removeClient(mPid, clientId); } bool MediaCodec::ResourceManagerServiceProxy::reclaimResource( @@ -517,9 +528,6 @@ MediaCodec::MediaCodec(const sp<ALooper> &looper, pid_t pid, uid_t uid) mStickyError(OK), mSoftRenderer(NULL), mAnalyticsItem(NULL), - mResourceManagerClient(new ResourceManagerClient(this)), - mResourceManagerService(new ResourceManagerServiceProxy(pid)), - mBatteryStatNotified(false), mIsVideo(false), mVideoWidth(0), mVideoHeight(0), @@ -537,13 +545,15 @@ MediaCodec::MediaCodec(const sp<ALooper> &looper, pid_t pid, uid_t uid) } else { mUid = uid; } + mResourceManagerClient = new ResourceManagerClient(this); + mResourceManagerService = new ResourceManagerServiceProxy(pid, mUid); initAnalyticsItem(); } MediaCodec::~MediaCodec() { CHECK_EQ(mState, UNINITIALIZED); - mResourceManagerService->removeResource(getId(mResourceManagerClient)); + mResourceManagerService->removeClient(getId(mResourceManagerClient)); flushAnalyticsItem(); } @@ -742,6 +752,12 @@ void MediaCodec::statsBufferSent(int64_t presentationUs) { return; } + if (mBatteryChecker != nullptr) { + mBatteryChecker->onCodecActivity([this] () { + addResource(MediaResource::kBattery, MediaResource::kVideoCodec, 1); + }); + } + const int64_t nowNs = systemTime(SYSTEM_TIME_MONOTONIC); BufferFlightTiming_t startdata = { presentationUs, nowNs }; @@ -776,6 +792,12 @@ void MediaCodec::statsBufferReceived(int64_t presentationUs) { return; } + if (mBatteryChecker != nullptr) { + mBatteryChecker->onCodecActivity([this] () { + addResource(MediaResource::kBattery, MediaResource::kVideoCodec, 1); + }); + } + BufferFlightTiming_t startdata; bool valid = false; while (mBuffersInFlight.size() > 0) { @@ -964,6 +986,10 @@ status_t MediaCodec::init(const AString &name) { mAnalyticsItem->setCString(kCodecMode, mIsVideo ? kCodecModeVideo : kCodecModeAudio); } + if (mIsVideo) { + mBatteryChecker = new BatteryChecker(new AMessage(kWhatCheckBatteryStats, this)); + } + status_t err; Vector<MediaResource> resources; MediaResource::Type type = @@ -1221,6 +1247,13 @@ void MediaCodec::addResource( getId(mResourceManagerClient), mResourceManagerClient, resources); } +void MediaCodec::removeResource( + MediaResource::Type type, MediaResource::SubType subtype, uint64_t value) { + Vector<MediaResource> resources; + resources.push_back(MediaResource(type, subtype, value)); + mResourceManagerService->removeResource(getId(mResourceManagerClient), resources); +} + status_t MediaCodec::start() { sp<AMessage> msg = new AMessage(kWhatStart, this); @@ -1682,6 +1715,59 @@ void MediaCodec::requestCpuBoostIfNeeded() { } } +BatteryChecker::BatteryChecker(const sp<AMessage> &msg, int64_t timeoutUs) + : mTimeoutUs(timeoutUs) + , mLastActivityTimeUs(-1ll) + , mBatteryStatNotified(false) + , mBatteryCheckerGeneration(0) + , mIsExecuting(false) + , mBatteryCheckerMsg(msg) {} + +void BatteryChecker::onCodecActivity(std::function<void()> batteryOnCb) { + if (!isExecuting()) { + // ignore if not executing + return; + } + if (!mBatteryStatNotified) { + batteryOnCb(); + mBatteryStatNotified = true; + sp<AMessage> msg = mBatteryCheckerMsg->dup(); + msg->setInt32("generation", mBatteryCheckerGeneration); + + // post checker and clear last activity time + msg->post(mTimeoutUs); + mLastActivityTimeUs = -1ll; + } else { + // update last activity time + mLastActivityTimeUs = ALooper::GetNowUs(); + } +} + +void BatteryChecker::onCheckBatteryTimer( + const sp<AMessage> &msg, std::function<void()> batteryOffCb) { + // ignore if this checker already expired because the client resource was removed + int32_t generation; + if (!msg->findInt32("generation", &generation) + || generation != mBatteryCheckerGeneration) { + return; + } + + if (mLastActivityTimeUs < 0ll) { + // timed out inactive, do not repost checker + batteryOffCb(); + mBatteryStatNotified = false; + } else { + // repost checker and clear last activity time + msg->post(mTimeoutUs + mLastActivityTimeUs - ALooper::GetNowUs()); + mLastActivityTimeUs = -1ll; + } +} + +void BatteryChecker::onClientRemoved() { + mBatteryStatNotified = false; + mBatteryCheckerGeneration++; +} + //////////////////////////////////////////////////////////////////////////////// void MediaCodec::cancelPendingDequeueOperations() { @@ -2318,7 +2404,12 @@ void MediaCodec::onMessageReceived(const sp<AMessage> &msg) { mFlags &= ~kFlagIsComponentAllocated; - mResourceManagerService->removeResource(getId(mResourceManagerClient)); + // off since we're removing all resources including the battery on + if (mBatteryChecker != nullptr) { + mBatteryChecker->onClientRemoved(); + } + + mResourceManagerService->removeClient(getId(mResourceManagerClient)); (new AMessage)->postReply(mReplyID); break; @@ -3029,6 +3120,16 @@ void MediaCodec::onMessageReceived(const sp<AMessage> &msg) { break; } + case kWhatCheckBatteryStats: + { + if (mBatteryChecker != nullptr) { + mBatteryChecker->onCheckBatteryTimer(msg, [this] () { + removeResource(MediaResource::kBattery, MediaResource::kVideoCodec, 1); + }); + } + break; + } + default: TRESPASS(); } @@ -3125,9 +3226,11 @@ void MediaCodec::setState(State newState) { mState = newState; - cancelPendingDequeueOperations(); + if (mBatteryChecker != nullptr) { + mBatteryChecker->setExecuting(isExecuting()); + } - updateBatteryStat(); + cancelPendingDequeueOperations(); } void MediaCodec::returnBuffersToCodec(bool isReclaim) { @@ -3631,20 +3734,6 @@ status_t MediaCodec::amendOutputFormatWithCodecSpecificData( return OK; } -void MediaCodec::updateBatteryStat() { - if (!mIsVideo) { - return; - } - - if (mState == CONFIGURED && !mBatteryStatNotified) { - BatteryNotifier::getInstance().noteStartVideo(mUid); - mBatteryStatNotified = true; - } else if (mState == UNINITIALIZED && mBatteryStatNotified) { - BatteryNotifier::getInstance().noteStopVideo(mUid); - mBatteryStatNotified = false; - } -} - std::string MediaCodec::stateString(State state) { const char *rval = NULL; char rawbuffer[16]; // room for "%d" diff --git a/media/libstagefright/MediaMuxer.cpp b/media/libstagefright/MediaMuxer.cpp index 9ba2add491..7ebdb1aaa3 100644 --- a/media/libstagefright/MediaMuxer.cpp +++ b/media/libstagefright/MediaMuxer.cpp @@ -96,10 +96,18 @@ ssize_t MediaMuxer::addTrack(const sp<AMessage> &format) { sp<MediaAdapter> newTrack = new MediaAdapter(trackMeta); status_t result = mWriter->addSource(newTrack); - if (result == OK) { - return mTrackList.add(newTrack); + if (result != OK) { + return -1; + } + float captureFps = -1.0; + if (format->findAsFloat("time-lapse-fps", &captureFps)) { + ALOGV("addTrack() time-lapse-fps: %f", captureFps); + result = mWriter->setCaptureRate(captureFps); + if (result != OK) { + ALOGW("addTrack() setCaptureRate failed :%d", result); + } } - return -1; + return mTrackList.add(newTrack); } status_t MediaMuxer::setOrientationHint(int degrees) { diff --git a/media/libstagefright/TEST_MAPPING b/media/libstagefright/TEST_MAPPING index 96818eb70d..c1b270cddf 100644 --- a/media/libstagefright/TEST_MAPPING +++ b/media/libstagefright/TEST_MAPPING @@ -7,6 +7,9 @@ "include-annotation": "android.platform.test.annotations.RequiresDevice" } ] + }, + { + "name": "BatteryChecker_test" } ] } diff --git a/media/libstagefright/codecs/vorbis/dec/SoftVorbis.cpp b/media/libstagefright/codecs/vorbis/dec/SoftVorbis.cpp index 76a400c0d8..1293a74ddf 100644 --- a/media/libstagefright/codecs/vorbis/dec/SoftVorbis.cpp +++ b/media/libstagefright/codecs/vorbis/dec/SoftVorbis.cpp @@ -290,7 +290,7 @@ OMX_ERRORTYPE SoftVorbis::internalSetParameter( } bool SoftVorbis::isConfigured() const { - return mInputBufferCount >= 2; + return (mState != NULL && mVi != NULL); } static void makeBitReader( diff --git a/media/libstagefright/foundation/OpusHeader.cpp b/media/libstagefright/foundation/OpusHeader.cpp index 513e41f4a9..f5687e0d60 100644 --- a/media/libstagefright/foundation/OpusHeader.cpp +++ b/media/libstagefright/foundation/OpusHeader.cpp @@ -292,6 +292,10 @@ bool GetOpusHeaderBuffers(const uint8_t *data, size_t data_size, *opusHeadSize = data_size; return true; } else if (memcmp(AOPUS_CSD_MARKER_PREFIX, data, AOPUS_CSD_MARKER_PREFIX_SIZE) == 0) { + if (data_size < AOPUS_UNIFIED_CSD_MINSIZE || data_size > AOPUS_UNIFIED_CSD_MAXSIZE) { + ALOGD("Unexpected size for unified opus csd %zu", data_size); + return false; + } size_t i = 0; bool found = false; while (i <= data_size - AOPUS_MARKER_SIZE - AOPUS_LENGTH_SIZE) { diff --git a/media/libstagefright/include/media/stagefright/BatteryChecker.h b/media/libstagefright/include/media/stagefright/BatteryChecker.h new file mode 100644 index 0000000000..2ec4ac046a --- /dev/null +++ b/media/libstagefright/include/media/stagefright/BatteryChecker.h @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef BATTERY_CHECKER_H_ +#define BATTERY_CHECKER_H_ + +#include <media/stagefright/foundation/AMessage.h> + +namespace android { + +struct BatteryChecker : public RefBase { + BatteryChecker(const sp<AMessage> &msg, int64_t timeout = 3000000ll); + + void setExecuting(bool executing) { mIsExecuting = executing; } + void onCodecActivity(std::function<void()> batteryOnCb); + void onCheckBatteryTimer(const sp<AMessage>& msg, std::function<void()> batteryOffCb); + void onClientRemoved(); + +private: + const int64_t mTimeoutUs; + int64_t mLastActivityTimeUs; + bool mBatteryStatNotified; + int32_t mBatteryCheckerGeneration; + bool mIsExecuting; + sp<AMessage> mBatteryCheckerMsg; + + bool isExecuting() { return mIsExecuting; } + + DISALLOW_EVIL_CONSTRUCTORS(BatteryChecker); +}; + +} // namespace android + +#endif // BATTERY_CHECKER_H_ diff --git a/media/libstagefright/include/media/stagefright/MediaCodec.h b/media/libstagefright/include/media/stagefright/MediaCodec.h index 89cca63dc6..cd303477cb 100644 --- a/media/libstagefright/include/media/stagefright/MediaCodec.h +++ b/media/libstagefright/include/media/stagefright/MediaCodec.h @@ -36,6 +36,7 @@ struct ABuffer; struct AMessage; struct AReplyToken; struct AString; +struct BatteryChecker; class BufferChannelBase; struct CodecBase; class IBatteryStats; @@ -257,6 +258,7 @@ private: kWhatSetCallback = 'setC', kWhatSetNotification = 'setN', kWhatDrmReleaseCrypto = 'rDrm', + kWhatCheckBatteryStats = 'chkB', }; enum { @@ -283,7 +285,7 @@ private: }; struct ResourceManagerServiceProxy : public IBinder::DeathRecipient { - ResourceManagerServiceProxy(pid_t pid); + ResourceManagerServiceProxy(pid_t pid, uid_t uid); ~ResourceManagerServiceProxy(); void init(); @@ -296,7 +298,11 @@ private: const sp<IResourceManagerClient> &client, const Vector<MediaResource> &resources); - void removeResource(int64_t clientId); + void removeResource( + int64_t clientId, + const Vector<MediaResource> &resources); + + void removeClient(int64_t clientId); bool reclaimResource(const Vector<MediaResource> &resources); @@ -304,6 +310,7 @@ private: Mutex mLock; sp<IResourceManagerService> mService; pid_t mPid; + uid_t mUid; }; State mState; @@ -335,7 +342,6 @@ private: sp<IResourceManagerClient> mResourceManagerClient; sp<ResourceManagerServiceProxy> mResourceManagerService; - bool mBatteryStatNotified; bool mIsVideo; int32_t mVideoWidth; int32_t mVideoHeight; @@ -425,11 +431,11 @@ private: status_t onSetParameters(const sp<AMessage> ¶ms); status_t amendOutputFormatWithCodecSpecificData(const sp<MediaCodecBuffer> &buffer); - void updateBatteryStat(); bool isExecuting() const; uint64_t getGraphicBufferSize(); void addResource(MediaResource::Type type, MediaResource::SubType subtype, uint64_t value); + void removeResource(MediaResource::Type type, MediaResource::SubType subtype, uint64_t value); void requestCpuBoostIfNeeded(); bool hasPendingBuffer(int portIndex); @@ -458,6 +464,8 @@ private: Mutex mLatencyLock; int64_t mLatencyUnknown; // buffers for which we couldn't calculate latency + sp<BatteryChecker> mBatteryChecker; + void statsBufferSent(int64_t presentationUs); void statsBufferReceived(int64_t presentationUs); diff --git a/media/libstagefright/include/media/stagefright/MediaWriter.h b/media/libstagefright/include/media/stagefright/MediaWriter.h index 2c12a87f36..972ae1d218 100644 --- a/media/libstagefright/include/media/stagefright/MediaWriter.h +++ b/media/libstagefright/include/media/stagefright/MediaWriter.h @@ -35,6 +35,10 @@ struct MediaWriter : public RefBase { virtual status_t start(MetaData *params = NULL) = 0; virtual status_t stop() = 0; virtual status_t pause() = 0; + virtual status_t setCaptureRate(float /* captureFps */) { + ALOGW("setCaptureRate unsupported"); + return ERROR_UNSUPPORTED; + } virtual void setMaxFileSize(int64_t bytes) { mMaxFileSizeLimitBytes = bytes; } virtual void setMaxFileDuration(int64_t durationUs) { mMaxFileDurationLimitUs = durationUs; } diff --git a/media/libstagefright/rtsp/AMPEG4ElementaryAssembler.cpp b/media/libstagefright/rtsp/AMPEG4ElementaryAssembler.cpp index 1e434cbcea..9df3508d85 100644 --- a/media/libstagefright/rtsp/AMPEG4ElementaryAssembler.cpp +++ b/media/libstagefright/rtsp/AMPEG4ElementaryAssembler.cpp @@ -338,6 +338,12 @@ ARTPAssembler::AssemblyStatus AMPEG4ElementaryAssembler::addPacket( ABitReader bits(buffer->data() + offset, buffer->size() - offset); unsigned auxSize = bits.getBits(mAuxiliaryDataSizeLength); + if (buffer->size() < auxSize) { + ALOGE("b/123940919 auxSize %u", auxSize); + android_errorWriteLog(0x534e4554, "123940919"); + queue->erase(queue->begin()); + return MALFORMED_PACKET; + } offset += (mAuxiliaryDataSizeLength + auxSize + 7) / 8; } @@ -346,6 +352,12 @@ ARTPAssembler::AssemblyStatus AMPEG4ElementaryAssembler::addPacket( it != headers.end(); ++it) { const AUHeader &header = *it; + if (buffer->size() < header.mSize) { + ALOGE("b/123940919 AU_size %u", header.mSize); + android_errorWriteLog(0x534e4554, "123940919"); + queue->erase(queue->begin()); + return MALFORMED_PACKET; + } if (buffer->size() < offset + header.mSize) { return MALFORMED_PACKET; } diff --git a/media/libstagefright/tests/Android.bp b/media/libstagefright/tests/Android.bp index be10fdc942..a7f94c1365 100644 --- a/media/libstagefright/tests/Android.bp +++ b/media/libstagefright/tests/Android.bp @@ -27,3 +27,21 @@ cc_test { "-Wall", ], } + +cc_test { + name: "BatteryChecker_test", + srcs: ["BatteryChecker_test.cpp"], + test_suites: ["device-tests"], + + shared_libs: [ + "libstagefright", + "libstagefright_foundation", + "libutils", + "liblog", + ], + + cflags: [ + "-Werror", + "-Wall", + ], +}
\ No newline at end of file diff --git a/media/libstagefright/tests/BatteryChecker_test.cpp b/media/libstagefright/tests/BatteryChecker_test.cpp new file mode 100644 index 0000000000..0c5ee9b462 --- /dev/null +++ b/media/libstagefright/tests/BatteryChecker_test.cpp @@ -0,0 +1,242 @@ +/* + * Copyright 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// #define LOG_NDEBUG 0 +#define LOG_TAG "BatteryChecker_test" +#include <utils/Log.h> + +#include <gtest/gtest.h> + +#include <media/stagefright/BatteryChecker.h> +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AHandler.h> + +#include <vector> + +namespace android { + +static const int kBatteryTimeoutUs = 1000000ll; // 1 seconds +static const int kTestMarginUs = 50000ll; // 50ms +static const int kWaitStatusChangeUs = kBatteryTimeoutUs + kTestMarginUs; +static const int kSparseFrameIntervalUs = kBatteryTimeoutUs - kTestMarginUs; + +class BatteryCheckerTestHandler : public AHandler { + enum EventType { + // Events simulating MediaCodec + kWhatStart = 0, // codec entering executing state + kWhatStop, // codec exiting executing state + kWhatActivity, // codec queue input or dequeue output + kWhatReleased, // codec released + kWhatCheckpoint, // test checkpoing with expected values on On/Off + + // Message for battery checker monitor (not for testing through runTest()) + kWhatBatteryChecker, + }; + + struct Operation { + int32_t event; + int64_t delay = 0; + uint32_t repeatCount = 0; + int32_t expectedOnCounter = 0; + int32_t expectedOffCounter = 0; + }; + + std::vector<Operation> mOps; + sp<BatteryChecker> mBatteryChecker; + int32_t mOnCounter; + int32_t mOffCounter; + Condition mDone; + Mutex mLock; + + BatteryCheckerTestHandler() : mOnCounter(0), mOffCounter(0) {} + + void runTest(const std::vector<Operation> &ops, int64_t timeoutUs) { + mOps = ops; + + mBatteryChecker = new BatteryChecker( + new AMessage(kWhatBatteryChecker, this), kBatteryTimeoutUs); + + (new AMessage(ops[0].event, this))->post(); + + // wait for done + AutoMutex lock(mLock); + EXPECT_NE(TIMED_OUT, mDone.waitRelative(mLock, timeoutUs * 1000ll)); + } + + virtual void onMessageReceived(const sp<AMessage> &msg); + + friend class BatteryCheckerTest; +}; + +class BatteryCheckerTest : public ::testing::Test { +public: + BatteryCheckerTest() + : mLooper(new ALooper) + , mHandler(new BatteryCheckerTestHandler()) { + mLooper->setName("BatterCheckerLooper"); + mLooper->start(false, false, ANDROID_PRIORITY_AUDIO); + mLooper->registerHandler(mHandler); + } + +protected: + using EventType = BatteryCheckerTestHandler::EventType; + using Operation = BatteryCheckerTestHandler::Operation; + + virtual ~BatteryCheckerTest() { + mLooper->stop(); + mLooper->unregisterHandler(mHandler->id()); + } + + void runTest(const std::vector<Operation> &ops, int64_t timeoutUs) { + mHandler->runTest(ops, timeoutUs); + } + + sp<ALooper> mLooper; + sp<BatteryCheckerTestHandler> mHandler; +}; + +void BatteryCheckerTestHandler::onMessageReceived(const sp<AMessage> &msg) { + switch(msg->what()) { + case kWhatStart: + mBatteryChecker->setExecuting(true); + break; + case kWhatStop: + mBatteryChecker->setExecuting(false); + break; + case kWhatActivity: + mBatteryChecker->onCodecActivity([this] () { mOnCounter++; }); + break; + case kWhatReleased: + mBatteryChecker->onClientRemoved(); + break; + case kWhatBatteryChecker: + mBatteryChecker->onCheckBatteryTimer(msg, [this] () { mOffCounter++; }); + break; + case kWhatCheckpoint: + // verify ON/OFF state and total events + EXPECT_EQ(mOnCounter, mOps[0].expectedOnCounter); + EXPECT_EQ(mOffCounter, mOps[0].expectedOffCounter); + break; + default: + TRESPASS(); + } + if (msg->what() != kWhatBatteryChecker) { + EXPECT_EQ(msg->what(), mOps[0].event); + // post next message + if (!mOps[0].repeatCount) { + mOps.erase(mOps.begin()); + } else { + mOps[0].repeatCount--; + } + int64_t duration = mOps[0].delay; + if (!mOps.empty()) { + (new AMessage(mOps[0].event, this))->post(duration); + } else { + AutoMutex lock(mLock); + mDone.signal(); + } + } +} + +TEST_F(BatteryCheckerTest, testNormalOperations) { + runTest({ + {EventType::kWhatStart, 0ll}, + {EventType::kWhatCheckpoint, 0ll, 0, 0, 0}, + {EventType::kWhatActivity, 33333ll}, + {EventType::kWhatCheckpoint, 0ll, 0, 1, 0}, // ON + {EventType::kWhatActivity, 33333ll, 2*kWaitStatusChangeUs/33333ll}, + {EventType::kWhatCheckpoint, 0ll, 0, 1, 0}, + {EventType::kWhatCheckpoint, kWaitStatusChangeUs, 0, 1, 1}, // OFF + }, 10000000ll); +} + +TEST_F(BatteryCheckerTest, testPauseResume) { + runTest({ + {EventType::kWhatStart, 0ll}, + {EventType::kWhatCheckpoint, 0ll, 0, 0, 0}, + {EventType::kWhatActivity, 33333ll}, + {EventType::kWhatCheckpoint, 0ll, 0, 1, 0}, // ON + {EventType::kWhatCheckpoint, kWaitStatusChangeUs, 0, 1, 1}, // OFF + {EventType::kWhatActivity, 33333ll}, + {EventType::kWhatCheckpoint, 0ll, 0, 2, 1}, // ON + {EventType::kWhatCheckpoint, kWaitStatusChangeUs, 0, 2, 2}, // OFF + }, 10000000ll); +} + +TEST_F(BatteryCheckerTest, testClientRemovedAndRestart) { + runTest({ + {EventType::kWhatStart, 0ll}, + {EventType::kWhatActivity, 33333ll, kWaitStatusChangeUs/33333ll}, + {EventType::kWhatCheckpoint, 0ll, 0, 1, 0}, + + // stop executing state itself shouldn't trigger any calls + {EventType::kWhatStop, 0ll}, + {EventType::kWhatCheckpoint, 0ll, 0, 1, 0}, + + // release shouldn't trigger any calls either, + // client resource will be removed entirely + {EventType::kWhatReleased, 0ll}, + {EventType::kWhatCheckpoint, 0ll, 0, 1, 0}, + {EventType::kWhatCheckpoint, kWaitStatusChangeUs, 0, 1, 0}, + + // start pushing buffers again, On should be received without any Off + {EventType::kWhatStart, 0ll}, + {EventType::kWhatActivity, 33333ll}, + {EventType::kWhatCheckpoint, 0ll, 0, 2, 0}, + + // double check that only new checker msg triggers OFF, + // left-over checker msg from stale generate discarded + {EventType::kWhatCheckpoint, kWaitStatusChangeUs, 0, 2, 1}, + }, 10000000ll); +} + +TEST_F(BatteryCheckerTest, testActivityWhileNotExecuting) { + runTest({ + // activity before start shouldn't trigger + {EventType::kWhatActivity, 0ll}, + {EventType::kWhatCheckpoint, 0ll, 0, 0, 0}, + + {EventType::kWhatStart, 0ll}, + {EventType::kWhatCheckpoint, 0ll, 0, 0, 0}, + + // activity after start before stop should trigger + {EventType::kWhatActivity, 33333ll}, + {EventType::kWhatCheckpoint, 0ll, 0, 1, 0}, + + // stop executing state itself shouldn't trigger any calls + {EventType::kWhatStop, 0ll}, + {EventType::kWhatCheckpoint, 0ll, 0, 1, 0}, + + // keep pushing another 3 seconds after stop, expected to OFF + {EventType::kWhatActivity, 33333ll, kWaitStatusChangeUs/33333ll}, + {EventType::kWhatCheckpoint, 0ll, 0, 1, 1}, + }, 10000000ll); +} + +TEST_F(BatteryCheckerTest, testSparseActivity) { + runTest({ + {EventType::kWhatStart, 0ll}, + {EventType::kWhatCheckpoint, 0ll, 0, 0, 0}, + + // activity arrives sparsely with interval only slightly small than timeout + // should only trigger 1 ON + {EventType::kWhatActivity, kSparseFrameIntervalUs, 2}, + {EventType::kWhatCheckpoint, 0ll, 0, 1, 0}, + {EventType::kWhatCheckpoint, kSparseFrameIntervalUs, 0, 1, 0}, + {EventType::kWhatCheckpoint, kTestMarginUs, 0, 1, 1}, // OFF + }, 10000000ll); +} +} // namespace android diff --git a/media/libstagefright/timedtext/TextDescriptions2.cpp b/media/libstagefright/timedtext/TextDescriptions2.cpp index fd42d3ac61..f48eaccbe3 100644 --- a/media/libstagefright/timedtext/TextDescriptions2.cpp +++ b/media/libstagefright/timedtext/TextDescriptions2.cpp @@ -145,7 +145,7 @@ status_t TextDescriptions2::extract3GPPGlobalDescriptions( tmpData += 8; size_t remaining = size - 8; - if (chunkSize <= 8 || size < chunkSize) { + if (size < chunkSize) { return OK; } switch(chunkType) { diff --git a/media/libwatchdog/Android.bp b/media/libwatchdog/Android.bp new file mode 100644 index 0000000000..2bdf17262c --- /dev/null +++ b/media/libwatchdog/Android.bp @@ -0,0 +1,35 @@ +// Copyright 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +cc_library { + name: "libwatchdog", + srcs: [ + "Watchdog.cpp", + ], + export_include_dirs: ["include"], + shared_libs: [ + "liblog", + ], + static_libs: [ + "libbase", + ], + target: { + windows: { + enabled: false, + }, + darwin: { + enabled: false, + }, + }, +} diff --git a/media/libwatchdog/Watchdog.cpp b/media/libwatchdog/Watchdog.cpp new file mode 100644 index 0000000000..bb012b9429 --- /dev/null +++ b/media/libwatchdog/Watchdog.cpp @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "Watchdog" + +#include <watchdog/Watchdog.h> + +#include <android-base/logging.h> +#include <android-base/threads.h> +#include <signal.h> +#include <time.h> +#include <cstring> +#include <utils/Log.h> + +namespace android { + +Watchdog::Watchdog(::std::chrono::steady_clock::duration timeout) { + // Create the timer. + struct sigevent sev; + sev.sigev_notify = SIGEV_THREAD_ID; + sev.sigev_notify_thread_id = base::GetThreadId(); + sev.sigev_signo = SIGABRT; + sev.sigev_value.sival_ptr = &mTimerId; + int err = timer_create(CLOCK_MONOTONIC, &sev, &mTimerId); + if (err != 0) { + PLOG(FATAL) << "Failed to create timer"; + } + + // Start the timer. + struct itimerspec spec; + memset(&spec, 0, sizeof(spec)); + auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(timeout); + LOG_ALWAYS_FATAL_IF(timeout.count() <= 0, "Duration must be positive"); + spec.it_value.tv_sec = ns.count() / 1000000000; + spec.it_value.tv_nsec = ns.count() % 1000000000; + err = timer_settime(mTimerId, 0, &spec, nullptr); + if (err != 0) { + PLOG(FATAL) << "Failed to start timer"; + } +} + +Watchdog::~Watchdog() { + // Delete the timer. + int err = timer_delete(mTimerId); + if (err != 0) { + PLOG(FATAL) << "Failed to delete timer"; + } +} + +} // namespace android diff --git a/media/libwatchdog/include/watchdog/Watchdog.h b/media/libwatchdog/include/watchdog/Watchdog.h new file mode 100644 index 0000000000..2819f8a270 --- /dev/null +++ b/media/libwatchdog/include/watchdog/Watchdog.h @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ANDROID_WATCHDOG_H +#define ANDROID_WATCHDOG_H + +#include <chrono> +#include <time.h> + +namespace android { + +/* + * An RAII-style object, which would crash the process if a timeout expires + * before the object is destroyed. + * The calling thread would be sent a SIGABORT, which would typically result in + * a stack trace. + * + * Sample usage: + * { + * Watchdog watchdog(std::chrono::milliseconds(10)); + * DoSomething(); + * } + * // If we got here, the function completed in time. + */ +class Watchdog final { +public: + Watchdog(std::chrono::steady_clock::duration timeout); + ~Watchdog(); + +private: + timer_t mTimerId; +}; + +} // namespace android + +#endif // ANDROID_WATCHDOG_H diff --git a/media/mtp/MtpServer.cpp b/media/mtp/MtpServer.cpp index ca8cb785e9..dd69496160 100644 --- a/media/mtp/MtpServer.cpp +++ b/media/mtp/MtpServer.cpp @@ -44,6 +44,7 @@ #include "MtpStringBuffer.h" namespace android { +static const int SN_EVENT_LOG_ID = 0x534e4554; static const MtpOperationCode kSupportedOperationCodes[] = { MTP_OPERATION_GET_DEVICE_INFO, @@ -961,6 +962,17 @@ MtpResponseCode MtpServer::doSendObjectInfo() { if (!parseDateTime(modified, modifiedTime)) modifiedTime = 0; + if ((strcmp(name, ".") == 0) || (strcmp(name, "..") == 0) || + (strchr(name, '/') != NULL)) { + char errMsg[80]; + + snprintf(errMsg, sizeof(errMsg), "Invalid name: %s", (const char *) name); + ALOGE("%s (b/130656917)", errMsg); + android_errorWriteWithInfoLog(SN_EVENT_LOG_ID, "130656917", -1, errMsg, + strlen(errMsg)); + + return MTP_RESPONSE_INVALID_PARAMETER; + } if (path[path.size() - 1] != '/') path.append("/"); path.append(name); diff --git a/media/ndk/Android.bp b/media/ndk/Android.bp index a3cabd877a..9c7a630c0f 100644 --- a/media/ndk/Android.bp +++ b/media/ndk/Android.bp @@ -110,10 +110,6 @@ cc_library_shared { symbol_file: "libmediandk.map.txt", versions: ["29"], }, - - // Bug: http://b/124522995 libmediandk has linker errors when built with - // coverage - native_coverage: false, } llndk_library { diff --git a/media/ndk/NdkImage.cpp b/media/ndk/NdkImage.cpp index 1883f6356d..1145b7b79f 100644 --- a/media/ndk/NdkImage.cpp +++ b/media/ndk/NdkImage.cpp @@ -35,6 +35,7 @@ AImage::AImage(AImageReader* reader, int32_t format, uint64_t usage, BufferItem* int64_t timestamp, int32_t width, int32_t height, int32_t numPlanes) : mReader(reader), mFormat(format), mUsage(usage), mBuffer(buffer), mLockedBuffer(nullptr), mTimestamp(timestamp), mWidth(width), mHeight(height), mNumPlanes(numPlanes) { + LOG_FATAL_IF(reader == nullptr, "AImageReader shouldn't be null while creating AImage"); } AImage::~AImage() { @@ -57,14 +58,9 @@ AImage::close(int releaseFenceFd) { if (mIsClosed) { return; } - sp<AImageReader> reader = mReader.promote(); - if (reader != nullptr) { - reader->releaseImageLocked(this, releaseFenceFd); - } else if (mBuffer != nullptr) { - LOG_ALWAYS_FATAL("%s: parent AImageReader closed without releasing image %p", - __FUNCTION__, this); + if (!mReader->mIsClosed) { + mReader->releaseImageLocked(this, releaseFenceFd); } - // Should have been set to nullptr in releaseImageLocked // Set to nullptr here for extra safety only mBuffer = nullptr; @@ -83,22 +79,12 @@ AImage::free() { void AImage::lockReader() const { - sp<AImageReader> reader = mReader.promote(); - if (reader == nullptr) { - // Reader has been closed - return; - } - reader->mLock.lock(); + mReader->mLock.lock(); } void AImage::unlockReader() const { - sp<AImageReader> reader = mReader.promote(); - if (reader == nullptr) { - // Reader has been closed - return; - } - reader->mLock.unlock(); + mReader->mLock.unlock(); } media_status_t diff --git a/media/ndk/NdkImagePriv.h b/media/ndk/NdkImagePriv.h index e0f16dadf9..0e8cbcbf78 100644 --- a/media/ndk/NdkImagePriv.h +++ b/media/ndk/NdkImagePriv.h @@ -72,7 +72,7 @@ struct AImage { uint32_t getJpegSize() const; // When reader is close, AImage will only accept close API call - wp<AImageReader> mReader; + const sp<AImageReader> mReader; const int32_t mFormat; const uint64_t mUsage; // AHARDWAREBUFFER_USAGE_* flags. BufferItem* mBuffer; diff --git a/media/ndk/NdkImageReader.cpp b/media/ndk/NdkImageReader.cpp index baa4fc77ab..c0ceb3d70b 100644 --- a/media/ndk/NdkImageReader.cpp +++ b/media/ndk/NdkImageReader.cpp @@ -113,12 +113,12 @@ AImageReader::getNumPlanesForFormat(int32_t format) { void AImageReader::FrameListener::onFrameAvailable(const BufferItem& /*item*/) { - Mutex::Autolock _l(mLock); sp<AImageReader> reader = mReader.promote(); if (reader == nullptr) { ALOGW("A frame is available after AImageReader closed!"); return; // reader has been closed } + Mutex::Autolock _l(mLock); if (mListener.onImageAvailable == nullptr) { return; // No callback registered } @@ -143,12 +143,12 @@ AImageReader::FrameListener::setImageListener(AImageReader_ImageListener* listen void AImageReader::BufferRemovedListener::onBufferFreed(const wp<GraphicBuffer>& graphicBuffer) { - Mutex::Autolock _l(mLock); sp<AImageReader> reader = mReader.promote(); if (reader == nullptr) { ALOGW("A frame is available after AImageReader closed!"); return; // reader has been closed } + Mutex::Autolock _l(mLock); if (mListener.onBufferRemoved == nullptr) { return; // No callback registered } @@ -272,6 +272,11 @@ AImageReader::AImageReader(int32_t width, mFrameListener(new FrameListener(this)), mBufferRemovedListener(new BufferRemovedListener(this)) {} +AImageReader::~AImageReader() { + Mutex::Autolock _l(mLock); + LOG_FATAL_IF("AImageReader not closed before destruction", mIsClosed != true); +} + media_status_t AImageReader::init() { PublicFormat publicFormat = static_cast<PublicFormat>(mFormat); @@ -347,8 +352,12 @@ AImageReader::init() { return AMEDIA_OK; } -AImageReader::~AImageReader() { +void AImageReader::close() { Mutex::Autolock _l(mLock); + if (mIsClosed) { + return; + } + mIsClosed = true; AImageReader_ImageListener nullListener = {nullptr, nullptr}; setImageListenerLocked(&nullListener); @@ -741,6 +750,7 @@ EXPORT void AImageReader_delete(AImageReader* reader) { ALOGV("%s", __FUNCTION__); if (reader != nullptr) { + reader->close(); reader->decStrong((void*) AImageReader_delete); } return; diff --git a/media/ndk/NdkImageReaderPriv.h b/media/ndk/NdkImageReaderPriv.h index e328cb1429..0779a716bf 100644 --- a/media/ndk/NdkImageReaderPriv.h +++ b/media/ndk/NdkImageReaderPriv.h @@ -76,6 +76,7 @@ struct AImageReader : public RefBase { int32_t getHeight() const { return mHeight; }; int32_t getFormat() const { return mFormat; }; int32_t getMaxImages() const { return mMaxImages; }; + void close(); private: @@ -134,7 +135,7 @@ struct AImageReader : public RefBase { private: AImageReader_ImageListener mListener = {nullptr, nullptr}; - wp<AImageReader> mReader; + const wp<AImageReader> mReader; Mutex mLock; }; sp<FrameListener> mFrameListener; @@ -149,7 +150,7 @@ struct AImageReader : public RefBase { private: AImageReader_BufferRemovedListener mListener = {nullptr, nullptr}; - wp<AImageReader> mReader; + const wp<AImageReader> mReader; Mutex mLock; }; sp<BufferRemovedListener> mBufferRemovedListener; @@ -165,6 +166,7 @@ struct AImageReader : public RefBase { native_handle_t* mWindowHandle = nullptr; List<AImage*> mAcquiredImages; + bool mIsClosed = false; Mutex mLock; }; diff --git a/media/ndk/NdkMediaCodec.cpp b/media/ndk/NdkMediaCodec.cpp index c23f19b9d2..e041533586 100644 --- a/media/ndk/NdkMediaCodec.cpp +++ b/media/ndk/NdkMediaCodec.cpp @@ -221,7 +221,13 @@ void CodecHandler::onMessageReceived(const sp<AMessage> &msg) { break; } - AMediaFormat *aMediaFormat = AMediaFormat_fromMsg(&format); + // Here format is MediaCodec's internal copy of output format. + // Make a copy since the client might modify it. + sp<AMessage> copy; + if (format != nullptr) { + copy = format->dup(); + } + AMediaFormat *aMediaFormat = AMediaFormat_fromMsg(©); Mutex::Autolock _l(mCodec->mAsyncCallbackLock); if (mCodec->mAsyncCallbackUserData != NULL diff --git a/media/ndk/NdkMediaDrm.cpp b/media/ndk/NdkMediaDrm.cpp index 85dbffe16d..cd5a23aa32 100644 --- a/media/ndk/NdkMediaDrm.cpp +++ b/media/ndk/NdkMediaDrm.cpp @@ -89,7 +89,7 @@ struct AMediaDrm { }; void DrmListener::notify(DrmPlugin::EventType eventType, int extra, const Parcel *obj) { - if (!mEventListener && !mExpirationUpdateListener && !mKeysChangeListener) { + if (!mEventListener || !mExpirationUpdateListener || !mKeysChangeListener) { ALOGE("No listeners are specified"); return; } diff --git a/media/utils/Android.bp b/media/utils/Android.bp index 3adb40f5a3..e2cd4e3483 100644 --- a/media/utils/Android.bp +++ b/media/utils/Android.bp @@ -27,6 +27,7 @@ cc_library { ], shared_libs: [ "libbinder", + "libcutils", "liblog", "libutils", "libmemunreachable", diff --git a/media/utils/ServiceUtilities.cpp b/media/utils/ServiceUtilities.cpp index b824212b51..bc8fff6ef2 100644 --- a/media/utils/ServiceUtilities.cpp +++ b/media/utils/ServiceUtilities.cpp @@ -63,7 +63,10 @@ static bool checkRecordingInternal(const String16& opPackageName, pid_t pid, uid_t uid, bool start) { // Okay to not track in app ops as audio server is us and if // device is rooted security model is considered compromised. - if (isAudioServerOrRootUid(uid)) return true; + // system_server loses its RECORD_AUDIO permission when a secondary + // user is active, but it is a core system service so let it through. + // TODO(b/141210120): UserManager.DISALLOW_RECORD_AUDIO should not affect system user 0 + if (isAudioServerOrSystemServerOrRootUid(uid)) return true; // We specify a pid and uid here as mediaserver (aka MediaRecorder or StageFrightRecorder) // may open a record track on behalf of a client. Note that pid may be a tid. diff --git a/media/utils/TimeCheck.cpp b/media/utils/TimeCheck.cpp index 59cf4ef393..f16776fd96 100644 --- a/media/utils/TimeCheck.cpp +++ b/media/utils/TimeCheck.cpp @@ -82,10 +82,10 @@ bool TimeCheck::TimeCheckThread::threadLoop() if (waitTimeNs > 0) { status = mCond.waitRelative(mMutex, waitTimeNs); } - } - if (status != NO_ERROR) { - LOG_EVENT_STRING(LOGTAG_AUDIO_BINDER_TIMEOUT, tag); - LOG_ALWAYS_FATAL("TimeCheck timeout for %s", tag); + if (status != NO_ERROR) { + LOG_EVENT_STRING(LOGTAG_AUDIO_BINDER_TIMEOUT, tag); + LOG_ALWAYS_FATAL("TimeCheck timeout for %s", tag); + } } return true; } diff --git a/media/utils/include/mediautils/ServiceUtilities.h b/media/utils/include/mediautils/ServiceUtilities.h index 2a6e60966f..e1089d5bbc 100644 --- a/media/utils/include/mediautils/ServiceUtilities.h +++ b/media/utils/include/mediautils/ServiceUtilities.h @@ -58,6 +58,12 @@ static inline bool isAudioServerOrSystemServerUid(uid_t uid) { return multiuser_get_app_id(uid) == AID_SYSTEM || uid == AID_AUDIOSERVER; } +// used for calls that should come from system_server or audio_server and +// include AID_ROOT for command-line tests. +static inline bool isAudioServerOrSystemServerOrRootUid(uid_t uid) { + return multiuser_get_app_id(uid) == AID_SYSTEM || uid == AID_AUDIOSERVER || uid == AID_ROOT; +} + // Mediaserver may forward the client PID and UID as part of a binder interface call; // otherwise the calling UID must be equal to the client UID. static inline bool isAudioServerOrMediaServerUid(uid_t uid) { diff --git a/services/audioflinger/AudioFlinger.cpp b/services/audioflinger/AudioFlinger.cpp index d38190d95e..8bbdc69c0f 100644 --- a/services/audioflinger/AudioFlinger.cpp +++ b/services/audioflinger/AudioFlinger.cpp @@ -810,7 +810,33 @@ sp<IAudioTrack> AudioFlinger::createTrack(const CreateTrackInput& input, continue; } - size_t frameCount = std::lcm(thread->frameCount(), secondaryThread->frameCount()); + size_t sourceFrameCount = thread->frameCount() * output.sampleRate + / thread->sampleRate(); + size_t sinkFrameCount = secondaryThread->frameCount() * output.sampleRate + / secondaryThread->sampleRate(); + // If the secondary output has just been opened, the first secondaryThread write + // will not block as it will fill the empty startup buffer of the HAL, + // so a second sink buffer needs to be ready for the immediate next blocking write. + // Additionally, have a margin of one main thread buffer as the scheduling jitter + // can reorder the writes (eg if thread A&B have the same write intervale, + // the scheduler could schedule AB...BA) + size_t frameCountToBeReady = 2 * sinkFrameCount + sourceFrameCount; + // Total secondary output buffer must be at least as the read frames plus + // the margin of a few buffers on both sides in case the + // threads scheduling has some jitter. + // That value should not impact latency as the secondary track is started before + // its buffer is full, see frameCountToBeReady. + size_t frameCount = frameCountToBeReady + 2 * (sourceFrameCount + sinkFrameCount); + // The frameCount should also not be smaller than the secondary thread min frame + // count + size_t minFrameCount = AudioSystem::calculateMinFrameCount( + [&] { Mutex::Autolock _l(secondaryThread->mLock); + return secondaryThread->latency_l(); }(), + secondaryThread->mNormalFrameCount, + secondaryThread->mSampleRate, + output.sampleRate, + input.speed); + frameCount = std::max(frameCount, minFrameCount); using namespace std::chrono_literals; auto inChannelMask = audio_channel_mask_out_to_in(input.config.channel_mask); @@ -843,7 +869,8 @@ sp<IAudioTrack> AudioFlinger::createTrack(const CreateTrackInput& input, patchRecord->buffer(), patchRecord->bufferSize(), outputFlags, - 0ns /* timeout */); + 0ns /* timeout */, + frameCountToBeReady); status = patchTrack->initCheck(); if (status != NO_ERROR) { ALOGE("Secondary output patchTrack init failed: %d", status); diff --git a/services/audioflinger/Effects.cpp b/services/audioflinger/Effects.cpp index 3c4fbba156..13152d0b84 100644 --- a/services/audioflinger/Effects.cpp +++ b/services/audioflinger/Effects.cpp @@ -24,6 +24,7 @@ #include "Configuration.h" #include <utils/Log.h> #include <system/audio_effects/effect_aec.h> +#include <system/audio_effects/effect_dynamicsprocessing.h> #include <system/audio_effects/effect_ns.h> #include <system/audio_effects/effect_visualizer.h> #include <audio_utils/channels.h> @@ -2569,7 +2570,8 @@ bool AudioFlinger::EffectChain::isEffectEligibleForSuspend(const effect_descript if ((mSessionId == AUDIO_SESSION_OUTPUT_MIX) && (((desc.flags & EFFECT_FLAG_TYPE_MASK) == EFFECT_FLAG_TYPE_AUXILIARY) || (memcmp(&desc.type, SL_IID_VISUALIZATION, sizeof(effect_uuid_t)) == 0) || - (memcmp(&desc.type, SL_IID_VOLUME, sizeof(effect_uuid_t)) == 0))) { + (memcmp(&desc.type, SL_IID_VOLUME, sizeof(effect_uuid_t)) == 0) || + (memcmp(&desc.type, SL_IID_DYNAMICSPROCESSING, sizeof(effect_uuid_t)) == 0))) { return false; } return true; diff --git a/services/audioflinger/PlaybackTracks.h b/services/audioflinger/PlaybackTracks.h index a093893fac..17adba5bd5 100644 --- a/services/audioflinger/PlaybackTracks.h +++ b/services/audioflinger/PlaybackTracks.h @@ -74,7 +74,10 @@ public: uid_t uid, audio_output_flags_t flags, track_type type, - audio_port_handle_t portId = AUDIO_PORT_HANDLE_NONE); + audio_port_handle_t portId = AUDIO_PORT_HANDLE_NONE, + /** default behaviour is to start when there are as many frames + * ready as possible (aka. Buffer is full). */ + size_t frameCountToBeReady = SIZE_MAX); virtual ~Track(); virtual status_t initCheck() const; @@ -263,6 +266,8 @@ protected: }; sp<AudioVibrationController> mAudioVibrationController; sp<os::ExternalVibration> mExternalVibration; + /** How many frames should be in the buffer before the track is considered ready */ + const size_t mFrameCountToBeReady; private: void interceptBuffer(const AudioBufferProvider::Buffer& buffer); @@ -384,7 +389,11 @@ public: void *buffer, size_t bufferSize, audio_output_flags_t flags, - const Timeout& timeout = {}); + const Timeout& timeout = {}, + size_t frameCountToBeReady = 1 /** Default behaviour is to start + * as soon as possible to have + * the lowest possible latency + * even if it might glitch. */); virtual ~PatchTrack(); virtual status_t start(AudioSystem::sync_event_t event = @@ -402,5 +411,4 @@ public: private: void restartIfDisabled(); - }; // end of PatchTrack diff --git a/services/audioflinger/Threads.cpp b/services/audioflinger/Threads.cpp index 873227394a..039e4f7638 100644 --- a/services/audioflinger/Threads.cpp +++ b/services/audioflinger/Threads.cpp @@ -3956,6 +3956,32 @@ status_t AudioFlinger::PlaybackThread::getTimestamp_l(AudioTimestamp& timestamp) return INVALID_OPERATION; } +// For dedicated VoIP outputs, let the HAL apply the stream volume. Track volume is +// still applied by the mixer. +// All tracks attached to a mixer with flag VOIP_RX are tied to the same +// stream type STREAM_VOICE_CALL so this will only change the HAL volume once even +// if more than one track are active +status_t AudioFlinger::PlaybackThread::handleVoipVolume_l(float *volume) +{ + status_t result = NO_ERROR; + if ((mOutput->flags & AUDIO_OUTPUT_FLAG_VOIP_RX) != 0) { + if (*volume != mLeftVolFloat) { + result = mOutput->stream->setVolume(*volume, *volume); + ALOGE_IF(result != OK, + "Error when setting output stream volume: %d", result); + if (result == NO_ERROR) { + mLeftVolFloat = *volume; + } + } + // if stream volume was successfully sent to the HAL, mLeftVolFloat == v here and we + // remove stream volume contribution from software volume. + if (mLeftVolFloat == *volume) { + *volume = 1.0f; + } + } + return result; +} + status_t AudioFlinger::MixerThread::createAudioPatch_l(const struct audio_patch *patch, audio_patch_handle_t *handle) { @@ -4758,22 +4784,25 @@ AudioFlinger::PlaybackThread::mixer_state AudioFlinger::MixerThread::prepareTrac // no acknowledgement required for newly active tracks } sp<AudioTrackServerProxy> proxy = track->mAudioTrackServerProxy; - // cache the combined master volume and stream type volume for fast mixer; this - // lacks any synchronization or barrier so VolumeProvider may read a stale value - const float vh = track->getVolumeHandler()->getVolume( - proxy->framesReleased()).first; float volume; - if (track->isPlaybackRestricted()) { + if (track->isPlaybackRestricted() || mStreamTypes[track->streamType()].mute) { volume = 0.f; } else { - volume = masterVolume - * mStreamTypes[track->streamType()].volume - * vh; + volume = masterVolume * mStreamTypes[track->streamType()].volume; } + + handleVoipVolume_l(&volume); + + // cache the combined master volume and stream type volume for fast mixer; this + // lacks any synchronization or barrier so VolumeProvider may read a stale value + const float vh = track->getVolumeHandler()->getVolume( + proxy->framesReleased()).first; + volume *= vh; track->mCachedVolume = volume; gain_minifloat_packed_t vlr = proxy->getVolumeLR(); float vlf = volume * float_from_gain(gain_minifloat_unpack_left(vlr)); float vrf = volume * float_from_gain(gain_minifloat_unpack_right(vlr)); + track->setFinalVolume((vlf + vrf) / 2.f); ++fastTracks; } else { @@ -4916,20 +4945,22 @@ AudioFlinger::PlaybackThread::mixer_state AudioFlinger::MixerThread::prepareTrac uint32_t vl, vr; // in U8.24 integer format float vlf, vrf, vaf; // in [0.0, 1.0] float format // read original volumes with volume control - float typeVolume = mStreamTypes[track->streamType()].volume; - float v = masterVolume * typeVolume; + float v = masterVolume * mStreamTypes[track->streamType()].volume; // Always fetch volumeshaper volume to ensure state is updated. const sp<AudioTrackServerProxy> proxy = track->mAudioTrackServerProxy; const float vh = track->getVolumeHandler()->getVolume( track->mAudioTrackServerProxy->framesReleased()).first; - if (track->isPausing() || mStreamTypes[track->streamType()].mute - || track->isPlaybackRestricted()) { + if (mStreamTypes[track->streamType()].mute || track->isPlaybackRestricted()) { + v = 0; + } + + handleVoipVolume_l(&v); + + if (track->isPausing()) { vl = vr = 0; vlf = vrf = vaf = 0.; - if (track->isPausing()) { - track->setPaused(); - } + track->setPaused(); } else { gain_minifloat_packed_t vlr = proxy->getVolumeLR(); vlf = float_from_gain(gain_minifloat_unpack_left(vlr)); @@ -4981,25 +5012,6 @@ AudioFlinger::PlaybackThread::mixer_state AudioFlinger::MixerThread::prepareTrac track->mHasVolumeController = false; } - // For dedicated VoIP outputs, let the HAL apply the stream volume. Track volume is - // still applied by the mixer. - if ((mOutput->flags & AUDIO_OUTPUT_FLAG_VOIP_RX) != 0) { - v = mStreamTypes[track->streamType()].mute ? 0.0f : v; - if (v != mLeftVolFloat) { - status_t result = mOutput->stream->setVolume(v, v); - ALOGE_IF(result != OK, "Error when setting output stream volume: %d", result); - if (result == OK) { - mLeftVolFloat = v; - } - } - // if stream volume was successfully sent to the HAL, mLeftVolFloat == v here and we - // remove stream volume contribution from software volume. - if (v != 0.0f && mLeftVolFloat == v) { - vlf = min(1.0f, vlf / v); - vrf = min(1.0f, vrf / v); - vaf = min(1.0f, vaf / v); - } - } // XXX: these things DON'T need to be done each time mAudioMixer->setBufferProvider(trackId, track); mAudioMixer->enable(trackId); diff --git a/services/audioflinger/Threads.h b/services/audioflinger/Threads.h index c6a82017c5..87bebf3d89 100644 --- a/services/audioflinger/Threads.h +++ b/services/audioflinger/Threads.h @@ -747,6 +747,7 @@ protected: // is safe to do so. That will drop the final ref count and destroy the tracks. virtual mixer_state prepareTracks_l(Vector< sp<Track> > *tracksToRemove) = 0; void removeTracks_l(const Vector< sp<Track> >& tracksToRemove); + status_t handleVoipVolume_l(float *volume); // StreamOutHalInterfaceCallback implementation virtual void onWriteReady(); diff --git a/services/audioflinger/Tracks.cpp b/services/audioflinger/Tracks.cpp index 2a5a713ca3..51e57b51cb 100644 --- a/services/audioflinger/Tracks.cpp +++ b/services/audioflinger/Tracks.cpp @@ -511,7 +511,8 @@ AudioFlinger::PlaybackThread::Track::Track( uid_t uid, audio_output_flags_t flags, track_type type, - audio_port_handle_t portId) + audio_port_handle_t portId, + size_t frameCountToBeReady) : TrackBase(thread, client, attr, sampleRate, format, channelMask, frameCount, (sharedBuffer != 0) ? sharedBuffer->pointer() : buffer, (sharedBuffer != 0) ? sharedBuffer->size() : bufferSize, @@ -530,6 +531,7 @@ AudioFlinger::PlaybackThread::Track::Track( mVolumeHandler(new media::VolumeHandler(sampleRate)), mOpPlayAudioMonitor(OpPlayAudioMonitor::createIfNeeded(uid, attr, id(), streamType)), // mSinkTimestamp + mFrameCountToBeReady(frameCountToBeReady), mFastIndex(-1), mCachedVolume(1.0), /* The track might not play immediately after being active, similarly as if its volume was 0. @@ -837,7 +839,7 @@ void AudioFlinger::PlaybackThread::Track::interceptBuffer( auto spent = ceil<std::chrono::microseconds>(std::chrono::steady_clock::now() - start); using namespace std::chrono_literals; // Average is ~20us per track, this should virtually never be logged (Logging takes >200us) - ALOGD_IF(spent > 200us, "%s: took %lldus to intercept %zu tracks", __func__, + ALOGD_IF(spent > 500us, "%s: took %lldus to intercept %zu tracks", __func__, spent.count(), mTeePatches.size()); } @@ -910,8 +912,12 @@ bool AudioFlinger::PlaybackThread::Track::isReady() const { return true; } - if (framesReady() >= mServerProxy->getBufferSizeInFrames() || - (mCblk->mFlags & CBLK_FORCEREADY)) { + size_t bufferSizeInFrames = mServerProxy->getBufferSizeInFrames(); + size_t framesToBeReady = std::min(mFrameCountToBeReady, bufferSizeInFrames); + + if (framesReady() >= framesToBeReady || (mCblk->mFlags & CBLK_FORCEREADY)) { + ALOGV("%s(%d): consider track ready with %zu/%zu, target was %zu)", + __func__, mId, framesReady(), bufferSizeInFrames, framesToBeReady); mFillingUpStatus = FS_FILLED; android_atomic_and(~CBLK_FORCEREADY, &mCblk->mFlags); return true; @@ -1413,6 +1419,7 @@ void AudioFlinger::PlaybackThread::Track::invalidate() void AudioFlinger::PlaybackThread::Track::disable() { + // TODO(b/142394888): the filling status should also be reset to filling signalClientFlag(CBLK_DISABLED); } @@ -1790,12 +1797,14 @@ AudioFlinger::PlaybackThread::PatchTrack::PatchTrack(PlaybackThread *playbackThr void *buffer, size_t bufferSize, audio_output_flags_t flags, - const Timeout& timeout) + const Timeout& timeout, + size_t frameCountToBeReady) : Track(playbackThread, NULL, streamType, audio_attributes_t{} /* currently unused for patch track */, sampleRate, format, channelMask, frameCount, buffer, bufferSize, nullptr /* sharedBuffer */, - AUDIO_SESSION_NONE, getpid(), AID_AUDIOSERVER, flags, TYPE_PATCH), + AUDIO_SESSION_NONE, getpid(), AID_AUDIOSERVER, flags, TYPE_PATCH, + AUDIO_PORT_HANDLE_NONE, frameCountToBeReady), PatchTrackBase(new ClientProxy(mCblk, mBuffer, frameCount, mFrameSize, true, true), *playbackThread, timeout) { @@ -1869,7 +1878,6 @@ void AudioFlinger::PlaybackThread::PatchTrack::releaseBuffer(Proxy::Buffer* buff { mProxy->releaseBuffer(buffer); restartIfDisabled(); - android_atomic_or(CBLK_FORCEREADY, &mCblk->mFlags); } void AudioFlinger::PlaybackThread::PatchTrack::restartIfDisabled() diff --git a/services/audiopolicy/common/managerdefinitions/src/HwModule.cpp b/services/audiopolicy/common/managerdefinitions/src/HwModule.cpp index 96a83377e1..1f9b725a24 100644 --- a/services/audiopolicy/common/managerdefinitions/src/HwModule.cpp +++ b/services/audiopolicy/common/managerdefinitions/src/HwModule.cpp @@ -333,9 +333,10 @@ sp<DeviceDescriptor> HwModuleCollection::getDeviceDescriptor(const audio_devices if (encodedFormat != AUDIO_FORMAT_DEFAULT) { moduleDevice->setEncodedFormat(encodedFormat); } - moduleDevice->setAddress(devAddress); if (allowToCreate) { moduleDevice->attach(hwModule); + moduleDevice->setAddress(devAddress); + moduleDevice->setName(String8(name)); } return moduleDevice; } diff --git a/services/audiopolicy/managerdefault/AudioPolicyManager.cpp b/services/audiopolicy/managerdefault/AudioPolicyManager.cpp index 1d4dacd4e9..a984b10c03 100644 --- a/services/audiopolicy/managerdefault/AudioPolicyManager.cpp +++ b/services/audiopolicy/managerdefault/AudioPolicyManager.cpp @@ -5535,7 +5535,10 @@ uint32_t AudioPolicyManager::setOutputDevices(const sp<SwAudioOutputDescriptor>& patchBuilder.addSink(filteredDevice); } - installPatch(__func__, patchHandle, outputDesc.get(), patchBuilder.patch(), delayMs); + // Add half reported latency to delayMs when muteWaitMs is null in order + // to avoid disordered sequence of muting volume and changing devices. + installPatch(__func__, patchHandle, outputDesc.get(), patchBuilder.patch(), + muteWaitMs == 0 ? (delayMs + (outputDesc->latency() / 2)) : delayMs); } // update stream volumes according to new device @@ -5690,8 +5693,9 @@ float AudioPolicyManager::computeVolume(IVolumeCurves &curves, const auto ringVolumeSrc = toVolumeSource(AUDIO_STREAM_RING); const auto musicVolumeSrc = toVolumeSource(AUDIO_STREAM_MUSIC); const auto alarmVolumeSrc = toVolumeSource(AUDIO_STREAM_ALARM); + const auto a11yVolumeSrc = toVolumeSource(AUDIO_STREAM_ACCESSIBILITY); - if (volumeSource == toVolumeSource(AUDIO_STREAM_ACCESSIBILITY) + if (volumeSource == a11yVolumeSrc && (AUDIO_MODE_RINGTONE == mEngine->getPhoneState()) && mOutputs.isActive(ringVolumeSrc, 0)) { auto &ringCurves = getVolumeCurves(AUDIO_STREAM_RING); @@ -5708,7 +5712,7 @@ float AudioPolicyManager::computeVolume(IVolumeCurves &curves, volumeSource == toVolumeSource(AUDIO_STREAM_NOTIFICATION) || volumeSource == toVolumeSource(AUDIO_STREAM_ENFORCED_AUDIBLE) || volumeSource == toVolumeSource(AUDIO_STREAM_DTMF) || - volumeSource == toVolumeSource(AUDIO_STREAM_ACCESSIBILITY))) { + volumeSource == a11yVolumeSrc)) { auto &voiceCurves = getVolumeCurves(callVolumeSrc); int voiceVolumeIndex = voiceCurves.getVolumeIndex(device); const float maxVoiceVolDb = @@ -5720,7 +5724,9 @@ float AudioPolicyManager::computeVolume(IVolumeCurves &curves, // VOICE_CALL stream has minVolumeIndex > 0 : Users cannot set the volume of voice calls to // 0. We don't want to cap volume when the system has programmatically muted the voice call // stream. See setVolumeCurveIndex() for more information. - bool exemptFromCapping = (volumeSource == ringVolumeSrc) && (voiceVolumeIndex == 0); + bool exemptFromCapping = + ((volumeSource == ringVolumeSrc) || (volumeSource == a11yVolumeSrc)) + && (voiceVolumeIndex == 0); ALOGV_IF(exemptFromCapping, "%s volume source %d at vol=%f not capped", __func__, volumeSource, volumeDb); if ((volumeDb > maxVoiceVolDb) && !exemptFromCapping) { diff --git a/services/audiopolicy/managerdefault/AudioPolicyManager.h b/services/audiopolicy/managerdefault/AudioPolicyManager.h index 612bd8fecf..dc548d67cb 100644 --- a/services/audiopolicy/managerdefault/AudioPolicyManager.h +++ b/services/audiopolicy/managerdefault/AudioPolicyManager.h @@ -346,7 +346,7 @@ protected: } virtual const DeviceVector getAvailableOutputDevices() const { - return mAvailableOutputDevices; + return mAvailableOutputDevices.filterForEngine(); } virtual const DeviceVector getAvailableInputDevices() const { diff --git a/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp b/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp index fa8da8916e..20e1c9e8ba 100644 --- a/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp +++ b/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp @@ -976,11 +976,6 @@ status_t AudioPolicyService::setAllowedCapturePolicy(uid_t uid, audio_flags_mask ALOGV("%s() mAudioPolicyManager == NULL", __func__); return NO_INIT; } - uint_t callingUid = IPCThreadState::self()->getCallingUid(); - if (uid != callingUid) { - ALOGD("%s() uid invalid %d != %d", __func__, uid, callingUid); - return PERMISSION_DENIED; - } return mAudioPolicyManager->setAllowedCapturePolicy(uid, capturePolicy); } diff --git a/services/audiopolicy/service/AudioPolicyService.cpp b/services/audiopolicy/service/AudioPolicyService.cpp index 77f7997a2a..a6cda20798 100644 --- a/services/audiopolicy/service/AudioPolicyService.cpp +++ b/services/audiopolicy/service/AudioPolicyService.cpp @@ -410,12 +410,17 @@ void AudioPolicyService::updateUidStates_l() // Another client in the same UID has already been allowed to capture // OR The client is the assistant // AND an accessibility service is on TOP or a RTT call is active -// AND the source is VOICE_RECOGNITION or HOTWORD -// OR uses VOICE_RECOGNITION AND is on TOP -// OR uses HOTWORD +// AND the source is VOICE_RECOGNITION or HOTWORD +// OR uses VOICE_RECOGNITION AND is on TOP +// OR uses HOTWORD // AND there is no active privacy sensitive capture or call // OR client has CAPTURE_AUDIO_OUTPUT privileged permission // OR The client is an accessibility service +// AND Is on TOP +// AND the source is VOICE_RECOGNITION or HOTWORD +// OR The assistant is not on TOP +// AND there is no active privacy sensitive capture or call +// OR client has CAPTURE_AUDIO_OUTPUT privileged permission // AND is on TOP // AND the source is VOICE_RECOGNITION or HOTWORD // OR the client source is virtual (remote submix, call audio TX or RX...) @@ -423,7 +428,7 @@ void AudioPolicyService::updateUidStates_l() // AND The assistant is not on TOP // AND is on TOP or latest started // AND there is no active privacy sensitive capture or call -// OR client has CAPTURE_AUDIO_OUTPUT privileged permission +// OR client has CAPTURE_AUDIO_OUTPUT privileged permission sp<AudioRecordClient> topActive; sp<AudioRecordClient> latestActive; @@ -459,16 +464,24 @@ void AudioPolicyService::updateUidStates_l() continue; } - if (appState == APP_STATE_TOP) { + bool isAssistant = mUidPolicy->isAssistantUid(current->uid); + bool isAccessibility = mUidPolicy->isA11yUid(current->uid); + if (appState == APP_STATE_TOP && !isAccessibility) { if (current->startTimeNs > topStartNs) { topActive = current; topStartNs = current->startTimeNs; } - if (mUidPolicy->isAssistantUid(current->uid)) { + if (isAssistant) { isAssistantOnTop = true; } } - if (current->startTimeNs > latestStartNs) { + // Assistant capturing for HOTWORD or Accessibility services not considered + // for latest active to avoid masking regular clients started before + if (current->startTimeNs > latestStartNs + && !((current->attributes.source == AUDIO_SOURCE_HOTWORD + || isA11yOnTop || rttCallActive) + && isAssistant) + && !isAccessibility) { latestActive = current; latestStartNs = current->startTimeNs; } @@ -541,10 +554,20 @@ void AudioPolicyService::updateUidStates_l() } else if (mUidPolicy->isA11yUid(current->uid)) { // For accessibility service allow capture if: // Is on TOP - // AND the source is VOICE_RECOGNITION or HOTWORD - if (isA11yOnTop && - (source == AUDIO_SOURCE_VOICE_RECOGNITION || source == AUDIO_SOURCE_HOTWORD)) { - allowCapture = true; + // AND the source is VOICE_RECOGNITION or HOTWORD + // Or + // The assistant is not on TOP + // AND there is no active privacy sensitive capture or call + // OR client has CAPTURE_AUDIO_OUTPUT privileged permission + if (isA11yOnTop) { + if (source == AUDIO_SOURCE_VOICE_RECOGNITION || source == AUDIO_SOURCE_HOTWORD) { + allowCapture = true; + } + } else { + if (!isAssistantOnTop + && (!(isSensitiveActive || isInCall) || current->canCaptureOutput)) { + allowCapture = true; + } } } setAppState_l(current->uid, @@ -703,11 +726,11 @@ status_t AudioPolicyService::shellCommand(int in, int out, int err, Vector<Strin if (in == BAD_TYPE || out == BAD_TYPE || err == BAD_TYPE) { return BAD_VALUE; } - if (args.size() == 3 && args[0] == String16("set-uid-state")) { + if (args.size() >= 3 && args[0] == String16("set-uid-state")) { return handleSetUidState(args, err); - } else if (args.size() == 2 && args[0] == String16("reset-uid-state")) { + } else if (args.size() >= 2 && args[0] == String16("reset-uid-state")) { return handleResetUidState(args, err); - } else if (args.size() == 2 && args[0] == String16("get-uid-state")) { + } else if (args.size() >= 2 && args[0] == String16("get-uid-state")) { return handleGetUidState(args, out, err); } else if (args.size() == 1 && args[0] == String16("help")) { printHelp(out); @@ -717,14 +740,32 @@ status_t AudioPolicyService::shellCommand(int in, int out, int err, Vector<Strin return BAD_VALUE; } -status_t AudioPolicyService::handleSetUidState(Vector<String16>& args, int err) { +static status_t getUidForPackage(String16 packageName, int userId, /*inout*/uid_t& uid, int err) { + if (userId < 0) { + ALOGE("Invalid user: %d", userId); + dprintf(err, "Invalid user: %d\n", userId); + return BAD_VALUE; + } + PermissionController pc; - int uid = pc.getPackageUid(args[1], 0); + uid = pc.getPackageUid(packageName, 0); if (uid <= 0) { - ALOGE("Unknown package: '%s'", String8(args[1]).string()); - dprintf(err, "Unknown package: '%s'\n", String8(args[1]).string()); + ALOGE("Unknown package: '%s'", String8(packageName).string()); + dprintf(err, "Unknown package: '%s'\n", String8(packageName).string()); + return BAD_VALUE; + } + + uid = multiuser_get_uid(userId, uid); + return NO_ERROR; +} + +status_t AudioPolicyService::handleSetUidState(Vector<String16>& args, int err) { + // Valid arg.size() is 3 or 5, args.size() is 5 with --user option. + if (!(args.size() == 3 || args.size() == 5)) { + printHelp(err); return BAD_VALUE; } + bool active = false; if (args[2] == String16("active")) { active = true; @@ -732,30 +773,59 @@ status_t AudioPolicyService::handleSetUidState(Vector<String16>& args, int err) ALOGE("Expected active or idle but got: '%s'", String8(args[2]).string()); return BAD_VALUE; } + + int userId = 0; + if (args.size() >= 5 && args[3] == String16("--user")) { + userId = atoi(String8(args[4])); + } + + uid_t uid; + if (getUidForPackage(args[1], userId, uid, err) == BAD_VALUE) { + return BAD_VALUE; + } + mUidPolicy->addOverrideUid(uid, active); return NO_ERROR; } status_t AudioPolicyService::handleResetUidState(Vector<String16>& args, int err) { - PermissionController pc; - int uid = pc.getPackageUid(args[1], 0); - if (uid < 0) { - ALOGE("Unknown package: '%s'", String8(args[1]).string()); - dprintf(err, "Unknown package: '%s'\n", String8(args[1]).string()); + // Valid arg.size() is 2 or 4, args.size() is 4 with --user option. + if (!(args.size() == 2 || args.size() == 4)) { + printHelp(err); return BAD_VALUE; } + + int userId = 0; + if (args.size() >= 4 && args[2] == String16("--user")) { + userId = atoi(String8(args[3])); + } + + uid_t uid; + if (getUidForPackage(args[1], userId, uid, err) == BAD_VALUE) { + return BAD_VALUE; + } + mUidPolicy->removeOverrideUid(uid); return NO_ERROR; } status_t AudioPolicyService::handleGetUidState(Vector<String16>& args, int out, int err) { - PermissionController pc; - int uid = pc.getPackageUid(args[1], 0); - if (uid < 0) { - ALOGE("Unknown package: '%s'", String8(args[1]).string()); - dprintf(err, "Unknown package: '%s'\n", String8(args[1]).string()); + // Valid arg.size() is 2 or 4, args.size() is 4 with --user option. + if (!(args.size() == 2 || args.size() == 4)) { + printHelp(err); return BAD_VALUE; } + + int userId = 0; + if (args.size() >= 4 && args[2] == String16("--user")) { + userId = atoi(String8(args[3])); + } + + uid_t uid; + if (getUidForPackage(args[1], userId, uid, err) == BAD_VALUE) { + return BAD_VALUE; + } + if (mUidPolicy->isUidActive(uid)) { return dprintf(out, "active\n"); } else { @@ -765,9 +835,9 @@ status_t AudioPolicyService::handleGetUidState(Vector<String16>& args, int out, status_t AudioPolicyService::printHelp(int out) { return dprintf(out, "Audio policy service commands:\n" - " get-uid-state <PACKAGE> gets the uid state\n" - " set-uid-state <PACKAGE> <active|idle> overrides the uid state\n" - " reset-uid-state <PACKAGE> clears the uid state override\n" + " get-uid-state <PACKAGE> [--user USER_ID] gets the uid state\n" + " set-uid-state <PACKAGE> <active|idle> [--user USER_ID] overrides the uid state\n" + " reset-uid-state <PACKAGE> [--user USER_ID] clears the uid state override\n" " help print this message\n"); } diff --git a/services/camera/libcameraservice/CameraService.cpp b/services/camera/libcameraservice/CameraService.cpp index 3e6210294b..670e026183 100644 --- a/services/camera/libcameraservice/CameraService.cpp +++ b/services/camera/libcameraservice/CameraService.cpp @@ -118,6 +118,8 @@ static void setLogLevel(int level) { // ---------------------------------------------------------------------------- static const String16 sManageCameraPermission("android.permission.MANAGE_CAMERA"); +static const String16 sCameraOpenCloseListenerPermission( + "android.permission.CAMERA_OPEN_CLOSE_LISTENER"); // Matches with PERCEPTIBLE_APP_ADJ in ProcessList.java static constexpr int32_t kVendorClientScore = 200; @@ -153,8 +155,6 @@ void CameraService::onFirstRef() mInitialized = true; } - CameraService::pingCameraServiceProxy(); - mUidPolicy = new UidPolicy(this); mUidPolicy->registerSelf(); mSensorPrivacyPolicy = new SensorPrivacyPolicy(this); @@ -164,6 +164,11 @@ void CameraService::onFirstRef() ALOGE("%s: Failed to register default android.frameworks.cameraservice.service@1.0", __FUNCTION__); } + + // This needs to be last call in this function, so that it's as close to + // ServiceManager::addService() as possible. + CameraService::pingCameraServiceProxy(); + ALOGI("CameraService pinged cameraservice proxy"); } status_t CameraService::enumerateProviders() { @@ -253,6 +258,15 @@ void CameraService::onNewProviderRegistered() { enumerateProviders(); } +bool CameraService::isPublicallyHiddenSecureCamera(const String8& cameraId) { + auto state = getCameraState(cameraId); + if (state != nullptr) { + return state->isPublicallyHiddenSecureCamera(); + } + // Hidden physical camera ids won't have CameraState + return mCameraProviderManager->isPublicallyHiddenSecureCamera(cameraId.c_str()); +} + void CameraService::updateCameraNumAndIds() { Mutex::Autolock l(mServiceLock); mNumberOfCameras = mCameraProviderManager->getCameraCount(); @@ -268,6 +282,8 @@ void CameraService::addStates(const String8 id) { ALOGE("Failed to query device resource cost: %s (%d)", strerror(-res), res); return; } + bool isPublicallyHiddenSecureCamera = + mCameraProviderManager->isPublicallyHiddenSecureCamera(id.string()); std::set<String8> conflicting; for (size_t i = 0; i < cost.conflictingDevices.size(); i++) { conflicting.emplace(String8(cost.conflictingDevices[i].c_str())); @@ -276,7 +292,8 @@ void CameraService::addStates(const String8 id) { { Mutex::Autolock lock(mCameraStatesLock); mCameraStates.emplace(id, std::make_shared<CameraState>(id, cost.resourceCost, - conflicting)); + conflicting, + isPublicallyHiddenSecureCamera)); } if (mFlashlight->hasFlashUnit(id)) { @@ -514,8 +531,16 @@ Status CameraService::getCameraCharacteristics(const String16& cameraId, "Camera subsystem is not available");; } - Status ret{}; + if (shouldRejectHiddenCameraConnection(String8(cameraId))) { + ALOGW("Attempting to retrieve characteristics for system-only camera id %s, rejected", + String8(cameraId).string()); + return STATUS_ERROR_FMT(ERROR_DISCONNECTED, + "No camera device with ID \"%s\" currently available", + String8(cameraId).string()); + + } + Status ret{}; status_t res = mCameraProviderManager->getCameraCharacteristics( String8(cameraId).string(), cameraInfo); if (res != OK) { @@ -1149,6 +1174,8 @@ status_t CameraService::handleEvictionsLocked(const String8& cameraId, int clien clientPid, states[states.size() - 1]); + resource_policy::ClientPriority clientPriority = clientDescriptor->getPriority(); + // Find clients that would be evicted auto evicted = mActiveClientManager.wouldEvict(clientDescriptor); @@ -1166,8 +1193,7 @@ status_t CameraService::handleEvictionsLocked(const String8& cameraId, int clien String8 msg = String8::format("%s : DENIED connect device %s client for package %s " "(PID %d, score %d state %d) due to eviction policy", curTime.string(), cameraId.string(), packageName.string(), clientPid, - priorityScores[priorityScores.size() - 1], - states[states.size() - 1]); + clientPriority.getScore(), clientPriority.getState()); for (auto& i : incompatibleClients) { msg.appendFormat("\n - Blocked by existing device %s client for package %s" @@ -1212,9 +1238,8 @@ status_t CameraService::handleEvictionsLocked(const String8& cameraId, int clien i->getKey().string(), String8{clientSp->getPackageName()}.string(), i->getOwnerId(), i->getPriority().getScore(), i->getPriority().getState(), cameraId.string(), - packageName.string(), clientPid, - priorityScores[priorityScores.size() - 1], - states[states.size() - 1])); + packageName.string(), clientPid, clientPriority.getScore(), + clientPriority.getState())); // Notify the client of disconnection clientSp->notifyError(hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_DISCONNECTED, @@ -1330,7 +1355,7 @@ bool CameraService::shouldRejectHiddenCameraConnection(const String8 & cameraId) // publically hidden, we should reject the connection. if (!hardware::IPCThreadState::self()->isServingCall() && CameraThreadState::getCallingPid() != getpid() && - mCameraProviderManager->isPublicallyHiddenSecureCamera(cameraId.c_str())) { + isPublicallyHiddenSecureCamera(cameraId)) { return true; } return false; @@ -1348,14 +1373,19 @@ Status CameraService::connectDevice( Status ret = Status::ok(); String8 id = String8(cameraId); sp<CameraDeviceClient> client = nullptr; - + String16 clientPackageNameAdj = clientPackageName; + if (hardware::IPCThreadState::self()->isServingCall()) { + std::string vendorClient = + StringPrintf("vendor.client.pid<%d>", CameraThreadState::getCallingPid()); + clientPackageNameAdj = String16(vendorClient.c_str()); + } ret = connectHelper<hardware::camera2::ICameraDeviceCallbacks,CameraDeviceClient>(cameraCb, id, /*api1CameraId*/-1, - CAMERA_HAL_API_VERSION_UNSPECIFIED, clientPackageName, + CAMERA_HAL_API_VERSION_UNSPECIFIED, clientPackageNameAdj, clientUid, USE_CALLING_PID, API_2, /*shimUpdateOnly*/ false, /*out*/client); if(!ret.isOk()) { - logRejected(id, CameraThreadState::getCallingPid(), String8(clientPackageName), + logRejected(id, CameraThreadState::getCallingPid(), String8(clientPackageNameAdj), ret.toString8()); return ret; } @@ -1783,7 +1813,11 @@ Status CameraService::addListenerHelper(const sp<ICameraServiceListener>& listen } auto clientUid = CameraThreadState::getCallingUid(); - sp<ServiceListener> serviceListener = new ServiceListener(this, listener, clientUid); + auto clientPid = CameraThreadState::getCallingPid(); + bool openCloseCallbackAllowed = checkPermission(sCameraOpenCloseListenerPermission, + clientPid, clientUid); + sp<ServiceListener> serviceListener = new ServiceListener(this, listener, + clientUid, clientPid, openCloseCallbackAllowed); auto ret = serviceListener->initialize(); if (ret != NO_ERROR) { String8 msg = String8::format("Failed to initialize service listener: %s (%d)", @@ -1799,16 +1833,25 @@ Status CameraService::addListenerHelper(const sp<ICameraServiceListener>& listen { Mutex::Autolock lock(mCameraStatesLock); for (auto& i : mCameraStates) { - if (!isVendorListener && - mCameraProviderManager->isPublicallyHiddenSecureCamera(i.first.c_str())) { - ALOGV("Cannot add public listener for hidden system-only %s for pid %d", - i.first.c_str(), CameraThreadState::getCallingPid()); - continue; - } cameraStatuses->emplace_back(i.first, mapToInterface(i.second->getStatus())); } } + // Remove the camera statuses that should be hidden from the client, we do + // this after collecting the states in order to avoid holding + // mCameraStatesLock and mInterfaceLock (held in + // isPublicallyHiddenSecureCamera()) at the same time. + cameraStatuses->erase(std::remove_if(cameraStatuses->begin(), cameraStatuses->end(), + [this, &isVendorListener](const hardware::CameraStatus& s) { + bool ret = !isVendorListener && isPublicallyHiddenSecureCamera(s.cameraId); + if (ret) { + ALOGV("Cannot add public listener for hidden system-only %s for pid %d", + s.cameraId.c_str(), CameraThreadState::getCallingPid()); + } + return ret; + }), + cameraStatuses->end()); + /* * Immediately signal current torch status to this listener only * This may be a subset of all the devices, so don't include it in the response directly @@ -2368,11 +2411,7 @@ CameraService::BasicClient::BasicClient(const sp<CameraService>& cameraService, } mClientPackageName = packages[0]; } - if (hardware::IPCThreadState::self()->isServingCall()) { - std::string vendorClient = - StringPrintf("vendor.client.pid<%d>", CameraThreadState::getCallingPid()); - mClientPackageName = String16(vendorClient.c_str()); - } else { + if (!hardware::IPCThreadState::self()->isServingCall()) { mAppOpsManager = std::make_unique<AppOpsManager>(); } } @@ -2482,6 +2521,9 @@ status_t CameraService::BasicClient::startCameraOps() { sCameraService->mUidPolicy->registerMonitorUid(mClientUid); + // Notify listeners of camera open/close status + sCameraService->updateOpenCloseStatus(mCameraIdStr, true/*open*/, mClientPackageName); + return OK; } @@ -2522,6 +2564,9 @@ status_t CameraService::BasicClient::finishCameraOps() { sCameraService->mUidPolicy->unregisterMonitorUid(mClientUid); + // Notify listeners of camera open/close status + sCameraService->updateOpenCloseStatus(mCameraIdStr, false/*open*/, mClientPackageName); + return OK; } @@ -2872,8 +2917,9 @@ void CameraService::SensorPrivacyPolicy::binderDied(const wp<IBinder>& /*who*/) // ---------------------------------------------------------------------------- CameraService::CameraState::CameraState(const String8& id, int cost, - const std::set<String8>& conflicting) : mId(id), - mStatus(StatusInternal::NOT_PRESENT), mCost(cost), mConflicting(conflicting) {} + const std::set<String8>& conflicting, bool isHidden) : mId(id), + mStatus(StatusInternal::NOT_PRESENT), mCost(cost), mConflicting(conflicting), + mIsPublicallyHiddenSecureCamera(isHidden) {} CameraService::CameraState::~CameraState() {} @@ -2902,6 +2948,10 @@ String8 CameraService::CameraState::getId() const { return mId; } +bool CameraService::CameraState::isPublicallyHiddenSecureCamera() const { + return mIsPublicallyHiddenSecureCamera; +} + // ---------------------------------------------------------------------------- // ClientEventListener // ---------------------------------------------------------------------------- @@ -3237,10 +3287,10 @@ void CameraService::updateStatus(StatusInternal status, const String8& cameraId, cameraId.string()); return; } - + bool isHidden = isPublicallyHiddenSecureCamera(cameraId); // Update the status for this camera state, then send the onStatusChangedCallbacks to each // of the listeners with both the mStatusStatus and mStatusListenerLock held - state->updateStatus(status, cameraId, rejectSourceStates, [this] + state->updateStatus(status, cameraId, rejectSourceStates, [this,&isHidden] (const String8& cameraId, StatusInternal status) { if (status != StatusInternal::ENUMERATING) { @@ -3262,8 +3312,7 @@ void CameraService::updateStatus(StatusInternal status, const String8& cameraId, Mutex::Autolock lock(mStatusListenerLock); for (auto& listener : mListenerList) { - if (!listener.first && - mCameraProviderManager->isPublicallyHiddenSecureCamera(cameraId.c_str())) { + if (!listener.first && isHidden) { ALOGV("Skipping camera discovery callback for system-only camera %s", cameraId.c_str()); continue; @@ -3274,6 +3323,29 @@ void CameraService::updateStatus(StatusInternal status, const String8& cameraId, }); } +void CameraService::updateOpenCloseStatus(const String8& cameraId, bool open, + const String16& clientPackageName) { + Mutex::Autolock lock(mStatusListenerLock); + + for (const auto& it : mListenerList) { + if (!it.second->isOpenCloseCallbackAllowed()) { + continue; + } + + binder::Status ret; + String16 cameraId64(cameraId); + if (open) { + ret = it.second->getListener()->onCameraOpened(cameraId64, clientPackageName); + } else { + ret = it.second->getListener()->onCameraClosed(cameraId64); + } + if (!ret.isOk()) { + ALOGE("%s: Failed to trigger onCameraOpened/onCameraClosed callback: %d", __FUNCTION__, + ret.exceptionCode()); + } + } +} + template<class Func> void CameraService::CameraState::updateStatus(StatusInternal status, const String8& cameraId, diff --git a/services/camera/libcameraservice/CameraService.h b/services/camera/libcameraservice/CameraService.h index 065157dad4..bcaca9ffcb 100644 --- a/services/camera/libcameraservice/CameraService.h +++ b/services/camera/libcameraservice/CameraService.h @@ -470,7 +470,8 @@ private: * Make a new CameraState and set the ID, cost, and conflicting devices using the values * returned in the HAL's camera_info struct for each device. */ - CameraState(const String8& id, int cost, const std::set<String8>& conflicting); + CameraState(const String8& id, int cost, const std::set<String8>& conflicting, + bool isHidden); virtual ~CameraState(); /** @@ -522,6 +523,11 @@ private: */ String8 getId() const; + /** + * Return if the camera device is a publically hidden secure camera + */ + bool isPublicallyHiddenSecureCamera() const; + private: const String8 mId; StatusInternal mStatus; // protected by mStatusLock @@ -529,6 +535,7 @@ private: std::set<String8> mConflicting; mutable Mutex mStatusLock; CameraParameters mShimParams; + const bool mIsPublicallyHiddenSecureCamera; }; // class CameraState // Observer for UID lifecycle enforcing that UIDs in idle @@ -633,7 +640,9 @@ private: // Should an operation attempt on a cameraId be rejected, if the camera id is // advertised as a publically hidden secure camera, by the camera HAL ? - bool shouldRejectHiddenCameraConnection(const String8 & cameraId); + bool shouldRejectHiddenCameraConnection(const String8& cameraId); + + bool isPublicallyHiddenSecureCamera(const String8& cameraId); // Single implementation shared between the various connect calls template<class CALLBACK, class CLIENT> @@ -808,7 +817,9 @@ private: class ServiceListener : public virtual IBinder::DeathRecipient { public: ServiceListener(sp<CameraService> parent, sp<hardware::ICameraServiceListener> listener, - int uid) : mParent(parent), mListener(listener), mListenerUid(uid) {} + int uid, int pid, bool openCloseCallbackAllowed) : mParent(parent), + mListener(listener), mListenerUid(uid), mListenerPid(pid), + mOpenCloseCallbackAllowed(openCloseCallbackAllowed) {} status_t initialize() { return IInterface::asBinder(mListener)->linkToDeath(this); @@ -822,12 +833,16 @@ private: } int getListenerUid() { return mListenerUid; } + int getListenerPid() { return mListenerPid; } sp<hardware::ICameraServiceListener> getListener() { return mListener; } + bool isOpenCloseCallbackAllowed() { return mOpenCloseCallbackAllowed; } private: wp<CameraService> mParent; sp<hardware::ICameraServiceListener> mListener; int mListenerUid; + int mListenerPid; + bool mOpenCloseCallbackAllowed = false; }; // Guarded by mStatusListenerMutex @@ -850,6 +865,13 @@ private: void updateStatus(StatusInternal status, const String8& cameraId); + /** + * Update the opened/closed status of the given camera id. + * + * This method acqiures mStatusListenerLock. + */ + void updateOpenCloseStatus(const String8& cameraId, bool open, const String16& packageName); + // flashlight control sp<CameraFlashlight> mFlashlight; // guard mTorchStatusMap diff --git a/services/camera/libcameraservice/api2/CameraDeviceClient.cpp b/services/camera/libcameraservice/api2/CameraDeviceClient.cpp index c7a4f2bfad..3587db33d9 100644 --- a/services/camera/libcameraservice/api2/CameraDeviceClient.cpp +++ b/services/camera/libcameraservice/api2/CameraDeviceClient.cpp @@ -2004,6 +2004,15 @@ void CameraDeviceClient::detachDevice() { } } + for (size_t i = 0; i < mCompositeStreamMap.size(); i++) { + auto ret = mCompositeStreamMap.valueAt(i)->deleteInternalStreams(); + if (ret != OK) { + ALOGE("%s: Failed removing composite stream %s (%d)", __FUNCTION__, + strerror(-ret), ret); + } + } + mCompositeStreamMap.clear(); + Camera2ClientBase::detachDevice(); } diff --git a/services/camera/libcameraservice/api2/DepthCompositeStream.cpp b/services/camera/libcameraservice/api2/DepthCompositeStream.cpp index 8ebaa2b2c7..0b9101619d 100644 --- a/services/camera/libcameraservice/api2/DepthCompositeStream.cpp +++ b/services/camera/libcameraservice/api2/DepthCompositeStream.cpp @@ -247,7 +247,7 @@ int64_t DepthCompositeStream::getNextFailingInputLocked(int64_t *currentTs /*ino return ret; } -status_t DepthCompositeStream::processInputFrame(const InputFrame &inputFrame) { +status_t DepthCompositeStream::processInputFrame(nsecs_t ts, const InputFrame &inputFrame) { status_t res; sp<ANativeWindow> outputANW = mOutputSurface; ANativeWindowBuffer *anb; @@ -370,6 +370,13 @@ status_t DepthCompositeStream::processInputFrame(const InputFrame &inputFrame) { return NO_MEMORY; } + res = native_window_set_buffers_timestamp(mOutputSurface.get(), ts); + if (res != OK) { + ALOGE("%s: Stream %d: Error setting timestamp: %s (%d)", __FUNCTION__, + getStreamId(), strerror(-res), res); + return res; + } + ALOGV("%s: Final jpeg size: %zu", __func__, finalJpegSize); uint8_t* header = static_cast<uint8_t *> (dstBuffer) + (gb->getWidth() - sizeof(struct camera3_jpeg_blob)); @@ -459,7 +466,7 @@ bool DepthCompositeStream::threadLoop() { } } - auto res = processInputFrame(mPendingInputFrames[currentTs]); + auto res = processInputFrame(currentTs, mPendingInputFrames[currentTs]); Mutex::Autolock l(mMutex); if (res != OK) { ALOGE("%s: Failed processing frame with timestamp: %" PRIu64 ": %s (%d)", __FUNCTION__, diff --git a/services/camera/libcameraservice/api2/DepthCompositeStream.h b/services/camera/libcameraservice/api2/DepthCompositeStream.h index 975c59bc5c..28a7826c5d 100644 --- a/services/camera/libcameraservice/api2/DepthCompositeStream.h +++ b/services/camera/libcameraservice/api2/DepthCompositeStream.h @@ -97,7 +97,7 @@ private: size_t maxJpegSize, uint8_t jpegQuality, std::vector<std::unique_ptr<Item>>* items /*out*/); std::unique_ptr<ImagingModel> getImagingModel(); - status_t processInputFrame(const InputFrame &inputFrame); + status_t processInputFrame(nsecs_t ts, const InputFrame &inputFrame); // Buffer/Results handling void compilePendingInputLocked(); diff --git a/services/camera/libcameraservice/api2/HeicCompositeStream.cpp b/services/camera/libcameraservice/api2/HeicCompositeStream.cpp index 5a87134743..9f15be02f7 100644 --- a/services/camera/libcameraservice/api2/HeicCompositeStream.cpp +++ b/services/camera/libcameraservice/api2/HeicCompositeStream.cpp @@ -31,7 +31,6 @@ #include <media/ICrypto.h> #include <media/MediaCodecBuffer.h> #include <media/stagefright/foundation/ABuffer.h> -#include <media/stagefright/foundation/AMessage.h> #include <media/stagefright/foundation/MediaDefs.h> #include <media/stagefright/MediaCodecConstants.h> @@ -61,12 +60,13 @@ HeicCompositeStream::HeicCompositeStream(wp<CameraDeviceBase> device, mUseGrid(false), mAppSegmentStreamId(-1), mAppSegmentSurfaceId(-1), - mAppSegmentBufferAcquired(false), mMainImageStreamId(-1), mMainImageSurfaceId(-1), mYuvBufferAcquired(false), mProducerListener(new ProducerListener()), - mOutputBufferCounter(0), + mDequeuedOutputBufferCnt(0), + mLockedAppSegmentBufferCnt(0), + mCodecOutputCounter(0), mGridTimestampUs(0) { } @@ -132,7 +132,7 @@ status_t HeicCompositeStream::createInternalStreams(const std::vector<sp<Surface sp<IGraphicBufferProducer> producer; sp<IGraphicBufferConsumer> consumer; BufferQueue::createBufferQueue(&producer, &consumer); - mAppSegmentConsumer = new CpuConsumer(consumer, 1); + mAppSegmentConsumer = new CpuConsumer(consumer, kMaxAcquiredAppSegment); mAppSegmentConsumer->setFrameAvailableListener(this); mAppSegmentConsumer->setName(String8("Camera3-HeicComposite-AppSegmentStream")); mAppSegmentSurface = new Surface(producer); @@ -231,6 +231,8 @@ void HeicCompositeStream::onBufferReleased(const BufferInfo& bufferInfo) { if (bufferInfo.mError) return; mCodecOutputBufferTimestamps.push(bufferInfo.mTimestamp); + ALOGV("%s: [%" PRId64 "]: Adding codecOutputBufferTimestamp (%zu timestamps in total)", + __FUNCTION__, bufferInfo.mTimestamp, mCodecOutputBufferTimestamps.size()); } // We need to get the settings early to handle the case where the codec output @@ -361,6 +363,8 @@ void HeicCompositeStream::onHeicOutputFrameAvailable( mCodecOutputBuffers.push_back(outputBufferInfo); mInputReadyCondition.signal(); } else { + ALOGV("%s: Releasing output buffer: size %d flags: 0x%x ", __FUNCTION__, + outputBufferInfo.size, outputBufferInfo.flags); mCodec->releaseOutputBuffer(outputBufferInfo.index); } } else { @@ -414,8 +418,10 @@ void HeicCompositeStream::onHeicFormatChanged(sp<AMessage>& newFormat) { mNumOutputTiles = 1; } - ALOGV("%s: mNumOutputTiles is %zu", __FUNCTION__, mNumOutputTiles); mFormat = newFormat; + + ALOGV("%s: mNumOutputTiles is %zu", __FUNCTION__, mNumOutputTiles); + mInputReadyCondition.signal(); } void HeicCompositeStream::onHeicCodecError() { @@ -459,9 +465,8 @@ status_t HeicCompositeStream::configureStream() { // Cannot use SourceSurface buffer count since it could be codec's 512*512 tile // buffer count. - int maxProducerBuffers = 1; if ((res = native_window_set_buffer_count( - anwConsumer, maxProducerBuffers + maxConsumerBuffers)) != OK) { + anwConsumer, kMaxOutputSurfaceProducerCount + maxConsumerBuffers)) != OK) { ALOGE("%s: Unable to set buffer count for stream %d", __FUNCTION__, mMainImageStreamId); return res; } @@ -505,6 +510,8 @@ void HeicCompositeStream::onShutter(const CaptureResultExtras& resultExtras, nse } if (mSettingsByFrameNumber.find(resultExtras.frameNumber) != mSettingsByFrameNumber.end()) { + ALOGV("%s: [%" PRId64 "]: frameNumber %" PRId64, __FUNCTION__, + timestamp, resultExtras.frameNumber); mFrameNumberMap.emplace(resultExtras.frameNumber, timestamp); mSettingsByTimestamp[timestamp] = mSettingsByFrameNumber[resultExtras.frameNumber]; mSettingsByFrameNumber.erase(resultExtras.frameNumber); @@ -520,12 +527,12 @@ void HeicCompositeStream::compilePendingInputLocked() { mSettingsByTimestamp.erase(it); } - while (!mInputAppSegmentBuffers.empty() && !mAppSegmentBufferAcquired) { + while (!mInputAppSegmentBuffers.empty()) { CpuConsumer::LockedBuffer imgBuffer; auto it = mInputAppSegmentBuffers.begin(); auto res = mAppSegmentConsumer->lockNextBuffer(&imgBuffer); if (res == NOT_ENOUGH_DATA) { - // Canot not lock any more buffers. + // Can not lock any more buffers. break; } else if ((res != OK) || (*it != imgBuffer.timestamp)) { if (res != OK) { @@ -535,6 +542,7 @@ void HeicCompositeStream::compilePendingInputLocked() { ALOGE("%s: Expecting JPEG_APP_SEGMENTS buffer with time stamp: %" PRId64 " received buffer with time stamp: %" PRId64, __FUNCTION__, *it, imgBuffer.timestamp); + mAppSegmentConsumer->unlockBuffer(imgBuffer); } mPendingInputFrames[*it].error = true; mInputAppSegmentBuffers.erase(it); @@ -546,7 +554,7 @@ void HeicCompositeStream::compilePendingInputLocked() { mAppSegmentConsumer->unlockBuffer(imgBuffer); } else { mPendingInputFrames[imgBuffer.timestamp].appSegmentBuffer = imgBuffer; - mAppSegmentBufferAcquired = true; + mLockedAppSegmentBufferCnt++; } mInputAppSegmentBuffers.erase(it); } @@ -556,7 +564,7 @@ void HeicCompositeStream::compilePendingInputLocked() { auto it = mInputYuvBuffers.begin(); auto res = mMainImageConsumer->lockNextBuffer(&imgBuffer); if (res == NOT_ENOUGH_DATA) { - // Canot not lock any more buffers. + // Can not lock any more buffers. break; } else if (res != OK) { ALOGE("%s: Error locking YUV_888 image buffer: %s (%d)", __FUNCTION__, @@ -589,17 +597,20 @@ void HeicCompositeStream::compilePendingInputLocked() { // to look up timestamp. int64_t bufferTime = -1; if (mCodecOutputBufferTimestamps.empty()) { - ALOGE("%s: Failed to find buffer timestamp for codec output buffer!", __FUNCTION__); + ALOGV("%s: Failed to find buffer timestamp for codec output buffer!", __FUNCTION__); + break; } else { // Direct mapping between camera timestamp (in ns) and codec timestamp (in us). bufferTime = mCodecOutputBufferTimestamps.front(); - mOutputBufferCounter++; - if (mOutputBufferCounter == mNumOutputTiles) { + mCodecOutputCounter++; + if (mCodecOutputCounter == mNumOutputTiles) { mCodecOutputBufferTimestamps.pop(); - mOutputBufferCounter = 0; + mCodecOutputCounter = 0; } mPendingInputFrames[bufferTime].codecOutputBuffers.push_back(*it); + ALOGV("%s: [%" PRId64 "]: Pushing codecOutputBuffers (time %" PRId64 " us)", + __FUNCTION__, bufferTime, it->timeUs); } mCodecOutputBuffers.erase(it); } @@ -607,6 +618,7 @@ void HeicCompositeStream::compilePendingInputLocked() { while (!mFrameNumberMap.empty()) { auto it = mFrameNumberMap.begin(); mPendingInputFrames[it->second].frameNumber = it->first; + ALOGV("%s: [%" PRId64 "]: frameNumber is %" PRId64, __FUNCTION__, it->second, it->first); mFrameNumberMap.erase(it); } @@ -675,16 +687,29 @@ bool HeicCompositeStream::getNextReadyInputLocked(int64_t *currentTs /*out*/) { } bool newInputAvailable = false; - for (const auto& it : mPendingInputFrames) { + for (auto& it : mPendingInputFrames) { + // New input is considered to be available only if: + // 1. input buffers are ready, or + // 2. App segment and muxer is created, or + // 3. A codec output tile is ready, and an output buffer is available. + // This makes sure that muxer gets created only when an output tile is + // generated, because right now we only handle 1 HEIC output buffer at a + // time (max dequeued buffer count is 1). bool appSegmentReady = (it.second.appSegmentBuffer.data != nullptr) && - !it.second.appSegmentWritten && it.second.result != nullptr; + !it.second.appSegmentWritten && it.second.result != nullptr && + it.second.muxer != nullptr; bool codecOutputReady = !it.second.codecOutputBuffers.empty(); bool codecInputReady = (it.second.yuvBuffer.data != nullptr) && (!it.second.codecInputBuffers.empty()); + bool hasOutputBuffer = it.second.muxer != nullptr || + (mDequeuedOutputBufferCnt < kMaxOutputSurfaceProducerCount); if ((!it.second.error) && (it.first < *currentTs) && - (appSegmentReady || codecOutputReady || codecInputReady)) { + (appSegmentReady || (codecOutputReady && hasOutputBuffer) || codecInputReady)) { *currentTs = it.first; + if (it.second.format == nullptr && mFormat != nullptr) { + it.second.format = mFormat->dup(); + } newInputAvailable = true; break; } @@ -716,15 +741,17 @@ status_t HeicCompositeStream::processInputFrame(nsecs_t timestamp, status_t res = OK; bool appSegmentReady = inputFrame.appSegmentBuffer.data != nullptr && - !inputFrame.appSegmentWritten && inputFrame.result != nullptr; + !inputFrame.appSegmentWritten && inputFrame.result != nullptr && + inputFrame.muxer != nullptr; bool codecOutputReady = inputFrame.codecOutputBuffers.size() > 0; bool codecInputReady = inputFrame.yuvBuffer.data != nullptr && - !inputFrame.codecInputBuffers.empty(); + !inputFrame.codecInputBuffers.empty(); + bool hasOutputBuffer = inputFrame.muxer != nullptr || + (mDequeuedOutputBufferCnt < kMaxOutputSurfaceProducerCount); - if (!appSegmentReady && !codecOutputReady && !codecInputReady) { - ALOGW("%s: No valid appSegmentBuffer/codec input/outputBuffer available!", __FUNCTION__); - return OK; - } + ALOGV("%s: [%" PRId64 "]: appSegmentReady %d, codecOutputReady %d, codecInputReady %d," + " dequeuedOutputBuffer %d", __FUNCTION__, timestamp, appSegmentReady, + codecOutputReady, codecInputReady, mDequeuedOutputBufferCnt); // Handle inputs for Hevc tiling if (codecInputReady) { @@ -736,7 +763,13 @@ status_t HeicCompositeStream::processInputFrame(nsecs_t timestamp, } } - // Initialize and start muxer if not yet done so + if (!(codecOutputReady && hasOutputBuffer) && !appSegmentReady) { + return OK; + } + + // Initialize and start muxer if not yet done so. In this case, + // codecOutputReady must be true. Otherwise, appSegmentReady is guaranteed + // to be false, and the function must have returned early. if (inputFrame.muxer == nullptr) { res = startMuxerForInputFrame(timestamp, inputFrame); if (res != OK) { @@ -747,7 +780,7 @@ status_t HeicCompositeStream::processInputFrame(nsecs_t timestamp, } // Write JPEG APP segments data to the muxer. - if (appSegmentReady && inputFrame.muxer != nullptr) { + if (appSegmentReady) { res = processAppSegment(timestamp, inputFrame); if (res != OK) { ALOGE("%s: Failed to process JPEG APP segments: %s (%d)", __FUNCTION__, @@ -766,12 +799,18 @@ status_t HeicCompositeStream::processInputFrame(nsecs_t timestamp, } } - if (inputFrame.appSegmentWritten && inputFrame.pendingOutputTiles == 0) { - res = processCompletedInputFrame(timestamp, inputFrame); - if (res != OK) { - ALOGE("%s: Failed to process completed input frame: %s (%d)", __FUNCTION__, - strerror(-res), res); - return res; + if (inputFrame.pendingOutputTiles == 0) { + if (inputFrame.appSegmentWritten) { + res = processCompletedInputFrame(timestamp, inputFrame); + if (res != OK) { + ALOGE("%s: Failed to process completed input frame: %s (%d)", __FUNCTION__, + strerror(-res), res); + return res; + } + } else if (mLockedAppSegmentBufferCnt == kMaxAcquiredAppSegment) { + ALOGE("%s: Out-of-order app segment buffers reaches limit %u", __FUNCTION__, + kMaxAcquiredAppSegment); + return INVALID_OPERATION; } } @@ -780,11 +819,6 @@ status_t HeicCompositeStream::processInputFrame(nsecs_t timestamp, status_t HeicCompositeStream::startMuxerForInputFrame(nsecs_t timestamp, InputFrame &inputFrame) { sp<ANativeWindow> outputANW = mOutputSurface; - if (inputFrame.codecOutputBuffers.size() == 0) { - // No single codec output buffer has been generated. Continue to - // wait. - return OK; - } auto res = outputANW->dequeueBuffer(mOutputSurface.get(), &inputFrame.anb, &inputFrame.fenceFd); if (res != OK) { @@ -792,6 +826,7 @@ status_t HeicCompositeStream::startMuxerForInputFrame(nsecs_t timestamp, InputFr res); return res; } + mDequeuedOutputBufferCnt++; // Combine current thread id, stream id and timestamp to uniquely identify image. std::ostringstream tempOutputFile; @@ -828,7 +863,7 @@ status_t HeicCompositeStream::startMuxerForInputFrame(nsecs_t timestamp, InputFr } } - ssize_t trackId = inputFrame.muxer->addTrack(mFormat); + ssize_t trackId = inputFrame.muxer->addTrack(inputFrame.format); if (trackId < 0) { ALOGE("%s: Failed to addTrack to the muxer: %zd", __FUNCTION__, trackId); return NO_INIT; @@ -844,6 +879,8 @@ status_t HeicCompositeStream::startMuxerForInputFrame(nsecs_t timestamp, InputFr return res; } + ALOGV("%s: [%" PRId64 "]: Muxer started for inputFrame", __FUNCTION__, + timestamp); return OK; } @@ -852,9 +889,6 @@ status_t HeicCompositeStream::processAppSegment(nsecs_t timestamp, InputFrame &i auto appSegmentSize = findAppSegmentsSize(inputFrame.appSegmentBuffer.data, inputFrame.appSegmentBuffer.width * inputFrame.appSegmentBuffer.height, &app1Size); - ALOGV("%s: appSegmentSize is %zu, width %d, height %d, app1Size %zu", __FUNCTION__, - appSegmentSize, inputFrame.appSegmentBuffer.width, - inputFrame.appSegmentBuffer.height, app1Size); if (appSegmentSize == 0) { ALOGE("%s: Failed to find JPEG APP segment size", __FUNCTION__); return NO_INIT; @@ -910,7 +944,16 @@ status_t HeicCompositeStream::processAppSegment(nsecs_t timestamp, InputFrame &i __FUNCTION__, strerror(-res), res); return res; } + + ALOGV("%s: [%" PRId64 "]: appSegmentSize is %zu, width %d, height %d, app1Size %zu", + __FUNCTION__, timestamp, appSegmentSize, inputFrame.appSegmentBuffer.width, + inputFrame.appSegmentBuffer.height, app1Size); + inputFrame.appSegmentWritten = true; + // Release the buffer now so any pending input app segments can be processed + mAppSegmentConsumer->unlockBuffer(inputFrame.appSegmentBuffer); + inputFrame.appSegmentBuffer.data = nullptr; + mLockedAppSegmentBufferCnt--; return OK; } @@ -934,8 +977,9 @@ status_t HeicCompositeStream::processCodecInputFrame(InputFrame &inputFrame) { mOutputWidth - tileX * mGridWidth : mGridWidth; size_t height = (tileY == static_cast<size_t>(mGridRows) - 1) ? mOutputHeight - tileY * mGridHeight : mGridHeight; - ALOGV("%s: inputBuffer tileIndex [%zu, %zu], top %zu, left %zu, width %zu, height %zu", - __FUNCTION__, tileX, tileY, top, left, width, height); + ALOGV("%s: inputBuffer tileIndex [%zu, %zu], top %zu, left %zu, width %zu, height %zu," + " timeUs %" PRId64, __FUNCTION__, tileX, tileY, top, left, width, height, + inputBuffer.timeUs); res = copyOneYuvTile(buffer, inputFrame.yuvBuffer, top, left, width, height); if (res != OK) { @@ -990,6 +1034,9 @@ status_t HeicCompositeStream::processOneCodecOutputFrame(nsecs_t timestamp, } inputFrame.codecOutputBuffers.erase(inputFrame.codecOutputBuffers.begin()); + + ALOGV("%s: [%" PRId64 "]: Output buffer index %d", + __FUNCTION__, timestamp, it->index); return OK; } @@ -1046,7 +1093,9 @@ status_t HeicCompositeStream::processCompletedInputFrame(nsecs_t timestamp, return res; } inputFrame.anb = nullptr; + mDequeuedOutputBufferCnt--; + ALOGV("%s: [%" PRId64 "]", __FUNCTION__, timestamp); ATRACE_ASYNC_END("HEIC capture", inputFrame.frameNumber); return OK; } @@ -1060,7 +1109,6 @@ void HeicCompositeStream::releaseInputFrameLocked(InputFrame *inputFrame /*out*/ if (inputFrame->appSegmentBuffer.data != nullptr) { mAppSegmentConsumer->unlockBuffer(inputFrame->appSegmentBuffer); inputFrame->appSegmentBuffer.data = nullptr; - mAppSegmentBufferAcquired = false; } while (!inputFrame->codecOutputBuffers.empty()) { @@ -1098,11 +1146,13 @@ void HeicCompositeStream::releaseInputFrameLocked(InputFrame *inputFrame /*out*/ } } -void HeicCompositeStream::releaseInputFramesLocked(int64_t currentTs) { +void HeicCompositeStream::releaseInputFramesLocked() { auto it = mPendingInputFrames.begin(); while (it != mPendingInputFrames.end()) { - if (it->first <= currentTs) { - releaseInputFrameLocked(&it->second); + auto& inputFrame = it->second; + if (inputFrame.error || + (inputFrame.appSegmentWritten && inputFrame.pendingOutputTiles == 0)) { + releaseInputFrameLocked(&inputFrame); it = mPendingInputFrames.erase(it); } else { it++; @@ -1506,7 +1556,7 @@ bool HeicCompositeStream::threadLoop() { // In case we landed in error state, return any pending buffers and // halt all further processing. compilePendingInputLocked(); - releaseInputFramesLocked(currentTs); + releaseInputFramesLocked(); return false; } @@ -1548,11 +1598,7 @@ bool HeicCompositeStream::threadLoop() { mPendingInputFrames[currentTs].error = true; } - if (mPendingInputFrames[currentTs].error || - (mPendingInputFrames[currentTs].appSegmentWritten && - mPendingInputFrames[currentTs].pendingOutputTiles == 0)) { - releaseInputFramesLocked(currentTs); - } + releaseInputFramesLocked(); return true; } @@ -1671,8 +1717,13 @@ void HeicCompositeStream::CodecCallbackHandler::onMessageReceived(const sp<AMess ALOGE("CB_OUTPUT_FORMAT_CHANGED: format is expected."); break; } - - parent->onHeicFormatChanged(format); + // Here format is MediaCodec's internal copy of output format. + // Make a copy since onHeicFormatChanged() might modify it. + sp<AMessage> formatCopy; + if (format != nullptr) { + formatCopy = format->dup(); + } + parent->onHeicFormatChanged(formatCopy); break; } diff --git a/services/camera/libcameraservice/api2/HeicCompositeStream.h b/services/camera/libcameraservice/api2/HeicCompositeStream.h index 260c68e081..04e7b83c20 100644 --- a/services/camera/libcameraservice/api2/HeicCompositeStream.h +++ b/services/camera/libcameraservice/api2/HeicCompositeStream.h @@ -25,6 +25,7 @@ #include <media/hardware/VideoAPI.h> #include <media/MediaCodecBuffer.h> #include <media/stagefright/foundation/ALooper.h> +#include <media/stagefright/foundation/AMessage.h> #include <media/stagefright/MediaCodec.h> #include <media/stagefright/MediaMuxer.h> @@ -157,6 +158,7 @@ private: bool errorNotified; int64_t frameNumber; + sp<AMessage> format; sp<MediaMuxer> muxer; int fenceFd; int fileFd; @@ -187,7 +189,7 @@ private: status_t processCompletedInputFrame(nsecs_t timestamp, InputFrame &inputFrame); void releaseInputFrameLocked(InputFrame *inputFrame /*out*/); - void releaseInputFramesLocked(int64_t currentTs); + void releaseInputFramesLocked(); size_t findAppSegmentsSize(const uint8_t* appSegmentBuffer, size_t maxSize, size_t* app1SegmentSize); @@ -205,11 +207,13 @@ private: static_cast<android_dataspace>(HAL_DATASPACE_JPEG_APP_SEGMENTS); static const android_dataspace kHeifDataSpace = static_cast<android_dataspace>(HAL_DATASPACE_HEIF); + // Use the limit of pipeline depth in the API sepc as maximum number of acquired + // app segment buffers. + static const uint32_t kMaxAcquiredAppSegment = 8; int mAppSegmentStreamId, mAppSegmentSurfaceId; sp<CpuConsumer> mAppSegmentConsumer; sp<Surface> mAppSegmentSurface; - bool mAppSegmentBufferAcquired; size_t mAppSegmentMaxSize; CameraMetadata mStaticInfo; @@ -218,9 +222,10 @@ private: sp<CpuConsumer> mMainImageConsumer; // Only applicable for HEVC codec. bool mYuvBufferAcquired; // Only applicable to HEVC codec + static const int32_t kMaxOutputSurfaceProducerCount = 1; sp<Surface> mOutputSurface; sp<ProducerListener> mProducerListener; - + int32_t mDequeuedOutputBufferCnt; // Map from frame number to JPEG setting of orientation+quality std::map<int64_t, std::pair<int32_t, int32_t>> mSettingsByFrameNumber; @@ -229,11 +234,12 @@ private: // Keep all incoming APP segment Blob buffer pending further processing. std::vector<int64_t> mInputAppSegmentBuffers; + int32_t mLockedAppSegmentBufferCnt; // Keep all incoming HEIC blob buffer pending further processing. std::vector<CodecOutputBufferInfo> mCodecOutputBuffers; std::queue<int64_t> mCodecOutputBufferTimestamps; - size_t mOutputBufferCounter; + size_t mCodecOutputCounter; // Keep all incoming Yuv buffer pending tiling and encoding (for HEVC YUV tiling only) std::vector<int64_t> mInputYuvBuffers; diff --git a/services/camera/libcameraservice/common/CameraProviderManager.cpp b/services/camera/libcameraservice/common/CameraProviderManager.cpp index 09638d0289..fdb5657500 100644 --- a/services/camera/libcameraservice/common/CameraProviderManager.cpp +++ b/services/camera/libcameraservice/common/CameraProviderManager.cpp @@ -29,6 +29,7 @@ #include <future> #include <inttypes.h> #include <hardware/camera_common.h> +#include <android/hidl/manager/1.2/IServiceManager.h> #include <hidl/ServiceManagement.h> #include <functional> #include <camera_metadata_hidden.h> @@ -47,10 +48,6 @@ using namespace ::android::hardware::camera::common::V1_0; using std::literals::chrono_literals::operator""s; namespace { -// Hardcoded name for the passthrough HAL implementation, since it can't be discovered via the -// service manager -const std::string kLegacyProviderName("legacy/0"); -const std::string kExternalProviderName("external/0"); const bool kEnableLazyHal(property_get_bool("ro.camera.enableLazyHal", false)); } // anonymous namespace @@ -62,6 +59,19 @@ CameraProviderManager::sHardwareServiceInteractionProxy{}; CameraProviderManager::~CameraProviderManager() { } +hardware::hidl_vec<hardware::hidl_string> +CameraProviderManager::HardwareServiceInteractionProxy::listServices() { + hardware::hidl_vec<hardware::hidl_string> ret; + auto manager = hardware::defaultServiceManager1_2(); + if (manager != nullptr) { + manager->listManifestByInterface(provider::V2_4::ICameraProvider::descriptor, + [&ret](const hardware::hidl_vec<hardware::hidl_string> ®istered) { + ret = registered; + }); + } + return ret; +} + status_t CameraProviderManager::initialize(wp<CameraProviderManager::StatusListener> listener, ServiceInteractionProxy* proxy) { std::lock_guard<std::mutex> lock(mInterfaceMutex); @@ -84,9 +94,10 @@ status_t CameraProviderManager::initialize(wp<CameraProviderManager::StatusListe return INVALID_OPERATION; } - // See if there's a passthrough HAL, but let's not complain if there's not - addProviderLocked(kLegacyProviderName, /*expected*/ false); - addProviderLocked(kExternalProviderName, /*expected*/ false); + + for (const auto& instance : mServiceProxy->listServices()) { + this->addProviderLocked(instance); + } IPCThreadState::self()->flushCommands(); @@ -97,7 +108,13 @@ int CameraProviderManager::getCameraCount() const { std::lock_guard<std::mutex> lock(mInterfaceMutex); int count = 0; for (auto& provider : mProviders) { - count += provider->mUniqueCameraIds.size(); + for (auto& id : provider->mUniqueCameraIds) { + // Hidden secure camera ids are not to be exposed to camera1 api. + if (isPublicallyHiddenSecureCameraLocked(id)) { + continue; + } + count++; + } } return count; } @@ -123,7 +140,11 @@ std::vector<std::string> CameraProviderManager::getAPI1CompatibleCameraDeviceIds // for each camera facing, only take the first id advertised by HAL in // all [logical, physical1, physical2, ...] id combos, and filter out the rest. filterLogicalCameraIdsLocked(providerDeviceIds); - + // Hidden secure camera ids are not to be exposed to camera1 api. + providerDeviceIds.erase(std::remove_if(providerDeviceIds.begin(), providerDeviceIds.end(), + [this](const std::string& s) { + return this->isPublicallyHiddenSecureCameraLocked(s);}), + providerDeviceIds.end()); deviceIds.insert(deviceIds.end(), providerDeviceIds.begin(), providerDeviceIds.end()); } @@ -1035,23 +1056,42 @@ bool CameraProviderManager::isLogicalCamera(const std::string& id, return deviceInfo->mIsLogicalCamera; } -bool CameraProviderManager::isPublicallyHiddenSecureCamera(const std::string& id) { +bool CameraProviderManager::isPublicallyHiddenSecureCamera(const std::string& id) const { std::lock_guard<std::mutex> lock(mInterfaceMutex); + return isPublicallyHiddenSecureCameraLocked(id); +} +bool CameraProviderManager::isPublicallyHiddenSecureCameraLocked(const std::string& id) const { auto deviceInfo = findDeviceInfoLocked(id); - if (deviceInfo == nullptr) { - return false; + if (deviceInfo != nullptr) { + return deviceInfo->mIsPublicallyHiddenSecureCamera; + } + // If this is a hidden physical camera, we should return what kind of + // camera the enclosing logical camera is. + auto isHiddenAndParent = isHiddenPhysicalCameraInternal(id); + if (isHiddenAndParent.first) { + LOG_ALWAYS_FATAL_IF(id == isHiddenAndParent.second->mId, + "%s: hidden physical camera id %s and enclosing logical camera id %s are the same", + __FUNCTION__, id.c_str(), isHiddenAndParent.second->mId.c_str()); + return isPublicallyHiddenSecureCameraLocked(isHiddenAndParent.second->mId); } - return deviceInfo->mIsPublicallyHiddenSecureCamera; + // Invalid camera id + return true; +} + +bool CameraProviderManager::isHiddenPhysicalCamera(const std::string& cameraId) const { + return isHiddenPhysicalCameraInternal(cameraId).first; } -bool CameraProviderManager::isHiddenPhysicalCamera(const std::string& cameraId) { +std::pair<bool, CameraProviderManager::ProviderInfo::DeviceInfo *> +CameraProviderManager::isHiddenPhysicalCameraInternal(const std::string& cameraId) const { + auto falseRet = std::make_pair(false, nullptr); for (auto& provider : mProviders) { for (auto& deviceInfo : provider->mDevices) { if (deviceInfo->mId == cameraId) { // cameraId is found in public camera IDs advertised by the // provider. - return false; + return falseRet; } } } @@ -1063,7 +1103,7 @@ bool CameraProviderManager::isHiddenPhysicalCamera(const std::string& cameraId) if (res != OK) { ALOGE("%s: Failed to getCameraCharacteristics for id %s", __FUNCTION__, deviceInfo->mId.c_str()); - return false; + return falseRet; } std::vector<std::string> physicalIds; @@ -1075,19 +1115,19 @@ bool CameraProviderManager::isHiddenPhysicalCamera(const std::string& cameraId) if (deviceVersion < CAMERA_DEVICE_API_VERSION_3_5) { ALOGE("%s: Wrong deviceVersion %x for hiddenPhysicalCameraId %s", __FUNCTION__, deviceVersion, cameraId.c_str()); - return false; + return falseRet; } else { - return true; + return std::make_pair(true, deviceInfo.get()); } } } } } - return false; + return falseRet; } -status_t CameraProviderManager::addProviderLocked(const std::string& newProvider, bool expected) { +status_t CameraProviderManager::addProviderLocked(const std::string& newProvider) { for (const auto& providerInfo : mProviders) { if (providerInfo->mProviderName == newProvider) { ALOGW("%s: Camera provider HAL with name '%s' already registered", __FUNCTION__, @@ -1100,13 +1140,9 @@ status_t CameraProviderManager::addProviderLocked(const std::string& newProvider interface = mServiceProxy->getService(newProvider); if (interface == nullptr) { - if (expected) { - ALOGE("%s: Camera provider HAL '%s' is not actually available", __FUNCTION__, - newProvider.c_str()); - return BAD_VALUE; - } else { - return OK; - } + ALOGE("%s: Camera provider HAL '%s' is not actually available", __FUNCTION__, + newProvider.c_str()); + return BAD_VALUE; } sp<ProviderInfo> providerInfo = new ProviderInfo(newProvider, this); @@ -2058,6 +2094,13 @@ status_t CameraProviderManager::ProviderInfo::DeviceInfo3::getCameraInfo( return OK; } bool CameraProviderManager::ProviderInfo::DeviceInfo3::isAPI1Compatible() const { + // Do not advertise NIR cameras to API1 camera app. + camera_metadata_ro_entry cfa = mCameraCharacteristics.find( + ANDROID_SENSOR_INFO_COLOR_FILTER_ARRANGEMENT); + if (cfa.count == 1 && cfa.data.u8[0] == ANDROID_SENSOR_INFO_COLOR_FILTER_ARRANGEMENT_NIR) { + return false; + } + bool isBackwardCompatible = false; camera_metadata_ro_entry_t caps = mCameraCharacteristics.find( ANDROID_REQUEST_AVAILABLE_CAPABILITIES); diff --git a/services/camera/libcameraservice/common/CameraProviderManager.h b/services/camera/libcameraservice/common/CameraProviderManager.h index a42fb4d60b..3a4655c8cf 100644 --- a/services/camera/libcameraservice/common/CameraProviderManager.h +++ b/services/camera/libcameraservice/common/CameraProviderManager.h @@ -78,6 +78,7 @@ public: ¬ification) = 0; virtual sp<hardware::camera::provider::V2_4::ICameraProvider> getService( const std::string &serviceName) = 0; + virtual hardware::hidl_vec<hardware::hidl_string> listServices() = 0; virtual ~ServiceInteractionProxy() {} }; @@ -95,6 +96,8 @@ public: const std::string &serviceName) override { return hardware::camera::provider::V2_4::ICameraProvider::getService(serviceName); } + + virtual hardware::hidl_vec<hardware::hidl_string> listServices() override; }; /** @@ -269,8 +272,8 @@ public: */ bool isLogicalCamera(const std::string& id, std::vector<std::string>* physicalCameraIds); - bool isPublicallyHiddenSecureCamera(const std::string& id); - bool isHiddenPhysicalCamera(const std::string& cameraId); + bool isPublicallyHiddenSecureCamera(const std::string& id) const; + bool isHiddenPhysicalCamera(const std::string& cameraId) const; static const float kDepthARTolerance; private: @@ -567,7 +570,7 @@ private: hardware::hidl_version minVersion = hardware::hidl_version{0,0}, hardware::hidl_version maxVersion = hardware::hidl_version{1000,0}) const; - status_t addProviderLocked(const std::string& newProvider, bool expected = true); + status_t addProviderLocked(const std::string& newProvider); status_t removeProvider(const std::string& provider); sp<StatusListener> getStatusListener() const; @@ -591,7 +594,15 @@ private: status_t getCameraCharacteristicsLocked(const std::string &id, CameraMetadata* characteristics) const; + + bool isPublicallyHiddenSecureCameraLocked(const std::string& id) const; + void filterLogicalCameraIdsLocked(std::vector<std::string>& deviceIds) const; + + bool isPublicallyHiddenSecureCameraLocked(const std::string& id); + + std::pair<bool, CameraProviderManager::ProviderInfo::DeviceInfo *> + isHiddenPhysicalCameraInternal(const std::string& cameraId) const; }; } // namespace android diff --git a/services/camera/libcameraservice/common/DepthPhotoProcessor.cpp b/services/camera/libcameraservice/common/DepthPhotoProcessor.cpp index 3c90de0d6e..94541d87a9 100644 --- a/services/camera/libcameraservice/common/DepthPhotoProcessor.cpp +++ b/services/camera/libcameraservice/common/DepthPhotoProcessor.cpp @@ -419,7 +419,7 @@ extern "C" int processDepthPhotoFrame(DepthPhotoInputFrame inputFrame, size_t de std::vector<std::unique_ptr<Item>> items; std::vector<std::unique_ptr<Camera>> cameraList; - auto image = Image::FromDataForPrimaryImage("android/mainimage", &items); + auto image = Image::FromDataForPrimaryImage("image/jpeg", &items); std::unique_ptr<CameraParams> cameraParams(new CameraParams(std::move(image))); if (cameraParams == nullptr) { ALOGE("%s: Failed to initialize camera parameters", __FUNCTION__); diff --git a/services/camera/libcameraservice/device3/Camera3Device.cpp b/services/camera/libcameraservice/device3/Camera3Device.cpp index e347127a58..93e18cfee3 100644 --- a/services/camera/libcameraservice/device3/Camera3Device.cpp +++ b/services/camera/libcameraservice/device3/Camera3Device.cpp @@ -29,6 +29,9 @@ #define CLOGE(fmt, ...) ALOGE("Camera %s: %s: " fmt, mId.string(), __FUNCTION__, \ ##__VA_ARGS__) +#define CLOGW(fmt, ...) ALOGW("Camera %s: %s: " fmt, mId.string(), __FUNCTION__, \ + ##__VA_ARGS__) + // Convenience macros for transitioning to the error state #define SET_ERR(fmt, ...) setErrorState( \ "%s: " fmt, __FUNCTION__, \ @@ -3267,14 +3270,19 @@ void Camera3Device::removeInFlightRequestIfReadyLocked(int idx) { ALOGVV("%s: removed frame %d from InFlightMap", __FUNCTION__, frameNumber); } - // Sanity check - if we have too many in-flight frames, something has - // likely gone wrong - if (!mIsConstrainedHighSpeedConfiguration && mInFlightMap.size() > kInFlightWarnLimit) { - CLOGE("In-flight list too large: %zu", mInFlightMap.size()); - } else if (mIsConstrainedHighSpeedConfiguration && mInFlightMap.size() > - kInFlightWarnLimitHighSpeed) { - CLOGE("In-flight list too large for high speed configuration: %zu", - mInFlightMap.size()); + // Sanity check - if we have too many in-flight frames with long total inflight duration, + // something has likely gone wrong. This might still be legit only if application send in + // a long burst of long exposure requests. + if (mExpectedInflightDuration > kMinWarnInflightDuration) { + if (!mIsConstrainedHighSpeedConfiguration && mInFlightMap.size() > kInFlightWarnLimit) { + CLOGW("In-flight list too large: %zu, total inflight duration %" PRIu64, + mInFlightMap.size(), mExpectedInflightDuration); + } else if (mIsConstrainedHighSpeedConfiguration && mInFlightMap.size() > + kInFlightWarnLimitHighSpeed) { + CLOGW("In-flight list too large for high speed configuration: %zu," + "total inflight duration %" PRIu64, + mInFlightMap.size(), mExpectedInflightDuration); + } } } diff --git a/services/camera/libcameraservice/device3/Camera3Device.h b/services/camera/libcameraservice/device3/Camera3Device.h index 6e8ac849c0..cae34ce5c4 100644 --- a/services/camera/libcameraservice/device3/Camera3Device.h +++ b/services/camera/libcameraservice/device3/Camera3Device.h @@ -227,6 +227,7 @@ class Camera3Device : static const size_t kDumpLockAttempts = 10; static const size_t kDumpSleepDuration = 100000; // 0.10 sec static const nsecs_t kActiveTimeout = 500000000; // 500 ms + static const nsecs_t kMinWarnInflightDuration = 5000000000; // 5 s static const size_t kInFlightWarnLimit = 30; static const size_t kInFlightWarnLimitHighSpeed = 256; // batch size 32 * pipe depth 8 static const nsecs_t kDefaultExpectedDuration = 100000000; // 100 ms diff --git a/services/camera/libcameraservice/device3/Camera3OutputStream.cpp b/services/camera/libcameraservice/device3/Camera3OutputStream.cpp index acb8b3cac1..e1d35e816e 100644 --- a/services/camera/libcameraservice/device3/Camera3OutputStream.cpp +++ b/services/camera/libcameraservice/device3/Camera3OutputStream.cpp @@ -54,9 +54,8 @@ Camera3OutputStream::Camera3OutputStream(int id, mState = STATE_ERROR; } - if (setId > CAMERA3_STREAM_SET_ID_INVALID) { - mBufferReleasedListener = new BufferReleasedListener(this); - } + bool needsReleaseNotify = setId > CAMERA3_STREAM_SET_ID_INVALID; + mBufferProducerListener = new BufferProducerListener(this, needsReleaseNotify); } Camera3OutputStream::Camera3OutputStream(int id, @@ -87,9 +86,8 @@ Camera3OutputStream::Camera3OutputStream(int id, mState = STATE_ERROR; } - if (setId > CAMERA3_STREAM_SET_ID_INVALID) { - mBufferReleasedListener = new BufferReleasedListener(this); - } + bool needsReleaseNotify = setId > CAMERA3_STREAM_SET_ID_INVALID; + mBufferProducerListener = new BufferProducerListener(this, needsReleaseNotify); } Camera3OutputStream::Camera3OutputStream(int id, @@ -124,10 +122,8 @@ Camera3OutputStream::Camera3OutputStream(int id, } mConsumerName = String8("Deferred"); - if (setId > CAMERA3_STREAM_SET_ID_INVALID) { - mBufferReleasedListener = new BufferReleasedListener(this); - } - + bool needsReleaseNotify = setId > CAMERA3_STREAM_SET_ID_INVALID; + mBufferProducerListener = new BufferProducerListener(this, needsReleaseNotify); } Camera3OutputStream::Camera3OutputStream(int id, camera3_stream_type_t type, @@ -151,9 +147,8 @@ Camera3OutputStream::Camera3OutputStream(int id, camera3_stream_type_t type, mDropBuffers(false), mDequeueBufferLatency(kDequeueLatencyBinSize) { - if (setId > CAMERA3_STREAM_SET_ID_INVALID) { - mBufferReleasedListener = new BufferReleasedListener(this); - } + bool needsReleaseNotify = setId > CAMERA3_STREAM_SET_ID_INVALID; + mBufferProducerListener = new BufferProducerListener(this, needsReleaseNotify); // Subclasses expected to initialize mConsumer themselves } @@ -261,7 +256,7 @@ status_t Camera3OutputStream::returnBufferCheckedLocked( notifyBufferReleased(anwBuffer); if (mUseBufferManager) { // Return this buffer back to buffer manager. - mBufferReleasedListener->onBufferReleased(); + mBufferProducerListener->onBufferReleased(); } } else { if (mTraceFirstBuffer && (stream_type == CAMERA3_STREAM_OUTPUT)) { @@ -387,8 +382,8 @@ status_t Camera3OutputStream::configureConsumerQueueLocked() { // Configure consumer-side ANativeWindow interface. The listener may be used // to notify buffer manager (if it is used) of the returned buffers. res = mConsumer->connect(NATIVE_WINDOW_API_CAMERA, - /*listener*/mBufferReleasedListener, - /*reportBufferRemoval*/true); + /*reportBufferRemoval*/true, + /*listener*/mBufferProducerListener); if (res != OK) { ALOGE("%s: Unable to connect to native window for stream %d", __FUNCTION__, mId); @@ -790,7 +785,7 @@ status_t Camera3OutputStream::updateStream(const std::vector<sp<Surface>> &/*out return INVALID_OPERATION; } -void Camera3OutputStream::BufferReleasedListener::onBufferReleased() { +void Camera3OutputStream::BufferProducerListener::onBufferReleased() { sp<Camera3OutputStream> stream = mParent.promote(); if (stream == nullptr) { ALOGV("%s: Parent camera3 output stream was destroyed", __FUNCTION__); @@ -823,6 +818,25 @@ void Camera3OutputStream::BufferReleasedListener::onBufferReleased() { } } +void Camera3OutputStream::BufferProducerListener::onBuffersDiscarded( + const std::vector<sp<GraphicBuffer>>& buffers) { + sp<Camera3OutputStream> stream = mParent.promote(); + if (stream == nullptr) { + ALOGV("%s: Parent camera3 output stream was destroyed", __FUNCTION__); + return; + } + + if (buffers.size() > 0) { + Mutex::Autolock l(stream->mLock); + stream->onBuffersRemovedLocked(buffers); + if (stream->mUseBufferManager) { + stream->mBufferManager->onBuffersRemoved(stream->getId(), + stream->getStreamSetId(), buffers.size()); + } + ALOGV("Stream %d: %zu Buffers discarded.", stream->getId(), buffers.size()); + } +} + void Camera3OutputStream::onBuffersRemovedLocked( const std::vector<sp<GraphicBuffer>>& removedBuffers) { sp<Camera3StreamBufferFreedListener> callback = mBufferFreedListener.promote(); diff --git a/services/camera/libcameraservice/device3/Camera3OutputStream.h b/services/camera/libcameraservice/device3/Camera3OutputStream.h index 729c655d57..b4e49f914e 100644 --- a/services/camera/libcameraservice/device3/Camera3OutputStream.h +++ b/services/camera/libcameraservice/device3/Camera3OutputStream.h @@ -146,18 +146,22 @@ class Camera3OutputStream : */ virtual status_t setConsumers(const std::vector<sp<Surface>>& consumers); - class BufferReleasedListener : public BnProducerListener { + class BufferProducerListener : public SurfaceListener { public: - BufferReleasedListener(wp<Camera3OutputStream> parent) : mParent(parent) {} + BufferProducerListener(wp<Camera3OutputStream> parent, bool needsReleaseNotify) + : mParent(parent), mNeedsReleaseNotify(needsReleaseNotify) {} - /** - * Implementation of IProducerListener, used to notify this stream that the consumer - * has returned a buffer and it is ready to return to Camera3BufferManager for reuse. - */ - virtual void onBufferReleased(); + /** + * Implementation of IProducerListener, used to notify this stream that the consumer + * has returned a buffer and it is ready to return to Camera3BufferManager for reuse. + */ + virtual void onBufferReleased(); + virtual bool needsReleaseNotify() { return mNeedsReleaseNotify; } + virtual void onBuffersDiscarded(const std::vector<sp<GraphicBuffer>>& buffers); private: - wp<Camera3OutputStream> mParent; + wp<Camera3OutputStream> mParent; + bool mNeedsReleaseNotify; }; virtual status_t detachBuffer(sp<GraphicBuffer>* buffer, int* fenceFd); @@ -262,10 +266,10 @@ class Camera3OutputStream : sp<Camera3BufferManager> mBufferManager; /** - * Buffer released listener, used to notify the buffer manager that a buffer is released - * from consumer side. + * Buffer producer listener, used to handle notification when a buffer is released + * from consumer side, or a set of buffers are discarded by the consumer. */ - sp<BufferReleasedListener> mBufferReleasedListener; + sp<BufferProducerListener> mBufferProducerListener; /** * Flag indicating if the buffer manager is used to allocate the stream buffers diff --git a/services/camera/libcameraservice/device3/Camera3StreamSplitter.cpp b/services/camera/libcameraservice/device3/Camera3StreamSplitter.cpp index 84c2ec7544..3089181890 100644 --- a/services/camera/libcameraservice/device3/Camera3StreamSplitter.cpp +++ b/services/camera/libcameraservice/device3/Camera3StreamSplitter.cpp @@ -139,7 +139,9 @@ void Camera3StreamSplitter::disconnect() { mOutputSlots.clear(); mConsumerBufferCount.clear(); - mConsumer->consumerDisconnect(); + if (mConsumer.get() != nullptr) { + mConsumer->consumerDisconnect(); + } if (mBuffers.size() > 0) { SP_LOGW("%zu buffers still being tracked", mBuffers.size()); diff --git a/services/camera/libcameraservice/hidl/AidlCameraServiceListener.h b/services/camera/libcameraservice/hidl/AidlCameraServiceListener.h index 0f6be79be0..b9e58574c6 100644 --- a/services/camera/libcameraservice/hidl/AidlCameraServiceListener.h +++ b/services/camera/libcameraservice/hidl/AidlCameraServiceListener.h @@ -54,6 +54,15 @@ struct H2BCameraServiceListener : // TODO: no implementation yet. return binder::Status::ok(); } + virtual binder::Status onCameraOpened(const ::android::String16& /*cameraId*/, + const ::android::String16& /*clientPackageId*/) { + // empty implementation + return binder::Status::ok(); + } + virtual binder::Status onCameraClosed(const ::android::String16& /*cameraId*/) { + // empty implementation + return binder::Status::ok(); + } }; } // implementation diff --git a/services/camera/libcameraservice/tests/CameraProviderManagerTest.cpp b/services/camera/libcameraservice/tests/CameraProviderManagerTest.cpp index f47e5a581e..78d737d1b2 100644 --- a/services/camera/libcameraservice/tests/CameraProviderManagerTest.cpp +++ b/services/camera/libcameraservice/tests/CameraProviderManagerTest.cpp @@ -205,6 +205,11 @@ struct TestInteractionProxy : public CameraProviderManager::ServiceInteractionPr return mTestCameraProvider; } + virtual hardware::hidl_vec<hardware::hidl_string> listServices() override { + hardware::hidl_vec<hardware::hidl_string> ret = {"test/0"}; + return ret; + } + }; struct TestStatusListener : public CameraProviderManager::StatusListener { @@ -231,37 +236,24 @@ TEST(CameraProviderManagerTest, InitializeTest) { vendorSection); serviceProxy.setProvider(provider); + int numProviders = static_cast<int>(serviceProxy.listServices().size()); + res = providerManager->initialize(statusListener, &serviceProxy); ASSERT_EQ(res, OK) << "Unable to initialize provider manager"; // Check that both "legacy" and "external" providers (really the same object) are called // once for all the init methods - EXPECT_EQ(provider->mCalledCounter[TestICameraProvider::SET_CALLBACK], 2) << + EXPECT_EQ(provider->mCalledCounter[TestICameraProvider::SET_CALLBACK], numProviders) << "Only one call to setCallback per provider expected during init"; - EXPECT_EQ(provider->mCalledCounter[TestICameraProvider::GET_VENDOR_TAGS], 2) << + EXPECT_EQ(provider->mCalledCounter[TestICameraProvider::GET_VENDOR_TAGS], numProviders) << "Only one call to getVendorTags per provider expected during init"; - EXPECT_EQ(provider->mCalledCounter[TestICameraProvider::IS_SET_TORCH_MODE_SUPPORTED], 2) << + EXPECT_EQ(provider->mCalledCounter[TestICameraProvider::IS_SET_TORCH_MODE_SUPPORTED], + numProviders) << "Only one call to isSetTorchModeSupported per provider expected during init"; - EXPECT_EQ(provider->mCalledCounter[TestICameraProvider::GET_CAMERA_ID_LIST], 2) << + EXPECT_EQ(provider->mCalledCounter[TestICameraProvider::GET_CAMERA_ID_LIST], numProviders) << "Only one call to getCameraIdList per provider expected during init"; - EXPECT_EQ(provider->mCalledCounter[TestICameraProvider::NOTIFY_DEVICE_STATE], 2) << + EXPECT_EQ(provider->mCalledCounter[TestICameraProvider::NOTIFY_DEVICE_STATE], numProviders) << "Only one call to notifyDeviceState per provider expected during init"; - std::string legacyInstanceName = "legacy/0"; - std::string externalInstanceName = "external/0"; - bool gotLegacy = false; - bool gotExternal = false; - EXPECT_EQ(2u, serviceProxy.mLastRequestedServiceNames.size()) << - "Only two service queries expected to be seen by hardware service manager"; - - for (auto& serviceName : serviceProxy.mLastRequestedServiceNames) { - if (serviceName == legacyInstanceName) gotLegacy = true; - if (serviceName == externalInstanceName) gotExternal = true; - } - ASSERT_TRUE(gotLegacy) << - "Legacy instance not requested from service manager"; - ASSERT_TRUE(gotExternal) << - "External instance not requested from service manager"; - hardware::hidl_string testProviderFqInterfaceName = "android.hardware.camera.provider@2.4::ICameraProvider"; hardware::hidl_string testProviderInstanceName = "test/0"; diff --git a/services/mediacodec/main_codecservice.cpp b/services/mediacodec/main_codecservice.cpp index f668c3381b..6a82b1b6b8 100644 --- a/services/mediacodec/main_codecservice.cpp +++ b/services/mediacodec/main_codecservice.cpp @@ -21,6 +21,7 @@ #include "minijail.h" #include <binder/ProcessState.h> +#include <cutils/properties.h> #include <hidl/HidlTransportSupport.h> #include <media/stagefright/omx/1.0/Omx.h> #include <media/stagefright/omx/1.0/OmxStore.h> @@ -57,7 +58,8 @@ int main(int argc __unused, char** argv) } else { LOG(INFO) << "IOmx HAL service created."; } - sp<IOmxStore> omxStore = new implementation::OmxStore(omx); + sp<IOmxStore> omxStore = new implementation::OmxStore( + property_get_int64("vendor.media.omx", 1) ? omx : nullptr); if (omxStore == nullptr) { LOG(ERROR) << "Cannot create IOmxStore HAL service."; } else if (omxStore->registerAsService() != OK) { diff --git a/services/mediacodec/registrant/Android.bp b/services/mediacodec/registrant/Android.bp index 17c2e0261b..fa5bc4a15a 100644 --- a/services/mediacodec/registrant/Android.bp +++ b/services/mediacodec/registrant/Android.bp @@ -42,7 +42,8 @@ cc_library_shared { "libcodec2_soft_opusenc", "libcodec2_soft_vp8dec", "libcodec2_soft_vp9dec", - "libcodec2_soft_av1dec", + // "libcodec2_soft_av1dec_aom", // replaced by the gav1 implementation + "libcodec2_soft_av1dec_gav1", "libcodec2_soft_vp8enc", "libcodec2_soft_vp9enc", "libcodec2_soft_rawdec", diff --git a/services/mediaextractor/seccomp_policy/mediaextractor-arm.policy b/services/mediaextractor/seccomp_policy/mediaextractor-arm.policy index 38f9be6c7a..118072ec65 100644 --- a/services/mediaextractor/seccomp_policy/mediaextractor-arm.policy +++ b/services/mediaextractor/seccomp_policy/mediaextractor-arm.policy @@ -41,6 +41,9 @@ getegid32: 1 getgroups32: 1 nanosleep: 1 getrandom: 1 +timer_create: 1 +timer_settime: 1 +timer_delete: 1 # for dynamically loading extractors pread64: 1 diff --git a/services/mediaextractor/seccomp_policy/mediaextractor-arm64.policy b/services/mediaextractor/seccomp_policy/mediaextractor-arm64.policy index 8fd8787419..481e29e253 100644 --- a/services/mediaextractor/seccomp_policy/mediaextractor-arm64.policy +++ b/services/mediaextractor/seccomp_policy/mediaextractor-arm64.policy @@ -30,6 +30,9 @@ rt_sigreturn: 1 getrlimit: 1 nanosleep: 1 getrandom: 1 +timer_create: 1 +timer_settime: 1 +timer_delete: 1 # for FileSource readlinkat: 1 diff --git a/services/mediaextractor/seccomp_policy/mediaextractor-x86.policy b/services/mediaextractor/seccomp_policy/mediaextractor-x86.policy index 05915d11d9..15fb24e5ea 100644 --- a/services/mediaextractor/seccomp_policy/mediaextractor-x86.policy +++ b/services/mediaextractor/seccomp_policy/mediaextractor-x86.policy @@ -39,6 +39,9 @@ getegid32: 1 getgroups32: 1 nanosleep: 1 getrandom: 1 +timer_create: 1 +timer_settime: 1 +timer_delete: 1 # for dynamically loading extractors getdents64: 1 diff --git a/services/mediaextractor/seccomp_policy/mediaextractor-x86_64.policy b/services/mediaextractor/seccomp_policy/mediaextractor-x86_64.policy index e6a55d0586..4f2646c9cd 100644 --- a/services/mediaextractor/seccomp_policy/mediaextractor-x86_64.policy +++ b/services/mediaextractor/seccomp_policy/mediaextractor-x86_64.policy @@ -34,6 +34,9 @@ sched_setscheduler: 1 getrlimit: 1 nanosleep: 1 getrandom: 1 +timer_create: 1 +timer_settime: 1 +timer_delete: 1 # for dynamically loading extractors getdents64: 1 diff --git a/services/mediaresourcemanager/ResourceManagerService.cpp b/services/mediaresourcemanager/ResourceManagerService.cpp index 28bfd3ff51..bdcd5e4381 100644 --- a/services/mediaresourcemanager/ResourceManagerService.cpp +++ b/services/mediaresourcemanager/ResourceManagerService.cpp @@ -21,8 +21,11 @@ #include <binder/IMediaResourceMonitor.h> #include <binder/IServiceManager.h> +#include <cutils/sched_policy.h> #include <dirent.h> #include <media/stagefright/ProcessInfo.h> +#include <mediautils/BatteryNotifier.h> +#include <mediautils/SchedulingPolicyService.h> #include <string.h> #include <sys/types.h> #include <sys/stat.h> @@ -31,8 +34,7 @@ #include "ResourceManagerService.h" #include "ServiceLog.h" -#include "mediautils/SchedulingPolicyService.h" -#include <cutils/sched_policy.h> + namespace android { namespace { @@ -69,9 +71,9 @@ static String8 getString(const Vector<T> &items) { return itemsStr; } -static bool hasResourceType(MediaResource::Type type, const Vector<MediaResource>& resources) { - for (size_t i = 0; i < resources.size(); ++i) { - if (resources[i].mType == type) { +static bool hasResourceType(MediaResource::Type type, const ResourceList& resources) { + for (auto it = resources.begin(); it != resources.end(); it++) { + if (it->second.mType == type) { return true; } } @@ -101,20 +103,22 @@ static ResourceInfos& getResourceInfosForEdit( } static ResourceInfo& getResourceInfoForEdit( + uid_t uid, int64_t clientId, const sp<IResourceManagerClient>& client, ResourceInfos& infos) { - for (size_t i = 0; i < infos.size(); ++i) { - if (infos[i].clientId == clientId) { - return infos.editItemAt(i); - } + ssize_t index = infos.indexOfKey(clientId); + + if (index < 0) { + ResourceInfo info; + info.uid = uid; + info.clientId = clientId; + info.client = client; + + index = infos.add(clientId, info); } - ResourceInfo info; - info.clientId = clientId; - info.client = client; - info.cpuBoost = false; - infos.push_back(info); - return infos.editItemAt(infos.size() - 1); + + return infos.editValueAt(index); } static void notifyResourceGranted(int pid, const Vector<MediaResource> &resources) { @@ -181,10 +185,10 @@ status_t ResourceManagerService::dump(int fd, const Vector<String16>& /* args */ snprintf(buffer, SIZE, " Name: %s\n", infos[j].client->getName().string()); result.append(buffer); - Vector<MediaResource> resources = infos[j].resources; + const ResourceList &resources = infos[j].resources; result.append(" Resources:\n"); - for (size_t k = 0; k < resources.size(); ++k) { - snprintf(buffer, SIZE, " %s\n", resources[k].toString().string()); + for (auto it = resources.begin(); it != resources.end(); it++) { + snprintf(buffer, SIZE, " %s\n", it->second.toString().string()); result.append(buffer); } } @@ -196,15 +200,45 @@ status_t ResourceManagerService::dump(int fd, const Vector<String16>& /* args */ return OK; } +struct SystemCallbackImpl : + public ResourceManagerService::SystemCallbackInterface { + SystemCallbackImpl() {} + + virtual void noteStartVideo(int uid) override { + BatteryNotifier::getInstance().noteStartVideo(uid); + } + virtual void noteStopVideo(int uid) override { + BatteryNotifier::getInstance().noteStopVideo(uid); + } + virtual void noteResetVideo() override { + BatteryNotifier::getInstance().noteResetVideo(); + } + virtual bool requestCpusetBoost( + bool enable, const sp<IInterface> &client) override { + return android::requestCpusetBoost(enable, client); + } + +protected: + virtual ~SystemCallbackImpl() {} + +private: + DISALLOW_EVIL_CONSTRUCTORS(SystemCallbackImpl); +}; + ResourceManagerService::ResourceManagerService() - : ResourceManagerService(new ProcessInfo()) {} + : ResourceManagerService(new ProcessInfo(), new SystemCallbackImpl()) {} -ResourceManagerService::ResourceManagerService(sp<ProcessInfoInterface> processInfo) +ResourceManagerService::ResourceManagerService( + const sp<ProcessInfoInterface> &processInfo, + const sp<SystemCallbackInterface> &systemResource) : mProcessInfo(processInfo), + mSystemCB(systemResource), mServiceLog(new ServiceLog()), mSupportsMultipleSecureCodecs(true), mSupportsSecureWithNonSecureCodec(true), - mCpuBoostCount(0) {} + mCpuBoostCount(0) { + mSystemCB->noteResetVideo(); +} ResourceManagerService::~ResourceManagerService() {} @@ -224,8 +258,41 @@ void ResourceManagerService::config(const Vector<MediaResourcePolicy> &policies) } } +void ResourceManagerService::onFirstAdded( + const MediaResource& resource, const ResourceInfo& clientInfo) { + // first time added + if (resource.mType == MediaResource::kCpuBoost + && resource.mSubType == MediaResource::kUnspecifiedSubType) { + // Request it on every new instance of kCpuBoost, as the media.codec + // could have died, if we only do it the first time subsequent instances + // never gets the boost. + if (mSystemCB->requestCpusetBoost(true, this) != OK) { + ALOGW("couldn't request cpuset boost"); + } + mCpuBoostCount++; + } else if (resource.mType == MediaResource::kBattery + && resource.mSubType == MediaResource::kVideoCodec) { + mSystemCB->noteStartVideo(clientInfo.uid); + } +} + +void ResourceManagerService::onLastRemoved( + const MediaResource& resource, const ResourceInfo& clientInfo) { + if (resource.mType == MediaResource::kCpuBoost + && resource.mSubType == MediaResource::kUnspecifiedSubType + && mCpuBoostCount > 0) { + if (--mCpuBoostCount == 0) { + mSystemCB->requestCpusetBoost(false, this); + } + } else if (resource.mType == MediaResource::kBattery + && resource.mSubType == MediaResource::kVideoCodec) { + mSystemCB->noteStopVideo(clientInfo.uid); + } +} + void ResourceManagerService::addResource( int pid, + int uid, int64_t clientId, const sp<IResourceManagerClient> client, const Vector<MediaResource> &resources) { @@ -239,20 +306,15 @@ void ResourceManagerService::addResource( return; } ResourceInfos& infos = getResourceInfosForEdit(pid, mMap); - ResourceInfo& info = getResourceInfoForEdit(clientId, client, infos); - // TODO: do the merge instead of append. - info.resources.appendVector(resources); + ResourceInfo& info = getResourceInfoForEdit(uid, clientId, client, infos); for (size_t i = 0; i < resources.size(); ++i) { - if (resources[i].mType == MediaResource::kCpuBoost && !info.cpuBoost) { - info.cpuBoost = true; - // Request it on every new instance of kCpuBoost, as the media.codec - // could have died, if we only do it the first time subsequent instances - // never gets the boost. - if (requestCpusetBoost(true, this) != OK) { - ALOGW("couldn't request cpuset boost"); - } - mCpuBoostCount++; + const auto resType = std::make_pair(resources[i].mType, resources[i].mSubType); + if (info.resources.find(resType) == info.resources.end()) { + onFirstAdded(resources[i], info); + info.resources[resType] = resources[i]; + } else { + info.resources[resType].mValue += resources[i].mValue; } } if (info.deathNotifier == nullptr) { @@ -262,7 +324,48 @@ void ResourceManagerService::addResource( notifyResourceGranted(pid, resources); } -void ResourceManagerService::removeResource(int pid, int64_t clientId) { +void ResourceManagerService::removeResource(int pid, int64_t clientId, + const Vector<MediaResource> &resources) { + String8 log = String8::format("removeResource(pid %d, clientId %lld, resources %s)", + pid, (long long) clientId, getString(resources).string()); + mServiceLog->add(log); + + Mutex::Autolock lock(mLock); + if (!mProcessInfo->isValidPid(pid)) { + ALOGE("Rejected removeResource call with invalid pid."); + return; + } + ssize_t index = mMap.indexOfKey(pid); + if (index < 0) { + ALOGV("removeResource: didn't find pid %d for clientId %lld", pid, (long long) clientId); + return; + } + ResourceInfos &infos = mMap.editValueAt(index); + + index = infos.indexOfKey(clientId); + if (index < 0) { + ALOGV("removeResource: didn't find clientId %lld", (long long) clientId); + return; + } + + ResourceInfo &info = infos.editValueAt(index); + + for (size_t i = 0; i < resources.size(); ++i) { + const auto resType = std::make_pair(resources[i].mType, resources[i].mSubType); + // ignore if we don't have it + if (info.resources.find(resType) != info.resources.end()) { + MediaResource &resource = info.resources[resType]; + if (resource.mValue > resources[i].mValue) { + resource.mValue -= resources[i].mValue; + } else { + onLastRemoved(resources[i], info); + info.resources.erase(resType); + } + } + } +} + +void ResourceManagerService::removeClient(int pid, int64_t clientId) { removeResource(pid, clientId, true); } @@ -282,24 +385,22 @@ void ResourceManagerService::removeResource(int pid, int64_t clientId, bool chec ALOGV("removeResource: didn't find pid %d for clientId %lld", pid, (long long) clientId); return; } - bool found = false; ResourceInfos &infos = mMap.editValueAt(index); - for (size_t j = 0; j < infos.size(); ++j) { - if (infos[j].clientId == clientId) { - if (infos[j].cpuBoost && mCpuBoostCount > 0) { - if (--mCpuBoostCount == 0) { - requestCpusetBoost(false, this); - } - } - IInterface::asBinder(infos[j].client)->unlinkToDeath(infos[j].deathNotifier); - j = infos.removeAt(j); - found = true; - break; - } + + index = infos.indexOfKey(clientId); + if (index < 0) { + ALOGV("removeResource: didn't find clientId %lld", (long long) clientId); + return; } - if (!found) { - ALOGV("didn't find client"); + + const ResourceInfo &info = infos[index]; + for (auto it = info.resources.begin(); it != info.resources.end(); it++) { + onLastRemoved(it->second, info); } + + IInterface::asBinder(info.client)->unlinkToDeath(info.deathNotifier); + + infos.removeItemsAt(index); } void ResourceManagerService::getClientForResource_l( @@ -410,7 +511,7 @@ bool ResourceManagerService::reclaimResource( ResourceInfos &infos = mMap.editValueAt(i); for (size_t j = 0; j < infos.size();) { if (infos[j].client == failedClient) { - j = infos.removeAt(j); + j = infos.removeItemsAt(j); found = true; } else { ++j; @@ -538,11 +639,12 @@ bool ResourceManagerService::getBiggestClient_l( uint64_t largestValue = 0; const ResourceInfos &infos = mMap.valueAt(index); for (size_t i = 0; i < infos.size(); ++i) { - Vector<MediaResource> resources = infos[i].resources; - for (size_t j = 0; j < resources.size(); ++j) { - if (resources[j].mType == type) { - if (resources[j].mValue > largestValue) { - largestValue = resources[j].mValue; + const ResourceList &resources = infos[i].resources; + for (auto it = resources.begin(); it != resources.end(); it++) { + const MediaResource &resource = it->second; + if (resource.mType == type) { + if (resource.mValue > largestValue) { + largestValue = resource.mValue; clientTemp = infos[i].client; } } diff --git a/services/mediaresourcemanager/ResourceManagerService.h b/services/mediaresourcemanager/ResourceManagerService.h index 82d2a0b369..f086dc3b22 100644 --- a/services/mediaresourcemanager/ResourceManagerService.h +++ b/services/mediaresourcemanager/ResourceManagerService.h @@ -33,15 +33,17 @@ namespace android { class ServiceLog; struct ProcessInfoInterface; +typedef std::map<std::pair<MediaResource::Type, MediaResource::SubType>, MediaResource> ResourceList; struct ResourceInfo { int64_t clientId; + uid_t uid; sp<IResourceManagerClient> client; sp<IBinder::DeathRecipient> deathNotifier; - Vector<MediaResource> resources; - bool cpuBoost; + ResourceList resources; }; -typedef Vector<ResourceInfo> ResourceInfos; +// TODO: convert these to std::map +typedef KeyedVector<int64_t, ResourceInfo> ResourceInfos; typedef KeyedVector<int, ResourceInfos> PidResourceInfosMap; class ResourceManagerService @@ -49,23 +51,37 @@ class ResourceManagerService public BnResourceManagerService { public: + struct SystemCallbackInterface : public RefBase { + virtual void noteStartVideo(int uid) = 0; + virtual void noteStopVideo(int uid) = 0; + virtual void noteResetVideo() = 0; + virtual bool requestCpusetBoost( + bool enable, const sp<IInterface> &client) = 0; + }; + static char const *getServiceName() { return "media.resource_manager"; } virtual status_t dump(int fd, const Vector<String16>& args); ResourceManagerService(); - explicit ResourceManagerService(sp<ProcessInfoInterface> processInfo); + explicit ResourceManagerService( + const sp<ProcessInfoInterface> &processInfo, + const sp<SystemCallbackInterface> &systemResource); // IResourceManagerService interface virtual void config(const Vector<MediaResourcePolicy> &policies); virtual void addResource( int pid, + int uid, int64_t clientId, const sp<IResourceManagerClient> client, const Vector<MediaResource> &resources); - virtual void removeResource(int pid, int64_t clientId); + virtual void removeResource(int pid, int64_t clientId, + const Vector<MediaResource> &resources); + + virtual void removeClient(int pid, int64_t clientId); // Tries to reclaim resource from processes with lower priority than the calling process // according to the requested resources. @@ -107,8 +123,12 @@ private: void getClientForResource_l( int callingPid, const MediaResource *res, Vector<sp<IResourceManagerClient>> *clients); + void onFirstAdded(const MediaResource& res, const ResourceInfo& clientInfo); + void onLastRemoved(const MediaResource& res, const ResourceInfo& clientInfo); + mutable Mutex mLock; sp<ProcessInfoInterface> mProcessInfo; + sp<SystemCallbackInterface> mSystemCB; sp<ServiceLog> mServiceLog; PidResourceInfosMap mMap; bool mSupportsMultipleSecureCodecs; diff --git a/services/mediaresourcemanager/TEST_MAPPING b/services/mediaresourcemanager/TEST_MAPPING new file mode 100644 index 0000000000..418b159820 --- /dev/null +++ b/services/mediaresourcemanager/TEST_MAPPING @@ -0,0 +1,10 @@ +{ + "presubmit": [ + { + "name": "ResourceManagerService_test" + }, + { + "name": "ServiceLog_test" + } + ] +} diff --git a/services/mediaresourcemanager/test/Android.bp b/services/mediaresourcemanager/test/Android.bp index 70e883300a..543c87c544 100644 --- a/services/mediaresourcemanager/test/Android.bp +++ b/services/mediaresourcemanager/test/Android.bp @@ -2,6 +2,7 @@ cc_test { name: "ResourceManagerService_test", srcs: ["ResourceManagerService_test.cpp"], + test_suites: ["device-tests"], shared_libs: [ "libbinder", "liblog", @@ -23,6 +24,7 @@ cc_test { cc_test { name: "ServiceLog_test", srcs: ["ServiceLog_test.cpp"], + test_suites: ["device-tests"], shared_libs: [ "liblog", "libmedia", diff --git a/services/mediaresourcemanager/test/ResourceManagerService_test.cpp b/services/mediaresourcemanager/test/ResourceManagerService_test.cpp index ed0b0efa63..ae97ec8324 100644 --- a/services/mediaresourcemanager/test/ResourceManagerService_test.cpp +++ b/services/mediaresourcemanager/test/ResourceManagerService_test.cpp @@ -52,13 +52,69 @@ private: DISALLOW_EVIL_CONSTRUCTORS(TestProcessInfo); }; +struct TestSystemCallback : + public ResourceManagerService::SystemCallbackInterface { + TestSystemCallback() : + mLastEvent({EventType::INVALID, 0}), mEventCount(0) {} + + enum EventType { + INVALID = -1, + VIDEO_ON = 0, + VIDEO_OFF = 1, + VIDEO_RESET = 2, + CPUSET_ENABLE = 3, + CPUSET_DISABLE = 4, + }; + + struct EventEntry { + EventType type; + int arg; + }; + + virtual void noteStartVideo(int uid) override { + mLastEvent = {EventType::VIDEO_ON, uid}; + mEventCount++; + } + + virtual void noteStopVideo(int uid) override { + mLastEvent = {EventType::VIDEO_OFF, uid}; + mEventCount++; + } + + virtual void noteResetVideo() override { + mLastEvent = {EventType::VIDEO_RESET, 0}; + mEventCount++; + } + + virtual bool requestCpusetBoost( + bool enable, const sp<IInterface> &/*client*/) override { + mLastEvent = {enable ? EventType::CPUSET_ENABLE : EventType::CPUSET_DISABLE, 0}; + mEventCount++; + return true; + } + + size_t eventCount() { return mEventCount; } + EventType lastEventType() { return mLastEvent.type; } + EventEntry lastEvent() { return mLastEvent; } + +protected: + virtual ~TestSystemCallback() {} + +private: + EventEntry mLastEvent; + size_t mEventCount; + + DISALLOW_EVIL_CONSTRUCTORS(TestSystemCallback); +}; + + struct TestClient : public BnResourceManagerClient { TestClient(int pid, sp<ResourceManagerService> service) : mReclaimed(false), mPid(pid), mService(service) {} virtual bool reclaimResource() { sp<IResourceManagerClient> client(this); - mService->removeResource(mPid, (int64_t) client.get()); + mService->removeClient(mPid, (int64_t) client.get()); mReclaimed = true; return true; } @@ -86,16 +142,26 @@ private: }; static const int kTestPid1 = 30; +static const int kTestUid1 = 1010; + static const int kTestPid2 = 20; +static const int kTestUid2 = 1011; static const int kLowPriorityPid = 40; static const int kMidPriorityPid = 25; static const int kHighPriorityPid = 10; +using EventType = TestSystemCallback::EventType; +using EventEntry = TestSystemCallback::EventEntry; +bool operator== (const EventEntry& lhs, const EventEntry& rhs) { + return lhs.type == rhs.type && lhs.arg == rhs.arg; +} + class ResourceManagerServiceTest : public ::testing::Test { public: ResourceManagerServiceTest() - : mService(new ResourceManagerService(new TestProcessInfo)), + : mSystemCB(new TestSystemCallback()), + mService(new ResourceManagerService(new TestProcessInfo, mSystemCB)), mTestClient1(new TestClient(kTestPid1, mService)), mTestClient2(new TestClient(kTestPid2, mService)), mTestClient3(new TestClient(kTestPid2, mService)) { @@ -103,20 +169,21 @@ public: protected: static bool isEqualResources(const Vector<MediaResource> &resources1, - const Vector<MediaResource> &resources2) { - if (resources1.size() != resources2.size()) { - return false; - } + const ResourceList &resources2) { + // convert resource1 to ResourceList + ResourceList r1; for (size_t i = 0; i < resources1.size(); ++i) { - if (resources1[i] != resources2[i]) { - return false; - } + const auto resType = std::make_pair(resources1[i].mType, resources1[i].mSubType); + r1[resType] = resources1[i]; } - return true; + return r1 == resources2; } - static void expectEqResourceInfo(const ResourceInfo &info, sp<IResourceManagerClient> client, + static void expectEqResourceInfo(const ResourceInfo &info, + int uid, + sp<IResourceManagerClient> client, const Vector<MediaResource> &resources) { + EXPECT_EQ(uid, info.uid); EXPECT_EQ(client, info.client); EXPECT_TRUE(isEqualResources(resources, info.resources)); } @@ -153,24 +220,24 @@ protected: // kTestPid1 mTestClient1 Vector<MediaResource> resources1; resources1.push_back(MediaResource(MediaResource::kSecureCodec, 1)); - mService->addResource(kTestPid1, getId(mTestClient1), mTestClient1, resources1); + mService->addResource(kTestPid1, kTestUid1, getId(mTestClient1), mTestClient1, resources1); resources1.push_back(MediaResource(MediaResource::kGraphicMemory, 200)); Vector<MediaResource> resources11; resources11.push_back(MediaResource(MediaResource::kGraphicMemory, 200)); - mService->addResource(kTestPid1, getId(mTestClient1), mTestClient1, resources11); + mService->addResource(kTestPid1, kTestUid1, getId(mTestClient1), mTestClient1, resources11); // kTestPid2 mTestClient2 Vector<MediaResource> resources2; resources2.push_back(MediaResource(MediaResource::kNonSecureCodec, 1)); resources2.push_back(MediaResource(MediaResource::kGraphicMemory, 300)); - mService->addResource(kTestPid2, getId(mTestClient2), mTestClient2, resources2); + mService->addResource(kTestPid2, kTestUid2, getId(mTestClient2), mTestClient2, resources2); // kTestPid2 mTestClient3 Vector<MediaResource> resources3; - mService->addResource(kTestPid2, getId(mTestClient3), mTestClient3, resources3); + mService->addResource(kTestPid2, kTestUid2, getId(mTestClient3), mTestClient3, resources3); resources3.push_back(MediaResource(MediaResource::kSecureCodec, 1)); resources3.push_back(MediaResource(MediaResource::kGraphicMemory, 100)); - mService->addResource(kTestPid2, getId(mTestClient3), mTestClient3, resources3); + mService->addResource(kTestPid2, kTestUid2, getId(mTestClient3), mTestClient3, resources3); const PidResourceInfosMap &map = mService->mMap; EXPECT_EQ(2u, map.size()); @@ -178,14 +245,14 @@ protected: ASSERT_GE(index1, 0); const ResourceInfos &infos1 = map[index1]; EXPECT_EQ(1u, infos1.size()); - expectEqResourceInfo(infos1[0], mTestClient1, resources1); + expectEqResourceInfo(infos1.valueFor(getId(mTestClient1)), kTestUid1, mTestClient1, resources1); ssize_t index2 = map.indexOfKey(kTestPid2); ASSERT_GE(index2, 0); const ResourceInfos &infos2 = map[index2]; EXPECT_EQ(2u, infos2.size()); - expectEqResourceInfo(infos2[0], mTestClient2, resources2); - expectEqResourceInfo(infos2[1], mTestClient3, resources3); + expectEqResourceInfo(infos2.valueFor(getId(mTestClient2)), kTestUid2, mTestClient2, resources2); + expectEqResourceInfo(infos2.valueFor(getId(mTestClient3)), kTestUid2, mTestClient3, resources3); } void testConfig() { @@ -219,10 +286,84 @@ protected: EXPECT_TRUE(mService->mSupportsSecureWithNonSecureCodec); } + void testCombineResource() { + // kTestPid1 mTestClient1 + Vector<MediaResource> resources1; + resources1.push_back(MediaResource(MediaResource::kSecureCodec, 1)); + mService->addResource(kTestPid1, kTestUid1, getId(mTestClient1), mTestClient1, resources1); + + Vector<MediaResource> resources11; + resources11.push_back(MediaResource(MediaResource::kGraphicMemory, 200)); + mService->addResource(kTestPid1, kTestUid1, getId(mTestClient1), mTestClient1, resources11); + + const PidResourceInfosMap &map = mService->mMap; + EXPECT_EQ(1u, map.size()); + ssize_t index1 = map.indexOfKey(kTestPid1); + ASSERT_GE(index1, 0); + const ResourceInfos &infos1 = map[index1]; + EXPECT_EQ(1u, infos1.size()); + + // test adding existing types to combine values + resources1.push_back(MediaResource(MediaResource::kGraphicMemory, 100)); + mService->addResource(kTestPid1, kTestUid1, getId(mTestClient1), mTestClient1, resources1); + + Vector<MediaResource> expected; + expected.push_back(MediaResource(MediaResource::kSecureCodec, 2)); + expected.push_back(MediaResource(MediaResource::kGraphicMemory, 300)); + expectEqResourceInfo(infos1.valueFor(getId(mTestClient1)), kTestUid1, mTestClient1, expected); + + // test adding new types (including types that differs only in subType) + resources11.push_back(MediaResource(MediaResource::kNonSecureCodec, 1)); + resources11.push_back(MediaResource(MediaResource::kSecureCodec, MediaResource::kVideoCodec, 1)); + mService->addResource(kTestPid1, kTestUid1, getId(mTestClient1), mTestClient1, resources11); + + expected.clear(); + expected.push_back(MediaResource(MediaResource::kSecureCodec, 2)); + expected.push_back(MediaResource(MediaResource::kNonSecureCodec, 1)); + expected.push_back(MediaResource(MediaResource::kSecureCodec, MediaResource::kVideoCodec, 1)); + expected.push_back(MediaResource(MediaResource::kGraphicMemory, 500)); + expectEqResourceInfo(infos1.valueFor(getId(mTestClient1)), kTestUid1, mTestClient1, expected); + } + void testRemoveResource() { + // kTestPid1 mTestClient1 + Vector<MediaResource> resources1; + resources1.push_back(MediaResource(MediaResource::kSecureCodec, 1)); + mService->addResource(kTestPid1, kTestUid1, getId(mTestClient1), mTestClient1, resources1); + + Vector<MediaResource> resources11; + resources11.push_back(MediaResource(MediaResource::kGraphicMemory, 200)); + mService->addResource(kTestPid1, kTestUid1, getId(mTestClient1), mTestClient1, resources11); + + const PidResourceInfosMap &map = mService->mMap; + EXPECT_EQ(1u, map.size()); + ssize_t index1 = map.indexOfKey(kTestPid1); + ASSERT_GE(index1, 0); + const ResourceInfos &infos1 = map[index1]; + EXPECT_EQ(1u, infos1.size()); + + // test partial removal + resources11.editItemAt(0).mValue = 100; + mService->removeResource(kTestPid1, getId(mTestClient1), resources11); + + Vector<MediaResource> expected; + expected.push_back(MediaResource(MediaResource::kSecureCodec, 1)); + expected.push_back(MediaResource(MediaResource::kGraphicMemory, 100)); + expectEqResourceInfo(infos1.valueFor(getId(mTestClient1)), kTestUid1, mTestClient1, expected); + + // test complete removal with overshoot value + resources11.editItemAt(0).mValue = 1000; + mService->removeResource(kTestPid1, getId(mTestClient1), resources11); + + expected.clear(); + expected.push_back(MediaResource(MediaResource::kSecureCodec, 1)); + expectEqResourceInfo(infos1.valueFor(getId(mTestClient1)), kTestUid1, mTestClient1, expected); + } + + void testRemoveClient() { addResource(); - mService->removeResource(kTestPid2, getId(mTestClient2)); + mService->removeClient(kTestPid2, getId(mTestClient2)); const PidResourceInfosMap &map = mService->mMap; EXPECT_EQ(2u, map.size()); @@ -231,6 +372,7 @@ protected: EXPECT_EQ(1u, infos1.size()); EXPECT_EQ(1u, infos2.size()); // mTestClient2 has been removed. + // (OK to use infos2[0] as there is only 1 entry) EXPECT_EQ(mTestClient3, infos2[0].client); } @@ -246,6 +388,7 @@ protected: EXPECT_TRUE(mService->getAllClients_l(kHighPriorityPid, type, &clients)); EXPECT_EQ(2u, clients.size()); + // (OK to require ordering in clients[], as the pid map is sorted) EXPECT_EQ(mTestClient3, clients[0]); EXPECT_EQ(mTestClient1, clients[1]); } @@ -438,7 +581,7 @@ protected: verifyClients(true /* c1 */, false /* c2 */, false /* c3 */); // clean up client 3 which still left - mService->removeResource(kTestPid2, getId(mTestClient3)); + mService->removeClient(kTestPid2, getId(mTestClient3)); } } @@ -498,6 +641,84 @@ protected: EXPECT_TRUE(mService->isCallingPriorityHigher_l(99, 100)); } + void testBatteryStats() { + // reset should always be called when ResourceManagerService is created (restarted) + EXPECT_EQ(1u, mSystemCB->eventCount()); + EXPECT_EQ(EventType::VIDEO_RESET, mSystemCB->lastEventType()); + + // new client request should cause VIDEO_ON + Vector<MediaResource> resources1; + resources1.push_back(MediaResource(MediaResource::kBattery, MediaResource::kVideoCodec, 1)); + mService->addResource(kTestPid1, kTestUid1, getId(mTestClient1), mTestClient1, resources1); + EXPECT_EQ(2u, mSystemCB->eventCount()); + EXPECT_EQ(EventEntry({EventType::VIDEO_ON, kTestUid1}), mSystemCB->lastEvent()); + + // each client should only cause 1 VIDEO_ON + mService->addResource(kTestPid1, kTestUid1, getId(mTestClient1), mTestClient1, resources1); + EXPECT_EQ(2u, mSystemCB->eventCount()); + + // new client request should cause VIDEO_ON + Vector<MediaResource> resources2; + resources2.push_back(MediaResource(MediaResource::kBattery, MediaResource::kVideoCodec, 2)); + mService->addResource(kTestPid2, kTestUid2, getId(mTestClient2), mTestClient2, resources2); + EXPECT_EQ(3u, mSystemCB->eventCount()); + EXPECT_EQ(EventEntry({EventType::VIDEO_ON, kTestUid2}), mSystemCB->lastEvent()); + + // partially remove mTestClient1's request, shouldn't be any VIDEO_OFF + mService->removeResource(kTestPid1, getId(mTestClient1), resources1); + EXPECT_EQ(3u, mSystemCB->eventCount()); + + // remove mTestClient1's request, should be VIDEO_OFF for kTestUid1 + // (use resource2 to test removing more instances than previously requested) + mService->removeResource(kTestPid1, getId(mTestClient1), resources2); + EXPECT_EQ(4u, mSystemCB->eventCount()); + EXPECT_EQ(EventEntry({EventType::VIDEO_OFF, kTestUid1}), mSystemCB->lastEvent()); + + // remove mTestClient2, should be VIDEO_OFF for kTestUid2 + mService->removeClient(kTestPid2, getId(mTestClient2)); + EXPECT_EQ(5u, mSystemCB->eventCount()); + EXPECT_EQ(EventEntry({EventType::VIDEO_OFF, kTestUid2}), mSystemCB->lastEvent()); + } + + void testCpusetBoost() { + // reset should always be called when ResourceManagerService is created (restarted) + EXPECT_EQ(1u, mSystemCB->eventCount()); + EXPECT_EQ(EventType::VIDEO_RESET, mSystemCB->lastEventType()); + + // new client request should cause CPUSET_ENABLE + Vector<MediaResource> resources1; + resources1.push_back(MediaResource(MediaResource::kCpuBoost, 1)); + mService->addResource(kTestPid1, kTestUid1, getId(mTestClient1), mTestClient1, resources1); + EXPECT_EQ(2u, mSystemCB->eventCount()); + EXPECT_EQ(EventType::CPUSET_ENABLE, mSystemCB->lastEventType()); + + // each client should only cause 1 CPUSET_ENABLE + mService->addResource(kTestPid1, kTestUid1, getId(mTestClient1), mTestClient1, resources1); + EXPECT_EQ(2u, mSystemCB->eventCount()); + + // new client request should cause CPUSET_ENABLE + Vector<MediaResource> resources2; + resources2.push_back(MediaResource(MediaResource::kCpuBoost, 2)); + mService->addResource(kTestPid2, kTestUid2, getId(mTestClient2), mTestClient2, resources2); + EXPECT_EQ(3u, mSystemCB->eventCount()); + EXPECT_EQ(EventType::CPUSET_ENABLE, mSystemCB->lastEventType()); + + // remove mTestClient2 should not cause CPUSET_DISABLE, mTestClient1 still active + mService->removeClient(kTestPid2, getId(mTestClient2)); + EXPECT_EQ(3u, mSystemCB->eventCount()); + + // remove 1 cpuboost from mTestClient1, should not be CPUSET_DISABLE (still 1 left) + mService->removeResource(kTestPid1, getId(mTestClient1), resources1); + EXPECT_EQ(3u, mSystemCB->eventCount()); + + // remove 2 cpuboost from mTestClient1, should be CPUSET_DISABLE + // (use resource2 to test removing more than previously requested) + mService->removeResource(kTestPid1, getId(mTestClient1), resources2); + EXPECT_EQ(4u, mSystemCB->eventCount()); + EXPECT_EQ(EventType::CPUSET_DISABLE, mSystemCB->lastEventType()); + } + + sp<TestSystemCallback> mSystemCB; sp<ResourceManagerService> mService; sp<IResourceManagerClient> mTestClient1; sp<IResourceManagerClient> mTestClient2; @@ -512,10 +733,18 @@ TEST_F(ResourceManagerServiceTest, addResource) { addResource(); } +TEST_F(ResourceManagerServiceTest, combineResource) { + testCombineResource(); +} + TEST_F(ResourceManagerServiceTest, removeResource) { testRemoveResource(); } +TEST_F(ResourceManagerServiceTest, removeClient) { + testRemoveClient(); +} + TEST_F(ResourceManagerServiceTest, reclaimResource) { testReclaimResourceSecure(); testReclaimResourceNonSecure(); @@ -541,4 +770,12 @@ TEST_F(ResourceManagerServiceTest, isCallingPriorityHigher_l) { testIsCallingPriorityHigher(); } +TEST_F(ResourceManagerServiceTest, testBatteryStats) { + testBatteryStats(); +} + +TEST_F(ResourceManagerServiceTest, testCpusetBoost) { + testCpusetBoost(); +} + } // namespace android diff --git a/services/soundtrigger/SoundTriggerHwService.cpp b/services/soundtrigger/SoundTriggerHwService.cpp index 377d30bcde..51afdcd82e 100644 --- a/services/soundtrigger/SoundTriggerHwService.cpp +++ b/services/soundtrigger/SoundTriggerHwService.cpp @@ -40,6 +40,32 @@ #define HW_MODULE_PREFIX "primary" namespace android { +namespace { + +// Given an IMemory, returns a copy of its content along with its size. +// Returns nullptr on failure or if input is nullptr. +std::pair<std::unique_ptr<uint8_t[]>, + size_t> CopyToArray(const sp<IMemory>& mem) { + if (mem == nullptr) { + return std::make_pair(nullptr, 0); + } + + const size_t size = mem->size(); + if (size == 0) { + return std::make_pair(nullptr, 0); + } + + std::unique_ptr<uint8_t[]> ar = std::make_unique<uint8_t[]>(size); + if (ar == nullptr) { + return std::make_pair(nullptr, 0); + } + + memcpy(ar.get(), mem->pointer(), size); + return std::make_pair(std::move(ar), size); +} + +} + SoundTriggerHwService::SoundTriggerHwService() : BnSoundTriggerHwService(), mNextUniqueId(1), @@ -557,8 +583,13 @@ status_t SoundTriggerHwService::Module::loadSoundModel(const sp<IMemory>& modelM return NO_INIT; } - struct sound_trigger_sound_model *sound_model = - (struct sound_trigger_sound_model *)modelMemory->pointer(); + auto immutableMemory = CopyToArray(modelMemory); + if (immutableMemory.first == nullptr) { + return NO_MEMORY; + } + + struct sound_trigger_sound_model* sound_model = + (struct sound_trigger_sound_model*) immutableMemory.first.get(); size_t structSize; if (sound_model->type == SOUND_MODEL_TYPE_KEYPHRASE) { @@ -568,9 +599,10 @@ status_t SoundTriggerHwService::Module::loadSoundModel(const sp<IMemory>& modelM } if (sound_model->data_offset < structSize || - sound_model->data_size > (UINT_MAX - sound_model->data_offset) || - modelMemory->size() < sound_model->data_offset || - sound_model->data_size > (modelMemory->size() - sound_model->data_offset)) { + sound_model->data_size > (UINT_MAX - sound_model->data_offset) || + immutableMemory.second < sound_model->data_offset || + sound_model->data_size > + (immutableMemory.second - sound_model->data_offset)) { android_errorWriteLog(0x534e4554, "30148546"); ALOGE("loadSoundModel() data_size is too big"); return BAD_VALUE; @@ -651,13 +683,19 @@ status_t SoundTriggerHwService::Module::startRecognition(sound_model_handle_t ha return NO_INIT; } - struct sound_trigger_recognition_config *config = - (struct sound_trigger_recognition_config *)dataMemory->pointer(); + auto immutableMemory = CopyToArray(dataMemory); + if (immutableMemory.first == nullptr) { + return NO_MEMORY; + } + + struct sound_trigger_recognition_config* config = + (struct sound_trigger_recognition_config*) immutableMemory.first.get(); if (config->data_offset < sizeof(struct sound_trigger_recognition_config) || - config->data_size > (UINT_MAX - config->data_offset) || - dataMemory->size() < config->data_offset || - config->data_size > (dataMemory->size() - config->data_offset)) { + config->data_size > (UINT_MAX - config->data_offset) || + immutableMemory.second < config->data_offset || + config->data_size > + (immutableMemory.second - config->data_offset)) { ALOGE("startRecognition() data_size is too big"); return BAD_VALUE; } |