camera: Add physical camera metadata in CaptureResult

- When physical stream of a logical multi-camera is requested, HAL needs
to generate metadata for the physical cameras.
- In case no physical stream is requested for the logical multi-camera, no
capture result metadata is required for physical camera.
- Batch physical and logical metadata within one capture_result call.

Test: testLogicalCamera CTS test
Bug: 64691172
Change-Id: Id040620f3f0c350711d49341ab31ab88ecd94888

1 files changed, 261 insertions, 3 deletions

diff --git a/camera/device/3.4/default/CameraDeviceSession.cpp b/camera/device/3.4/default/CameraDeviceSession.cpp
index d0547882c2..f6c6b2b0fe 100644
--- a/camera/device/3.4/default/CameraDeviceSession.cpp
+++ b/camera/device/3.4/default/CameraDeviceSession.cpp
@@ -34,7 +34,23 @@ CameraDeviceSession::CameraDeviceSession(
     camera3_device_t* device,
     const camera_metadata_t* deviceInfo,
     const sp<V3_2::ICameraDeviceCallback>& callback) :
-        V3_3::implementation::CameraDeviceSession(device, deviceInfo, callback) {
+        V3_3::implementation::CameraDeviceSession(device, deviceInfo, callback),
+        mResultBatcher_3_4(callback) {
+
+    mHasCallback_3_4 = false;
+
+    auto castResult = ICameraDeviceCallback::castFrom(callback);
+    if (castResult.isOk()) {
+        sp<ICameraDeviceCallback> callback3_4 = castResult;
+        if (callback3_4 != nullptr) {
+            process_capture_result = sProcessCaptureResult_3_4;
+            notify = sNotify_3_4;
+            mHasCallback_3_4 = true;
+            if (!mInitFail) {
+                mResultBatcher_3_4.setResultMetadataQueue(mResultMetadataQueue);
+            }
+        }
+    }
 }
 
 CameraDeviceSession::~CameraDeviceSession() {
@@ -54,6 +70,18 @@ Return<void> CameraDeviceSession::configureStreams_3_4(
     Status status = initStatus();
     HalStreamConfiguration outStreams;
 
+    // If callback is 3.2, make sure no physical stream is configured
+    if (!mHasCallback_3_4) {
+        for (size_t i = 0; i < requestedConfiguration.streams.size(); i++) {
+            if (requestedConfiguration.streams[i].physicalCameraId.size() > 0) {
+                ALOGE("%s: trying to configureStreams with physical camera id with V3.2 callback",
+                        __FUNCTION__);
+                _hidl_cb(Status::INTERNAL_ERROR, outStreams);
+                return Void();
+            }
+        }
+    }
+
     // hold the inflight lock for entire configureStreams scope since there must not be any
     // inflight request/results during stream configuration.
     Mutex::Autolock _l(mInflightLock);
@@ -205,7 +233,7 @@ void CameraDeviceSession::postProcessConfigurationLocked_3_4(
             mVideoStreamIds.push_back(stream.v3_2.id);
         }
     }
-    mResultBatcher.setBatchedStreams(mVideoStreamIds);
+    mResultBatcher_3_4.setBatchedStreams(mVideoStreamIds);
 }
 
 Return<void> CameraDeviceSession::processCaptureRequest_3_4(
@@ -224,7 +252,7 @@ Return<void> CameraDeviceSession::processCaptureRequest_3_4(
     }
 
     if (s == Status::OK && requests.size() > 1) {
-        mResultBatcher.registerBatch(requests[0].v3_2.frameNumber, requests.size());
+        mResultBatcher_3_4.registerBatch(requests[0].v3_2.frameNumber, requests.size());
     }
 
     _hidl_cb(s, numRequestProcessed);
@@ -237,6 +265,14 @@ Status CameraDeviceSession::processOneCaptureRequest_3_4(const V3_4::CaptureRequ
         ALOGE("%s: camera init failed or disconnected", __FUNCTION__);
         return status;
     }
+    // If callback is 3.2, make sure there are no physical settings.
+    if (!mHasCallback_3_4) {
+        if (request.physicalCameraSettings.size() > 0) {
+            ALOGE("%s: trying to call processCaptureRequest_3_4 with physical camera id "
+                    "and V3.2 callback", __FUNCTION__);
+            return Status::INTERNAL_ERROR;
+        }
+    }
 
     camera3_capture_request_t halRequest;
     halRequest.frame_number = request.v3_2.frameNumber;
@@ -407,6 +443,228 @@ Status CameraDeviceSession::processOneCaptureRequest_3_4(const V3_4::CaptureRequ
     return Status::OK;
 }
 
+/**
+ * Static callback forwarding methods from HAL to instance
+ */
+void CameraDeviceSession::sProcessCaptureResult_3_4(
+        const camera3_callback_ops *cb,
+        const camera3_capture_result *hal_result) {
+    CameraDeviceSession *d =
+            const_cast<CameraDeviceSession*>(static_cast<const CameraDeviceSession*>(cb));
+
+    CaptureResult result;
+    d->constructCaptureResult(result.v3_2, hal_result);
+    result.physicalCameraMetadata.resize(hal_result->num_physcam_metadata);
+    for (uint32_t i = 0; i < hal_result->num_physcam_metadata; i++) {
+        std::string physicalId = hal_result->physcam_ids[i];
+        V3_2::CameraMetadata physicalMetadata;
+        V3_2::implementation::convertToHidl(hal_result->physcam_metadata[i], &physicalMetadata);
+        PhysicalCameraMetadata physicalCameraMetadata = {
+                .fmqMetadataSize = 0,
+                .physicalCameraId = physicalId,
+                .metadata = physicalMetadata };
+        result.physicalCameraMetadata[i] = physicalCameraMetadata;
+    }
+    d->mResultBatcher_3_4.processCaptureResult_3_4(result);
+}
+
+void CameraDeviceSession::sNotify_3_4(
+        const camera3_callback_ops *cb,
+        const camera3_notify_msg *msg) {
+    CameraDeviceSession *d =
+            const_cast<CameraDeviceSession*>(static_cast<const CameraDeviceSession*>(cb));
+    V3_2::NotifyMsg hidlMsg;
+    V3_2::implementation::convertToHidl(msg, &hidlMsg);
+
+    if (hidlMsg.type == (V3_2::MsgType) CAMERA3_MSG_ERROR &&
+            hidlMsg.msg.error.errorStreamId != -1) {
+        if (d->mStreamMap.count(hidlMsg.msg.error.errorStreamId) != 1) {
+            ALOGE("%s: unknown stream ID %d reports an error!",
+                    __FUNCTION__, hidlMsg.msg.error.errorStreamId);
+            return;
+        }
+    }
+
+    if (static_cast<camera3_msg_type_t>(hidlMsg.type) == CAMERA3_MSG_ERROR) {
+        switch (hidlMsg.msg.error.errorCode) {
+            case V3_2::ErrorCode::ERROR_DEVICE:
+            case V3_2::ErrorCode::ERROR_REQUEST:
+            case V3_2::ErrorCode::ERROR_RESULT: {
+                Mutex::Autolock _l(d->mInflightLock);
+                auto entry = d->mInflightAETriggerOverrides.find(
+                        hidlMsg.msg.error.frameNumber);
+                if (d->mInflightAETriggerOverrides.end() != entry) {
+                    d->mInflightAETriggerOverrides.erase(
+                            hidlMsg.msg.error.frameNumber);
+                }
+
+                auto boostEntry = d->mInflightRawBoostPresent.find(
+                        hidlMsg.msg.error.frameNumber);
+                if (d->mInflightRawBoostPresent.end() != boostEntry) {
+                    d->mInflightRawBoostPresent.erase(
+                            hidlMsg.msg.error.frameNumber);
+                }
+
+            }
+                break;
+            case V3_2::ErrorCode::ERROR_BUFFER:
+            default:
+                break;
+        }
+
+    }
+
+    d->mResultBatcher_3_4.notify(hidlMsg);
+}
+
+CameraDeviceSession::ResultBatcher_3_4::ResultBatcher_3_4(
+        const sp<V3_2::ICameraDeviceCallback>& callback) :
+        V3_3::implementation::CameraDeviceSession::ResultBatcher(callback) {
+    auto castResult = ICameraDeviceCallback::castFrom(callback);
+    if (castResult.isOk()) {
+        mCallback_3_4 = castResult;
+    }
+}
+
+void CameraDeviceSession::ResultBatcher_3_4::processCaptureResult_3_4(CaptureResult& result) {
+    auto pair = getBatch(result.v3_2.frameNumber);
+    int batchIdx = pair.first;
+    if (batchIdx == NOT_BATCHED) {
+        processOneCaptureResult_3_4(result);
+        return;
+    }
+    std::shared_ptr<InflightBatch> batch = pair.second;
+    {
+        Mutex::Autolock _l(batch->mLock);
+        // Check if the batch is removed (mostly by notify error) before lock was acquired
+        if (batch->mRemoved) {
+            // Fall back to non-batch path
+            processOneCaptureResult_3_4(result);
+            return;
+        }
+
+        // queue metadata
+        if (result.v3_2.result.size() != 0) {
+            // Save a copy of metadata
+            batch->mResultMds[result.v3_2.partialResult].mMds.push_back(
+                    std::make_pair(result.v3_2.frameNumber, result.v3_2.result));
+        }
+
+        // queue buffer
+        std::vector<int> filledStreams;
+        std::vector<V3_2::StreamBuffer> nonBatchedBuffers;
+        for (auto& buffer : result.v3_2.outputBuffers) {
+            auto it = batch->mBatchBufs.find(buffer.streamId);
+            if (it != batch->mBatchBufs.end()) {
+                InflightBatch::BufferBatch& bb = it->second;
+                pushStreamBuffer(std::move(buffer), bb.mBuffers);
+                filledStreams.push_back(buffer.streamId);
+            } else {
+                pushStreamBuffer(std::move(buffer), nonBatchedBuffers);
+            }
+        }
+
+        // send non-batched buffers up
+        if (nonBatchedBuffers.size() > 0 || result.v3_2.inputBuffer.streamId != -1) {
+            CaptureResult nonBatchedResult;
+            nonBatchedResult.v3_2.frameNumber = result.v3_2.frameNumber;
+            nonBatchedResult.v3_2.fmqResultSize = 0;
+            nonBatchedResult.v3_2.outputBuffers.resize(nonBatchedBuffers.size());
+            for (size_t i = 0; i < nonBatchedBuffers.size(); i++) {
+                moveStreamBuffer(
+                        std::move(nonBatchedBuffers[i]), nonBatchedResult.v3_2.outputBuffers[i]);
+            }
+            moveStreamBuffer(std::move(result.v3_2.inputBuffer), nonBatchedResult.v3_2.inputBuffer);
+            nonBatchedResult.v3_2.partialResult = 0; // 0 for buffer only results
+            processOneCaptureResult_3_4(nonBatchedResult);
+        }
+
+        if (result.v3_2.frameNumber == batch->mLastFrame) {
+            // Send data up
+            if (result.v3_2.partialResult > 0) {
+                sendBatchMetadataLocked(batch, result.v3_2.partialResult);
+            }
+            // send buffer up
+            if (filledStreams.size() > 0) {
+                sendBatchBuffersLocked(batch, filledStreams);
+            }
+        }
+    } // end of batch lock scope
+
+    // see if the batch is complete
+    if (result.v3_2.frameNumber == batch->mLastFrame) {
+        checkAndRemoveFirstBatch();
+    }
+}
+
+void CameraDeviceSession::ResultBatcher_3_4::processOneCaptureResult_3_4(CaptureResult& result) {
+    hidl_vec<CaptureResult> results;
+    results.resize(1);
+    results[0] = std::move(result);
+    invokeProcessCaptureResultCallback_3_4(results, /* tryWriteFmq */true);
+    freeReleaseFences_3_4(results);
+    return;
+}
+
+void CameraDeviceSession::ResultBatcher_3_4::invokeProcessCaptureResultCallback_3_4(
+        hidl_vec<CaptureResult> &results, bool tryWriteFmq) {
+    if (mProcessCaptureResultLock.tryLock() != OK) {
+        ALOGV("%s: previous call is not finished! waiting 1s...", __FUNCTION__);
+        if (mProcessCaptureResultLock.timedLock(1000000000 /* 1s */) != OK) {
+            ALOGE("%s: cannot acquire lock in 1s, cannot proceed",
+                    __FUNCTION__);
+            return;
+        }
+    }
+    if (tryWriteFmq && mResultMetadataQueue->availableToWrite() > 0) {
+        for (CaptureResult &result : results) {
+            if (result.v3_2.result.size() > 0) {
+                if (mResultMetadataQueue->write(result.v3_2.result.data(),
+                        result.v3_2.result.size())) {
+                    result.v3_2.fmqResultSize = result.v3_2.result.size();
+                    result.v3_2.result.resize(0);
+                } else {
+                    ALOGW("%s: couldn't utilize fmq, fall back to hwbinder", __FUNCTION__);
+                    result.v3_2.fmqResultSize = 0;
+                }
+            }
+
+            for (auto& onePhysMetadata : result.physicalCameraMetadata) {
+                if (mResultMetadataQueue->write(onePhysMetadata.metadata.data(),
+                        onePhysMetadata.metadata.size())) {
+                    onePhysMetadata.fmqMetadataSize = onePhysMetadata.metadata.size();
+                    onePhysMetadata.metadata.resize(0);
+                } else {
+                    ALOGW("%s: couldn't utilize fmq, fall back to hwbinder", __FUNCTION__);
+                    onePhysMetadata.fmqMetadataSize = 0;
+                }
+            }
+        }
+    }
+    mCallback_3_4->processCaptureResult_3_4(results);
+    mProcessCaptureResultLock.unlock();
+}
+
+void CameraDeviceSession::ResultBatcher_3_4::freeReleaseFences_3_4(hidl_vec<CaptureResult>& results) {
+    for (auto& result : results) {
+        if (result.v3_2.inputBuffer.releaseFence.getNativeHandle() != nullptr) {
+            native_handle_t* handle = const_cast<native_handle_t*>(
+                    result.v3_2.inputBuffer.releaseFence.getNativeHandle());
+            native_handle_close(handle);
+            native_handle_delete(handle);
+        }
+        for (auto& buf : result.v3_2.outputBuffers) {
+            if (buf.releaseFence.getNativeHandle() != nullptr) {
+                native_handle_t* handle = const_cast<native_handle_t*>(
+                        buf.releaseFence.getNativeHandle());
+                native_handle_close(handle);
+                native_handle_delete(handle);
+            }
+        }
+    }
+    return;
+}
+
 } // namespace implementation
 }  // namespace V3_4
 }  // namespace device