Camera: Various Camera2 extension updates

Update Camera2 extensions to:
 - Query advanced API support
 - Be able to query latency ranges when using
   legacy extensions
 - Trigger the camera extension initializer
   for both the legacy and advanced cases

Bug: 188579227
Test: Camera CTS

Change-Id: I8db505d0c649f6c664f60c1c1e3cd17de54abe3a

6 files changed, 105 insertions, 43 deletions

diff --git a/core/java/android/hardware/camera2/CameraExtensionCharacteristics.java b/core/java/android/hardware/camera2/CameraExtensionCharacteristics.java
index 8e3de616f07b..41272437c407 100644
--- a/core/java/android/hardware/camera2/CameraExtensionCharacteristics.java
+++ b/core/java/android/hardware/camera2/CameraExtensionCharacteristics.java
@@ -693,20 +693,42 @@ public final class CameraExtensionCharacteristics {
                 throw new IllegalArgumentException("Unsupported extension");
             }
 
+            android.hardware.camera2.extension.Size sz =
+                    new android.hardware.camera2.extension.Size();
+            sz.width = captureOutputSize.getWidth();
+            sz.height = captureOutputSize.getHeight();
             if (areAdvancedExtensionsSupported()) {
                 IAdvancedExtenderImpl extender = initializeAdvancedExtension(extension);
                 extender.init(mCameraId);
-                android.hardware.camera2.extension.Size sz =
-                        new android.hardware.camera2.extension.Size();
-                sz.width = captureOutputSize.getWidth();
-                sz.height = captureOutputSize.getHeight();
                 LatencyRange latencyRange = extender.getEstimatedCaptureLatencyRange(mCameraId,
                         sz, format);
                 if (latencyRange != null) {
                     return new Range(latencyRange.min, latencyRange.max);
                 }
-            }
-        } catch (RemoteException e) {
+            } else {
+                Pair<IPreviewExtenderImpl, IImageCaptureExtenderImpl> extenders =
+                        initializeExtension(extension);
+                extenders.second.init(mCameraId, mChars.getNativeMetadata());
+                if ((format == ImageFormat.YUV_420_888) &&
+                        (extenders.second.getCaptureProcessor() == null) ){
+                    // Extensions that don't implement any capture processor are limited to
+                    // JPEG only!
+                    return null;
+                }
+                if ((format == ImageFormat.JPEG) &&
+                        (extenders.second.getCaptureProcessor() != null)) {
+                    // The framework will perform the additional encoding pass on the
+                    // processed YUV_420 buffers. Latency in this case is very device
+                    // specific and cannot be estimated accurately enough.
+                    return  null;
+                }
+
+                LatencyRange latencyRange = extenders.second.getEstimatedCaptureLatencyRange(sz);
+                if (latencyRange != null) {
+                    return new Range(latencyRange.min, latencyRange.max);
+                }
+        }
+    } catch (RemoteException e) {
             Log.e(TAG, "Failed to query the extension capture latency! Extension service does"
                     + " not respond!");
         } finally {
diff --git a/core/java/android/hardware/camera2/extension/IImageCaptureExtenderImpl.aidl b/core/java/android/hardware/camera2/extension/IImageCaptureExtenderImpl.aidl
index c04e75ea6758..3ebf63793b79 100644
--- a/core/java/android/hardware/camera2/extension/IImageCaptureExtenderImpl.aidl
+++ b/core/java/android/hardware/camera2/extension/IImageCaptureExtenderImpl.aidl
@@ -19,6 +19,8 @@ import android.hardware.camera2.impl.CameraMetadataNative;
 
 import android.hardware.camera2.extension.CaptureStageImpl;
 import android.hardware.camera2.extension.ICaptureProcessorImpl;
+import android.hardware.camera2.extension.LatencyRange;
+import android.hardware.camera2.extension.Size;
 import android.hardware.camera2.extension.SizeList;
 
 /** @hide */
@@ -36,4 +38,5 @@ interface IImageCaptureExtenderImpl
     @nullable List<CaptureStageImpl> getCaptureStages();
     int getMaxCaptureStage();
     @nullable List<SizeList> getSupportedResolutions();
+    LatencyRange getEstimatedCaptureLatencyRange(in Size outputSize);
 }
diff --git a/core/java/android/hardware/camera2/extension/IImageProcessorImpl.aidl b/core/java/android/hardware/camera2/extension/IImageProcessorImpl.aidl
index f365469eaecc..4c85966c10dc 100644
--- a/core/java/android/hardware/camera2/extension/IImageProcessorImpl.aidl
+++ b/core/java/android/hardware/camera2/extension/IImageProcessorImpl.aidl
@@ -21,5 +21,6 @@ import android.hardware.camera2.extension.ParcelImage;
 /** @hide */
 interface IImageProcessorImpl
 {
-    void onNextImageAvailable(in OutputConfigId outputConfigId, in ParcelImage image);
+    void onNextImageAvailable(in OutputConfigId outputConfigId, in ParcelImage image,
+            in String physicalCameraId);
 }
diff --git a/core/java/android/hardware/camera2/impl/CameraAdvancedExtensionSessionImpl.java b/core/java/android/hardware/camera2/impl/CameraAdvancedExtensionSessionImpl.java
index abc487db5bdb..5cf50a25de78 100644
--- a/core/java/android/hardware/camera2/impl/CameraAdvancedExtensionSessionImpl.java
+++ b/core/java/android/hardware/camera2/impl/CameraAdvancedExtensionSessionImpl.java
@@ -79,8 +79,8 @@ public final class CameraAdvancedExtensionSessionImpl extends CameraExtensionSes
     private final HandlerThread mHandlerThread;
     private final CameraExtensionSession.StateCallback mCallbacks;
     private final IAdvancedExtenderImpl mAdvancedExtender;
-    // maps camera outputs to extension output ids
-    private final HashMap<Surface, Integer> mSurfaceIdMap = new HashMap<>();
+    // maps registered camera surfaces to extension output configs
+    private final HashMap<Surface, CameraOutputConfig> mCameraConfigMap = new HashMap<>();
     // maps camera extension output ids to camera registered image readers
     private final HashMap<Integer, ImageReader> mReaderMap = new HashMap<>();
     private final RequestProcessor mRequestProcessor = new RequestProcessor();
@@ -226,7 +226,7 @@ public final class CameraAdvancedExtensionSessionImpl extends CameraExtensionSes
                             reader.getSurface());
                     break;
                 case CameraOutputConfig.TYPE_MULTIRES_IMAGEREADER:
-                    // TBD
+                    // Support for multi-resolution outputs to be added in future releases
                 default:
                     throw new IllegalArgumentException("Unsupported output config type: " +
                             output.type);
@@ -251,7 +251,7 @@ public final class CameraAdvancedExtensionSessionImpl extends CameraExtensionSes
                 }
             }
             outputList.add(outConfig);
-            mSurfaceIdMap.put(outConfig.getSurface(), output.outputId.id);
+            mCameraConfigMap.put(outConfig.getSurface(), output);
         }
 
         SessionConfiguration sessionConfiguration = new SessionConfiguration(
@@ -629,7 +629,7 @@ public final class CameraAdvancedExtensionSessionImpl extends CameraExtensionSes
                 if (request.getTag() instanceof Integer) {
                     Integer requestId = (Integer) request.getTag();
                     mCallback.onCaptureBufferLost(requestId, frameNumber,
-                            mSurfaceIdMap.get(target));
+                            mCameraConfigMap.get(target).outputId.id);
                 } else {
                     Log.e(TAG, "Invalid capture request tag!");
                 }
@@ -736,12 +736,14 @@ public final class CameraAdvancedExtensionSessionImpl extends CameraExtensionSes
     private static final class ImageReaderHandler implements ImageReader.OnImageAvailableListener {
         private final OutputConfigId mOutputConfigId;
         private final IImageProcessorImpl mIImageProcessor;
+        private final String mPhysicalCameraId;
 
         private ImageReaderHandler(int outputConfigId,
-                IImageProcessorImpl iImageProcessor) {
+                IImageProcessorImpl iImageProcessor, String physicalCameraId) {
             mOutputConfigId = new OutputConfigId();
             mOutputConfigId.id = outputConfigId;
             mIImageProcessor = iImageProcessor;
+            mPhysicalCameraId = physicalCameraId;
         }
 
         @Override
@@ -787,7 +789,8 @@ public final class CameraAdvancedExtensionSessionImpl extends CameraExtensionSes
             parcelImage.crop = img.getCropRect();
 
             try {
-                mIImageProcessor.onNextImageAvailable(mOutputConfigId, parcelImage);
+                mIImageProcessor.onNextImageAvailable(mOutputConfigId, parcelImage,
+                        mPhysicalCameraId);
             } catch (RemoteException e) {
                 Log.e(TAG, "Failed to propagate image buffer on output surface id: " +
                         mOutputConfigId + " extension service does not respond!");
@@ -804,8 +807,17 @@ public final class CameraAdvancedExtensionSessionImpl extends CameraExtensionSes
                 IImageProcessorImpl imageProcessor) {
             synchronized (mInterfaceLock) {
                 if (mReaderMap.containsKey(outputConfigId.id)) {
-                    mReaderMap.get(outputConfigId.id).setOnImageAvailableListener(
-                            new ImageReaderHandler(outputConfigId.id, imageProcessor), mHandler);
+                    ImageReader reader = mReaderMap.get(outputConfigId.id);
+                    String physicalCameraId = null;
+                    if (mCameraConfigMap.containsKey(reader.getSurface())) {
+                        physicalCameraId =
+                                mCameraConfigMap.get(reader.getSurface()).physicalCameraId;
+                        reader.setOnImageAvailableListener(new ImageReaderHandler(outputConfigId.id,
+                                    imageProcessor, physicalCameraId), mHandler);
+                    } else {
+                        Log.e(TAG, "Camera output configuration for ImageReader with " +
+                                        " config Id " + outputConfigId.id + " not found!");
+                    }
                 } else {
                     Log.e(TAG, "ImageReader with output config id: " + outputConfigId.id +
                             " not found!");
@@ -828,7 +840,7 @@ public final class CameraAdvancedExtensionSessionImpl extends CameraExtensionSes
                     ArrayList<CaptureRequest> captureRequests = new ArrayList<>();
                     for (Request request : requests) {
                         captureRequests.add(initializeCaptureRequest(mCameraDevice, request,
-                                mSurfaceIdMap));
+                                mCameraConfigMap));
                     }
                     mCaptureSession.captureBurstRequests(captureRequests,
                             new CameraExtensionUtils.HandlerExecutor(mHandler), captureCallback);
@@ -848,7 +860,7 @@ public final class CameraAdvancedExtensionSessionImpl extends CameraExtensionSes
             synchronized (mInterfaceLock) {
                 try {
                     CaptureRequest repeatingRequest = initializeCaptureRequest(mCameraDevice,
-                                request, mSurfaceIdMap);
+                                request, mCameraConfigMap);
                     CaptureCallbackHandler captureCallback = new CaptureCallbackHandler(callback);
                     mCaptureSession.setSingleRepeatingRequest(repeatingRequest,
                             new CameraExtensionUtils.HandlerExecutor(mHandler), captureCallback);
@@ -891,12 +903,13 @@ public final class CameraAdvancedExtensionSessionImpl extends CameraExtensionSes
     }
 
     private static CaptureRequest initializeCaptureRequest(CameraDevice cameraDevice,
-            Request request, HashMap<Surface, Integer> surfaceIdMap) throws CameraAccessException {
+            Request request, HashMap<Surface, CameraOutputConfig> surfaceIdMap)
+            throws CameraAccessException {
         CaptureRequest.Builder builder = cameraDevice.createCaptureRequest(request.templateId);
         for (OutputConfigId configId : request.targetOutputConfigIds) {
             boolean found = false;
-            for (Map.Entry<Surface, Integer> entry : surfaceIdMap.entrySet()) {
-                if (entry.getValue() == configId.id) {
+            for (Map.Entry<Surface, CameraOutputConfig> entry : surfaceIdMap.entrySet()) {
+                if (entry.getValue().outputId.id == configId.id) {
                     builder.addTarget(entry.getKey());
                     found = true;
                     break;
diff --git a/core/java/android/hardware/camera2/impl/CameraExtensionSessionImpl.java b/core/java/android/hardware/camera2/impl/CameraExtensionSessionImpl.java
index 5339f41ecdd6..4bcc4942d8a0 100644
--- a/core/java/android/hardware/camera2/impl/CameraExtensionSessionImpl.java
+++ b/core/java/android/hardware/camera2/impl/CameraExtensionSessionImpl.java
@@ -18,7 +18,6 @@ package android.hardware.camera2.impl;
 
 import android.content.Context;
 import android.graphics.ImageFormat;
-import android.graphics.PixelFormat;
 import android.graphics.SurfaceTexture;
 import android.hardware.HardwareBuffer;
 import android.hardware.camera2.CameraAccessException;
@@ -42,7 +41,6 @@ import android.hardware.camera2.TotalCaptureResult;
 import android.hardware.camera2.params.ExtensionSessionConfiguration;
 import android.hardware.camera2.params.OutputConfiguration;
 import android.hardware.camera2.params.SessionConfiguration;
-import android.hardware.camera2.params.StreamConfigurationMap;
 import android.hardware.camera2.utils.SurfaceUtils;
 import android.media.Image;
 import android.media.ImageReader;
@@ -68,7 +66,6 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.Executor;
-import java.util.concurrent.RejectedExecutionException;
 
 public final class CameraExtensionSessionImpl extends CameraExtensionSession {
     private static final int PREVIEW_QUEUE_SIZE = 3;
diff --git a/packages/services/CameraExtensionsProxy/src/com/android/camera/CameraExtensionsProxyService.java b/packages/services/CameraExtensionsProxy/src/com/android/camera/CameraExtensionsProxyService.java
index d44a4173fc12..7b9ca2ddc345 100644
--- a/packages/services/CameraExtensionsProxy/src/com/android/camera/CameraExtensionsProxyService.java
+++ b/packages/services/CameraExtensionsProxy/src/com/android/camera/CameraExtensionsProxyService.java
@@ -119,20 +119,32 @@ public class CameraExtensionsProxyService extends Service {
     private static final String CAMERA_EXTENSION_VERSION_NAME =
             "androidx.camera.extensions.impl.ExtensionVersionImpl";
     private static final String LATEST_VERSION = "1.2.0";
-    private static final String LEGACY_VERSION_PREFIX = "1.1";
+    private static final String NON_INIT_VERSION_PREFIX = "1.0";
     private static final String ADVANCED_VERSION_PREFIX = "1.2";
     private static final String[] SUPPORTED_VERSION_PREFIXES = {ADVANCED_VERSION_PREFIX,
-            LEGACY_VERSION_PREFIX, "1.0."};
+            "1.1", NON_INIT_VERSION_PREFIX};
     private static final boolean EXTENSIONS_PRESENT = checkForExtensions();
     private static final String EXTENSIONS_VERSION = EXTENSIONS_PRESENT ?
             (new ExtensionVersionImpl()).checkApiVersion(LATEST_VERSION) : null;
-    private static final boolean LEGACY_VERSION_SUPPORTED =
-            EXTENSIONS_PRESENT && EXTENSIONS_VERSION.startsWith(LEGACY_VERSION_PREFIX);
-    private static final boolean ADVANCED_VERSION_SUPPORTED =
-            EXTENSIONS_PRESENT && EXTENSIONS_VERSION.startsWith(ADVANCED_VERSION_PREFIX);
+    private static final boolean ADVANCED_API_SUPPORTED = checkForAdvancedAPI();
+    private static final boolean INIT_API_SUPPORTED = EXTENSIONS_PRESENT &&
+            (!EXTENSIONS_VERSION.startsWith(NON_INIT_VERSION_PREFIX));
 
     private HashMap<String, CameraCharacteristics> mCharacteristicsHashMap = new HashMap<>();
 
+    private static boolean checkForAdvancedAPI() {
+        if (EXTENSIONS_PRESENT && EXTENSIONS_VERSION.startsWith(ADVANCED_VERSION_PREFIX)) {
+            try {
+                return (new ExtensionVersionImpl()).isAdvancedExtenderImplemented();
+            } catch (NoSuchMethodError e) {
+                // This could happen in case device specific extension implementations are using an
+                // older extension API but incorrectly set the extension version.
+            }
+        }
+
+        return false;
+    }
+
     private static boolean checkForExtensions() {
         try {
             Class.forName(CAMERA_EXTENSION_VERSION_NAME);
@@ -265,7 +277,7 @@ public class CameraExtensionsProxyService extends Service {
 
         public long registerClient(Context ctx) {
             synchronized (mLock) {
-                if (LEGACY_VERSION_SUPPORTED) {
+                if (INIT_API_SUPPORTED) {
                     if (mActiveClients.isEmpty()) {
                         InitializerFuture status = new InitializerFuture();
                         InitializerImpl.init(EXTENSIONS_VERSION, ctx, new InitializeHandler(status),
@@ -299,7 +311,7 @@ public class CameraExtensionsProxyService extends Service {
         public void unregisterClient(long clientId) {
             synchronized (mLock) {
                 if (mActiveClients.remove(clientId) && mActiveClients.isEmpty() &&
-                        LEGACY_VERSION_SUPPORTED) {
+                        INIT_API_SUPPORTED) {
                     InitializerFuture status = new InitializerFuture();
                     InitializerImpl.deinit(new ReleaseHandler(status),
                             new HandlerExecutor(mHandler));
@@ -528,7 +540,7 @@ public class CameraExtensionsProxyService extends Service {
 
         @Override
         public boolean advancedExtensionsSupported() {
-            return ADVANCED_VERSION_SUPPORTED;
+            return ADVANCED_API_SUPPORTED;
         }
 
         @Override
@@ -819,10 +831,11 @@ public class CameraExtensionsProxyService extends Service {
         }
 
         @Override
-        public void onNextImageAvailable(OutputConfigId outputConfigId, ParcelImage img) {
+        public void onNextImageAvailable(OutputConfigId outputConfigId, ParcelImage img,
+                String physicalCameraId) {
             if (mImageProcessor != null) {
                 mImageProcessor.onNextImageAvailable(outputConfigId.id, img.timestamp,
-                        new ImageReferenceImpl(img));
+                        new ImageReferenceImpl(img), physicalCameraId);
             }
         }
     }
@@ -1072,9 +1085,7 @@ public class CameraExtensionsProxyService extends Service {
 
         @Override
         public void init(String cameraId, CameraMetadataNative chars) {
-            if (LEGACY_VERSION_SUPPORTED) {
-                mPreviewExtender.init(cameraId, new CameraCharacteristics(chars));
-            }
+            mPreviewExtender.init(cameraId, new CameraCharacteristics(chars));
         }
 
         @Override
@@ -1136,7 +1147,7 @@ public class CameraExtensionsProxyService extends Service {
 
         @Override
         public List<SizeList> getSupportedResolutions() {
-            if (LEGACY_VERSION_SUPPORTED) {
+            if (INIT_API_SUPPORTED) {
                 List<Pair<Integer, android.util.Size[]>> sizes =
                         mPreviewExtender.getSupportedResolutions();
                 if ((sizes != null) && !sizes.isEmpty()) {
@@ -1182,9 +1193,7 @@ public class CameraExtensionsProxyService extends Service {
 
         @Override
         public void init(String cameraId, CameraMetadataNative chars) {
-            if (LEGACY_VERSION_SUPPORTED) {
-                mImageExtender.init(cameraId, new CameraCharacteristics(chars));
-            }
+            mImageExtender.init(cameraId, new CameraCharacteristics(chars));
         }
 
         @Override
@@ -1228,7 +1237,7 @@ public class CameraExtensionsProxyService extends Service {
 
         @Override
         public List<SizeList> getSupportedResolutions() {
-            if (LEGACY_VERSION_SUPPORTED) {
+            if (INIT_API_SUPPORTED) {
                 List<Pair<Integer, android.util.Size[]>> sizes =
                         mImageExtender.getSupportedResolutions();
                 if ((sizes != null) && !sizes.isEmpty()) {
@@ -1238,6 +1247,23 @@ public class CameraExtensionsProxyService extends Service {
 
             return null;
         }
+
+        @Override
+        public LatencyRange getEstimatedCaptureLatencyRange(
+                android.hardware.camera2.extension.Size outputSize) {
+            if (EXTENSIONS_VERSION.startsWith(ADVANCED_VERSION_PREFIX)) {
+                Size sz = new Size(outputSize.width, outputSize.height);
+                Range<Long> latencyRange = mImageExtender.getEstimatedCaptureLatencyRange(sz);
+                if (latencyRange != null) {
+                    LatencyRange ret = new LatencyRange();
+                    ret.min = latencyRange.getLower();
+                    ret.max = latencyRange.getUpper();
+                    return ret;
+                }
+            }
+
+            return null;
+        }
     }
 
     private class CaptureProcessorImplStub extends ICaptureProcessorImpl.Stub {