diff options
| -rw-r--r-- | api/current.txt | 42 | ||||
| -rw-r--r-- | media/java/android/media/MediaCodec.java | 1017 | ||||
| -rw-r--r-- | media/jni/Android.bp | 2 | ||||
| -rw-r--r-- | media/jni/android_media_MediaCodec.cpp | 1321 | ||||
| -rw-r--r-- | media/jni/android_media_MediaCodec.h | 37 |
5 files changed, 2184 insertions, 235 deletions
diff --git a/api/current.txt b/api/current.txt index afbb427ed288..2027734124fb 100644 --- a/api/current.txt +++ b/api/current.txt @@ -24737,7 +24737,9 @@ package android.media { method @Deprecated @NonNull public java.nio.ByteBuffer[] getOutputBuffers(); method @NonNull public android.media.MediaFormat getOutputFormat(); method @NonNull public android.media.MediaFormat getOutputFormat(int); + method @NonNull public android.media.MediaCodec.OutputFrame getOutputFrame(int); method @Nullable public android.media.Image getOutputImage(int); + method @NonNull public android.media.MediaCodec.QueueRequest getQueueRequest(int); method public void queueInputBuffer(int, int, int, long, int) throws android.media.MediaCodec.CryptoException; method public void queueSecureInputBuffer(int, int, @NonNull android.media.MediaCodec.CryptoInfo, long, int) throws android.media.MediaCodec.CryptoException; method public void release(); @@ -24761,6 +24763,7 @@ package android.media { field public static final int BUFFER_FLAG_PARTIAL_FRAME = 8; // 0x8 field @Deprecated public static final int BUFFER_FLAG_SYNC_FRAME = 1; // 0x1 field public static final int CONFIGURE_FLAG_ENCODE = 1; // 0x1 + field public static final int CONFIGURE_FLAG_USE_BLOCK_MODEL = 2; // 0x2 field public static final int CRYPTO_MODE_AES_CBC = 2; // 0x2 field public static final int CRYPTO_MODE_AES_CTR = 1; // 0x1 field public static final int CRYPTO_MODE_UNENCRYPTED = 0; // 0x0 @@ -24837,6 +24840,24 @@ package android.media { method public void set(int, int); } + public static final class MediaCodec.GraphicBlock { + method protected void finalize(); + method public static boolean isCodecCopyFreeCompatible(@NonNull String[]); + method public boolean isMappable(); + method @NonNull public android.media.Image map(); + method @NonNull public static android.media.MediaCodec.GraphicBlock obtain(int, int, int, long, @NonNull String[]); + method public void recycle(); + } + + public static final class MediaCodec.LinearBlock { + method protected void finalize(); + method public static boolean isCodecCopyFreeCompatible(@NonNull String[]); + method public boolean isMappable(); + method @NonNull public java.nio.ByteBuffer map(); + method @Nullable public static android.media.MediaCodec.LinearBlock obtain(int, @NonNull String[]); + method public void recycle(); + } + public static final class MediaCodec.MetricsConstants { field public static final String CODEC = "android.media.mediacodec.codec"; field public static final String ENCODER = "android.media.mediacodec.encoder"; @@ -24854,6 +24875,27 @@ package android.media { method public void onFrameRendered(@NonNull android.media.MediaCodec, long, long); } + public static final class MediaCodec.OutputFrame { + method public void getChangedKeys(@NonNull java.util.Set<java.lang.String>); + method public int getFlags(); + method @NonNull public android.media.MediaFormat getFormat(); + method @Nullable public android.media.MediaCodec.GraphicBlock getGraphicBlock(); + method @Nullable public android.media.MediaCodec.LinearBlock getLinearBlock(); + method public long getPresentationTimeUs(); + } + + public final class MediaCodec.QueueRequest { + method public void queue(); + method @NonNull public android.media.MediaCodec.QueueRequest setByteBufferParameter(@NonNull String, @NonNull java.nio.ByteBuffer); + method @NonNull public android.media.MediaCodec.QueueRequest setEncryptedLinearBlock(@NonNull android.media.MediaCodec.LinearBlock, int, @NonNull android.media.MediaCodec.CryptoInfo, long, int); + method @NonNull public android.media.MediaCodec.QueueRequest setFloatParameter(@NonNull String, float); + method @NonNull public android.media.MediaCodec.QueueRequest setGraphicBlock(@NonNull android.media.MediaCodec.GraphicBlock, long, int); + method @NonNull public android.media.MediaCodec.QueueRequest setIntegerParameter(@NonNull String, int); + method @NonNull public android.media.MediaCodec.QueueRequest setLinearBlock(@NonNull android.media.MediaCodec.LinearBlock, int, int, long, int); + method @NonNull public android.media.MediaCodec.QueueRequest setLongParameter(@NonNull String, long); + method @NonNull public android.media.MediaCodec.QueueRequest setStringParameter(@NonNull String, @NonNull String); + } + public final class MediaCodecInfo { method @NonNull public String getCanonicalName(); method public android.media.MediaCodecInfo.CodecCapabilities getCapabilitiesForType(String); diff --git a/media/java/android/media/MediaCodec.java b/media/java/android/media/MediaCodec.java index f780d40d349b..abc7e0b7be0e 100644 --- a/media/java/android/media/MediaCodec.java +++ b/media/java/android/media/MediaCodec.java @@ -23,6 +23,7 @@ import android.compat.annotation.UnsupportedAppUsage; import android.graphics.ImageFormat; import android.graphics.Rect; import android.graphics.SurfaceTexture; +import android.hardware.HardwareBuffer; import android.media.MediaCodecInfo.CodecCapabilities; import android.os.Build; import android.os.Bundle; @@ -39,9 +40,13 @@ import java.lang.annotation.RetentionPolicy; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.ReadOnlyBufferException; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.Map; +import java.util.Set; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -1736,7 +1741,25 @@ final public class MediaCodec { { int index = msg.arg2; synchronized(mBufferLock) { - validateInputByteBuffer(mCachedInputBuffers, index); + switch (mBufferMode) { + case BUFFER_MODE_LEGACY: + validateInputByteBuffer(mCachedInputBuffers, index); + break; + case BUFFER_MODE_BLOCK: + while (mQueueRequests.size() <= index) { + mQueueRequests.add(null); + } + QueueRequest request = mQueueRequests.get(index); + if (request == null) { + request = new QueueRequest(mCodec, index); + mQueueRequests.set(index, request); + } + request.setAccessible(true); + break; + default: + throw new IllegalStateException( + "Unrecognized buffer mode: " + mBufferMode); + } } mCallback.onInputBufferAvailable(mCodec, index); break; @@ -1747,7 +1770,26 @@ final public class MediaCodec { int index = msg.arg2; BufferInfo info = (MediaCodec.BufferInfo) msg.obj; synchronized(mBufferLock) { - validateOutputByteBuffer(mCachedOutputBuffers, index, info); + switch (mBufferMode) { + case BUFFER_MODE_LEGACY: + validateOutputByteBuffer(mCachedOutputBuffers, index, info); + break; + case BUFFER_MODE_BLOCK: + while (mOutputFrames.size() <= index) { + mOutputFrames.add(null); + } + OutputFrame frame = mOutputFrames.get(index); + if (frame == null) { + frame = new OutputFrame(index); + mOutputFrames.set(index, frame); + } + frame.setBufferInfo(info); + frame.setAccessible(true); + break; + default: + throw new IllegalStateException( + "Unrecognized buffer mode: " + mBufferMode); + } } mCallback.onOutputBufferAvailable( mCodec, index, info); @@ -1913,8 +1955,33 @@ final public class MediaCodec { */ public static final int CONFIGURE_FLAG_ENCODE = 1; + /** + * If this codec is to be used with {@link LinearBlock} and/or {@link + * GraphicBlock}, pass this flag. + * <p> + * When this flag is set, the following APIs throw IllegalStateException. + * <ul> + * <li>{@link #getInputBuffer} + * <li>{@link #getInputImage} + * <li>{@link #getInputBuffers} + * <li>{@link #getOutputBuffer} + * <li>{@link #getOutputImage} + * <li>{@link #getOutputBuffers} + * <li>{@link #queueInputBuffer} + * <li>{@link #queueSecureInputBuffer} + * <li>{@link #dequeueInputBuffer} + * <li>{@link #dequeueOutputBuffer} + * </ul> + */ + public static final int CONFIGURE_FLAG_USE_BLOCK_MODEL = 2; + /** @hide */ - @IntDef(flag = true, value = { CONFIGURE_FLAG_ENCODE }) + @IntDef( + flag = true, + value = { + CONFIGURE_FLAG_ENCODE, + CONFIGURE_FLAG_USE_BLOCK_MODEL, + }) @Retention(RetentionPolicy.SOURCE) public @interface ConfigureFlag {} @@ -1984,6 +2051,11 @@ final public class MediaCodec { descrambler != null ? descrambler.getBinder() : null, flags); } + private static final int BUFFER_MODE_INVALID = -1; + private static final int BUFFER_MODE_LEGACY = 0; + private static final int BUFFER_MODE_BLOCK = 1; + private int mBufferMode = BUFFER_MODE_INVALID; + private void configure( @Nullable MediaFormat format, @Nullable Surface surface, @Nullable MediaCrypto crypto, @Nullable IHwBinder descramblerBinder, @@ -2022,6 +2094,13 @@ final public class MediaCodec { mHasSurface = surface != null; mCrypto = crypto; + synchronized (mBufferLock) { + if ((flags & CONFIGURE_FLAG_USE_BLOCK_MODEL) != 0) { + mBufferMode = BUFFER_MODE_BLOCK; + } else { + mBufferMode = BUFFER_MODE_LEGACY; + } + } native_configure(keys, values, surface, crypto, descramblerBinder, flags); } @@ -2446,6 +2525,9 @@ final public class MediaCodec { int offset, int size, long presentationTimeUs, int flags) throws CryptoException { synchronized(mBufferLock) { + if (mBufferMode != BUFFER_MODE_LEGACY) { + throw new IllegalStateException(); + } invalidateByteBuffer(mCachedInputBuffers, index); mDequeuedInputBuffers.remove(index); } @@ -2695,6 +2777,9 @@ final public class MediaCodec { long presentationTimeUs, int flags) throws CryptoException { synchronized(mBufferLock) { + if (mBufferMode != BUFFER_MODE_LEGACY) { + throw new IllegalStateException(); + } invalidateByteBuffer(mCachedInputBuffers, index); mDequeuedInputBuffers.remove(index); } @@ -2726,6 +2811,11 @@ final public class MediaCodec { * @throws MediaCodec.CodecException upon codec error. */ public final int dequeueInputBuffer(long timeoutUs) { + synchronized (mBufferLock) { + if (mBufferMode != BUFFER_MODE_LEGACY) { + throw new IllegalStateException(); + } + } int res = native_dequeueInputBuffer(timeoutUs); if (res >= 0) { synchronized(mBufferLock) { @@ -2738,6 +2828,654 @@ final public class MediaCodec { private native final int native_dequeueInputBuffer(long timeoutUs); /** + * Section of memory that represents a linear block. Applications may + * acquire a block via {@link LinearBlock#obtain} and queue all or part + * of the block as an input buffer to a codec, or get a block allocated by + * codec as an output buffer from {@link OutputFrame}. + * + * {@see QueueRequest#setLinearBlock} + * {@see QueueRequest#setEncryptedLinearBlock} + * {@see OutputFrame#getLinearBlock} + */ + public static final class LinearBlock { + // No public constructors. + private LinearBlock() {} + + /** + * Returns true if the buffer is mappable. + * @throws IllegalStateException if invalid + */ + public boolean isMappable() { + synchronized (mLock) { + if (!mValid) { + throw new IllegalStateException(); + } + return mMappable; + } + } + + /** + * Map the memory and return the mapped region. + * <p> + * The returned memory region becomes inaccessible after + * {@link #recycle}, or the buffer is queued to the codecs and not + * returned to the client yet. + * + * @return mapped memory region as {@link ByteBuffer} object + * @throws IllegalStateException if not mappable or invalid + */ + public @NonNull ByteBuffer map() { + synchronized (mLock) { + if (!mValid) { + throw new IllegalStateException(); + } + if (!mMappable) { + throw new IllegalStateException(); + } + if (mMapped == null) { + mMapped = native_map(); + } + return mMapped; + } + } + + private native ByteBuffer native_map(); + + /** + * Mark this block as ready to be recycled by the framework once it is + * no longer in use. All operations to this object after + * this call will cause exceptions, as well as attempt to access the + * previously mapped memory region. Caller should clear all references + * to this object after this call. + * <p> + * To avoid excessive memory consumption, it is recommended that callers + * recycle buffers as soon as they no longer need the buffers + * + * @throws IllegalStateException if invalid + */ + public void recycle() { + synchronized (mLock) { + if (!mValid) { + throw new IllegalStateException(); + } + if (mMapped != null) { + mMapped.setAccessible(false); + mMapped = null; + } + native_recycle(); + mValid = false; + mNativeContext = 0; + } + sPool.offer(this); + } + + private native void native_recycle(); + + private native void native_obtain(int capacity, String[] codecNames); + + @Override + protected void finalize() { + native_recycle(); + } + + /** + * Returns true if it is possible to allocate a linear block that can be + * passed to all listed codecs as input buffers without copying the + * content. + * <p> + * Note that even if this function returns true, {@link #obtain} may + * still throw due to invalid arguments or allocation failure. + * + * @param codecNames list of codecs that the client wants to use a + * linear block without copying. Null entries are + * ignored. + */ + public static boolean isCodecCopyFreeCompatible(@NonNull String[] codecNames) { + return native_checkCompatible(codecNames); + } + + private static native boolean native_checkCompatible(@NonNull String[] codecNames); + + /** + * Obtain a linear block object no smaller than {@code capacity}. + * If {@link #isCodecCopyFreeCompatible} with the same + * {@code codecNames} returned true, the returned + * {@link LinearBlock} object can be queued to the listed codecs without + * copying. The returned {@link LinearBlock} object is always + * read/write mappable. + * + * @param capacity requested capacity of the linear block in bytes + * @param codecNames list of codecs that the client wants to use this + * linear block without copying. Null entries are + * ignored. + * @return a linear block object. + * @throws IllegalArgumentException if the capacity is invalid or + * codecNames contains invalid name + * @throws IOException if an error occurred while allocating a buffer + */ + public static @Nullable LinearBlock obtain( + int capacity, @NonNull String[] codecNames) { + LinearBlock buffer = sPool.poll(); + if (buffer == null) { + buffer = new LinearBlock(); + } + synchronized (buffer.mLock) { + buffer.native_obtain(capacity, codecNames); + } + return buffer; + } + + // Called from native + private void setInternalStateLocked(long context, boolean isMappable) { + mNativeContext = context; + mMappable = isMappable; + mValid = (context != 0); + } + + private static final BlockingQueue<LinearBlock> sPool = + new LinkedBlockingQueue<>(); + + private final Object mLock = new Object(); + private boolean mValid = false; + private boolean mMappable = false; + private ByteBuffer mMapped = null; + private long mNativeContext = 0; + } + + /** + * Section of memory that represents a graphic block. Applications may + * acquire a block via {@link GraphicBlock#obtain} and queue + * the block as an input buffer to a codec, or get a block allocated by + * codec as an output buffer from {@link OutputFrame}. + * + * {@see QueueRequest#setGraphicBlock} + * {@see OutputFrame#getGraphicBlock} + */ + public static final class GraphicBlock { + // No public constructors. + private GraphicBlock() {} + + /** + * Returns true if the buffer is mappable. + * @throws IllegalStateException if invalid + */ + public boolean isMappable() { + synchronized (mLock) { + if (!mValid) { + throw new IllegalStateException(); + } + return mMappable; + } + } + + /** + * Map the memory and return the mapped region. + * <p> + * Calling {@link #recycle} or + * {@link QueueRequest#setGraphicBlock} causes the returned + * {@link Image} object to be closed, if not already. + * + * @return mapped memory region as {@link Image} object + * @throws IllegalStateException if not mappable or invalid + */ + public @NonNull Image map() { + synchronized (mLock) { + if (!mValid) { + throw new IllegalStateException(); + } + if (!mMappable) { + throw new IllegalStateException(); + } + if (mMapped == null) { + mMapped = native_map(); + } + return mMapped; + } + } + + private native Image native_map(); + + /** + * Mark this block as ready to be recycled by the framework once it is + * no longer in use. All operations to this object after + * this call will cause exceptions, as well as attempt to access the + * previously mapped memory region. Caller should clear all references + * to this object after this call. + * <p> + * To avoid excessive memory consumption, it is recommended that callers + * recycle buffers as soon as they no longer need the buffers. + * + * @throws IllegalStateException if invalid + */ + public void recycle() { + synchronized (mLock) { + if (!mValid) { + throw new IllegalStateException(); + } + if (mMapped != null) { + mMapped.close(); + mMapped = null; + } + native_recycle(); + mValid = false; + mNativeContext = 0; + } + sPool.offer(this); + } + + private native void native_recycle(); + + /** + * Returns true if it is possible to allocate a graphic block that + * can be passed to all listed codecs as an input buffer without + * copying. + * <p> + * Note that even if this function returns true, {@link #obtain} + * may still throw due to invalid arguments or allocation failure. + * In addition, choosing a format that is not natively supported by the + * codec may cause color conversion. + * + * @param codecNames list of codecs that the client wants to use a + * graphic block without copying. Null entries are + * ignored. + */ + public static boolean isCodecCopyFreeCompatible(@NonNull String[] codecNames) { + return native_checkCompatible(codecNames); + } + + private static native boolean native_checkCompatible(@NonNull String[] codecNames); + + // Called from native + private void setInternalStateLocked(long context, boolean isMappable) { + mNativeContext = context; + mMappable = isMappable; + mValid = (context != 0); + } + + private static final BlockingQueue<GraphicBlock> sPool = + new LinkedBlockingQueue<>(); + + /** + * Obtain a graphic block object of dimension + * {@code width}x{@code height}. + * If {@link #isCodecCopyFreeCompatible} with the same + * {@code codecNames} returned true, the returned + * {@link GraphicBlock} object can be queued to the listed codecs + * without copying. The returned {@link GraphicBlock} object is always + * read/write mappable. + * + * @param width requested width of the graphic block + * @param height requested height of the graphic block + * @param format the format of pixels. One of the {@code COLOR_Format} + * values from {@link MediaCodecInfo.CodecCapabilities}. + * @param usage the usage of the buffer. @HardwareBuffer.Usage + * @param codecNames list of codecs that the client wants to use this + * graphic block without copying. Null entries are + * ignored. + * @return a graphic block object. + * @throws IllegalArgumentException if the parameters are invalid or + * not supported + * @throws IOException if an error occurred while allocating a buffer + */ + public static @NonNull GraphicBlock obtain( + int width, + int height, + int format, + @HardwareBuffer.Usage long usage, + @NonNull String[] codecNames) { + GraphicBlock buffer = sPool.poll(); + if (buffer == null) { + buffer = new GraphicBlock(); + } + if (width < 0 || height < 0) { + throw new IllegalArgumentException(); + } + synchronized (buffer.mLock) { + buffer.native_obtain(width, height, format, usage, codecNames); + } + return buffer; + } + + private native void native_obtain( + int width, + int height, + int format, + @HardwareBuffer.Usage long usage, + @NonNull String[] codecNames); + + @Override + protected void finalize() { + native_recycle(); + } + + private final Object mLock = new Object(); + private boolean mValid = false; + private boolean mMappable = false; + private Image mMapped = null; + private long mNativeContext = 0; + } + + /** + * Builder-like class for queue requests. Use this class to prepare a + * queue request and send it. + */ + public final class QueueRequest { + // No public constructor + private QueueRequest(@NonNull MediaCodec codec, int index) { + mCodec = codec; + mIndex = index; + } + + /** + * Set a linear block to this queue request. Exactly one buffer must be + * set for a queue request before calling {@link #queue}. It is possible + * to use the same {@link LinearBlock} object for multiple queue + * requests. The behavior is undefined if the range of the buffer + * overlaps for multiple requests, or the application writes into the + * region being processed by the codec. + * + * @param block The linear block object + * @param offset The byte offset into the input buffer at which the data starts. + * @param size The number of bytes of valid input data. + * @param presentationTimeUs The presentation timestamp in microseconds for this + * buffer. This is normally the media time at which this + * buffer should be presented (rendered). When using an output + * surface, this will be propagated as the {@link + * SurfaceTexture#getTimestamp timestamp} for the frame (after + * conversion to nanoseconds). + * @param flags A bitmask of flags + * {@link #BUFFER_FLAG_CODEC_CONFIG} and {@link #BUFFER_FLAG_END_OF_STREAM}. + * While not prohibited, most codecs do not use the + * {@link #BUFFER_FLAG_KEY_FRAME} flag for input buffers. + * @return this object + * @throws IllegalStateException if a buffer is already set + */ + public @NonNull QueueRequest setLinearBlock( + @NonNull LinearBlock block, + int offset, + int size, + long presentationTimeUs, + @BufferFlag int flags) { + if (!isAccessible()) { + throw new IllegalStateException(); + } + if (mLinearBlock != null || mGraphicBlock != null) { + throw new IllegalStateException(); + } + mLinearBlock = block; + mOffset = offset; + mSize = size; + mPresentationTimeUs = presentationTimeUs; + mFlags = flags; + return this; + } + + /** + * Set an encrypted linear block to this queue request. Exactly one + * buffer must be set for a queue request before calling {@link #queue}. + * + * @param block The linear block object + * @param offset The byte offset into the input buffer at which the data starts. + * @param presentationTimeUs The presentation timestamp in microseconds for this + * buffer. This is normally the media time at which this + * buffer should be presented (rendered). When using an output + * surface, this will be propagated as the {@link + * SurfaceTexture#getTimestamp timestamp} for the frame (after + * conversion to nanoseconds). + * @param cryptoInfo Metadata describing the structure of the encrypted input sample. + * @param flags A bitmask of flags + * {@link #BUFFER_FLAG_CODEC_CONFIG} and {@link #BUFFER_FLAG_END_OF_STREAM}. + * While not prohibited, most codecs do not use the + * {@link #BUFFER_FLAG_KEY_FRAME} flag for input buffers. + * @return this object + * @throws IllegalStateException if a buffer is already set + */ + public @NonNull QueueRequest setEncryptedLinearBlock( + @NonNull LinearBlock block, + int offset, + @NonNull MediaCodec.CryptoInfo cryptoInfo, + long presentationTimeUs, + @BufferFlag int flags) { + if (!isAccessible()) { + throw new IllegalStateException(); + } + if (mLinearBlock != null || mGraphicBlock != null) { + throw new IllegalStateException(); + } + mLinearBlock = block; + mOffset = offset; + mCryptoInfo = cryptoInfo; + mPresentationTimeUs = presentationTimeUs; + mFlags = flags; + return this; + } + + /** + * Set a graphic block to this queue request. Exactly one buffer must + * be set for a queue request before calling {@link #queue}. + * + * @param block The graphic block object + * @param presentationTimeUs The presentation timestamp in microseconds for this + * buffer. This is normally the media time at which this + * buffer should be presented (rendered). When using an output + * surface, this will be propagated as the {@link + * SurfaceTexture#getTimestamp timestamp} for the frame (after + * conversion to nanoseconds). + * @param flags A bitmask of flags + * {@link #BUFFER_FLAG_CODEC_CONFIG} and {@link #BUFFER_FLAG_END_OF_STREAM}. + * While not prohibited, most codecs do not use the + * {@link #BUFFER_FLAG_KEY_FRAME} flag for input buffers. + * @return this object + * @throws IllegalStateException if a buffer is already set + */ + public @NonNull QueueRequest setGraphicBlock( + @NonNull GraphicBlock block, + long presentationTimeUs, + @BufferFlag int flags) { + if (!isAccessible()) { + throw new IllegalStateException(); + } + if (mLinearBlock != null || mGraphicBlock != null) { + throw new IllegalStateException(); + } + mGraphicBlock = block; + mPresentationTimeUs = presentationTimeUs; + mFlags = flags; + return this; + } + + /** + * Add a integer parameter. See {@link MediaFormat} for the list of + * supported tunings. If there was {@link MediaCodec#setParameters} + * call with the same key which is not processed by the codec yet, the + * value set from this method will override the unprocessed value. + */ + public @NonNull QueueRequest setIntegerParameter( + @NonNull String key, int value) { + if (!isAccessible()) { + throw new IllegalStateException(); + } + mTuningKeys.add(key); + mTuningValues.add(Integer.valueOf(value)); + return this; + } + + /** + * Add a long parameter. See {@link MediaFormat} for the list of + * supported tunings. If there was {@link MediaCodec#setParameters} + * call with the same key which is not processed by the codec yet, the + * value set from this method will override the unprocessed value. + */ + public @NonNull QueueRequest setLongParameter( + @NonNull String key, long value) { + if (!isAccessible()) { + throw new IllegalStateException(); + } + mTuningKeys.add(key); + mTuningValues.add(Long.valueOf(value)); + return this; + } + + /** + * Add a float parameter. See {@link MediaFormat} for the list of + * supported tunings. If there was {@link MediaCodec#setParameters} + * call with the same key which is not processed by the codec yet, the + * value set from this method will override the unprocessed value. + */ + public @NonNull QueueRequest setFloatParameter( + @NonNull String key, float value) { + if (!isAccessible()) { + throw new IllegalStateException(); + } + mTuningKeys.add(key); + mTuningValues.add(Float.valueOf(value)); + return this; + } + + /** + * Add a {@link ByteBuffer} parameter. See {@link MediaFormat} for the list of + * supported tunings. If there was {@link MediaCodec#setParameters} + * call with the same key which is not processed by the codec yet, the + * value set from this method will override the unprocessed value. + */ + public @NonNull QueueRequest setByteBufferParameter( + @NonNull String key, @NonNull ByteBuffer value) { + if (!isAccessible()) { + throw new IllegalStateException(); + } + mTuningKeys.add(key); + mTuningValues.add(value); + return this; + } + + /** + * Add a string parameter. See {@link MediaFormat} for the list of + * supported tunings. If there was {@link MediaCodec#setParameters} + * call with the same key which is not processed by the codec yet, the + * value set from this method will override the unprocessed value. + */ + public @NonNull QueueRequest setStringParameter( + @NonNull String key, @NonNull String value) { + if (!isAccessible()) { + throw new IllegalStateException(); + } + mTuningKeys.add(key); + mTuningValues.add(value); + return this; + } + + /** + * Finish building a queue request and queue the buffers with tunings. + */ + public void queue() { + if (!isAccessible()) { + throw new IllegalStateException(); + } + if (mLinearBlock == null && mGraphicBlock == null) { + throw new IllegalStateException(); + } + setAccessible(false); + if (mLinearBlock != null) { + mCodec.native_queueLinearBlock( + mIndex, mLinearBlock, mOffset, mSize, mCryptoInfo, + mPresentationTimeUs, mFlags, + mTuningKeys, mTuningValues); + } else if (mGraphicBlock != null) { + mCodec.native_queueGraphicBlock( + mIndex, mGraphicBlock, mPresentationTimeUs, mFlags, + mTuningKeys, mTuningValues); + } + clear(); + } + + @NonNull QueueRequest clear() { + mLinearBlock = null; + mOffset = 0; + mSize = 0; + mCryptoInfo = null; + mGraphicBlock = null; + mPresentationTimeUs = 0; + mFlags = 0; + mTuningKeys.clear(); + mTuningValues.clear(); + return this; + } + + boolean isAccessible() { + return mAccessible; + } + + @NonNull QueueRequest setAccessible(boolean accessible) { + mAccessible = accessible; + return this; + } + + private final MediaCodec mCodec; + private final int mIndex; + private LinearBlock mLinearBlock = null; + private int mOffset = 0; + private int mSize = 0; + private MediaCodec.CryptoInfo mCryptoInfo = null; + private GraphicBlock mGraphicBlock = null; + private long mPresentationTimeUs = 0; + private @BufferFlag int mFlags = 0; + private final ArrayList<String> mTuningKeys = new ArrayList<>(); + private final ArrayList<Object> mTuningValues = new ArrayList<>(); + + private boolean mAccessible = false; + } + + private native void native_queueLinearBlock( + int index, + @NonNull LinearBlock block, + int offset, + int size, + @Nullable CryptoInfo cryptoInfo, + long presentationTimeUs, + int flags, + @NonNull ArrayList<String> keys, + @NonNull ArrayList<Object> values); + + private native void native_queueGraphicBlock( + int index, + @NonNull GraphicBlock block, + long presentationTimeUs, + int flags, + @NonNull ArrayList<String> keys, + @NonNull ArrayList<Object> values); + + private final ArrayList<QueueRequest> mQueueRequests = new ArrayList<>(); + + /** + * Return a clear {@link QueueRequest} object for an input slot index. + * + * @param index input slot index from + * {@link Callback#onInputBufferAvailable} + * @return queue request object + * @throws IllegalStateException if not using block model + * @throws IllegalArgumentException if the input slot is not available or + * the index is out of range + */ + public @NonNull QueueRequest getQueueRequest(int index) { + synchronized (mBufferLock) { + if (mBufferMode != BUFFER_MODE_BLOCK) { + throw new IllegalStateException(); + } + if (index < 0 || index >= mQueueRequests.size()) { + throw new IllegalArgumentException(); + } + QueueRequest request = mQueueRequests.get(index); + if (request == null) { + throw new IllegalArgumentException(); + } + if (!request.isAccessible()) { + throw new IllegalArgumentException(); + } + return request.clear(); + } + } + + /** * If a non-negative timeout had been specified in the call * to {@link #dequeueOutputBuffer}, indicates that the call timed out. */ @@ -2789,8 +3527,13 @@ final public class MediaCodec { @OutputBufferInfo public final int dequeueOutputBuffer( @NonNull BufferInfo info, long timeoutUs) { + synchronized (mBufferLock) { + if (mBufferMode != BUFFER_MODE_LEGACY) { + throw new IllegalStateException(); + } + } int res = native_dequeueOutputBuffer(info, timeoutUs); - synchronized(mBufferLock) { + synchronized (mBufferLock) { if (res == INFO_OUTPUT_BUFFERS_CHANGED) { cacheBuffers(false /* input */); } else if (res >= 0) { @@ -2826,15 +3569,7 @@ final public class MediaCodec { * @throws MediaCodec.CodecException upon codec error. */ public final void releaseOutputBuffer(int index, boolean render) { - BufferInfo info = null; - synchronized(mBufferLock) { - invalidateByteBuffer(mCachedOutputBuffers, index); - mDequeuedOutputBuffers.remove(index); - if (mHasSurface) { - info = mDequeuedOutputInfos.remove(index); - } - } - releaseOutputBuffer(index, render, false /* updatePTS */, 0 /* dummy */); + releaseOutputBufferInternal(index, render, false /* updatePTS */, 0 /* dummy */); } /** @@ -2887,16 +3622,33 @@ final public class MediaCodec { * @throws MediaCodec.CodecException upon codec error. */ public final void releaseOutputBuffer(int index, long renderTimestampNs) { + releaseOutputBufferInternal( + index, true /* render */, true /* updatePTS */, renderTimestampNs); + } + + private void releaseOutputBufferInternal( + int index, boolean render, boolean updatePts, long renderTimestampNs) { BufferInfo info = null; synchronized(mBufferLock) { - invalidateByteBuffer(mCachedOutputBuffers, index); - mDequeuedOutputBuffers.remove(index); - if (mHasSurface) { - info = mDequeuedOutputInfos.remove(index); + switch (mBufferMode) { + case BUFFER_MODE_LEGACY: + invalidateByteBuffer(mCachedOutputBuffers, index); + mDequeuedOutputBuffers.remove(index); + if (mHasSurface) { + info = mDequeuedOutputInfos.remove(index); + } + break; + case BUFFER_MODE_BLOCK: + OutputFrame frame = mOutputFrames.get(index); + frame.setAccessible(false); + frame.clear(); + break; + default: + throw new IllegalStateException(); } } releaseOutputBuffer( - index, true /* render */, true /* updatePTS */, renderTimestampNs); + index, render, updatePts, renderTimestampNs); } @UnsupportedAppUsage @@ -3116,6 +3868,8 @@ final public class MediaCodec { mCachedOutputBuffers = null; mDequeuedInputBuffers.clear(); mDequeuedOutputBuffers.clear(); + mQueueRequests.clear(); + mOutputFrames.clear(); } } @@ -3154,11 +3908,16 @@ final public class MediaCodec { */ @NonNull public ByteBuffer[] getInputBuffers() { - if (mCachedInputBuffers == null) { - throw new IllegalStateException(); + synchronized (mBufferLock) { + if (mBufferMode != BUFFER_MODE_LEGACY) { + throw new IllegalStateException(); + } + if (mCachedInputBuffers == null) { + throw new IllegalStateException(); + } + // FIXME: check codec status + return mCachedInputBuffers; } - // FIXME: check codec status - return mCachedInputBuffers; } /** @@ -3185,11 +3944,16 @@ final public class MediaCodec { */ @NonNull public ByteBuffer[] getOutputBuffers() { - if (mCachedOutputBuffers == null) { - throw new IllegalStateException(); + synchronized (mBufferLock) { + if (mBufferMode != BUFFER_MODE_LEGACY) { + throw new IllegalStateException(); + } + if (mCachedOutputBuffers == null) { + throw new IllegalStateException(); + } + // FIXME: check codec status + return mCachedOutputBuffers; } - // FIXME: check codec status - return mCachedOutputBuffers; } /** @@ -3212,8 +3976,13 @@ final public class MediaCodec { */ @Nullable public ByteBuffer getInputBuffer(int index) { + synchronized (mBufferLock) { + if (mBufferMode != BUFFER_MODE_LEGACY) { + throw new IllegalStateException(); + } + } ByteBuffer newBuffer = getBuffer(true /* input */, index); - synchronized(mBufferLock) { + synchronized (mBufferLock) { invalidateByteBuffer(mCachedInputBuffers, index); mDequeuedInputBuffers.put(index, newBuffer); } @@ -3241,8 +4010,13 @@ final public class MediaCodec { */ @Nullable public Image getInputImage(int index) { + synchronized (mBufferLock) { + if (mBufferMode != BUFFER_MODE_LEGACY) { + throw new IllegalStateException(); + } + } Image newImage = getImage(true /* input */, index); - synchronized(mBufferLock) { + synchronized (mBufferLock) { invalidateByteBuffer(mCachedInputBuffers, index); mDequeuedInputBuffers.put(index, newImage); } @@ -3270,8 +4044,13 @@ final public class MediaCodec { */ @Nullable public ByteBuffer getOutputBuffer(int index) { + synchronized (mBufferLock) { + if (mBufferMode != BUFFER_MODE_LEGACY) { + throw new IllegalStateException(); + } + } ByteBuffer newBuffer = getBuffer(false /* input */, index); - synchronized(mBufferLock) { + synchronized (mBufferLock) { invalidateByteBuffer(mCachedOutputBuffers, index); mDequeuedOutputBuffers.put(index, newBuffer); } @@ -3298,8 +4077,13 @@ final public class MediaCodec { */ @Nullable public Image getOutputImage(int index) { + synchronized (mBufferLock) { + if (mBufferMode != BUFFER_MODE_LEGACY) { + throw new IllegalStateException(); + } + } Image newImage = getImage(false /* input */, index); - synchronized(mBufferLock) { + synchronized (mBufferLock) { invalidateByteBuffer(mCachedOutputBuffers, index); mDequeuedOutputBuffers.put(index, newImage); } @@ -3307,6 +4091,149 @@ final public class MediaCodec { } /** + * A single output frame and its associated metadata. + */ + public static final class OutputFrame { + // No public constructor + OutputFrame(int index) { + mIndex = index; + } + + /** + * Returns the output linear block, or null if this frame is empty. + * + * @throws IllegalStateException if this output frame is not linear. + */ + public @Nullable LinearBlock getLinearBlock() { + if (mGraphicBlock != null) { + throw new IllegalStateException(); + } + return mLinearBlock; + } + + /** + * Returns the output graphic block, or null if this frame is empty. + * + * @throws IllegalStateException if this output frame is not graphic. + */ + public @Nullable GraphicBlock getGraphicBlock() { + if (mLinearBlock != null) { + throw new IllegalStateException(); + } + return mGraphicBlock; + } + + /** + * Returns the presentation timestamp in microseconds. + */ + public long getPresentationTimeUs() { + return mPresentationTimeUs; + } + + /** + * Returns the buffer flags. + */ + public @BufferFlag int getFlags() { + return mFlags; + } + + /** + * Returns a read-only {@link MediaFormat} for this frame. The returned + * object is valid only while the client is holding the output frame. + */ + public @NonNull MediaFormat getFormat() { + return mFormat; + } + + /** + * Populate {@code keys} with the name of entries that has changed from + * the previous frame. The entries may have been removed/changed/added. + * Client can find out what the change is by querying {@link MediaFormat} + * object returned from {@link #getFormat}. + */ + public void getChangedKeys(@NonNull Set<String> keys) { + keys.clear(); + keys.addAll(mChangedKeys); + } + + void clear() { + mLinearBlock = null; + mGraphicBlock = null; + mFormat = null; + mChangedKeys.clear(); + mLoaded = false; + } + + boolean isAccessible() { + return mAccessible; + } + + void setAccessible(boolean accessible) { + mAccessible = accessible; + } + + void setBufferInfo(MediaCodec.BufferInfo info) { + mPresentationTimeUs = info.presentationTimeUs; + mFlags = info.flags; + } + + boolean isLoaded() { + return mLoaded; + } + + void setLoaded(boolean loaded) { + mLoaded = loaded; + } + + private final int mIndex; + private LinearBlock mLinearBlock = null; + private GraphicBlock mGraphicBlock = null; + private long mPresentationTimeUs = 0; + private @BufferFlag int mFlags = 0; + private MediaFormat mFormat = null; + private final ArrayList<String> mChangedKeys = new ArrayList<>(); + private boolean mAccessible = false; + private boolean mLoaded = false; + } + + private final ArrayList<OutputFrame> mOutputFrames = new ArrayList<>(); + + /** + * Returns an {@link OutputFrame} object. + * + * @param index output buffer index from + * {@link Callback#onOutputBufferAvailable} + * @return {@link OutputFrame} object describing the output buffer + * @throws IllegalStateException if not using block model + * @throws IllegalArgumentException if the output buffer is not available or + * the index is out of range + */ + public @NonNull OutputFrame getOutputFrame(int index) { + synchronized (mBufferLock) { + if (mBufferMode != BUFFER_MODE_BLOCK) { + throw new IllegalStateException(); + } + if (index < 0 || index >= mOutputFrames.size()) { + throw new IllegalArgumentException(); + } + OutputFrame frame = mOutputFrames.get(index); + if (frame == null) { + throw new IllegalArgumentException(); + } + if (!frame.isAccessible()) { + throw new IllegalArgumentException(); + } + if (!frame.isLoaded()) { + native_getOutputFrame(frame, index); + frame.setLoaded(true); + } + return frame; + } + } + + private native void native_getOutputFrame(OutputFrame frame, int index); + + /** * The content is scaled to the surface dimensions */ public static final int VIDEO_SCALING_MODE_SCALE_TO_FIT = 1; @@ -3889,7 +4816,9 @@ final public class MediaCodec { @Override public void close() { if (mIsImageValid) { - java.nio.NioUtils.freeDirectBuffer(mBuffer); + if (mBuffer != null) { + java.nio.NioUtils.freeDirectBuffer(mBuffer); + } mIsImageValid = false; } } @@ -3908,7 +4837,6 @@ final public class MediaCodec { super.setCropRect(cropRect); } - public MediaImage( @NonNull ByteBuffer buffer, @NonNull ByteBuffer info, boolean readOnly, long timestamp, int xOffset, int yOffset, @Nullable Rect cropRect) { @@ -3963,7 +4891,6 @@ final public class MediaCodec { throw new UnsupportedOperationException("unexpected strides: " + colInc + " pixel, " + rowInc + " row on plane " + ix); } - buffer.clear(); buffer.position(mBuffer.position() + planeOffset + (xOffset / horiz) * colInc + (yOffset / vert) * rowInc); @@ -3983,6 +4910,30 @@ final public class MediaCodec { super.setCropRect(cropRect); } + public MediaImage( + @NonNull Image.Plane[] planes, int width, int height, int format, boolean readOnly, + long timestamp, int xOffset, int yOffset, @Nullable Rect cropRect) { + mWidth = width; + mHeight = height; + mFormat = format; + mTimestamp = timestamp; + mIsImageValid = true; + mIsReadOnly = readOnly; + mBuffer = null; + mInfo = null; + mPlanes = planes; + + // save offsets and info + mXOffset = xOffset; + mYOffset = yOffset; + + if (cropRect == null) { + cropRect = new Rect(0, 0, mWidth, mHeight); + } + cropRect.offset(-xOffset, -yOffset); + super.setCropRect(cropRect); + } + private class MediaPlane extends Plane { public MediaPlane(@NonNull ByteBuffer buffer, int rowInc, int colInc) { mData = buffer; diff --git a/media/jni/Android.bp b/media/jni/Android.bp index 536a061190d7..2305b44c87ad 100644 --- a/media/jni/Android.bp +++ b/media/jni/Android.bp @@ -1,6 +1,8 @@ cc_library_shared { name: "libmedia_jni", + defaults: ["libcodec2-internal-defaults"], + srcs: [ "android_media_ImageWriter.cpp", "android_media_ImageReader.cpp", diff --git a/media/jni/android_media_MediaCodec.cpp b/media/jni/android_media_MediaCodec.cpp index 20980a90e9fe..ab6966d5d1c3 100644 --- a/media/jni/android_media_MediaCodec.cpp +++ b/media/jni/android_media_MediaCodec.cpp @@ -18,6 +18,8 @@ #define LOG_TAG "MediaCodec-JNI" #include <utils/Log.h> +#include <type_traits> + #include "android_media_MediaCodec.h" #include "android_media_MediaCrypto.h" @@ -31,13 +33,20 @@ #include <nativehelper/JNIHelp.h> #include <nativehelper/ScopedLocalRef.h> +#include <C2Buffer.h> + #include <android/hardware/cas/native/1.0/IDescrambler.h> +#include <binder/MemoryHeapBase.h> + #include <cutils/compiler.h> #include <gui/Surface.h> +#include <hidlmemory/FrameworkUtils.h> + #include <media/MediaCodecBuffer.h> +#include <media/hardware/VideoAPI.h> #include <media/stagefright/MediaCodec.h> #include <media/stagefright/foundation/ABuffer.h> #include <media/stagefright/foundation/ADebug.h> @@ -47,7 +56,6 @@ #include <media/stagefright/MediaErrors.h> #include <media/stagefright/PersistentSurface.h> #include <mediadrm/ICrypto.h> -#include <nativehelper/ScopedLocalRef.h> #include <system/window.h> @@ -110,6 +118,39 @@ static struct { jfieldID levelField; } gCodecInfo; +static struct { + jclass clazz; + jobject nativeByteOrder; + jmethodID orderId; + jmethodID asReadOnlyBufferId; + jmethodID positionId; + jmethodID limitId; +} gByteBufferInfo; + +static struct { + jmethodID sizeId; + jmethodID getId; + jmethodID addId; +} gArrayListInfo; + +static struct { + jclass clazz; + jmethodID ctorId; + jmethodID setInternalStateId; + jfieldID contextId; + jfieldID validId; + jfieldID lockId; +} gLinearBlockInfo; + +static struct { + jclass clazz; + jmethodID ctorId; + jmethodID setInternalStateId; + jfieldID contextId; + jfieldID validId; + jfieldID lockId; +} gGraphicBlockInfo; + struct fields_t { jmethodID postEventFromNativeID; jmethodID lockAndGetContextID; @@ -123,11 +164,65 @@ struct fields_t { jfieldID cryptoInfoPatternID; jfieldID patternEncryptBlocksID; jfieldID patternSkipBlocksID; + jfieldID queueRequestIndexID; + jfieldID outputFrameLinearBlockID; + jfieldID outputFrameGraphicBlockID; + jfieldID outputFrameChangedKeysID; + jfieldID outputFrameFormatID; }; static fields_t gFields; static const void *sRefBaseOwner; +struct JMediaCodecLinearBlock { + std::shared_ptr<C2Buffer> mBuffer; + std::shared_ptr<C2ReadView> mReadonlyMapping; + + std::shared_ptr<C2LinearBlock> mBlock; + std::shared_ptr<C2WriteView> mReadWriteMapping; + + sp<IMemoryHeap> mHeap; + sp<hardware::HidlMemory> mMemory; + + sp<MediaCodecBuffer> mLegacyBuffer; + + std::once_flag mCopyWarningFlag; + + std::shared_ptr<C2Buffer> toC2Buffer(size_t offset, size_t size) { + if (mBuffer) { + if (mBuffer->data().type() != C2BufferData::LINEAR) { + return nullptr; + } + C2ConstLinearBlock block = mBuffer->data().linearBlocks().front(); + if (offset == 0 && size == block.capacity()) { + return mBuffer; + } + return C2Buffer::CreateLinearBuffer(block.subBlock(offset, size)); + } + if (mBlock) { + return C2Buffer::CreateLinearBuffer(mBlock->share(offset, size, C2Fence{})); + } + return nullptr; + } + + sp<hardware::HidlMemory> toHidlMemory() { + if (mMemory) { + return mMemory; + } + return nullptr; + } +}; + +struct JMediaCodecGraphicBlock { + std::shared_ptr<C2Buffer> mBuffer; + std::shared_ptr<const C2GraphicView> mReadonlyMapping; + + std::shared_ptr<C2GraphicBlock> mBlock; + std::shared_ptr<C2GraphicView> mReadWriteMapping; + + sp<MediaCodecBuffer> mLegacyBuffer; +}; + //////////////////////////////////////////////////////////////////////////////// JMediaCodec::JMediaCodec( @@ -141,8 +236,6 @@ JMediaCodec::JMediaCodec( mClass = (jclass)env->NewGlobalRef(clazz); mObject = env->NewWeakGlobalRef(thiz); - cacheJavaObjects(env); - mLooper = new ALooper; mLooper->setName("MediaCodec_looper"); @@ -163,45 +256,6 @@ JMediaCodec::JMediaCodec( CHECK((mCodec != NULL) != (mInitStatus != OK)); } -void JMediaCodec::cacheJavaObjects(JNIEnv *env) { - jclass clazz = (jclass)env->FindClass("java/nio/ByteBuffer"); - mByteBufferClass = (jclass)env->NewGlobalRef(clazz); - CHECK(mByteBufferClass != NULL); - - ScopedLocalRef<jclass> byteOrderClass( - env, env->FindClass("java/nio/ByteOrder")); - CHECK(byteOrderClass.get() != NULL); - - jmethodID nativeOrderID = env->GetStaticMethodID( - byteOrderClass.get(), "nativeOrder", "()Ljava/nio/ByteOrder;"); - CHECK(nativeOrderID != NULL); - - jobject nativeByteOrderObj = - env->CallStaticObjectMethod(byteOrderClass.get(), nativeOrderID); - mNativeByteOrderObj = env->NewGlobalRef(nativeByteOrderObj); - CHECK(mNativeByteOrderObj != NULL); - env->DeleteLocalRef(nativeByteOrderObj); - nativeByteOrderObj = NULL; - - mByteBufferOrderMethodID = env->GetMethodID( - mByteBufferClass, - "order", - "(Ljava/nio/ByteOrder;)Ljava/nio/ByteBuffer;"); - CHECK(mByteBufferOrderMethodID != NULL); - - mByteBufferAsReadOnlyBufferMethodID = env->GetMethodID( - mByteBufferClass, "asReadOnlyBuffer", "()Ljava/nio/ByteBuffer;"); - CHECK(mByteBufferAsReadOnlyBufferMethodID != NULL); - - mByteBufferPositionMethodID = env->GetMethodID( - mByteBufferClass, "position", "(I)Ljava/nio/Buffer;"); - CHECK(mByteBufferPositionMethodID != NULL); - - mByteBufferLimitMethodID = env->GetMethodID( - mByteBufferClass, "limit", "(I)Ljava/nio/Buffer;"); - CHECK(mByteBufferLimitMethodID != NULL); -} - status_t JMediaCodec::initCheck() const { return mInitStatus; } @@ -247,19 +301,6 @@ JMediaCodec::~JMediaCodec() { mObject = NULL; env->DeleteGlobalRef(mClass); mClass = NULL; - deleteJavaObjects(env); -} - -void JMediaCodec::deleteJavaObjects(JNIEnv *env) { - env->DeleteGlobalRef(mByteBufferClass); - mByteBufferClass = NULL; - env->DeleteGlobalRef(mNativeByteOrderObj); - mNativeByteOrderObj = NULL; - - mByteBufferOrderMethodID = NULL; - mByteBufferAsReadOnlyBufferMethodID = NULL; - mByteBufferPositionMethodID = NULL; - mByteBufferLimitMethodID = NULL; } status_t JMediaCodec::enableOnFrameRenderedListener(jboolean enable) { @@ -300,6 +341,12 @@ status_t JMediaCodec::configure( mSurfaceTextureClient.clear(); } + constexpr int32_t CONFIGURE_FLAG_ENCODE = 1; + AString mime; + CHECK(format->findString("mime", &mime)); + mGraphicOutput = (mime.startsWithIgnoreCase("video/") || mime.startsWithIgnoreCase("image/")) + && !(flags & CONFIGURE_FLAG_ENCODE); + return mCodec->configure( format, mSurfaceTextureClient, crypto, descrambler, flags); } @@ -370,6 +417,32 @@ status_t JMediaCodec::queueSecureInputBuffer( presentationTimeUs, flags, errorDetailMsg); } +status_t JMediaCodec::queueBuffer( + size_t index, const std::shared_ptr<C2Buffer> &buffer, int64_t timeUs, + uint32_t flags, const sp<AMessage> &tunings, AString *errorDetailMsg) { + return mCodec->queueBuffer( + index, buffer, timeUs, flags, tunings, errorDetailMsg); +} + +status_t JMediaCodec::queueEncryptedLinearBlock( + size_t index, + const sp<hardware::HidlMemory> &buffer, + size_t offset, + const CryptoPlugin::SubSample *subSamples, + size_t numSubSamples, + const uint8_t key[16], + const uint8_t iv[16], + CryptoPlugin::Mode mode, + const CryptoPlugin::Pattern &pattern, + int64_t presentationTimeUs, + uint32_t flags, + const sp<AMessage> &tunings, + AString *errorDetailMsg) { + return mCodec->queueEncryptedBuffer( + index, buffer, offset, subSamples, numSubSamples, key, iv, mode, pattern, + presentationTimeUs, flags, tunings, errorDetailMsg); +} + status_t JMediaCodec::dequeueInputBuffer(size_t *index, int64_t timeoutUs) { return mCodec->dequeueInputBuffer(index, timeoutUs); } @@ -444,7 +517,7 @@ status_t JMediaCodec::getBuffers( } *bufArray = (jobjectArray)env->NewObjectArray( - buffers.size(), mByteBufferClass, NULL); + buffers.size(), gByteBufferInfo.clazz, NULL); if (*bufArray == NULL) { return NO_MEMORY; } @@ -470,6 +543,39 @@ status_t JMediaCodec::getBuffers( return OK; } +template <typename T> +static jobject CreateByteBuffer( + JNIEnv *env, T *base, size_t capacity, size_t offset, size_t size, + bool readOnly, bool clearBuffer) { + jobject byteBuffer = + env->NewDirectByteBuffer( + const_cast<typename std::remove_const<T>::type *>(base), + capacity); + if (readOnly && byteBuffer != NULL) { + jobject readOnlyBuffer = env->CallObjectMethod( + byteBuffer, gByteBufferInfo.asReadOnlyBufferId); + env->DeleteLocalRef(byteBuffer); + byteBuffer = readOnlyBuffer; + } + if (byteBuffer == NULL) { + return nullptr; + } + jobject me = env->CallObjectMethod( + byteBuffer, gByteBufferInfo.orderId, gByteBufferInfo.nativeByteOrder); + env->DeleteLocalRef(me); + me = env->CallObjectMethod( + byteBuffer, gByteBufferInfo.limitId, + clearBuffer ? capacity : offset + size); + env->DeleteLocalRef(me); + me = env->CallObjectMethod( + byteBuffer, gByteBufferInfo.positionId, + clearBuffer ? 0 : offset); + env->DeleteLocalRef(me); + me = NULL; + return byteBuffer; +} + + // static template <typename T> status_t JMediaCodec::createByteBufferFromABuffer( @@ -488,29 +594,9 @@ status_t JMediaCodec::createByteBufferFromABuffer( return OK; } - jobject byteBuffer = - env->NewDirectByteBuffer(buffer->base(), buffer->capacity()); - if (readOnly && byteBuffer != NULL) { - jobject readOnlyBuffer = env->CallObjectMethod( - byteBuffer, mByteBufferAsReadOnlyBufferMethodID); - env->DeleteLocalRef(byteBuffer); - byteBuffer = readOnlyBuffer; - } - if (byteBuffer == NULL) { - return NO_MEMORY; - } - jobject me = env->CallObjectMethod( - byteBuffer, mByteBufferOrderMethodID, mNativeByteOrderObj); - env->DeleteLocalRef(me); - me = env->CallObjectMethod( - byteBuffer, mByteBufferLimitMethodID, - clearBuffer ? buffer->capacity() : (buffer->offset() + buffer->size())); - env->DeleteLocalRef(me); - me = env->CallObjectMethod( - byteBuffer, mByteBufferPositionMethodID, - clearBuffer ? 0 : buffer->offset()); - env->DeleteLocalRef(me); - me = NULL; + jobject byteBuffer = CreateByteBuffer( + env, buffer->base(), buffer->capacity(), buffer->offset(), buffer->size(), + readOnly, clearBuffer); *buf = byteBuffer; return OK; @@ -628,6 +714,92 @@ status_t JMediaCodec::getImage( return OK; } +status_t JMediaCodec::getOutputFrame( + JNIEnv *env, jobject frame, size_t index) const { + sp<MediaCodecBuffer> buffer; + + status_t err = mCodec->getOutputBuffer(index, &buffer); + if (err != OK) { + return err; + } + + if (buffer->size() > 0) { + // asC2Buffer clears internal reference, so set the reference again. + std::shared_ptr<C2Buffer> c2Buffer = buffer->asC2Buffer(); + buffer->copy(c2Buffer); + if (c2Buffer) { + switch (c2Buffer->data().type()) { + case C2BufferData::LINEAR: { + std::unique_ptr<JMediaCodecLinearBlock> context{new JMediaCodecLinearBlock}; + context->mBuffer = c2Buffer; + ScopedLocalRef<jobject> linearBlock{env, env->NewObject( + gLinearBlockInfo.clazz, gLinearBlockInfo.ctorId)}; + env->SetLongField( + linearBlock.get(), gLinearBlockInfo.contextId, (jlong)context.release()); + env->SetObjectField(frame, gFields.outputFrameLinearBlockID, linearBlock.get()); + break; + } + case C2BufferData::GRAPHIC: { + std::unique_ptr<JMediaCodecGraphicBlock> context{new JMediaCodecGraphicBlock}; + context->mBuffer = c2Buffer; + ScopedLocalRef<jobject> graphicBlock{env, env->NewObject( + gGraphicBlockInfo.clazz, gGraphicBlockInfo.ctorId)}; + env->SetLongField( + graphicBlock.get(), gGraphicBlockInfo.contextId, (jlong)context.release()); + env->SetObjectField(frame, gFields.outputFrameGraphicBlockID, graphicBlock.get()); + break; + } + case C2BufferData::LINEAR_CHUNKS: [[fallthrough]]; + case C2BufferData::GRAPHIC_CHUNKS: [[fallthrough]]; + case C2BufferData::INVALID: [[fallthrough]]; + default: + return INVALID_OPERATION; + } + } else { + if (!mGraphicOutput) { + std::unique_ptr<JMediaCodecLinearBlock> context{new JMediaCodecLinearBlock}; + context->mLegacyBuffer = buffer; + ScopedLocalRef<jobject> linearBlock{env, env->NewObject( + gLinearBlockInfo.clazz, gLinearBlockInfo.ctorId)}; + env->SetLongField( + linearBlock.get(), gLinearBlockInfo.contextId, (jlong)context.release()); + env->SetObjectField(frame, gFields.outputFrameLinearBlockID, linearBlock.get()); + } else { + std::unique_ptr<JMediaCodecGraphicBlock> context{new JMediaCodecGraphicBlock}; + context->mLegacyBuffer = buffer; + ScopedLocalRef<jobject> graphicBlock{env, env->NewObject( + gGraphicBlockInfo.clazz, gGraphicBlockInfo.ctorId)}; + env->SetLongField( + graphicBlock.get(), gGraphicBlockInfo.contextId, (jlong)context.release()); + env->SetObjectField(frame, gFields.outputFrameGraphicBlockID, graphicBlock.get()); + } + } + } + + jobject format; + err = getOutputFormat(env, index, &format); + if (err != OK) { + return err; + } + env->SetObjectField(frame, gFields.outputFrameFormatID, format); + env->DeleteLocalRef(format); + format = nullptr; + + sp<RefBase> obj; + if (buffer->meta()->findObject("changedKeys", &obj) && obj) { + sp<MediaCodec::WrapperObject<std::set<std::string>>> changedKeys{ + (decltype(changedKeys.get()))obj.get()}; + ScopedLocalRef<jobject> changedKeysObj{env, env->GetObjectField( + frame, gFields.outputFrameChangedKeysID)}; + for (const std::string &key : changedKeys->value) { + ScopedLocalRef<jstring> keyStr{env, env->NewStringUTF(key.c_str())}; + (void)env->CallBooleanMethod(changedKeysObj.get(), gArrayListInfo.addId, keyStr.get()); + } + } + return OK; +} + + status_t JMediaCodec::getName(JNIEnv *env, jstring *nameStr) const { AString name; @@ -1428,6 +1600,139 @@ static void android_media_MediaCodec_queueInputBuffer( env, err, ACTION_CODE_FATAL, errorDetailMsg.empty() ? NULL : errorDetailMsg.c_str()); } +struct NativeCryptoInfo { + NativeCryptoInfo(JNIEnv *env, jobject cryptoInfoObj) + : mEnv{env}, + mIvObj{env, (jbyteArray)env->GetObjectField(cryptoInfoObj, gFields.cryptoInfoIVID)}, + mKeyObj{env, (jbyteArray)env->GetObjectField(cryptoInfoObj, gFields.cryptoInfoKeyID)} { + mNumSubSamples = env->GetIntField(cryptoInfoObj, gFields.cryptoInfoNumSubSamplesID); + + ScopedLocalRef<jintArray> numBytesOfClearDataObj{env, (jintArray)env->GetObjectField( + cryptoInfoObj, gFields.cryptoInfoNumBytesOfClearDataID)}; + + ScopedLocalRef<jintArray> numBytesOfEncryptedDataObj{env, (jintArray)env->GetObjectField( + cryptoInfoObj, gFields.cryptoInfoNumBytesOfEncryptedDataID)}; + + jint jmode = env->GetIntField(cryptoInfoObj, gFields.cryptoInfoModeID); + if (jmode == gCryptoModes.Unencrypted) { + mMode = CryptoPlugin::kMode_Unencrypted; + } else if (jmode == gCryptoModes.AesCtr) { + mMode = CryptoPlugin::kMode_AES_CTR; + } else if (jmode == gCryptoModes.AesCbc) { + mMode = CryptoPlugin::kMode_AES_CBC; + } else { + throwExceptionAsNecessary(env, INVALID_OPERATION); + return; + } + + ScopedLocalRef<jobject> patternObj{ + env, env->GetObjectField(cryptoInfoObj, gFields.cryptoInfoPatternID)}; + + CryptoPlugin::Pattern pattern; + if (patternObj.get() == nullptr) { + pattern.mEncryptBlocks = 0; + pattern.mSkipBlocks = 0; + } else { + pattern.mEncryptBlocks = env->GetIntField( + patternObj.get(), gFields.patternEncryptBlocksID); + pattern.mSkipBlocks = env->GetIntField( + patternObj.get(), gFields.patternSkipBlocksID); + } + + mErr = OK; + if (mNumSubSamples <= 0) { + mErr = -EINVAL; + } else if (numBytesOfClearDataObj == nullptr + && numBytesOfEncryptedDataObj == nullptr) { + mErr = -EINVAL; + } else if (numBytesOfEncryptedDataObj != nullptr + && env->GetArrayLength(numBytesOfEncryptedDataObj.get()) < mNumSubSamples) { + mErr = -ERANGE; + } else if (numBytesOfClearDataObj != nullptr + && env->GetArrayLength(numBytesOfClearDataObj.get()) < mNumSubSamples) { + mErr = -ERANGE; + // subSamples array may silently overflow if number of samples are too large. Use + // INT32_MAX as maximum allocation size may be less than SIZE_MAX on some platforms + } else if (CC_UNLIKELY(mNumSubSamples >= (signed)(INT32_MAX / sizeof(*mSubSamples))) ) { + mErr = -EINVAL; + } else { + jint *numBytesOfClearData = + (numBytesOfClearDataObj == nullptr) + ? nullptr + : env->GetIntArrayElements(numBytesOfClearDataObj.get(), nullptr); + + jint *numBytesOfEncryptedData = + (numBytesOfEncryptedDataObj == nullptr) + ? nullptr + : env->GetIntArrayElements(numBytesOfEncryptedDataObj.get(), nullptr); + + mSubSamples = new CryptoPlugin::SubSample[mNumSubSamples]; + + for (jint i = 0; i < mNumSubSamples; ++i) { + mSubSamples[i].mNumBytesOfClearData = + (numBytesOfClearData == nullptr) ? 0 : numBytesOfClearData[i]; + + mSubSamples[i].mNumBytesOfEncryptedData = + (numBytesOfEncryptedData == nullptr) ? 0 : numBytesOfEncryptedData[i]; + } + + if (numBytesOfEncryptedData != nullptr) { + env->ReleaseIntArrayElements( + numBytesOfEncryptedDataObj.get(), numBytesOfEncryptedData, 0); + numBytesOfEncryptedData = nullptr; + } + + if (numBytesOfClearData != nullptr) { + env->ReleaseIntArrayElements( + numBytesOfClearDataObj.get(), numBytesOfClearData, 0); + numBytesOfClearData = nullptr; + } + } + + if (mErr == OK && mKeyObj.get() != nullptr) { + if (env->GetArrayLength(mKeyObj.get()) != 16) { + mErr = -EINVAL; + } else { + mKey = env->GetByteArrayElements(mKeyObj.get(), nullptr); + } + } + + if (mErr == OK && mIvObj.get() != nullptr) { + if (env->GetArrayLength(mIvObj.get()) != 16) { + mErr = -EINVAL; + } else { + mIv = env->GetByteArrayElements(mIvObj.get(), nullptr); + } + } + } + + ~NativeCryptoInfo() { + if (mIv != nullptr) { + mEnv->ReleaseByteArrayElements(mIvObj.get(), mIv, 0); + } + + if (mKey != nullptr) { + mEnv->ReleaseByteArrayElements(mKeyObj.get(), mKey, 0); + } + + if (mSubSamples != nullptr) { + delete[] mSubSamples; + } + } + + JNIEnv *mEnv{nullptr}; + ScopedLocalRef<jbyteArray> mIvObj; + ScopedLocalRef<jbyteArray> mKeyObj; + status_t mErr{OK}; + + CryptoPlugin::SubSample *mSubSamples{nullptr}; + int32_t mNumSubSamples{0}; + jbyte *mIv{nullptr}; + jbyte *mKey{nullptr}; + enum CryptoPlugin::Mode mMode; + CryptoPlugin::Pattern mPattern; +}; + static void android_media_MediaCodec_queueSecureInputBuffer( JNIEnv *env, jobject thiz, @@ -1445,152 +1750,274 @@ static void android_media_MediaCodec_queueSecureInputBuffer( return; } - jint numSubSamples = - env->GetIntField(cryptoInfoObj, gFields.cryptoInfoNumSubSamplesID); - - jintArray numBytesOfClearDataObj = - (jintArray)env->GetObjectField( - cryptoInfoObj, gFields.cryptoInfoNumBytesOfClearDataID); + NativeCryptoInfo cryptoInfo{env, cryptoInfoObj}; + AString errorDetailMsg; - jintArray numBytesOfEncryptedDataObj = - (jintArray)env->GetObjectField( - cryptoInfoObj, gFields.cryptoInfoNumBytesOfEncryptedDataID); + status_t err = cryptoInfo.mErr; + if (err == OK) { + err = codec->queueSecureInputBuffer( + index, offset, + cryptoInfo.mSubSamples, cryptoInfo.mNumSubSamples, + (const uint8_t *)cryptoInfo.mKey, (const uint8_t *)cryptoInfo.mIv, + cryptoInfo.mMode, + cryptoInfo.mPattern, + timestampUs, + flags, + &errorDetailMsg); + } - jbyteArray keyObj = - (jbyteArray)env->GetObjectField(cryptoInfoObj, gFields.cryptoInfoKeyID); + throwExceptionAsNecessary( + env, err, ACTION_CODE_FATAL, errorDetailMsg.empty() ? NULL : errorDetailMsg.c_str()); +} - jbyteArray ivObj = - (jbyteArray)env->GetObjectField(cryptoInfoObj, gFields.cryptoInfoIVID); +static status_t ConvertKeyValueListsToAMessage( + JNIEnv *env, jobject keys, jobject values, sp<AMessage> *msg) { + static struct Fields { + explicit Fields(JNIEnv *env) { + ScopedLocalRef<jclass> clazz{env, env->FindClass("java/lang/String")}; + CHECK(clazz.get() != NULL); + mStringClass = (jclass)env->NewGlobalRef(clazz.get()); - jint jmode = env->GetIntField(cryptoInfoObj, gFields.cryptoInfoModeID); - enum CryptoPlugin::Mode mode; - if (jmode == gCryptoModes.Unencrypted) { - mode = CryptoPlugin::kMode_Unencrypted; - } else if (jmode == gCryptoModes.AesCtr) { - mode = CryptoPlugin::kMode_AES_CTR; - } else if (jmode == gCryptoModes.AesCbc) { - mode = CryptoPlugin::kMode_AES_CBC; - } else { - throwExceptionAsNecessary(env, INVALID_OPERATION); - return; - } + clazz.reset(env->FindClass("java/lang/Integer")); + CHECK(clazz.get() != NULL); + mIntegerClass = (jclass)env->NewGlobalRef(clazz.get()); - jobject patternObj = env->GetObjectField(cryptoInfoObj, gFields.cryptoInfoPatternID); + mIntegerValueId = env->GetMethodID(clazz.get(), "intValue", "()I"); + CHECK(mIntegerValueId != NULL); - CryptoPlugin::Pattern pattern; - if (patternObj == NULL) { - pattern.mEncryptBlocks = 0; - pattern.mSkipBlocks = 0; - } else { - pattern.mEncryptBlocks = env->GetIntField(patternObj, gFields.patternEncryptBlocksID); - pattern.mSkipBlocks = env->GetIntField(patternObj, gFields.patternSkipBlocksID); - } - - status_t err = OK; - - CryptoPlugin::SubSample *subSamples = NULL; - jbyte *key = NULL; - jbyte *iv = NULL; - - if (numSubSamples <= 0) { - err = -EINVAL; - } else if (numBytesOfClearDataObj == NULL - && numBytesOfEncryptedDataObj == NULL) { - err = -EINVAL; - } else if (numBytesOfEncryptedDataObj != NULL - && env->GetArrayLength(numBytesOfEncryptedDataObj) < numSubSamples) { - err = -ERANGE; - } else if (numBytesOfClearDataObj != NULL - && env->GetArrayLength(numBytesOfClearDataObj) < numSubSamples) { - err = -ERANGE; - // subSamples array may silently overflow if number of samples are too large. Use - // INT32_MAX as maximum allocation size may be less than SIZE_MAX on some platforms - } else if ( CC_UNLIKELY(numSubSamples >= (signed)(INT32_MAX / sizeof(*subSamples))) ) { - err = -EINVAL; - } else { - jboolean isCopy; + clazz.reset(env->FindClass("java/lang/Long")); + CHECK(clazz.get() != NULL); + mLongClass = (jclass)env->NewGlobalRef(clazz.get()); - jint *numBytesOfClearData = - (numBytesOfClearDataObj == NULL) - ? NULL - : env->GetIntArrayElements(numBytesOfClearDataObj, &isCopy); + mLongValueId = env->GetMethodID(clazz.get(), "longValue", "()J"); + CHECK(mLongValueId != NULL); - jint *numBytesOfEncryptedData = - (numBytesOfEncryptedDataObj == NULL) - ? NULL - : env->GetIntArrayElements(numBytesOfEncryptedDataObj, &isCopy); + clazz.reset(env->FindClass("java/lang/Float")); + CHECK(clazz.get() != NULL); + mFloatClass = (jclass)env->NewGlobalRef(clazz.get()); - subSamples = new CryptoPlugin::SubSample[numSubSamples]; + mFloatValueId = env->GetMethodID(clazz.get(), "floatValue", "()F"); + CHECK(mFloatValueId != NULL); - for (jint i = 0; i < numSubSamples; ++i) { - subSamples[i].mNumBytesOfClearData = - (numBytesOfClearData == NULL) ? 0 : numBytesOfClearData[i]; + clazz.reset(env->FindClass("java/util/ArrayList")); + CHECK(clazz.get() != NULL); - subSamples[i].mNumBytesOfEncryptedData = - (numBytesOfEncryptedData == NULL) - ? 0 : numBytesOfEncryptedData[i]; + mByteBufferArrayId = env->GetMethodID(gByteBufferInfo.clazz, "array", "()[B"); + CHECK(mByteBufferArrayId != NULL); } - if (numBytesOfEncryptedData != NULL) { - env->ReleaseIntArrayElements( - numBytesOfEncryptedDataObj, numBytesOfEncryptedData, 0); - numBytesOfEncryptedData = NULL; + jclass mStringClass; + jclass mIntegerClass; + jmethodID mIntegerValueId; + jclass mLongClass; + jmethodID mLongValueId; + jclass mFloatClass; + jmethodID mFloatValueId; + jmethodID mByteBufferArrayId; + } sFields{env}; + + jint size = env->CallIntMethod(keys, gArrayListInfo.sizeId); + if (size != env->CallIntMethod(values, gArrayListInfo.sizeId)) { + return BAD_VALUE; + } + + sp<AMessage> result{new AMessage}; + for (jint i = 0; i < size; ++i) { + ScopedLocalRef<jstring> jkey{ + env, (jstring)env->CallObjectMethod(keys, gArrayListInfo.getId, i)}; + const char *tmp = env->GetStringUTFChars(jkey.get(), nullptr); + AString key; + if (tmp) { + key.setTo(tmp); + } + env->ReleaseStringUTFChars(jkey.get(), tmp); + if (key.empty()) { + return NO_MEMORY; } - if (numBytesOfClearData != NULL) { - env->ReleaseIntArrayElements( - numBytesOfClearDataObj, numBytesOfClearData, 0); - numBytesOfClearData = NULL; + ScopedLocalRef<jobject> jvalue{ + env, env->CallObjectMethod(values, gArrayListInfo.getId, i)}; + + if (env->IsInstanceOf(jvalue.get(), sFields.mStringClass)) { + const char *tmp = env->GetStringUTFChars((jstring)jvalue.get(), nullptr); + AString value; + if (tmp) { + value.setTo(tmp); + } + env->ReleaseStringUTFChars((jstring)jvalue.get(), tmp); + if (value.empty()) { + return NO_MEMORY; + } + result->setString(key.c_str(), value); + } else if (env->IsInstanceOf(jvalue.get(), sFields.mIntegerClass)) { + jint value = env->CallIntMethod(jvalue.get(), sFields.mIntegerValueId); + result->setInt32(key.c_str(), value); + } else if (env->IsInstanceOf(jvalue.get(), sFields.mLongClass)) { + jlong value = env->CallLongMethod(jvalue.get(), sFields.mLongValueId); + result->setInt64(key.c_str(), value); + } else if (env->IsInstanceOf(jvalue.get(), sFields.mFloatClass)) { + jfloat value = env->CallFloatMethod(jvalue.get(), sFields.mFloatValueId); + result->setFloat(key.c_str(), value); + } else if (env->IsInstanceOf(jvalue.get(), gByteBufferInfo.clazz)) { + jint position = env->CallIntMethod(jvalue.get(), gByteBufferInfo.positionId); + jint limit = env->CallIntMethod(jvalue.get(), gByteBufferInfo.limitId); + sp<ABuffer> buffer{new ABuffer(limit - position)}; + void *data = env->GetDirectBufferAddress(jvalue.get()); + if (data != nullptr) { + memcpy(buffer->data(), + static_cast<const uint8_t *>(data) + position, + buffer->size()); + } else { + ScopedLocalRef<jbyteArray> byteArray{env, (jbyteArray)env->CallObjectMethod( + jvalue.get(), sFields.mByteBufferArrayId)}; + env->GetByteArrayRegion(byteArray.get(), position, buffer->size(), + reinterpret_cast<jbyte *>(buffer->data())); + } + result->setBuffer(key.c_str(), buffer); } } - if (err == OK && keyObj != NULL) { - if (env->GetArrayLength(keyObj) != 16) { - err = -EINVAL; - } else { - jboolean isCopy; - key = env->GetByteArrayElements(keyObj, &isCopy); - } + *msg = result; + return OK; +} + +static void android_media_MediaCodec_native_queueLinearBlock( + JNIEnv *env, jobject thiz, jint index, jobject bufferObj, + jint offset, jint size, jobject cryptoInfoObj, + jlong presentationTimeUs, jint flags, jobject keys, jobject values) { + ALOGV("android_media_MediaCodec_native_queueLinearBlock"); + + sp<JMediaCodec> codec = getMediaCodec(env, thiz); + + if (codec == nullptr) { + throwExceptionAsNecessary(env, INVALID_OPERATION); + return; } - if (err == OK && ivObj != NULL) { - if (env->GetArrayLength(ivObj) != 16) { - err = -EINVAL; - } else { - jboolean isCopy; - iv = env->GetByteArrayElements(ivObj, &isCopy); + sp<AMessage> tunings; + status_t err = ConvertKeyValueListsToAMessage(env, keys, values, &tunings); + if (err != OK) { + throwExceptionAsNecessary(env, err); + return; + } + + std::shared_ptr<C2Buffer> buffer; + sp<hardware::HidlMemory> memory; + ScopedLocalRef<jobject> lock{env, env->GetObjectField(bufferObj, gLinearBlockInfo.lockId)}; + if (env->MonitorEnter(lock.get()) == JNI_OK) { + if (env->GetBooleanField(bufferObj, gLinearBlockInfo.validId)) { + JMediaCodecLinearBlock *context = + (JMediaCodecLinearBlock *)env->GetLongField(bufferObj, gLinearBlockInfo.contextId); + if (cryptoInfoObj != nullptr) { + memory = context->toHidlMemory(); + } else { + buffer = context->toC2Buffer(offset, size); + } } + env->MonitorExit(lock.get()); + } else { + throwExceptionAsNecessary(env, INVALID_OPERATION); + return; } AString errorDetailMsg; + if (cryptoInfoObj != nullptr) { + if (!memory) { + throwExceptionAsNecessary(env, BAD_VALUE); + return; + } - if (err == OK) { - err = codec->queueSecureInputBuffer( - index, offset, - subSamples, numSubSamples, - (const uint8_t *)key, (const uint8_t *)iv, - mode, - pattern, - timestampUs, + NativeCryptoInfo cryptoInfo{env, cryptoInfoObj}; + size_t cryptoSize = 0; + for (int i = 0; i < cryptoInfo.mNumSubSamples; ++i) { + cryptoSize += cryptoInfo.mSubSamples[i].mNumBytesOfClearData; + cryptoSize += cryptoInfo.mSubSamples[i].mNumBytesOfEncryptedData; + } + err = codec->queueEncryptedLinearBlock( + index, + memory, + offset, + cryptoInfo.mSubSamples, cryptoInfo.mNumSubSamples, + (const uint8_t *)cryptoInfo.mKey, (const uint8_t *)cryptoInfo.mIv, + cryptoInfo.mMode, + cryptoInfo.mPattern, + presentationTimeUs, flags, + tunings, &errorDetailMsg); + } else { + if (!buffer) { + throwExceptionAsNecessary(env, BAD_VALUE); + return; + } + err = codec->queueBuffer( + index, buffer, presentationTimeUs, flags, tunings, &errorDetailMsg); } + throwExceptionAsNecessary(env, err, ACTION_CODE_FATAL, errorDetailMsg.c_str()); +} + +static void android_media_MediaCodec_native_queueGraphicBlock( + JNIEnv *env, jobject thiz, jint index, jobject bufferObj, + jlong presentationTimeUs, jint flags, jobject keys, jobject values) { + ALOGV("android_media_MediaCodec_native_queueGraphicBlock"); + + sp<JMediaCodec> codec = getMediaCodec(env, thiz); - if (iv != NULL) { - env->ReleaseByteArrayElements(ivObj, iv, 0); - iv = NULL; + if (codec == NULL) { + throwExceptionAsNecessary(env, INVALID_OPERATION); + return; } - if (key != NULL) { - env->ReleaseByteArrayElements(keyObj, key, 0); - key = NULL; + sp<AMessage> tunings; + status_t err = ConvertKeyValueListsToAMessage(env, keys, values, &tunings); + if (err != OK) { + throwExceptionAsNecessary(env, err); + return; } - delete[] subSamples; - subSamples = NULL; + std::shared_ptr<C2Buffer> buffer; + std::shared_ptr<C2GraphicBlock> block; + ScopedLocalRef<jobject> lock{env, env->GetObjectField(bufferObj, gGraphicBlockInfo.lockId)}; + if (env->MonitorEnter(lock.get()) == JNI_OK) { + if (env->GetBooleanField(bufferObj, gGraphicBlockInfo.validId)) { + JMediaCodecGraphicBlock *context = (JMediaCodecGraphicBlock *)env->GetLongField( + bufferObj, gGraphicBlockInfo.contextId); + buffer = context->mBuffer; + block = context->mBlock; + } + env->MonitorExit(lock.get()); + } else { + throwExceptionAsNecessary(env, INVALID_OPERATION); + return; + } - throwExceptionAsNecessary( - env, err, ACTION_CODE_FATAL, errorDetailMsg.empty() ? NULL : errorDetailMsg.c_str()); + if (!block && !buffer) { + throwExceptionAsNecessary(env, BAD_VALUE); + return; + } + if (!buffer) { + buffer = C2Buffer::CreateGraphicBuffer(block->share(block->crop(), C2Fence{})); + } + AString errorDetailMsg; + err = codec->queueBuffer(index, buffer, presentationTimeUs, flags, tunings, &errorDetailMsg); + throwExceptionAsNecessary(env, err, ACTION_CODE_FATAL, errorDetailMsg.c_str()); +} + +static void android_media_MediaCodec_native_getOutputFrame( + JNIEnv *env, jobject thiz, jobject frame, jint index) { + ALOGV("android_media_MediaCodec_native_getOutputFrame"); + + sp<JMediaCodec> codec = getMediaCodec(env, thiz); + + if (codec == NULL) { + throwExceptionAsNecessary(env, INVALID_OPERATION); + return; + } + + status_t err = codec->getOutputFrame(env, frame, index); + if (err != OK) { + throwExceptionAsNecessary(env, err); + } } static jint android_media_MediaCodec_dequeueInputBuffer( @@ -1991,6 +2418,31 @@ static void android_media_MediaCodec_native_init(JNIEnv *env) { gFields.patternSkipBlocksID = env->GetFieldID(clazz.get(), "mSkipBlocks", "I"); CHECK(gFields.patternSkipBlocksID != NULL); + clazz.reset(env->FindClass("android/media/MediaCodec$QueueRequest")); + CHECK(clazz.get() != NULL); + + gFields.queueRequestIndexID = env->GetFieldID(clazz.get(), "mIndex", "I"); + CHECK(gFields.queueRequestIndexID != NULL); + + clazz.reset(env->FindClass("android/media/MediaCodec$OutputFrame")); + CHECK(clazz.get() != NULL); + + gFields.outputFrameLinearBlockID = + env->GetFieldID(clazz.get(), "mLinearBlock", "Landroid/media/MediaCodec$LinearBlock;"); + CHECK(gFields.outputFrameLinearBlockID != NULL); + + gFields.outputFrameGraphicBlockID = + env->GetFieldID(clazz.get(), "mGraphicBlock", "Landroid/media/MediaCodec$GraphicBlock;"); + CHECK(gFields.outputFrameGraphicBlockID != NULL); + + gFields.outputFrameChangedKeysID = + env->GetFieldID(clazz.get(), "mChangedKeys", "Ljava/util/ArrayList;"); + CHECK(gFields.outputFrameChangedKeysID != NULL); + + gFields.outputFrameFormatID = + env->GetFieldID(clazz.get(), "mFormat", "Landroid/media/MediaFormat;"); + CHECK(gFields.outputFrameFormatID != NULL); + clazz.reset(env->FindClass("android/media/MediaCodec$CryptoException")); CHECK(clazz.get() != NULL); @@ -2105,6 +2557,96 @@ static void android_media_MediaCodec_native_init(JNIEnv *env) { field = env->GetFieldID(clazz.get(), "level", "I"); CHECK(field != NULL); gCodecInfo.levelField = field; + + clazz.reset(env->FindClass("java/nio/ByteBuffer")); + CHECK(clazz.get() != NULL); + gByteBufferInfo.clazz = (jclass)env->NewGlobalRef(clazz.get()); + + ScopedLocalRef<jclass> byteOrderClass( + env, env->FindClass("java/nio/ByteOrder")); + CHECK(byteOrderClass.get() != NULL); + + jmethodID nativeOrderID = env->GetStaticMethodID( + byteOrderClass.get(), "nativeOrder", "()Ljava/nio/ByteOrder;"); + CHECK(nativeOrderID != NULL); + + ScopedLocalRef<jobject> nativeByteOrderObj{ + env, env->CallStaticObjectMethod(byteOrderClass.get(), nativeOrderID)}; + gByteBufferInfo.nativeByteOrder = env->NewGlobalRef(nativeByteOrderObj.get()); + CHECK(gByteBufferInfo.nativeByteOrder != NULL); + nativeByteOrderObj.reset(); + + gByteBufferInfo.orderId = env->GetMethodID( + clazz.get(), + "order", + "(Ljava/nio/ByteOrder;)Ljava/nio/ByteBuffer;"); + CHECK(gByteBufferInfo.orderId != NULL); + + gByteBufferInfo.asReadOnlyBufferId = env->GetMethodID( + clazz.get(), "asReadOnlyBuffer", "()Ljava/nio/ByteBuffer;"); + CHECK(gByteBufferInfo.asReadOnlyBufferId != NULL); + + gByteBufferInfo.positionId = env->GetMethodID( + clazz.get(), "position", "(I)Ljava/nio/Buffer;"); + CHECK(gByteBufferInfo.positionId != NULL); + + gByteBufferInfo.limitId = env->GetMethodID( + clazz.get(), "limit", "(I)Ljava/nio/Buffer;"); + CHECK(gByteBufferInfo.limitId != NULL); + + clazz.reset(env->FindClass("java/util/ArrayList")); + CHECK(clazz.get() != NULL); + + gArrayListInfo.sizeId = env->GetMethodID(clazz.get(), "size", "()I"); + CHECK(gArrayListInfo.sizeId != NULL); + + gArrayListInfo.getId = env->GetMethodID(clazz.get(), "get", "(I)Ljava/lang/Object;"); + CHECK(gArrayListInfo.getId != NULL); + + gArrayListInfo.addId = env->GetMethodID(clazz.get(), "add", "(Ljava/lang/Object;)Z"); + CHECK(gArrayListInfo.addId != NULL); + + clazz.reset(env->FindClass("android/media/MediaCodec$LinearBlock")); + CHECK(clazz.get() != NULL); + + gLinearBlockInfo.clazz = (jclass)env->NewGlobalRef(clazz.get()); + + gLinearBlockInfo.ctorId = env->GetMethodID(clazz.get(), "<init>", "()V"); + CHECK(gLinearBlockInfo.ctorId != NULL); + + gLinearBlockInfo.setInternalStateId = env->GetMethodID( + clazz.get(), "setInternalStateLocked", "(JZ)V"); + CHECK(gLinearBlockInfo.setInternalStateId != NULL); + + gLinearBlockInfo.contextId = env->GetFieldID(clazz.get(), "mNativeContext", "J"); + CHECK(gLinearBlockInfo.contextId != NULL); + + gLinearBlockInfo.validId = env->GetFieldID(clazz.get(), "mValid", "Z"); + CHECK(gLinearBlockInfo.validId != NULL); + + gLinearBlockInfo.lockId = env->GetFieldID(clazz.get(), "mLock", "Ljava/lang/Object;"); + CHECK(gLinearBlockInfo.lockId != NULL); + + clazz.reset(env->FindClass("android/media/MediaCodec$GraphicBlock")); + CHECK(clazz.get() != NULL); + + gGraphicBlockInfo.clazz = (jclass)env->NewGlobalRef(clazz.get()); + + gGraphicBlockInfo.ctorId = env->GetMethodID(clazz.get(), "<init>", "()V"); + CHECK(gGraphicBlockInfo.ctorId != NULL); + + gGraphicBlockInfo.setInternalStateId = env->GetMethodID( + clazz.get(), "setInternalStateLocked", "(JZ)V"); + CHECK(gGraphicBlockInfo.setInternalStateId != NULL); + + gGraphicBlockInfo.contextId = env->GetFieldID(clazz.get(), "mNativeContext", "J"); + CHECK(gGraphicBlockInfo.contextId != NULL); + + gGraphicBlockInfo.validId = env->GetFieldID(clazz.get(), "mValid", "Z"); + CHECK(gGraphicBlockInfo.validId != NULL); + + gGraphicBlockInfo.lockId = env->GetFieldID(clazz.get(), "mLock", "Ljava/lang/Object;"); + CHECK(gGraphicBlockInfo.lockId != NULL); } static void android_media_MediaCodec_native_setup( @@ -2155,6 +2697,345 @@ static void android_media_MediaCodec_native_finalize( android_media_MediaCodec_release(env, thiz); } +// MediaCodec.LinearBlock + +static jobject android_media_MediaCodec_LinearBlock_native_map( + JNIEnv *env, jobject thiz) { + JMediaCodecLinearBlock *context = + (JMediaCodecLinearBlock *)env->GetLongField(thiz, gLinearBlockInfo.contextId); + if (context->mBuffer) { + std::shared_ptr<C2Buffer> buffer = context->mBuffer; + if (!context->mReadonlyMapping) { + const C2BufferData data = buffer->data(); + if (data.type() != C2BufferData::LINEAR) { + throwExceptionAsNecessary(env, INVALID_OPERATION); + return nullptr; + } + if (data.linearBlocks().size() != 1u) { + throwExceptionAsNecessary(env, INVALID_OPERATION); + return nullptr; + } + C2ConstLinearBlock block = data.linearBlocks().front(); + context->mReadonlyMapping = + std::make_shared<C2ReadView>(block.map().get()); + } + return CreateByteBuffer( + env, + context->mReadonlyMapping->data(), // base + context->mReadonlyMapping->capacity(), // capacity + 0u, // offset + context->mReadonlyMapping->capacity(), // size + true, // readOnly + true /* clearBuffer */); + } else if (context->mBlock) { + std::shared_ptr<C2LinearBlock> block = context->mBlock; + if (!context->mReadWriteMapping) { + context->mReadWriteMapping = + std::make_shared<C2WriteView>(block->map().get()); + } + return CreateByteBuffer( + env, + context->mReadWriteMapping->base(), + context->mReadWriteMapping->capacity(), + context->mReadWriteMapping->offset(), + context->mReadWriteMapping->size(), + false, // readOnly + true /* clearBuffer */); + } else if (context->mLegacyBuffer) { + return CreateByteBuffer( + env, + context->mLegacyBuffer->base(), + context->mLegacyBuffer->capacity(), + context->mLegacyBuffer->offset(), + context->mLegacyBuffer->size(), + true, // readOnly + true /* clearBuffer */); + } else if (context->mHeap) { + return CreateByteBuffer( + env, + static_cast<uint8_t *>(context->mHeap->getBase()) + context->mHeap->getOffset(), + context->mHeap->getSize(), + 0, + context->mHeap->getSize(), + false, // readOnly + true /* clearBuffer */); + } + throwExceptionAsNecessary(env, INVALID_OPERATION); + return nullptr; +} + +static void android_media_MediaCodec_LinearBlock_native_recycle( + JNIEnv *env, jobject thiz) { + JMediaCodecLinearBlock *context = + (JMediaCodecLinearBlock *)env->GetLongField(thiz, gLinearBlockInfo.contextId); + env->CallVoidMethod(thiz, gLinearBlockInfo.setInternalStateId, 0, false); + delete context; +} + +static void PopulateNamesVector( + JNIEnv *env, jobjectArray codecNames, std::vector<std::string> *names) { + jsize length = env->GetArrayLength(codecNames); + for (jsize i = 0; i < length; ++i) { + jstring jstr = static_cast<jstring>(env->GetObjectArrayElement(codecNames, i)); + if (jstr == nullptr) { + // null entries are ignored + continue; + } + const char *cstr = env->GetStringUTFChars(jstr, nullptr); + if (cstr == nullptr) { + throwExceptionAsNecessary(env, BAD_VALUE); + return; + } + names->emplace_back(cstr); + env->ReleaseStringUTFChars(jstr, cstr); + } +} + +static void android_media_MediaCodec_LinearBlock_native_obtain( + JNIEnv *env, jobject thiz, jint capacity, jobjectArray codecNames) { + std::unique_ptr<JMediaCodecLinearBlock> context{new JMediaCodecLinearBlock}; + std::vector<std::string> names; + PopulateNamesVector(env, codecNames, &names); + bool hasSecure = false; + bool hasNonSecure = false; + for (const std::string &name : names) { + if (name.length() >= 7 && name.substr(name.length() - 7) == ".secure") { + hasSecure = true; + } else { + hasNonSecure = true; + } + } + if (hasSecure && !hasNonSecure) { + context->mHeap = new MemoryHeapBase(capacity); + context->mMemory = hardware::fromHeap(context->mHeap); + } else { + context->mBlock = MediaCodec::FetchLinearBlock(capacity, names); + if (!context->mBlock) { + jniThrowException(env, "java/io/IOException", nullptr); + return; + } + } + env->CallVoidMethod( + thiz, + gLinearBlockInfo.setInternalStateId, + (jlong)context.release(), + true /* isMappable */); +} + +static jboolean android_media_MediaCodec_LinearBlock_checkCompatible( + JNIEnv *env, jobjectArray codecNames) { + std::vector<std::string> names; + PopulateNamesVector(env, codecNames, &names); + bool isCompatible = false; + bool hasSecure = false; + bool hasNonSecure = false; + for (const std::string &name : names) { + if (name.length() >= 7 && name.substr(name.length() - 7) == ".secure") { + hasSecure = true; + } else { + hasNonSecure = true; + } + } + if (hasSecure && hasNonSecure) { + return false; + } + status_t err = MediaCodec::CanFetchLinearBlock(names, &isCompatible); + if (err != OK) { + throwExceptionAsNecessary(env, err); + } + return isCompatible; +} + +// MediaCodec.GraphicBlock + +template <class T> +static jobject CreateImage(JNIEnv *env, const std::shared_ptr<T> &view) { + bool readOnly = std::is_const<T>::value; + const C2PlanarLayout layout = view->layout(); + jint format = HAL_PIXEL_FORMAT_YCBCR_420_888; + switch (layout.type) { + case C2PlanarLayout::TYPE_YUV: { + if (layout.numPlanes != 3) { + throwExceptionAsNecessary(env, INVALID_OPERATION); + return nullptr; + } + const C2PlaneInfo & yPlane = layout.planes[C2PlanarLayout::PLANE_Y]; + const C2PlaneInfo & uPlane = layout.planes[C2PlanarLayout::PLANE_U]; + const C2PlaneInfo & vPlane = layout.planes[C2PlanarLayout::PLANE_V]; + if (yPlane.rowSampling != 1 || yPlane.colSampling != 1) { + throwExceptionAsNecessary(env, INVALID_OPERATION); + return nullptr; + } + if (uPlane.rowSampling != vPlane.rowSampling + || uPlane.colSampling != vPlane.colSampling) { + throwExceptionAsNecessary(env, INVALID_OPERATION); + return nullptr; + } + if (uPlane.rowSampling == 2 && uPlane.colSampling == 2) { + format = HAL_PIXEL_FORMAT_YCBCR_420_888; + break; + } else if (uPlane.rowSampling == 1 && uPlane.colSampling == 2) { + format = HAL_PIXEL_FORMAT_YCBCR_422_888; + break; + } else if (uPlane.rowSampling == 1 && uPlane.colSampling == 1) { + format = HAL_PIXEL_FORMAT_YCBCR_444_888; + break; + } + throwExceptionAsNecessary(env, INVALID_OPERATION); + return nullptr; + } + case C2PlanarLayout::TYPE_RGB: { + if (layout.numPlanes != 3) { + throwExceptionAsNecessary(env, INVALID_OPERATION); + return nullptr; + } + format = HAL_PIXEL_FORMAT_FLEX_RGB_888; + break; + } + case C2PlanarLayout::TYPE_RGBA: { + if (layout.numPlanes != 4) { + throwExceptionAsNecessary(env, INVALID_OPERATION); + return nullptr; + } + format = HAL_PIXEL_FORMAT_FLEX_RGBA_8888; + break; + } + case C2PlanarLayout::TYPE_YUVA: + [[fallthrough]]; + default: + throwExceptionAsNecessary(env, INVALID_OPERATION); + return nullptr; + } + + ScopedLocalRef<jclass> planeClazz( + env, env->FindClass("android/media/MediaCodec$MediaImage$MediaPlane")); + ScopedLocalRef<jobjectArray> planeArray{ + env, env->NewObjectArray(layout.numPlanes, planeClazz.get(), NULL)}; + CHECK(planeClazz.get() != NULL); + jmethodID planeConstructID = env->GetMethodID(planeClazz.get(), "<init>", + "([Ljava/nio/ByteBuffer;IIIII)V"); + + // plane indices are happened to be Y-U-V and R-G-B(-A) order. + for (uint32_t i = 0; i < layout.numPlanes; ++i) { + const C2PlaneInfo &plane = layout.planes[i]; + if (plane.rowInc < 0 || plane.colInc < 0) { + throwExceptionAsNecessary(env, INVALID_OPERATION); + return nullptr; + } + ssize_t minOffset = plane.minOffset(view->width(), view->height()); + ssize_t maxOffset = plane.maxOffset(view->width(), view->height()); + ScopedLocalRef<jobject> byteBuffer{env, CreateByteBuffer( + env, + view->data()[plane.rootIx] + plane.offset + minOffset, + maxOffset - minOffset + 1, + 0, + maxOffset - minOffset + 1, + readOnly, + true)}; + + ScopedLocalRef<jobject> jPlane{env, env->NewObject( + planeClazz.get(), planeConstructID, + byteBuffer.get(), plane.rowInc, plane.colInc)}; + } + + ScopedLocalRef<jclass> imageClazz( + env, env->FindClass("android/media/MediaCodec$MediaImage")); + CHECK(imageClazz.get() != NULL); + + jmethodID imageConstructID = env->GetMethodID(imageClazz.get(), "<init>", + "([Landroid/media/Image$Plane;IIIZJIILandroid/graphics/Rect;)V"); + + jobject img = env->NewObject(imageClazz.get(), imageConstructID, + planeArray.get(), + view->width(), + view->height(), + format, + (jboolean)readOnly /* readOnly */, + (jlong)0 /* timestamp */, + (jint)0 /* xOffset */, (jint)0 /* yOffset */, nullptr /* cropRect */); + + // if MediaImage creation fails, return null + if (env->ExceptionCheck()) { + env->ExceptionDescribe(); + env->ExceptionClear(); + return nullptr; + } + + return img; +} + +static jobject android_media_MediaCodec_GraphicBlock_native_map( + JNIEnv *env, jobject thiz) { + JMediaCodecGraphicBlock *context = + (JMediaCodecGraphicBlock *)env->GetLongField(thiz, gGraphicBlockInfo.contextId); + if (context->mBuffer) { + std::shared_ptr<C2Buffer> buffer = context->mBuffer; + if (!context->mReadonlyMapping) { + const C2BufferData data = buffer->data(); + if (data.type() != C2BufferData::GRAPHIC) { + throwExceptionAsNecessary(env, INVALID_OPERATION); + return nullptr; + } + if (data.graphicBlocks().size() != 1u) { + throwExceptionAsNecessary(env, INVALID_OPERATION); + return nullptr; + } + C2ConstGraphicBlock block = data.graphicBlocks().front(); + context->mReadonlyMapping = + std::make_shared<const C2GraphicView>(block.map().get()); + } + return CreateImage(env, context->mReadonlyMapping); + } else if (context->mBlock) { + std::shared_ptr<C2GraphicBlock> block = context->mBlock; + if (!context->mReadWriteMapping) { + context->mReadWriteMapping = + std::make_shared<C2GraphicView>(block->map().get()); + } + return CreateImage(env, context->mReadWriteMapping); + } else if (context->mLegacyBuffer) { + } + throwExceptionAsNecessary(env, INVALID_OPERATION); + return nullptr; +} + +static void android_media_MediaCodec_GraphicBlock_native_recycle( + JNIEnv *env, jobject thiz) { + JMediaCodecGraphicBlock *context = + (JMediaCodecGraphicBlock *)env->GetLongField(thiz, gGraphicBlockInfo.contextId); + env->CallVoidMethod(thiz, gGraphicBlockInfo.setInternalStateId, 0, false /* isMappable */); + delete context; +} + +static void android_media_MediaCodec_GraphicBlock_native_obtain( + JNIEnv *env, jobject thiz, + jint width, jint height, jint format, jlong usage, jobjectArray codecNames) { + std::unique_ptr<JMediaCodecGraphicBlock> context{new JMediaCodecGraphicBlock}; + std::vector<std::string> names; + PopulateNamesVector(env, codecNames, &names); + context->mBlock = MediaCodec::FetchGraphicBlock(width, height, format, usage, names); + if (!context->mBlock) { + jniThrowException(env, "java/io/IOException", nullptr); + return; + } + env->CallVoidMethod( + thiz, + gGraphicBlockInfo.setInternalStateId, + (jlong)context.release(), + true /*isMappable */); +} + +static jboolean android_media_MediaCodec_GraphicBlock_checkCompatible( + JNIEnv *env, jobjectArray codecNames) { + std::vector<std::string> names; + PopulateNamesVector(env, codecNames, &names); + bool isCompatible = false; + status_t err = MediaCodec::CanFetchGraphicBlock(names, &isCompatible); + if (err != OK) { + throwExceptionAsNecessary(env, err); + } + return isCompatible; +} + static const JNINativeMethod gMethods[] = { { "native_release", "()V", (void *)android_media_MediaCodec_release }, @@ -2200,6 +3081,19 @@ static const JNINativeMethod gMethods[] = { { "native_queueSecureInputBuffer", "(IILandroid/media/MediaCodec$CryptoInfo;JI)V", (void *)android_media_MediaCodec_queueSecureInputBuffer }, + { "native_queueLinearBlock", + "(ILandroid/media/MediaCodec$LinearBlock;IILandroid/media/MediaCodec$CryptoInfo;JI" + "Ljava/util/ArrayList;Ljava/util/ArrayList;)V", + (void *)android_media_MediaCodec_native_queueLinearBlock }, + + { "native_queueGraphicBlock", + "(ILandroid/media/MediaCodec$GraphicBlock;JILjava/util/ArrayList;Ljava/util/ArrayList;)V", + (void *)android_media_MediaCodec_native_queueGraphicBlock }, + + { "native_getOutputFrame", + "(Landroid/media/MediaCodec$OutputFrame;I)V", + (void *)android_media_MediaCodec_native_getOutputFrame }, + { "native_dequeueInputBuffer", "(J)I", (void *)android_media_MediaCodec_dequeueInputBuffer }, @@ -2254,7 +3148,50 @@ static const JNINativeMethod gMethods[] = { (void *)android_media_MediaCodec_native_finalize }, }; +static const JNINativeMethod gLinearBlockMethods[] = { + { "native_map", "()Ljava/nio/ByteBuffer;", + (void *)android_media_MediaCodec_LinearBlock_native_map }, + + { "native_recycle", "()V", + (void *)android_media_MediaCodec_LinearBlock_native_recycle }, + + { "native_obtain", "(I[Ljava/lang/String;)V", + (void *)android_media_MediaCodec_LinearBlock_native_obtain }, + + { "native_checkCompatible", "([Ljava/lang/String;)Z", + (void *)android_media_MediaCodec_LinearBlock_checkCompatible }, +}; + +static const JNINativeMethod gGraphicBlockMethods[] = { + { "native_map", "()Landroid/media/Image;", + (void *)android_media_MediaCodec_GraphicBlock_native_map }, + + { "native_recycle", "()V", + (void *)android_media_MediaCodec_GraphicBlock_native_recycle }, + + { "native_obtain", "(IIIJ[Ljava/lang/String;)V", + (void *)android_media_MediaCodec_GraphicBlock_native_obtain }, + + { "native_checkCompatible", "([Ljava/lang/String;)Z", + (void *)android_media_MediaCodec_GraphicBlock_checkCompatible }, +}; + int register_android_media_MediaCodec(JNIEnv *env) { - return AndroidRuntime::registerNativeMethods(env, + int result = AndroidRuntime::registerNativeMethods(env, "android/media/MediaCodec", gMethods, NELEM(gMethods)); + if (result != JNI_OK) { + return result; + } + result = AndroidRuntime::registerNativeMethods(env, + "android/media/MediaCodec$LinearBlock", + gLinearBlockMethods, + NELEM(gLinearBlockMethods)); + if (result != JNI_OK) { + return result; + } + result = AndroidRuntime::registerNativeMethods(env, + "android/media/MediaCodec$GraphicBlock", + gGraphicBlockMethods, + NELEM(gGraphicBlockMethods)); + return result; } diff --git a/media/jni/android_media_MediaCodec.h b/media/jni/android_media_MediaCodec.h index ce1c805b6366..1d12e776d43e 100644 --- a/media/jni/android_media_MediaCodec.h +++ b/media/jni/android_media_MediaCodec.h @@ -27,6 +27,8 @@ #include <media/stagefright/foundation/AHandler.h> #include <utils/Errors.h> +class C2Buffer; + namespace android { struct ABuffer; @@ -39,6 +41,7 @@ struct MediaCodec; struct PersistentSurface; class Surface; namespace hardware { +class HidlMemory; namespace cas { namespace native { namespace V1_0 { @@ -97,6 +100,26 @@ struct JMediaCodec : public AHandler { uint32_t flags, AString *errorDetailMsg); + status_t queueBuffer( + size_t index, const std::shared_ptr<C2Buffer> &buffer, + int64_t timeUs, uint32_t flags, const sp<AMessage> &tunings, + AString *errorDetailMsg); + + status_t queueEncryptedLinearBlock( + size_t index, + const sp<hardware::HidlMemory> &buffer, + size_t offset, + const CryptoPlugin::SubSample *subSamples, + size_t numSubSamples, + const uint8_t key[16], + const uint8_t iv[16], + CryptoPlugin::Mode mode, + const CryptoPlugin::Pattern &pattern, + int64_t presentationTimeUs, + uint32_t flags, + const sp<AMessage> &tunings, + AString *errorDetailMsg); + status_t dequeueInputBuffer(size_t *index, int64_t timeoutUs); status_t dequeueOutputBuffer( @@ -120,6 +143,9 @@ struct JMediaCodec : public AHandler { status_t getImage( JNIEnv *env, bool input, size_t index, jobject *image) const; + status_t getOutputFrame( + JNIEnv *env, jobject frame, size_t index) const; + status_t getName(JNIEnv *env, jstring *name) const; status_t getCodecInfo(JNIEnv *env, jobject *codecInfo) const; @@ -147,17 +173,10 @@ private: jweak mObject; sp<Surface> mSurfaceTextureClient; - // java objects cached - jclass mByteBufferClass; - jobject mNativeByteOrderObj; - jmethodID mByteBufferOrderMethodID; - jmethodID mByteBufferPositionMethodID; - jmethodID mByteBufferLimitMethodID; - jmethodID mByteBufferAsReadOnlyBufferMethodID; - sp<ALooper> mLooper; sp<MediaCodec> mCodec; AString mNameAtCreation; + bool mGraphicOutput{false}; std::once_flag mReleaseFlag; sp<AMessage> mCallbackNotification; @@ -170,8 +189,6 @@ private: JNIEnv *env, bool readOnly, bool clearBuffer, const sp<T> &buffer, jobject *buf) const; - void cacheJavaObjects(JNIEnv *env); - void deleteJavaObjects(JNIEnv *env); void handleCallback(const sp<AMessage> &msg); void handleFrameRenderedNotification(const sp<AMessage> &msg); |