diff options
107 files changed, 14694 insertions, 180 deletions
diff --git a/camera/Android.bp b/camera/Android.bp index fa36bb31b1..21ef9eed70 100644 --- a/camera/Android.bp +++ b/camera/Android.bp @@ -14,6 +14,7 @@ cc_library_shared { name: "libcamera_client", + defaults: ["camera_parameter_library_defaults"], aidl: { export_aidl_headers: true, @@ -32,7 +33,6 @@ cc_library_shared { // Source for camera interface parcelables, and manually-written interfaces "Camera.cpp", "CameraMetadata.cpp", - "CameraParameters.cpp", "CaptureResult.cpp", "CameraParameters2.cpp", "ICamera.cpp", @@ -77,6 +77,16 @@ cc_library_shared { } +cc_library_static { + name: "libcamera_parameters", + + export_include_dirs: [ + "include", + ], + srcs: ["CameraParameters.cpp"], + +} + // AIDL interface between camera clients and the camera service. filegroup { name: "libcamera_client_aidl", diff --git a/camera/CameraParameters.cpp b/camera/CameraParameters.cpp index 68969cf649..de8ac2ffce 100644 --- a/camera/CameraParameters.cpp +++ b/camera/CameraParameters.cpp @@ -237,6 +237,9 @@ void CameraParameters::unflatten(const String8 ¶ms) void CameraParameters::set(const char *key, const char *value) { + if (key == NULL || value == NULL) + return; + // XXX i think i can do this with strspn() if (strchr(key, '=') || strchr(key, ';')) { //XXX ALOGE("Key \"%s\"contains invalid character (= or ;)", key); diff --git a/camera/ICameraClient.cpp b/camera/ICameraClient.cpp index bef2ea0acd..be82ff4ec6 100644 --- a/camera/ICameraClient.cpp +++ b/camera/ICameraClient.cpp @@ -51,7 +51,11 @@ public: data.writeInterfaceToken(ICameraClient::getInterfaceDescriptor()); data.writeInt32(msgType); data.writeInt32(ext1); - data.writeInt32(ext2); + if ((msgType == CAMERA_MSG_PREVIEW_FRAME) && (ext1 == CAMERA_FRAME_DATA_FD)) { + data.writeFileDescriptor(ext2); + } else { + data.writeInt32(ext2); + } remote()->transact(NOTIFY_CALLBACK, data, &reply, IBinder::FLAG_ONEWAY); } @@ -129,8 +133,13 @@ status_t BnCameraClient::onTransact( ALOGV("NOTIFY_CALLBACK"); CHECK_INTERFACE(ICameraClient, data, reply); int32_t msgType = data.readInt32(); - int32_t ext1 = data.readInt32(); - int32_t ext2 = data.readInt32(); + int32_t ext1 = data.readInt32(); + int32_t ext2 = 0; + if ((msgType == CAMERA_MSG_PREVIEW_FRAME) && (ext1 == CAMERA_FRAME_DATA_FD)) { + ext2 = data.readFileDescriptor(); + } else { + ext2 = data.readInt32(); + } notifyCallback(msgType, ext1, ext2); return NO_ERROR; } break; diff --git a/camera/include/camera/CameraParameters2.h b/camera/include/camera/CameraParameters2.h index f691cd6ea0..5fae079376 100644 --- a/camera/include/camera/CameraParameters2.h +++ b/camera/include/camera/CameraParameters2.h @@ -19,7 +19,7 @@ #include <utils/Vector.h> #include <utils/String8.h> -#include "CameraParameters.h" +#include <camera/CameraParameters.h> namespace android { diff --git a/include/media/AudioSession.h b/include/media/AudioSession.h new file mode 120000 index 0000000000..005e48e7ed --- /dev/null +++ b/include/media/AudioSession.h @@ -0,0 +1 @@ +../../media/libaudioclient/include/media/AudioSession.h
\ No newline at end of file diff --git a/include/media/IHDCP.h b/include/media/IHDCP.h new file mode 120000 index 0000000000..9d4568eafd --- /dev/null +++ b/include/media/IHDCP.h @@ -0,0 +1 @@ +../../media/libmedia/include/media/IHDCP.h
\ No newline at end of file diff --git a/media/codec2/sfplugin/C2OMXNode.cpp b/media/codec2/sfplugin/C2OMXNode.cpp index c7588e9a51..dd1f4858a5 100644 --- a/media/codec2/sfplugin/C2OMXNode.cpp +++ b/media/codec2/sfplugin/C2OMXNode.cpp @@ -25,6 +25,7 @@ #include <C2AllocatorGralloc.h> #include <C2BlockInternal.h> #include <C2Component.h> +#include <C2Config.h> #include <C2PlatformSupport.h> #include <OMX_Component.h> @@ -44,6 +45,8 @@ namespace android { namespace { +constexpr OMX_U32 kPortIndexInput = 0; + class Buffer2D : public C2Buffer { public: explicit Buffer2D(C2ConstGraphicBlock block) : C2Buffer({ block }) {} @@ -200,11 +203,27 @@ status_t C2OMXNode::getParameter(OMX_INDEXTYPE index, void *params, size_t size) return BAD_VALUE; } OMX_PARAM_PORTDEFINITIONTYPE *pDef = (OMX_PARAM_PORTDEFINITIONTYPE *)params; - // TODO: read these from intf() + if (pDef->nPortIndex != kPortIndexInput) { + break; + } + pDef->nBufferCountActual = 16; + + std::shared_ptr<Codec2Client::Component> comp = mComp.lock(); + C2PortActualDelayTuning::input inputDelay(0); + C2ActualPipelineDelayTuning pipelineDelay(0); + c2_status_t c2err = comp->query( + {&inputDelay, &pipelineDelay}, {}, C2_DONT_BLOCK, nullptr); + if (c2err == C2_OK || c2err == C2_BAD_INDEX) { + pDef->nBufferCountActual = 4; + pDef->nBufferCountActual += (inputDelay ? inputDelay.value : 0u); + pDef->nBufferCountActual += (pipelineDelay ? pipelineDelay.value : 0u); + } + pDef->eDomain = OMX_PortDomainVideo; pDef->format.video.nFrameWidth = mWidth; pDef->format.video.nFrameHeight = mHeight; + pDef->format.video.eColorFormat = OMX_COLOR_FormatAndroidOpaque; err = OK; break; } diff --git a/media/codec2/sfplugin/CCodec.cpp b/media/codec2/sfplugin/CCodec.cpp index 54107bd852..55ff18f4a2 100644 --- a/media/codec2/sfplugin/CCodec.cpp +++ b/media/codec2/sfplugin/CCodec.cpp @@ -246,8 +246,24 @@ public: if (source == nullptr) { return NO_INIT; } - constexpr size_t kNumSlots = 16; - for (size_t i = 0; i < kNumSlots; ++i) { + + size_t numSlots = 16; + // WORKAROUND: having more slots improve performance while consuming + // more memory. This is a temporary workaround to reduce memory for + // larger-than-4K scenario. + if (mWidth * mHeight > 4096 * 2340) { + constexpr OMX_U32 kPortIndexInput = 0; + + OMX_PARAM_PORTDEFINITIONTYPE param; + param.nPortIndex = kPortIndexInput; + status_t err = mNode->getParameter(OMX_IndexParamPortDefinition, + ¶m, sizeof(param)); + if (err == OK) { + numSlots = param.nBufferCountActual; + } + } + + for (size_t i = 0; i < numSlots; ++i) { source->onInputBufferAdded(i); } diff --git a/media/codec2/sfplugin/CCodecBuffers.cpp b/media/codec2/sfplugin/CCodecBuffers.cpp index bddaa9f22b..566a18fbee 100644 --- a/media/codec2/sfplugin/CCodecBuffers.cpp +++ b/media/codec2/sfplugin/CCodecBuffers.cpp @@ -91,9 +91,14 @@ void CCodecBuffers::handleImageData(const sp<Codec2Buffer> &buffer) { newFormat->setInt32(KEY_STRIDE, stride); ALOGD("[%s] updating stride = %d", mName, stride); if (img->mNumPlanes > 1 && stride > 0) { - int32_t vstride = (img->mPlane[1].mOffset - img->mPlane[0].mOffset) / stride; + int64_t offsetDelta = + (int64_t)img->mPlane[1].mOffset - (int64_t)img->mPlane[0].mOffset; + int32_t vstride = int32_t(offsetDelta / stride); newFormat->setInt32(KEY_SLICE_HEIGHT, vstride); ALOGD("[%s] updating vstride = %d", mName, vstride); + buffer->setRange( + img->mPlane[0].mOffset, + buffer->size() - img->mPlane[0].mOffset); } } setFormat(newFormat); diff --git a/media/codec2/sfplugin/CCodecBuffers.h b/media/codec2/sfplugin/CCodecBuffers.h index 4772ab53eb..c383a7ce1a 100644 --- a/media/codec2/sfplugin/CCodecBuffers.h +++ b/media/codec2/sfplugin/CCodecBuffers.h @@ -33,8 +33,8 @@ class MemoryDealer; class SkipCutBuffer; constexpr size_t kLinearBufferSize = 1048576; -// This can fit 4K RGBA frame, and most likely client won't need more than this. -constexpr size_t kMaxLinearBufferSize = 4096 * 2304 * 4; +// This can fit an 8K frame. +constexpr size_t kMaxLinearBufferSize = 7680 * 4320 * 2; /** * Base class for representation of buffers at one port. diff --git a/media/codec2/sfplugin/Codec2Buffer.cpp b/media/codec2/sfplugin/Codec2Buffer.cpp index 25e7da9206..19414a0a0c 100644 --- a/media/codec2/sfplugin/Codec2Buffer.cpp +++ b/media/codec2/sfplugin/Codec2Buffer.cpp @@ -276,20 +276,22 @@ public: int32_t planeSize = 0; for (uint32_t i = 0; i < layout.numPlanes; ++i) { const C2PlaneInfo &plane = layout.planes[i]; - ssize_t minOffset = plane.minOffset(mWidth, mHeight); - ssize_t maxOffset = plane.maxOffset(mWidth, mHeight); + int64_t planeStride = std::abs(plane.rowInc / plane.colInc); + ssize_t minOffset = plane.minOffset( + mWidth / plane.colSampling, mHeight / plane.rowSampling); + ssize_t maxOffset = plane.maxOffset( + mWidth / plane.colSampling, mHeight / plane.rowSampling); if (minPtr > mView.data()[i] + minOffset) { minPtr = mView.data()[i] + minOffset; } if (maxPtr < mView.data()[i] + maxOffset) { maxPtr = mView.data()[i] + maxOffset; } - planeSize += std::abs(plane.rowInc) * align(mHeight, 64) - / plane.rowSampling / plane.colSampling - * divUp(mAllocatedDepth, 8u); + planeSize += planeStride * divUp(mAllocatedDepth, 8u) + * align(mHeight, 64) / plane.rowSampling; } - if ((maxPtr - minPtr + 1) <= planeSize) { + if (minPtr == mView.data()[0] && (maxPtr - minPtr + 1) <= planeSize) { // FIXME: this is risky as reading/writing data out of bound results // in an undefined behavior, but gralloc does assume a // contiguous mapping diff --git a/media/codec2/sfplugin/tests/CCodecBuffers_test.cpp b/media/codec2/sfplugin/tests/CCodecBuffers_test.cpp index 5bee605276..ad8f6e555b 100644 --- a/media/codec2/sfplugin/tests/CCodecBuffers_test.cpp +++ b/media/codec2/sfplugin/tests/CCodecBuffers_test.cpp @@ -18,22 +18,31 @@ #include <gtest/gtest.h> +#include <media/stagefright/foundation/AString.h> #include <media/stagefright/MediaCodecConstants.h> +#include <C2BlockInternal.h> #include <C2PlatformSupport.h> namespace android { +static std::shared_ptr<RawGraphicOutputBuffers> GetRawGraphicOutputBuffers( + int32_t width, int32_t height) { + std::shared_ptr<RawGraphicOutputBuffers> buffers = + std::make_shared<RawGraphicOutputBuffers>("test"); + sp<AMessage> format{new AMessage}; + format->setInt32(KEY_WIDTH, width); + format->setInt32(KEY_HEIGHT, height); + buffers->setFormat(format); + return buffers; +} + TEST(RawGraphicOutputBuffersTest, ChangeNumSlots) { constexpr int32_t kWidth = 3840; constexpr int32_t kHeight = 2160; std::shared_ptr<RawGraphicOutputBuffers> buffers = - std::make_shared<RawGraphicOutputBuffers>("test"); - sp<AMessage> format{new AMessage}; - format->setInt32("width", kWidth); - format->setInt32("height", kHeight); - buffers->setFormat(format); + GetRawGraphicOutputBuffers(kWidth, kHeight); std::shared_ptr<C2BlockPool> pool; ASSERT_EQ(OK, GetCodec2BlockPool(C2BlockPool::BASIC_GRAPHIC, nullptr, &pool)); @@ -96,4 +105,435 @@ TEST(RawGraphicOutputBuffersTest, ChangeNumSlots) { } } +class TestGraphicAllocation : public C2GraphicAllocation { +public: + TestGraphicAllocation( + uint32_t width, + uint32_t height, + const C2PlanarLayout &layout, + size_t capacity, + std::vector<size_t> offsets) + : C2GraphicAllocation(width, height), + mLayout(layout), + mMemory(capacity, 0xAA), + mOffsets(offsets) { + } + + c2_status_t map( + C2Rect rect, C2MemoryUsage usage, C2Fence *fence, + C2PlanarLayout *layout, uint8_t **addr) override { + (void)rect; + (void)usage; + (void)fence; + *layout = mLayout; + for (size_t i = 0; i < mLayout.numPlanes; ++i) { + addr[i] = mMemory.data() + mOffsets[i]; + } + return C2_OK; + } + + c2_status_t unmap(uint8_t **, C2Rect, C2Fence *) override { return C2_OK; } + + C2Allocator::id_t getAllocatorId() const override { return -1; } + + const C2Handle *handle() const override { return nullptr; } + + bool equals(const std::shared_ptr<const C2GraphicAllocation> &other) const override { + return other.get() == this; + } + +private: + C2PlanarLayout mLayout; + std::vector<uint8_t> mMemory; + std::vector<uint8_t *> mAddr; + std::vector<size_t> mOffsets; +}; + +class LayoutTest : public ::testing::TestWithParam<std::tuple<bool, std::string, bool, int32_t>> { +private: + static C2PlanarLayout YUVPlanarLayout(int32_t stride) { + C2PlanarLayout layout = { + C2PlanarLayout::TYPE_YUV, + 3, /* numPlanes */ + 3, /* rootPlanes */ + {}, /* planes --- to be filled below */ + }; + layout.planes[C2PlanarLayout::PLANE_Y] = { + C2PlaneInfo::CHANNEL_Y, + 1, /* colInc */ + stride, /* rowInc */ + 1, /* colSampling */ + 1, /* rowSampling */ + 8, /* allocatedDepth */ + 8, /* bitDepth */ + 0, /* rightShift */ + C2PlaneInfo::NATIVE, + C2PlanarLayout::PLANE_Y, /* rootIx */ + 0, /* offset */ + }; + layout.planes[C2PlanarLayout::PLANE_U] = { + C2PlaneInfo::CHANNEL_CB, + 1, /* colInc */ + stride / 2, /* rowInc */ + 2, /* colSampling */ + 2, /* rowSampling */ + 8, /* allocatedDepth */ + 8, /* bitDepth */ + 0, /* rightShift */ + C2PlaneInfo::NATIVE, + C2PlanarLayout::PLANE_U, /* rootIx */ + 0, /* offset */ + }; + layout.planes[C2PlanarLayout::PLANE_V] = { + C2PlaneInfo::CHANNEL_CR, + 1, /* colInc */ + stride / 2, /* rowInc */ + 2, /* colSampling */ + 2, /* rowSampling */ + 8, /* allocatedDepth */ + 8, /* bitDepth */ + 0, /* rightShift */ + C2PlaneInfo::NATIVE, + C2PlanarLayout::PLANE_V, /* rootIx */ + 0, /* offset */ + }; + return layout; + } + + static C2PlanarLayout YUVSemiPlanarLayout(int32_t stride) { + C2PlanarLayout layout = { + C2PlanarLayout::TYPE_YUV, + 3, /* numPlanes */ + 2, /* rootPlanes */ + {}, /* planes --- to be filled below */ + }; + layout.planes[C2PlanarLayout::PLANE_Y] = { + C2PlaneInfo::CHANNEL_Y, + 1, /* colInc */ + stride, /* rowInc */ + 1, /* colSampling */ + 1, /* rowSampling */ + 8, /* allocatedDepth */ + 8, /* bitDepth */ + 0, /* rightShift */ + C2PlaneInfo::NATIVE, + C2PlanarLayout::PLANE_Y, /* rootIx */ + 0, /* offset */ + }; + layout.planes[C2PlanarLayout::PLANE_U] = { + C2PlaneInfo::CHANNEL_CB, + 2, /* colInc */ + stride, /* rowInc */ + 2, /* colSampling */ + 2, /* rowSampling */ + 8, /* allocatedDepth */ + 8, /* bitDepth */ + 0, /* rightShift */ + C2PlaneInfo::NATIVE, + C2PlanarLayout::PLANE_U, /* rootIx */ + 0, /* offset */ + }; + layout.planes[C2PlanarLayout::PLANE_V] = { + C2PlaneInfo::CHANNEL_CR, + 2, /* colInc */ + stride, /* rowInc */ + 2, /* colSampling */ + 2, /* rowSampling */ + 8, /* allocatedDepth */ + 8, /* bitDepth */ + 0, /* rightShift */ + C2PlaneInfo::NATIVE, + C2PlanarLayout::PLANE_U, /* rootIx */ + 1, /* offset */ + }; + return layout; + } + + static C2PlanarLayout YVUSemiPlanarLayout(int32_t stride) { + C2PlanarLayout layout = { + C2PlanarLayout::TYPE_YUV, + 3, /* numPlanes */ + 2, /* rootPlanes */ + {}, /* planes --- to be filled below */ + }; + layout.planes[C2PlanarLayout::PLANE_Y] = { + C2PlaneInfo::CHANNEL_Y, + 1, /* colInc */ + stride, /* rowInc */ + 1, /* colSampling */ + 1, /* rowSampling */ + 8, /* allocatedDepth */ + 8, /* bitDepth */ + 0, /* rightShift */ + C2PlaneInfo::NATIVE, + C2PlanarLayout::PLANE_Y, /* rootIx */ + 0, /* offset */ + }; + layout.planes[C2PlanarLayout::PLANE_U] = { + C2PlaneInfo::CHANNEL_CB, + 2, /* colInc */ + stride, /* rowInc */ + 2, /* colSampling */ + 2, /* rowSampling */ + 8, /* allocatedDepth */ + 8, /* bitDepth */ + 0, /* rightShift */ + C2PlaneInfo::NATIVE, + C2PlanarLayout::PLANE_V, /* rootIx */ + 1, /* offset */ + }; + layout.planes[C2PlanarLayout::PLANE_V] = { + C2PlaneInfo::CHANNEL_CR, + 2, /* colInc */ + stride, /* rowInc */ + 2, /* colSampling */ + 2, /* rowSampling */ + 8, /* allocatedDepth */ + 8, /* bitDepth */ + 0, /* rightShift */ + C2PlaneInfo::NATIVE, + C2PlanarLayout::PLANE_V, /* rootIx */ + 0, /* offset */ + }; + return layout; + } + + static std::shared_ptr<C2GraphicBlock> CreateGraphicBlock( + uint32_t width, + uint32_t height, + const C2PlanarLayout &layout, + size_t capacity, + std::vector<size_t> offsets) { + std::shared_ptr<C2GraphicAllocation> alloc = std::make_shared<TestGraphicAllocation>( + width, + height, + layout, + capacity, + offsets); + + return _C2BlockFactory::CreateGraphicBlock(alloc); + } + + static constexpr uint8_t GetPixelValue(uint8_t value, uint32_t row, uint32_t col) { + return (uint32_t(value) * row + col) & 0xFF; + } + + static void FillPlane(C2GraphicView &view, size_t index, uint8_t value) { + C2PlanarLayout layout = view.layout(); + + uint8_t *rowPtr = view.data()[index]; + C2PlaneInfo plane = layout.planes[index]; + for (uint32_t row = 0; row < view.height() / plane.rowSampling; ++row) { + uint8_t *colPtr = rowPtr; + for (uint32_t col = 0; col < view.width() / plane.colSampling; ++col) { + *colPtr = GetPixelValue(value, row, col); + colPtr += plane.colInc; + } + rowPtr += plane.rowInc; + } + } + + static void FillBlock(const std::shared_ptr<C2GraphicBlock> &block) { + C2GraphicView view = block->map().get(); + + FillPlane(view, C2PlanarLayout::PLANE_Y, 'Y'); + FillPlane(view, C2PlanarLayout::PLANE_U, 'U'); + FillPlane(view, C2PlanarLayout::PLANE_V, 'V'); + } + + static bool VerifyPlane( + const MediaImage2 *mediaImage, + const uint8_t *base, + uint32_t index, + uint8_t value, + std::string *errorMsg) { + *errorMsg = ""; + MediaImage2::PlaneInfo plane = mediaImage->mPlane[index]; + const uint8_t *rowPtr = base + plane.mOffset; + for (uint32_t row = 0; row < mediaImage->mHeight / plane.mVertSubsampling; ++row) { + const uint8_t *colPtr = rowPtr; + for (uint32_t col = 0; col < mediaImage->mWidth / plane.mHorizSubsampling; ++col) { + if (GetPixelValue(value, row, col) != *colPtr) { + *errorMsg = AStringPrintf("row=%u col=%u expected=%02x actual=%02x", + row, col, GetPixelValue(value, row, col), *colPtr).c_str(); + return false; + } + colPtr += plane.mColInc; + } + rowPtr += plane.mRowInc; + } + return true; + } + +public: + static constexpr int32_t kWidth = 320; + static constexpr int32_t kHeight = 240; + static constexpr int32_t kGapLength = kWidth * kHeight * 10; + + static std::shared_ptr<C2Buffer> CreateAndFillBufferFromParam(const ParamType ¶m) { + bool contiguous = std::get<0>(param); + std::string planeOrderStr = std::get<1>(param); + bool planar = std::get<2>(param); + int32_t stride = std::get<3>(param); + + C2PlanarLayout::plane_index_t planeOrder[3]; + C2PlanarLayout layout; + + if (planeOrderStr.size() != 3) { + return nullptr; + } + for (size_t i = 0; i < 3; ++i) { + C2PlanarLayout::plane_index_t planeIndex; + switch (planeOrderStr[i]) { + case 'Y': planeIndex = C2PlanarLayout::PLANE_Y; break; + case 'U': planeIndex = C2PlanarLayout::PLANE_U; break; + case 'V': planeIndex = C2PlanarLayout::PLANE_V; break; + default: return nullptr; + } + planeOrder[i] = planeIndex; + } + + if (planar) { + layout = YUVPlanarLayout(stride); + } else { // semi-planar + for (size_t i = 0; i < 3; ++i) { + if (planeOrder[i] == C2PlanarLayout::PLANE_U) { + layout = YUVSemiPlanarLayout(stride); + break; + } + if (planeOrder[i] == C2PlanarLayout::PLANE_V) { + layout = YVUSemiPlanarLayout(stride); + break; + } + } + } + + size_t yPlaneSize = stride * kHeight; + size_t uvPlaneSize = stride * kHeight / 4; + size_t capacity = yPlaneSize + uvPlaneSize * 2; + std::vector<size_t> offsets(3); + + if (!contiguous) { + if (planar) { + capacity += kGapLength * 2; + } else { // semi-planar + capacity += kGapLength; + } + } + + offsets[planeOrder[0]] = 0; + size_t planeSize = (planeOrder[0] == C2PlanarLayout::PLANE_Y) ? yPlaneSize : uvPlaneSize; + for (size_t i = 1; i < 3; ++i) { + offsets[planeOrder[i]] = offsets[planeOrder[i - 1]] + planeSize; + if (!contiguous) { + offsets[planeOrder[i]] += kGapLength; + } + planeSize = (planeOrder[i] == C2PlanarLayout::PLANE_Y) ? yPlaneSize : uvPlaneSize; + if (!planar // semi-planar + && planeOrder[i - 1] != C2PlanarLayout::PLANE_Y + && planeOrder[i] != C2PlanarLayout::PLANE_Y) { + offsets[planeOrder[i]] = offsets[planeOrder[i - 1]] + 1; + planeSize = uvPlaneSize * 2 - 1; + } + } + + std::shared_ptr<C2GraphicBlock> block = CreateGraphicBlock( + kWidth, + kHeight, + layout, + capacity, + offsets); + FillBlock(block); + return C2Buffer::CreateGraphicBuffer( + block->share(block->crop(), C2Fence())); + } + + static bool VerifyClientBuffer( + const sp<MediaCodecBuffer> &buffer, std::string *errorMsg) { + *errorMsg = ""; + sp<ABuffer> imageData; + if (!buffer->format()->findBuffer("image-data", &imageData)) { + *errorMsg = "Missing image data"; + return false; + } + MediaImage2 *mediaImage = (MediaImage2 *)imageData->data(); + if (mediaImage->mType != MediaImage2::MEDIA_IMAGE_TYPE_YUV) { + *errorMsg = AStringPrintf("Unexpected type: %d", mediaImage->mType).c_str(); + return false; + } + std::string planeErrorMsg; + if (!VerifyPlane(mediaImage, buffer->base(), MediaImage2::Y, 'Y', &planeErrorMsg)) { + *errorMsg = "Y plane does not match: " + planeErrorMsg; + return false; + } + if (!VerifyPlane(mediaImage, buffer->base(), MediaImage2::U, 'U', &planeErrorMsg)) { + *errorMsg = "U plane does not match: " + planeErrorMsg; + return false; + } + if (!VerifyPlane(mediaImage, buffer->base(), MediaImage2::V, 'V', &planeErrorMsg)) { + *errorMsg = "V plane does not match: " + planeErrorMsg; + return false; + } + + int32_t width, height, stride; + buffer->format()->findInt32(KEY_WIDTH, &width); + buffer->format()->findInt32(KEY_HEIGHT, &height); + buffer->format()->findInt32(KEY_STRIDE, &stride); + + MediaImage2 legacyYLayout = { + MediaImage2::MEDIA_IMAGE_TYPE_Y, + 1, // mNumPlanes + uint32_t(width), + uint32_t(height), + 8, + 8, + {}, // mPlane + }; + legacyYLayout.mPlane[MediaImage2::Y] = { + 0, // mOffset + 1, // mColInc + stride, // mRowInc + 1, // mHorizSubsampling + 1, // mVertSubsampling + }; + if (!VerifyPlane(&legacyYLayout, buffer->data(), MediaImage2::Y, 'Y', &planeErrorMsg)) { + *errorMsg = "Y plane by legacy layout does not match: " + planeErrorMsg; + return false; + } + return true; + } + +}; + +TEST_P(LayoutTest, VerifyLayout) { + std::shared_ptr<RawGraphicOutputBuffers> buffers = + GetRawGraphicOutputBuffers(kWidth, kHeight); + + std::shared_ptr<C2Buffer> c2Buffer = CreateAndFillBufferFromParam(GetParam()); + ASSERT_NE(nullptr, c2Buffer); + sp<MediaCodecBuffer> clientBuffer; + size_t index; + ASSERT_EQ(OK, buffers->registerBuffer(c2Buffer, &index, &clientBuffer)); + ASSERT_NE(nullptr, clientBuffer); + std::string errorMsg; + ASSERT_TRUE(VerifyClientBuffer(clientBuffer, &errorMsg)) << errorMsg; +} + +INSTANTIATE_TEST_SUITE_P( + RawGraphicOutputBuffersTest, + LayoutTest, + ::testing::Combine( + ::testing::Bool(), /* contiguous */ + ::testing::Values("YUV", "YVU", "UVY", "VUY"), + ::testing::Bool(), /* planar */ + ::testing::Values(320, 512)), + [](const ::testing::TestParamInfo<LayoutTest::ParamType> &info) { + std::string contiguous = std::get<0>(info.param) ? "Contiguous" : "Noncontiguous"; + std::string planar = std::get<2>(info.param) ? "Planar" : "SemiPlanar"; + return contiguous + + std::get<1>(info.param) + + planar + + std::to_string(std::get<3>(info.param)); + }); + } // namespace android diff --git a/media/codec2/vndk/C2AllocatorBlob.cpp b/media/codec2/vndk/C2AllocatorBlob.cpp index 50c9e59af8..aa054998f8 100644 --- a/media/codec2/vndk/C2AllocatorBlob.cpp +++ b/media/codec2/vndk/C2AllocatorBlob.cpp @@ -17,6 +17,8 @@ // #define LOG_NDEBUG 0 #define LOG_TAG "C2AllocatorBlob" +#include <set> + #include <C2AllocatorBlob.h> #include <C2PlatformSupport.h> @@ -67,6 +69,10 @@ public: private: const std::shared_ptr<C2GraphicAllocation> mGraphicAllocation; const C2Allocator::id_t mAllocatorId; + + std::mutex mMapLock; + std::multiset<std::pair<size_t, size_t>> mMappedOffsetSize; + uint8_t *mMappedAddr; }; C2AllocationBlob::C2AllocationBlob( @@ -74,20 +80,74 @@ C2AllocationBlob::C2AllocationBlob( C2Allocator::id_t allocatorId) : C2LinearAllocation(capacity), mGraphicAllocation(std::move(graphicAllocation)), - mAllocatorId(allocatorId) {} + mAllocatorId(allocatorId), + mMappedAddr(nullptr) {} -C2AllocationBlob::~C2AllocationBlob() {} +C2AllocationBlob::~C2AllocationBlob() { + if (mMappedAddr) { + C2Rect rect(capacity(), kLinearBufferHeight); + mGraphicAllocation->unmap(&mMappedAddr, rect, nullptr); + } +} c2_status_t C2AllocationBlob::map(size_t offset, size_t size, C2MemoryUsage usage, C2Fence* fence, void** addr /* nonnull */) { + *addr = nullptr; + if (size > capacity() || offset > capacity() || offset > capacity() - size) { + ALOGV("C2AllocationBlob: map: bad offset / size: offset=%zu size=%zu capacity=%u", + offset, size, capacity()); + return C2_BAD_VALUE; + } + std::unique_lock<std::mutex> lock(mMapLock); + if (mMappedAddr) { + *addr = mMappedAddr + offset; + mMappedOffsetSize.insert({offset, size}); + ALOGV("C2AllocationBlob: mapped from existing mapping: offset=%zu size=%zu capacity=%u", + offset, size, capacity()); + return C2_OK; + } C2PlanarLayout layout; - C2Rect rect = C2Rect(size, kLinearBufferHeight).at(offset, 0u); - return mGraphicAllocation->map(rect, usage, fence, &layout, reinterpret_cast<uint8_t**>(addr)); + C2Rect rect = C2Rect(capacity(), kLinearBufferHeight); + c2_status_t err = mGraphicAllocation->map(rect, usage, fence, &layout, &mMappedAddr); + if (err != C2_OK) { + ALOGV("C2AllocationBlob: map failed: offset=%zu size=%zu capacity=%u err=%d", + offset, size, capacity(), err); + mMappedAddr = nullptr; + return err; + } + *addr = mMappedAddr + offset; + mMappedOffsetSize.insert({offset, size}); + ALOGV("C2AllocationBlob: new map succeeded: offset=%zu size=%zu capacity=%u", + offset, size, capacity()); + return C2_OK; } c2_status_t C2AllocationBlob::unmap(void* addr, size_t size, C2Fence* fenceFd) { - C2Rect rect(size, kLinearBufferHeight); - return mGraphicAllocation->unmap(reinterpret_cast<uint8_t**>(&addr), rect, fenceFd); + std::unique_lock<std::mutex> lock(mMapLock); + uint8_t *u8Addr = static_cast<uint8_t *>(addr); + if (u8Addr < mMappedAddr || mMappedAddr + capacity() < u8Addr + size) { + ALOGV("C2AllocationBlob: unmap: Bad addr / size: addr=%p size=%zu capacity=%u", + addr, size, capacity()); + return C2_BAD_VALUE; + } + auto it = mMappedOffsetSize.find(std::make_pair(u8Addr - mMappedAddr, size)); + if (it == mMappedOffsetSize.end()) { + ALOGV("C2AllocationBlob: unrecognized map: addr=%p size=%zu capacity=%u", + addr, size, capacity()); + return C2_BAD_VALUE; + } + mMappedOffsetSize.erase(it); + if (!mMappedOffsetSize.empty()) { + ALOGV("C2AllocationBlob: still maintain mapping: addr=%p size=%zu capacity=%u", + addr, size, capacity()); + return C2_OK; + } + C2Rect rect(capacity(), kLinearBufferHeight); + c2_status_t err = mGraphicAllocation->unmap(&mMappedAddr, rect, fenceFd); + ALOGV("C2AllocationBlob: last unmap: addr=%p size=%zu capacity=%u err=%d", + addr, size, capacity(), err); + mMappedAddr = nullptr; + return err; } /* ====================================== BLOB ALLOCATOR ====================================== */ diff --git a/media/libaudioclient/AudioSystem.cpp b/media/libaudioclient/AudioSystem.cpp index e26a83160a..bf98822b53 100644 --- a/media/libaudioclient/AudioSystem.cpp +++ b/media/libaudioclient/AudioSystem.cpp @@ -43,6 +43,7 @@ Mutex AudioSystem::gLockAPS; sp<IAudioFlinger> AudioSystem::gAudioFlinger; sp<AudioSystem::AudioFlingerClient> AudioSystem::gAudioFlingerClient; std::set<audio_error_callback> AudioSystem::gAudioErrorCallbacks; +audio_session_callback AudioSystem::gAudioSessionCallback = NULL; dynamic_policy_callback AudioSystem::gDynPolicyCallback = NULL; record_config_callback AudioSystem::gRecordConfigCallback = NULL; @@ -753,6 +754,17 @@ status_t AudioSystem::AudioFlingerClient::removeAudioDeviceCallback( gRecordConfigCallback = cb; } +/*static*/ status_t AudioSystem::setAudioSessionCallback(audio_session_callback cb) +{ + const sp<IAudioPolicyService>& aps = AudioSystem::get_audio_policy_service(); + if (aps == 0) return PERMISSION_DENIED; + + Mutex::Autolock _l(gLock); + gAudioSessionCallback = cb; + + return NO_ERROR; +} + // client singleton for AudioPolicyService binder interface // protected by gLockAPS sp<IAudioPolicyService> AudioSystem::gAudioPolicyService; @@ -1812,6 +1824,32 @@ void AudioSystem::AudioPolicyServiceClient::onRecordingConfigurationUpdate( } } +// --------------------------------------------------------------------------- + +status_t AudioSystem::listAudioSessions(audio_stream_type_t stream, + Vector< sp<AudioSessionInfo>> &sessions) +{ + const sp<IAudioPolicyService>& aps = AudioSystem::get_audio_policy_service(); + if (aps == 0) return PERMISSION_DENIED; + return aps->listAudioSessions(stream, sessions); +} + +void AudioSystem::AudioPolicyServiceClient::onOutputSessionEffectsUpdate( + sp<AudioSessionInfo>& info, bool added) +{ + ALOGV("AudioPolicyServiceClient::onOutputSessionEffectsUpdate(%d, %d, %d)", + info->mStream, info->mSessionId, added); + audio_session_callback cb = NULL; + { + Mutex::Autolock _l(AudioSystem::gLock); + cb = gAudioSessionCallback; + } + + if (cb != NULL) { + cb(AUDIO_OUTPUT_SESSION_EFFECTS_UPDATE, info, added); + } +} + void AudioSystem::AudioPolicyServiceClient::binderDied(const wp<IBinder>& who __unused) { { diff --git a/media/libaudioclient/IAudioPolicyService.cpp b/media/libaudioclient/IAudioPolicyService.cpp index 43a5369220..223ebbbfa4 100644 --- a/media/libaudioclient/IAudioPolicyService.cpp +++ b/media/libaudioclient/IAudioPolicyService.cpp @@ -120,6 +120,7 @@ enum { AUDIO_MODULES_UPDATED, // oneway SET_CURRENT_IME_UID, REGISTER_SOUNDTRIGGER_CAPTURE_STATE_LISTENER, + LIST_AUDIO_SESSIONS, }; #define MAX_ITEMS_PER_LIST 1024 @@ -1496,6 +1497,29 @@ public: if (status != NO_ERROR) return status; return NO_ERROR; } + + virtual status_t listAudioSessions(audio_stream_type_t streams, + Vector< sp<AudioSessionInfo>> &sessions) + { + Parcel data, reply; + data.writeInterfaceToken(IAudioPolicyService::getInterfaceDescriptor()); + data.writeInt32(streams); + status_t status = remote()->transact(LIST_AUDIO_SESSIONS, data, &reply); + if (status != NO_ERROR) { + return status; + } + + status = reply.readInt32(); + if (status == NO_ERROR) { + size_t size = (size_t)reply.readUint32(); + for (size_t i = 0; i < size && reply.dataAvail() > 0; i++) { + sp<AudioSessionInfo> info = new AudioSessionInfo(); + info->readFromParcel(reply); + sessions.push_back(info); + } + } + return status; + } }; IMPLEMENT_META_INTERFACE(AudioPolicyService, "android.media.IAudioPolicyService"); @@ -1570,7 +1594,8 @@ status_t BnAudioPolicyService::onTransact( case SET_ALLOWED_CAPTURE_POLICY: case AUDIO_MODULES_UPDATED: case SET_CURRENT_IME_UID: - case REGISTER_SOUNDTRIGGER_CAPTURE_STATE_LISTENER: { + case REGISTER_SOUNDTRIGGER_CAPTURE_STATE_LISTENER: + case LIST_AUDIO_SESSIONS: { if (!isServiceUid(IPCThreadState::self()->getCallingUid())) { ALOGW("%s: transaction %d received from PID %d unauthorized UID %d", __func__, code, IPCThreadState::self()->getCallingPid(), @@ -2231,6 +2256,23 @@ status_t BnAudioPolicyService::onTransact( return NO_ERROR; } break; + case LIST_AUDIO_SESSIONS: { + CHECK_INTERFACE(IAudioPolicyService, data, reply); + audio_stream_type_t streams = (audio_stream_type_t)data.readInt32(); + + Vector< sp<AudioSessionInfo>> sessions; + status_t status = listAudioSessions(streams, sessions); + + reply->writeInt32(status); + if (status == NO_ERROR) { + reply->writeUint32(static_cast<uint32_t>(sessions.size())); + for (size_t i = 0; i < sessions.size(); i++) { + sessions[i]->writeToParcel(reply); + } + } + return NO_ERROR; + } + case ACQUIRE_SOUNDTRIGGER_SESSION: { CHECK_INTERFACE(IAudioPolicyService, data, reply); audio_session_t session = AUDIO_SESSION_NONE; diff --git a/media/libaudioclient/IAudioPolicyServiceClient.cpp b/media/libaudioclient/IAudioPolicyServiceClient.cpp index 0f9580c3ff..eeefff9551 100644 --- a/media/libaudioclient/IAudioPolicyServiceClient.cpp +++ b/media/libaudioclient/IAudioPolicyServiceClient.cpp @@ -33,6 +33,7 @@ enum { MIX_STATE_UPDATE, RECORDING_CONFIGURATION_UPDATE, VOLUME_GROUP_CHANGED, + OUTPUT_SESSION_EFFECTS_UPDATE, }; // ---------------------------------------------------------------------- @@ -149,6 +150,19 @@ public: data.writeInt32((int32_t) source); remote()->transact(RECORDING_CONFIGURATION_UPDATE, data, &reply, IBinder::FLAG_ONEWAY); } + + void onOutputSessionEffectsUpdate(sp<AudioSessionInfo>& info, bool added) + { + Parcel data, reply; + data.writeInterfaceToken(IAudioPolicyServiceClient::getInterfaceDescriptor()); + data.writeInt32(info->mStream); + data.writeInt32(info->mSessionId); + data.writeInt32(info->mFlags); + data.writeInt32(info->mChannelMask); + data.writeInt32(info->mUid); + data.writeInt32(added ? 1 : 0); + remote()->transact(OUTPUT_SESSION_EFFECTS_UPDATE, data, &reply, IBinder::FLAG_ONEWAY); + } }; IMPLEMENT_META_INTERFACE(AudioPolicyServiceClient, "android.media.IAudioPolicyServiceClient"); @@ -202,6 +216,20 @@ status_t BnAudioPolicyServiceClient::onTransact( &deviceConfig, effects, patchHandle, source); return NO_ERROR; } break; + case OUTPUT_SESSION_EFFECTS_UPDATE: { + CHECK_INTERFACE(IAudioPolicyServiceClient, data, reply); + audio_stream_type_t stream = static_cast<audio_stream_type_t>(data.readInt32()); + audio_session_t sessionId = static_cast<audio_session_t>(data.readInt32()); + audio_output_flags_t flags = static_cast<audio_output_flags_t>(data.readInt32()); + audio_channel_mask_t channelMask = static_cast<audio_channel_mask_t>(data.readInt32()); + uid_t uid = static_cast<uid_t>(data.readInt32()); + bool added = data.readInt32() > 0; + + sp<AudioSessionInfo> info = new AudioSessionInfo( + sessionId, stream, flags, channelMask, uid); + onOutputSessionEffectsUpdate(info, added); + return NO_ERROR; + } break; default: return BBinder::onTransact(code, data, reply, flags); } diff --git a/media/libaudioclient/include/media/AudioPolicy.h b/media/libaudioclient/include/media/AudioPolicy.h index 00fe278951..b7474fbf0e 100644 --- a/media/libaudioclient/include/media/AudioPolicy.h +++ b/media/libaudioclient/include/media/AudioPolicy.h @@ -25,6 +25,7 @@ #include <utils/String8.h> #include <utils/Vector.h> #include <cutils/multiuser.h> +#include <media/AudioSession.h> namespace android { @@ -48,6 +49,7 @@ namespace android { // AudioSystem's implementation of the AudioPolicyClient interface // keep in sync with AudioSystem.java #define DYNAMIC_POLICY_EVENT_MIX_STATE_UPDATE 0 +#define AUDIO_OUTPUT_SESSION_EFFECTS_UPDATE 10 #define MIX_STATE_DISABLED (-1) #define MIX_STATE_IDLE 0 diff --git a/media/libaudioclient/include/media/AudioSession.h b/media/libaudioclient/include/media/AudioSession.h new file mode 100644 index 0000000000..2bae5212cf --- /dev/null +++ b/media/libaudioclient/include/media/AudioSession.h @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2016 The CyanogenMod Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ANDROID_AUDIOSESSION_H +#define ANDROID_AUDIOSESSION_H + +#include <stdint.h> +#include <sys/types.h> + +#include <system/audio.h> + +#include <utils/RefBase.h> +#include <utils/Errors.h> +#include <binder/Parcel.h> + +namespace android { + +// class to store streaminfo +class AudioSessionInfo : public RefBase { +public: + AudioSessionInfo(audio_session_t session, audio_stream_type_t stream, audio_output_flags_t flags, + audio_channel_mask_t channelMask, uid_t uid) : + mSessionId(session), mStream(stream), mFlags(flags), mChannelMask(channelMask), + mUid(uid), mRefCount(0) {} + + AudioSessionInfo() : mSessionId((audio_session_t) 0), mStream(AUDIO_STREAM_DEFAULT), mFlags(AUDIO_OUTPUT_FLAG_NONE), mChannelMask(AUDIO_CHANNEL_NONE), mUid(0) {} + + /*virtual*/ ~AudioSessionInfo() {} + + audio_session_t mSessionId; + audio_stream_type_t mStream; + audio_output_flags_t mFlags; + audio_channel_mask_t mChannelMask; + uid_t mUid; + + // AudioPolicyManager keeps mLock, no need for lock on reference count here + int mRefCount; + + void readFromParcel(const Parcel &parcel) { + mSessionId = (audio_session_t) parcel.readInt32(); + mStream = static_cast<audio_stream_type_t>(parcel.readInt32()); + mFlags = static_cast<audio_output_flags_t>(parcel.readInt32()); + mChannelMask = static_cast<audio_channel_mask_t>(parcel.readInt32()); + mUid = static_cast<uid_t>(parcel.readInt32()); + } + + void writeToParcel(Parcel *parcel) const { + parcel->writeInt32(mSessionId); + parcel->writeInt32(mStream); + parcel->writeInt32(mFlags); + parcel->writeInt32(mChannelMask); + parcel->writeInt32(mUid); + } +}; + +}; // namespace android + +#endif // ANDROID_AUDIOSESSION_H diff --git a/media/libaudioclient/include/media/AudioSystem.h b/media/libaudioclient/include/media/AudioSystem.h index 19c2cbd165..dfd8b45a25 100644 --- a/media/libaudioclient/include/media/AudioSystem.h +++ b/media/libaudioclient/include/media/AudioSystem.h @@ -47,6 +47,8 @@ typedef void (*record_config_callback)(int event, std::vector<effect_descriptor_t> effects, audio_patch_handle_t patchHandle, audio_source_t source); +typedef void (*audio_session_callback)(int event, + sp<AudioSessionInfo>& session, bool added); class IAudioFlinger; class IAudioPolicyService; @@ -120,6 +122,7 @@ public: static void setDynPolicyCallback(dynamic_policy_callback cb); static void setRecordConfigCallback(record_config_callback); + static status_t setAudioSessionCallback(audio_session_callback cb); // helper function to obtain AudioFlinger service handle static const sp<IAudioFlinger> get_audio_flinger(); @@ -460,6 +463,9 @@ public: static status_t registerSoundTriggerCaptureStateListener( const sp<CaptureStateListener>& listener); + static status_t listAudioSessions(audio_stream_type_t streams, + Vector< sp<AudioSessionInfo>> &sessions); + // ---------------------------------------------------------------------------- class AudioVolumeGroupCallback : public RefBase @@ -593,6 +599,7 @@ private: std::vector<effect_descriptor_t> effects, audio_patch_handle_t patchHandle, audio_source_t source); + virtual void onOutputSessionEffectsUpdate(sp<AudioSessionInfo>& info, bool added); private: Mutex mLock; @@ -619,6 +626,7 @@ private: static std::set<audio_error_callback> gAudioErrorCallbacks; static dynamic_policy_callback gDynPolicyCallback; static record_config_callback gRecordConfigCallback; + static audio_session_callback gAudioSessionCallback; static size_t gInBuffSize; // previous parameters for recording buffer size queries diff --git a/media/libaudioclient/include/media/IAudioPolicyService.h b/media/libaudioclient/include/media/IAudioPolicyService.h index 376c6eb6e6..b1001a89a2 100644 --- a/media/libaudioclient/include/media/IAudioPolicyService.h +++ b/media/libaudioclient/include/media/IAudioPolicyService.h @@ -254,6 +254,9 @@ public: virtual status_t registerSoundTriggerCaptureStateListener( const sp<media::ICaptureStateListener>& listener, bool* result) = 0; + + virtual status_t listAudioSessions(audio_stream_type_t streams, + Vector< sp<AudioSessionInfo>> &sessions) = 0; }; diff --git a/media/libaudioclient/include/media/IAudioPolicyServiceClient.h b/media/libaudioclient/include/media/IAudioPolicyServiceClient.h index 47b31eedc3..bc36f749d3 100644 --- a/media/libaudioclient/include/media/IAudioPolicyServiceClient.h +++ b/media/libaudioclient/include/media/IAudioPolicyServiceClient.h @@ -25,6 +25,7 @@ #include <system/audio_effect.h> #include <media/AudioPolicy.h> #include <media/AudioVolumeGroup.h> +#include <media/AudioSession.h> namespace android { @@ -65,6 +66,8 @@ public: std::vector<effect_descriptor_t> effects, audio_patch_handle_t patchHandle, audio_source_t source) = 0; + // Notifies when a default effect set is attached to a session/stream + virtual void onOutputSessionEffectsUpdate(sp<AudioSessionInfo>& info, bool added) = 0; }; diff --git a/media/libmedia/Android.bp b/media/libmedia/Android.bp index 3fd3fc35db..f339d98020 100644 --- a/media/libmedia/Android.bp +++ b/media/libmedia/Android.bp @@ -271,6 +271,7 @@ cc_library { srcs: [ ":mediaextractorservice_aidl", "IDataSource.cpp", + "IHDCP.cpp", "BufferingSettings.cpp", "mediaplayer.cpp", "IMediaHTTPConnection.cpp", @@ -330,6 +331,7 @@ cc_library { "libcamera_client", "libstagefright_foundation", "libgui", + "libui", "libdl", "libaudioclient", "libmedia_codeclist", diff --git a/media/libmedia/IHDCP.cpp b/media/libmedia/IHDCP.cpp new file mode 100644 index 0000000000..31e37535c2 --- /dev/null +++ b/media/libmedia/IHDCP.cpp @@ -0,0 +1,359 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "IHDCP" +#include <utils/Log.h> + +#include <binder/Parcel.h> +#include <media/IHDCP.h> +#include <media/stagefright/MediaErrors.h> +#include <media/stagefright/foundation/ADebug.h> + +namespace android { + +enum { + OBSERVER_NOTIFY = IBinder::FIRST_CALL_TRANSACTION, + HDCP_SET_OBSERVER, + HDCP_INIT_ASYNC, + HDCP_SHUTDOWN_ASYNC, + HDCP_GET_CAPS, + HDCP_ENCRYPT, + HDCP_ENCRYPT_NATIVE, + HDCP_DECRYPT, +}; + +struct BpHDCPObserver : public BpInterface<IHDCPObserver> { + explicit BpHDCPObserver(const sp<IBinder> &impl) + : BpInterface<IHDCPObserver>(impl) { + } + + virtual void notify( + int msg, int ext1, int ext2, const Parcel *obj) { + Parcel data, reply; + data.writeInterfaceToken(IHDCPObserver::getInterfaceDescriptor()); + data.writeInt32(msg); + data.writeInt32(ext1); + data.writeInt32(ext2); + if (obj && obj->dataSize() > 0) { + data.appendFrom(const_cast<Parcel *>(obj), 0, obj->dataSize()); + } + remote()->transact(OBSERVER_NOTIFY, data, &reply, IBinder::FLAG_ONEWAY); + } +}; + +DO_NOT_DIRECTLY_USE_ME_IMPLEMENT_META_INTERFACE(HDCPObserver, "android.hardware.IHDCPObserver"); + +struct BpHDCP : public BpInterface<IHDCP> { + explicit BpHDCP(const sp<IBinder> &impl) + : BpInterface<IHDCP>(impl) { + } + + virtual status_t setObserver(const sp<IHDCPObserver> &observer) { + Parcel data, reply; + data.writeInterfaceToken(IHDCP::getInterfaceDescriptor()); + data.writeStrongBinder(IInterface::asBinder(observer)); + remote()->transact(HDCP_SET_OBSERVER, data, &reply); + return reply.readInt32(); + } + + virtual status_t initAsync(const char *host, unsigned port) { + Parcel data, reply; + data.writeInterfaceToken(IHDCP::getInterfaceDescriptor()); + data.writeCString(host); + data.writeInt32(port); + remote()->transact(HDCP_INIT_ASYNC, data, &reply); + return reply.readInt32(); + } + + virtual status_t shutdownAsync() { + Parcel data, reply; + data.writeInterfaceToken(IHDCP::getInterfaceDescriptor()); + remote()->transact(HDCP_SHUTDOWN_ASYNC, data, &reply); + return reply.readInt32(); + } + + virtual uint32_t getCaps() { + Parcel data, reply; + data.writeInterfaceToken(IHDCP::getInterfaceDescriptor()); + remote()->transact(HDCP_GET_CAPS, data, &reply); + return reply.readInt32(); + } + + virtual status_t encrypt( + const void *inData, size_t size, uint32_t streamCTR, + uint64_t *outInputCTR, void *outData) { + Parcel data, reply; + data.writeInterfaceToken(IHDCP::getInterfaceDescriptor()); + data.writeInt32(size); + data.write(inData, size); + data.writeInt32(streamCTR); + remote()->transact(HDCP_ENCRYPT, data, &reply); + + status_t err = reply.readInt32(); + + if (err != OK) { + *outInputCTR = 0; + + return err; + } + + *outInputCTR = reply.readInt64(); + reply.read(outData, size); + + return err; + } + + virtual status_t encryptNative( + const sp<GraphicBuffer> &graphicBuffer, + size_t offset, size_t size, uint32_t streamCTR, + uint64_t *outInputCTR, void *outData) { + Parcel data, reply; + data.writeInterfaceToken(IHDCP::getInterfaceDescriptor()); + data.write(*graphicBuffer); + data.writeInt32(offset); + data.writeInt32(size); + data.writeInt32(streamCTR); + remote()->transact(HDCP_ENCRYPT_NATIVE, data, &reply); + + status_t err = reply.readInt32(); + + if (err != OK) { + *outInputCTR = 0; + return err; + } + + *outInputCTR = reply.readInt64(); + reply.read(outData, size); + + return err; + } + + virtual status_t decrypt( + const void *inData, size_t size, + uint32_t streamCTR, uint64_t inputCTR, + void *outData) { + Parcel data, reply; + data.writeInterfaceToken(IHDCP::getInterfaceDescriptor()); + data.writeInt32(size); + data.write(inData, size); + data.writeInt32(streamCTR); + data.writeInt64(inputCTR); + remote()->transact(HDCP_DECRYPT, data, &reply); + + status_t err = reply.readInt32(); + + if (err != OK) { + return err; + } + + reply.read(outData, size); + + return err; + } +}; + +DO_NOT_DIRECTLY_USE_ME_IMPLEMENT_META_INTERFACE(HDCP, "android.hardware.IHDCP"); + +status_t BnHDCPObserver::onTransact( + uint32_t code, const Parcel &data, Parcel *reply, uint32_t flags) { + switch (code) { + case OBSERVER_NOTIFY: + { + CHECK_INTERFACE(IHDCPObserver, data, reply); + + int msg = data.readInt32(); + int ext1 = data.readInt32(); + int ext2 = data.readInt32(); + + Parcel obj; + if (data.dataAvail() > 0) { + obj.appendFrom( + const_cast<Parcel *>(&data), + data.dataPosition(), + data.dataAvail()); + } + + notify(msg, ext1, ext2, &obj); + + return OK; + } + + default: + return BBinder::onTransact(code, data, reply, flags); + } +} + +status_t BnHDCP::onTransact( + uint32_t code, const Parcel &data, Parcel *reply, uint32_t flags) { + switch (code) { + case HDCP_SET_OBSERVER: + { + CHECK_INTERFACE(IHDCP, data, reply); + + sp<IHDCPObserver> observer = + interface_cast<IHDCPObserver>(data.readStrongBinder()); + + reply->writeInt32(setObserver(observer)); + return OK; + } + + case HDCP_INIT_ASYNC: + { + CHECK_INTERFACE(IHDCP, data, reply); + + const char *host = data.readCString(); + unsigned port = data.readInt32(); + + reply->writeInt32(initAsync(host, port)); + return OK; + } + + case HDCP_SHUTDOWN_ASYNC: + { + CHECK_INTERFACE(IHDCP, data, reply); + + reply->writeInt32(shutdownAsync()); + return OK; + } + + case HDCP_GET_CAPS: + { + CHECK_INTERFACE(IHDCP, data, reply); + + reply->writeInt32(getCaps()); + return OK; + } + + case HDCP_ENCRYPT: + { + CHECK_INTERFACE(IHDCP, data, reply); + + size_t size = data.readInt32(); + void *inData = NULL; + // watch out for overflow + if (size <= SIZE_MAX / 2) { + inData = malloc(2 * size); + } + if (inData == NULL) { + reply->writeInt32(ERROR_OUT_OF_RANGE); + return OK; + } + + void *outData = (uint8_t *)inData + size; + + status_t err = data.read(inData, size); + if (err != OK) { + free(inData); + reply->writeInt32(err); + return OK; + } + + uint32_t streamCTR = data.readInt32(); + uint64_t inputCTR; + err = encrypt(inData, size, streamCTR, &inputCTR, outData); + + reply->writeInt32(err); + + if (err == OK) { + reply->writeInt64(inputCTR); + reply->write(outData, size); + } + + free(inData); + inData = outData = NULL; + + return OK; + } + + case HDCP_ENCRYPT_NATIVE: + { + CHECK_INTERFACE(IHDCP, data, reply); + + sp<GraphicBuffer> graphicBuffer = new GraphicBuffer(); + data.read(*graphicBuffer); + size_t offset = data.readInt32(); + size_t size = data.readInt32(); + uint32_t streamCTR = data.readInt32(); + void *outData = NULL; + uint64_t inputCTR; + + status_t err = ERROR_OUT_OF_RANGE; + + outData = malloc(size); + + if (outData != NULL) { + err = encryptNative(graphicBuffer, offset, size, + streamCTR, &inputCTR, outData); + } + + reply->writeInt32(err); + + if (err == OK) { + reply->writeInt64(inputCTR); + reply->write(outData, size); + } + + free(outData); + outData = NULL; + + return OK; + } + + case HDCP_DECRYPT: + { + CHECK_INTERFACE(IHDCP, data, reply); + + size_t size = data.readInt32(); + size_t bufSize = 2 * size; + + // watch out for overflow + void *inData = NULL; + if (bufSize > size) { + inData = malloc(bufSize); + } + + if (inData == NULL) { + reply->writeInt32(ERROR_OUT_OF_RANGE); + return OK; + } + + void *outData = (uint8_t *)inData + size; + + data.read(inData, size); + + uint32_t streamCTR = data.readInt32(); + uint64_t inputCTR = data.readInt64(); + status_t err = decrypt(inData, size, streamCTR, inputCTR, outData); + + reply->writeInt32(err); + + if (err == OK) { + reply->write(outData, size); + } + + free(inData); + inData = outData = NULL; + + return OK; + } + + default: + return BBinder::onTransact(code, data, reply, flags); + } +} + +} // namespace android diff --git a/media/libmedia/IMediaPlayerService.cpp b/media/libmedia/IMediaPlayerService.cpp index 11005c6cdf..b17bd61dd2 100644 --- a/media/libmedia/IMediaPlayerService.cpp +++ b/media/libmedia/IMediaPlayerService.cpp @@ -20,6 +20,7 @@ #include <binder/Parcel.h> #include <binder/IMemory.h> +#include <media/IHDCP.h> #include <media/IMediaCodecList.h> #include <media/IMediaHTTPService.h> #include <media/IMediaPlayerService.h> @@ -39,6 +40,7 @@ enum { CREATE = IBinder::FIRST_CALL_TRANSACTION, CREATE_MEDIA_RECORDER, CREATE_METADATA_RETRIEVER, + MAKE_HDCP, ADD_BATTERY_DATA, PULL_BATTERY_DATA, LISTEN_FOR_REMOTE_DISPLAY, @@ -83,6 +85,14 @@ public: return interface_cast<IMediaRecorder>(reply.readStrongBinder()); } + virtual sp<IHDCP> makeHDCP(bool createEncryptionModule) { + Parcel data, reply; + data.writeInterfaceToken(IMediaPlayerService::getInterfaceDescriptor()); + data.writeInt32(createEncryptionModule); + remote()->transact(MAKE_HDCP, data, &reply); + return interface_cast<IHDCP>(reply.readStrongBinder()); + } + virtual void addBatteryData(uint32_t params) { Parcel data, reply; data.writeInterfaceToken(IMediaPlayerService::getInterfaceDescriptor()); @@ -151,6 +161,13 @@ status_t BnMediaPlayerService::onTransact( reply->writeStrongBinder(IInterface::asBinder(retriever)); return NO_ERROR; } break; + case MAKE_HDCP: { + CHECK_INTERFACE(IMediaPlayerService, data, reply); + bool createEncryptionModule = data.readInt32(); + sp<IHDCP> hdcp = makeHDCP(createEncryptionModule); + reply->writeStrongBinder(IInterface::asBinder(hdcp)); + return NO_ERROR; + } break; case ADD_BATTERY_DATA: { CHECK_INTERFACE(IMediaPlayerService, data, reply); uint32_t params = data.readInt32(); diff --git a/media/libmedia/MediaProfiles.cpp b/media/libmedia/MediaProfiles.cpp index 637322f60a..8be961c9fe 100644 --- a/media/libmedia/MediaProfiles.cpp +++ b/media/libmedia/MediaProfiles.cpp @@ -240,7 +240,10 @@ MediaProfiles::createVideoCodec(const char **atts, MediaProfiles *profiles) const size_t nMappings = sizeof(sVideoEncoderNameMap)/sizeof(sVideoEncoderNameMap[0]); const int codec = findTagForName(sVideoEncoderNameMap, nMappings, atts[1]); - CHECK(codec != -1); + if (codec == -1) { + ALOGE("MediaProfiles::createVideoCodec failed to locate codec %s", atts[1]); + return nullptr; + } MediaProfiles::VideoCodec *videoCodec = new MediaProfiles::VideoCodec(static_cast<video_encoder>(codec), @@ -262,7 +265,10 @@ MediaProfiles::createAudioCodec(const char **atts, MediaProfiles *profiles) !strcmp("channels", atts[6])); const size_t nMappings = sizeof(sAudioEncoderNameMap)/sizeof(sAudioEncoderNameMap[0]); const int codec = findTagForName(sAudioEncoderNameMap, nMappings, atts[1]); - CHECK(codec != -1); + if (codec == -1) { + ALOGE("MediaProfiles::createAudioCodec failed to locate codec %s", atts[1]); + return nullptr; + } MediaProfiles::AudioCodec *audioCodec = new MediaProfiles::AudioCodec(static_cast<audio_encoder>(codec), @@ -282,7 +288,10 @@ MediaProfiles::createAudioDecoderCap(const char **atts) const size_t nMappings = sizeof(sAudioDecoderNameMap)/sizeof(sAudioDecoderNameMap[0]); const int codec = findTagForName(sAudioDecoderNameMap, nMappings, atts[1]); - CHECK(codec != -1); + if (codec == -1) { + ALOGE("MediaProfiles::createAudioDecoderCap failed to locate codec %s", atts[1]); + return nullptr; + } MediaProfiles::AudioDecoderCap *cap = new MediaProfiles::AudioDecoderCap(static_cast<audio_decoder>(codec)); @@ -298,7 +307,10 @@ MediaProfiles::createVideoDecoderCap(const char **atts) const size_t nMappings = sizeof(sVideoDecoderNameMap)/sizeof(sVideoDecoderNameMap[0]); const int codec = findTagForName(sVideoDecoderNameMap, nMappings, atts[1]); - CHECK(codec != -1); + if (codec == -1) { + ALOGE("MediaProfiles::createVideoDecoderCap failed to locate codec %s", atts[1]); + return nullptr; + } MediaProfiles::VideoDecoderCap *cap = new MediaProfiles::VideoDecoderCap(static_cast<video_decoder>(codec)); @@ -322,7 +334,10 @@ MediaProfiles::createVideoEncoderCap(const char **atts) const size_t nMappings = sizeof(sVideoEncoderNameMap)/sizeof(sVideoEncoderNameMap[0]); const int codec = findTagForName(sVideoEncoderNameMap, nMappings, atts[1]); - CHECK(codec != -1); + if (codec == -1) { + ALOGE("MediaProfiles::createVideoEncoderCap failed to locate codec %s", atts[1]); + return nullptr; + } MediaProfiles::VideoEncoderCap *cap = new MediaProfiles::VideoEncoderCap(static_cast<video_encoder>(codec), @@ -346,7 +361,10 @@ MediaProfiles::createAudioEncoderCap(const char **atts) const size_t nMappings = sizeof(sAudioEncoderNameMap)/sizeof(sAudioEncoderNameMap[0]); const int codec = findTagForName(sAudioEncoderNameMap, nMappings, atts[1]); - CHECK(codec != -1); + if (codec == -1) { + ALOGE("MediaProfiles::createAudioEncoderCap failed to locate codec %s", atts[1]); + return nullptr; + } MediaProfiles::AudioEncoderCap *cap = new MediaProfiles::AudioEncoderCap(static_cast<audio_encoder>(codec), atoi(atts[5]), @@ -386,11 +404,17 @@ MediaProfiles::createCamcorderProfile(int cameraId, const char **atts, Vector<in const size_t nProfileMappings = sizeof(sCamcorderQualityNameMap)/ sizeof(sCamcorderQualityNameMap[0]); const int quality = findTagForName(sCamcorderQualityNameMap, nProfileMappings, atts[1]); - CHECK(quality != -1); + if (quality == -1) { + ALOGE("MediaProfiles::createCamcorderProfile failed to locate quality %s", atts[1]); + return nullptr; + } const size_t nFormatMappings = sizeof(sFileFormatMap)/sizeof(sFileFormatMap[0]); const int fileFormat = findTagForName(sFileFormatMap, nFormatMappings, atts[3]); - CHECK(fileFormat != -1); + if (fileFormat == -1) { + ALOGE("MediaProfiles::createCamcorderProfile failed to locate file format %s", atts[1]); + return nullptr; + } MediaProfiles::CamcorderProfile *profile = new MediaProfiles::CamcorderProfile; profile->mCameraId = cameraId; @@ -462,24 +486,39 @@ MediaProfiles::startElementHandler(void *userData, const char *name, const char createAudioCodec(atts, profiles); } else if (strcmp("VideoEncoderCap", name) == 0 && strcmp("true", atts[3]) == 0) { - profiles->mVideoEncoders.add(createVideoEncoderCap(atts)); + MediaProfiles::VideoEncoderCap* cap = createVideoEncoderCap(atts); + if (cap != nullptr) { + profiles->mVideoEncoders.add(cap); + } } else if (strcmp("AudioEncoderCap", name) == 0 && strcmp("true", atts[3]) == 0) { - profiles->mAudioEncoders.add(createAudioEncoderCap(atts)); + MediaProfiles::AudioEncoderCap* cap = createAudioEncoderCap(atts); + if (cap != nullptr) { + profiles->mAudioEncoders.add(cap); + } } else if (strcmp("VideoDecoderCap", name) == 0 && strcmp("true", atts[3]) == 0) { - profiles->mVideoDecoders.add(createVideoDecoderCap(atts)); + MediaProfiles::VideoDecoderCap* cap = createVideoDecoderCap(atts); + if (cap != nullptr) { + profiles->mVideoDecoders.add(cap); + } } else if (strcmp("AudioDecoderCap", name) == 0 && strcmp("true", atts[3]) == 0) { - profiles->mAudioDecoders.add(createAudioDecoderCap(atts)); + MediaProfiles::AudioDecoderCap* cap = createAudioDecoderCap(atts); + if (cap != nullptr) { + profiles->mAudioDecoders.add(cap); + } } else if (strcmp("EncoderOutputFileFormat", name) == 0) { profiles->mEncoderOutputFileFormats.add(createEncoderOutputFileFormat(atts)); } else if (strcmp("CamcorderProfiles", name) == 0) { profiles->mCurrentCameraId = getCameraId(atts); profiles->addStartTimeOffset(profiles->mCurrentCameraId, atts); } else if (strcmp("EncoderProfile", name) == 0) { - profiles->mCamcorderProfiles.add( - createCamcorderProfile(profiles->mCurrentCameraId, atts, profiles->mCameraIds)); + MediaProfiles::CamcorderProfile* profile = createCamcorderProfile( + profiles->mCurrentCameraId, atts, profiles->mCameraIds); + if (profile != nullptr) { + profiles->mCamcorderProfiles.add(profile); + } } else if (strcmp("ImageEncoding", name) == 0) { profiles->addImageEncodingQualityLevel(profiles->mCurrentCameraId, atts); } diff --git a/media/libmedia/include/media/IHDCP.h b/media/libmedia/include/media/IHDCP.h new file mode 100644 index 0000000000..352561ecbb --- /dev/null +++ b/media/libmedia/include/media/IHDCP.h @@ -0,0 +1,120 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <binder/IInterface.h> +#include <media/hardware/HDCPAPI.h> +#include <media/stagefright/foundation/ABase.h> +#include <ui/GraphicBuffer.h> + +namespace android { + +struct IHDCPObserver : public IInterface { + DECLARE_META_INTERFACE(HDCPObserver); + + virtual void notify( + int msg, int ext1, int ext2, const Parcel *obj) = 0; + +private: + DISALLOW_EVIL_CONSTRUCTORS(IHDCPObserver); +}; + +struct IHDCP : public IInterface { + DECLARE_META_INTERFACE(HDCP); + + // Called to specify the observer that receives asynchronous notifications + // from the HDCP implementation to signal completion/failure of asynchronous + // operations (such as initialization) or out of band events. + virtual status_t setObserver(const sp<IHDCPObserver> &observer) = 0; + + // Request to setup an HDCP session with the specified host listening + // on the specified port. + virtual status_t initAsync(const char *host, unsigned port) = 0; + + // Request to shutdown the active HDCP session. + virtual status_t shutdownAsync() = 0; + + // Returns the capability bitmask of this HDCP session. + // Possible return values (please refer to HDCAPAPI.h): + // HDCP_CAPS_ENCRYPT: mandatory, meaning the HDCP module can encrypt + // from an input byte-array buffer to an output byte-array buffer + // HDCP_CAPS_ENCRYPT_NATIVE: the HDCP module supports encryption from + // a native buffer to an output byte-array buffer. The format of the + // input native buffer is specific to vendor's encoder implementation. + // It is the same format as that used by the encoder when + // "storeMetaDataInBuffers" extension is enabled on its output port. + virtual uint32_t getCaps() = 0; + + // ENCRYPTION only: + // Encrypt data according to the HDCP spec. "size" bytes of data are + // available at "inData" (virtual address), "size" may not be a multiple + // of 128 bits (16 bytes). An equal number of encrypted bytes should be + // written to the buffer at "outData" (virtual address). + // This operation is to be synchronous, i.e. this call does not return + // until outData contains size bytes of encrypted data. + // streamCTR will be assigned by the caller (to 0 for the first PES stream, + // 1 for the second and so on) + // inputCTR _will_be_maintained_by_the_callee_ for each PES stream. + virtual status_t encrypt( + const void *inData, size_t size, uint32_t streamCTR, + uint64_t *outInputCTR, void *outData) = 0; + + // Encrypt data according to the HDCP spec. "size" bytes of data starting + // at location "offset" are available in "buffer" (buffer handle). "size" + // may not be a multiple of 128 bits (16 bytes). An equal number of + // encrypted bytes should be written to the buffer at "outData" (virtual + // address). This operation is to be synchronous, i.e. this call does not + // return until outData contains size bytes of encrypted data. + // streamCTR will be assigned by the caller (to 0 for the first PES stream, + // 1 for the second and so on) + // inputCTR _will_be_maintained_by_the_callee_ for each PES stream. + virtual status_t encryptNative( + const sp<GraphicBuffer> &graphicBuffer, + size_t offset, size_t size, uint32_t streamCTR, + uint64_t *outInputCTR, void *outData) = 0; + + // DECRYPTION only: + // Decrypt data according to the HDCP spec. + // "size" bytes of encrypted data are available at "inData" + // (virtual address), "size" may not be a multiple of 128 bits (16 bytes). + // An equal number of decrypted bytes should be written to the buffer + // at "outData" (virtual address). + // This operation is to be synchronous, i.e. this call does not return + // until outData contains size bytes of decrypted data. + // Both streamCTR and inputCTR will be provided by the caller. + virtual status_t decrypt( + const void *inData, size_t size, + uint32_t streamCTR, uint64_t inputCTR, + void *outData) = 0; + +private: + DISALLOW_EVIL_CONSTRUCTORS(IHDCP); +}; + +struct BnHDCPObserver : public BnInterface<IHDCPObserver> { + virtual status_t onTransact( + uint32_t code, const Parcel &data, Parcel *reply, + uint32_t flags = 0); +}; + +struct BnHDCP : public BnInterface<IHDCP> { + virtual status_t onTransact( + uint32_t code, const Parcel &data, Parcel *reply, + uint32_t flags = 0); +}; + +} // namespace android + + diff --git a/media/libmedia/include/media/IMediaPlayerService.h b/media/libmedia/include/media/IMediaPlayerService.h index a4207eb533..c9efb02db4 100644 --- a/media/libmedia/include/media/IMediaPlayerService.h +++ b/media/libmedia/include/media/IMediaPlayerService.h @@ -33,6 +33,7 @@ namespace android { class IMediaPlayer; +struct IHDCP; class IMediaCodecList; struct IMediaHTTPService; class IMediaRecorder; @@ -51,6 +52,7 @@ public: virtual sp<IMediaPlayer> create(const sp<IMediaPlayerClient>& client, audio_session_t audioSessionId = AUDIO_SESSION_ALLOCATE, const std::string opPackage = "") = 0; + virtual sp<IHDCP> makeHDCP(bool createEncryptionModule) = 0; virtual sp<IMediaCodecList> getCodecList() const = 0; // Connects to a remote display. diff --git a/media/libmediaplayerservice/Android.bp b/media/libmediaplayerservice/Android.bp index 5301f5c571..21da3ce113 100644 --- a/media/libmediaplayerservice/Android.bp +++ b/media/libmediaplayerservice/Android.bp @@ -3,11 +3,13 @@ cc_library_shared { srcs: [ "ActivityManager.cpp", "DeathNotifier.cpp", + "HDCP.cpp", "MediaPlayerFactory.cpp", "MediaPlayerService.cpp", "MediaRecorderClient.cpp", "MetadataRetrieverClient.cpp", "StagefrightMetadataRetriever.cpp", + "RemoteDisplay.cpp", "StagefrightRecorder.cpp", "TestPlayerStub.cpp", ], @@ -38,6 +40,7 @@ cc_library_shared { "libnetd_client", "libpowermanager", "libstagefright", + "libstagefright_wfd", "libstagefright_foundation", "libstagefright_httplive", "libutils", @@ -59,6 +62,7 @@ cc_library_shared { include_dirs: [ "frameworks/av/media/libstagefright/rtsp", "frameworks/av/media/libstagefright/webm", + "frameworks/av/media/libstagefright/wifi-display", ], local_include_dirs: ["include"], diff --git a/media/libmediaplayerservice/HDCP.cpp b/media/libmediaplayerservice/HDCP.cpp new file mode 100644 index 0000000000..afe39367fb --- /dev/null +++ b/media/libmediaplayerservice/HDCP.cpp @@ -0,0 +1,175 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "HDCP" +#include <utils/Log.h> + +#include "HDCP.h" + +#include <media/stagefright/foundation/ADebug.h> + +#include <dlfcn.h> + +namespace android { + +HDCP::HDCP(bool createEncryptionModule) + : mIsEncryptionModule(createEncryptionModule), + mLibHandle(NULL), + mHDCPModule(NULL) { + mLibHandle = dlopen("libstagefright_hdcp.so", RTLD_NOW); + + if (mLibHandle == NULL) { + ALOGE("Unable to locate libstagefright_hdcp.so"); + return; + } + + typedef HDCPModule *(*CreateHDCPModuleFunc)( + void *, HDCPModule::ObserverFunc); + + CreateHDCPModuleFunc createHDCPModule = + mIsEncryptionModule + ? (CreateHDCPModuleFunc)dlsym(mLibHandle, "createHDCPModule") + : (CreateHDCPModuleFunc)dlsym( + mLibHandle, "createHDCPModuleForDecryption"); + + if (createHDCPModule == NULL) { + ALOGE("Unable to find symbol 'createHDCPModule'."); + } else if ((mHDCPModule = createHDCPModule( + this, &HDCP::ObserveWrapper)) == NULL) { + ALOGE("createHDCPModule failed."); + } +} + +HDCP::~HDCP() { + Mutex::Autolock autoLock(mLock); + + if (mHDCPModule != NULL) { + delete mHDCPModule; + mHDCPModule = NULL; + } + + if (mLibHandle != NULL) { + dlclose(mLibHandle); + mLibHandle = NULL; + } +} + +status_t HDCP::setObserver(const sp<IHDCPObserver> &observer) { + Mutex::Autolock autoLock(mLock); + + if (mHDCPModule == NULL) { + return NO_INIT; + } + + mObserver = observer; + + return OK; +} + +status_t HDCP::initAsync(const char *host, unsigned port) { + Mutex::Autolock autoLock(mLock); + + if (mHDCPModule == NULL) { + return NO_INIT; + } + + return mHDCPModule->initAsync(host, port); +} + +status_t HDCP::shutdownAsync() { + Mutex::Autolock autoLock(mLock); + + if (mHDCPModule == NULL) { + return NO_INIT; + } + + return mHDCPModule->shutdownAsync(); +} + +uint32_t HDCP::getCaps() { + Mutex::Autolock autoLock(mLock); + + if (mHDCPModule == NULL) { + return NO_INIT; + } + + return mHDCPModule->getCaps(); +} + +status_t HDCP::encrypt( + const void *inData, size_t size, uint32_t streamCTR, + uint64_t *outInputCTR, void *outData) { + Mutex::Autolock autoLock(mLock); + + CHECK(mIsEncryptionModule); + + if (mHDCPModule == NULL) { + *outInputCTR = 0; + + return NO_INIT; + } + + return mHDCPModule->encrypt(inData, size, streamCTR, outInputCTR, outData); +} + +status_t HDCP::encryptNative( + const sp<GraphicBuffer> &graphicBuffer, + size_t offset, size_t size, uint32_t streamCTR, + uint64_t *outInputCTR, void *outData) { + Mutex::Autolock autoLock(mLock); + + CHECK(mIsEncryptionModule); + + if (mHDCPModule == NULL) { + *outInputCTR = 0; + + return NO_INIT; + } + + return mHDCPModule->encryptNative(graphicBuffer->handle, + offset, size, streamCTR, outInputCTR, outData); +} + +status_t HDCP::decrypt( + const void *inData, size_t size, + uint32_t streamCTR, uint64_t outInputCTR, void *outData) { + Mutex::Autolock autoLock(mLock); + + CHECK(!mIsEncryptionModule); + + if (mHDCPModule == NULL) { + return NO_INIT; + } + + return mHDCPModule->decrypt(inData, size, streamCTR, outInputCTR, outData); +} + +// static +void HDCP::ObserveWrapper(void *me, int msg, int ext1, int ext2) { + static_cast<HDCP *>(me)->observe(msg, ext1, ext2); +} + +void HDCP::observe(int msg, int ext1, int ext2) { + Mutex::Autolock autoLock(mLock); + + if (mObserver != NULL) { + mObserver->notify(msg, ext1, ext2, NULL /* obj */); + } +} + +} // namespace android + diff --git a/media/libmediaplayerservice/HDCP.h b/media/libmediaplayerservice/HDCP.h new file mode 100644 index 0000000000..83c61b56c0 --- /dev/null +++ b/media/libmediaplayerservice/HDCP.h @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef HDCP_H_ + +#define HDCP_H_ + +#include <media/IHDCP.h> +#include <utils/Mutex.h> + +namespace android { + +struct HDCP : public BnHDCP { + explicit HDCP(bool createEncryptionModule); + virtual ~HDCP(); + + virtual status_t setObserver(const sp<IHDCPObserver> &observer); + virtual status_t initAsync(const char *host, unsigned port); + virtual status_t shutdownAsync(); + virtual uint32_t getCaps(); + + virtual status_t encrypt( + const void *inData, size_t size, uint32_t streamCTR, + uint64_t *outInputCTR, void *outData); + + virtual status_t encryptNative( + const sp<GraphicBuffer> &graphicBuffer, + size_t offset, size_t size, uint32_t streamCTR, + uint64_t *outInputCTR, void *outData); + + virtual status_t decrypt( + const void *inData, size_t size, + uint32_t streamCTR, uint64_t outInputCTR, void *outData); + +private: + Mutex mLock; + + bool mIsEncryptionModule; + + void *mLibHandle; + HDCPModule *mHDCPModule; + sp<IHDCPObserver> mObserver; + + static void ObserveWrapper(void *me, int msg, int ext1, int ext2); + void observe(int msg, int ext1, int ext2); + + DISALLOW_EVIL_CONSTRUCTORS(HDCP); +}; + +} // namespace android + +#endif // HDCP_H_ + diff --git a/media/libmediaplayerservice/MediaPlayerService.cpp b/media/libmediaplayerservice/MediaPlayerService.cpp index 016f622fb2..95985890a3 100644 --- a/media/libmediaplayerservice/MediaPlayerService.cpp +++ b/media/libmediaplayerservice/MediaPlayerService.cpp @@ -82,6 +82,8 @@ #include "TestPlayerStub.h" #include "nuplayer/NuPlayerDriver.h" +#include "HDCP.h" +#include "RemoteDisplay.h" static const int kDumpLockRetries = 50; static const int kDumpLockSleepUs = 20000; @@ -504,13 +506,18 @@ sp<IMediaCodecList> MediaPlayerService::getCodecList() const { return MediaCodecList::getLocalInstance(); } +sp<IHDCP> MediaPlayerService::makeHDCP(bool createEncryptionModule) { + return new HDCP(createEncryptionModule); +} + sp<IRemoteDisplay> MediaPlayerService::listenForRemoteDisplay( - const String16 &/*opPackageName*/, - const sp<IRemoteDisplayClient>& /*client*/, - const String8& /*iface*/) { - ALOGE("listenForRemoteDisplay is no longer supported!"); + const String16 &opPackageName, + const sp<IRemoteDisplayClient>& client, const String8& iface) { + if (!checkPermission("android.permission.CONTROL_WIFI_DISPLAY")) { + return NULL; + } - return NULL; + return new RemoteDisplay(opPackageName, client, iface.string()); } status_t MediaPlayerService::AudioOutput::dump(int fd, const Vector<String16>& args) const diff --git a/media/libmediaplayerservice/MediaPlayerService.h b/media/libmediaplayerservice/MediaPlayerService.h index a7de3f3bbf..d992895684 100644 --- a/media/libmediaplayerservice/MediaPlayerService.h +++ b/media/libmediaplayerservice/MediaPlayerService.h @@ -242,6 +242,7 @@ public: const std::string opPackageName); virtual sp<IMediaCodecList> getCodecList() const; + virtual sp<IHDCP> makeHDCP(bool createEncryptionModule); virtual sp<IRemoteDisplay> listenForRemoteDisplay(const String16 &opPackageName, const sp<IRemoteDisplayClient>& client, const String8& iface); diff --git a/media/libmediaplayerservice/RemoteDisplay.cpp b/media/libmediaplayerservice/RemoteDisplay.cpp new file mode 100644 index 0000000000..80cca265f3 --- /dev/null +++ b/media/libmediaplayerservice/RemoteDisplay.cpp @@ -0,0 +1,66 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "RemoteDisplay.h" + +#include "source/WifiDisplaySource.h" + +#include <media/IRemoteDisplayClient.h> +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> +#include <media/stagefright/ANetworkSession.h> + +namespace android { + +RemoteDisplay::RemoteDisplay( + const String16 &opPackageName, + const sp<IRemoteDisplayClient> &client, + const char *iface) + : mLooper(new ALooper), + mNetSession(new ANetworkSession) { + mLooper->setName("wfd_looper"); + + mSource = new WifiDisplaySource(opPackageName, mNetSession, client); + mLooper->registerHandler(mSource); + + mNetSession->start(); + mLooper->start(); + + mSource->start(iface); +} + +RemoteDisplay::~RemoteDisplay() { +} + +status_t RemoteDisplay::pause() { + return mSource->pause(); +} + +status_t RemoteDisplay::resume() { + return mSource->resume(); +} + +status_t RemoteDisplay::dispose() { + mSource->stop(); + mSource.clear(); + + mLooper->stop(); + mNetSession->stop(); + + return OK; +} + +} // namespace android diff --git a/media/libmediaplayerservice/RemoteDisplay.h b/media/libmediaplayerservice/RemoteDisplay.h new file mode 100644 index 0000000000..d4573e9a39 --- /dev/null +++ b/media/libmediaplayerservice/RemoteDisplay.h @@ -0,0 +1,59 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef REMOTE_DISPLAY_H_ + +#define REMOTE_DISPLAY_H_ + +#include <media/IMediaPlayerService.h> +#include <media/IRemoteDisplay.h> +#include <media/stagefright/foundation/ABase.h> +#include <utils/Errors.h> +#include <utils/RefBase.h> + +namespace android { + +struct ALooper; +struct ANetworkSession; +class IRemoteDisplayClient; +struct WifiDisplaySource; + +struct RemoteDisplay : public BnRemoteDisplay { + RemoteDisplay( + const String16 &opPackageName, + const sp<IRemoteDisplayClient> &client, + const char *iface); + + virtual status_t pause(); + virtual status_t resume(); + virtual status_t dispose(); + +protected: + virtual ~RemoteDisplay(); + +private: + sp<ALooper> mNetLooper; + sp<ALooper> mLooper; + sp<ANetworkSession> mNetSession; + sp<WifiDisplaySource> mSource; + + DISALLOW_EVIL_CONSTRUCTORS(RemoteDisplay); +}; + +} // namespace android + +#endif // REMOTE_DISPLAY_H_ + diff --git a/media/libmediaplayerservice/StagefrightRecorder.cpp b/media/libmediaplayerservice/StagefrightRecorder.cpp index 7897959137..a7344de402 100644 --- a/media/libmediaplayerservice/StagefrightRecorder.cpp +++ b/media/libmediaplayerservice/StagefrightRecorder.cpp @@ -1663,7 +1663,7 @@ status_t StagefrightRecorder::setupCameraSource( Size videoSize; videoSize.width = mVideoWidth; videoSize.height = mVideoHeight; - if (mCaptureFpsEnable) { + if (mCaptureFpsEnable && mCaptureFps != mFrameRate) { if (!(mCaptureFps > 0.)) { ALOGE("Invalid mCaptureFps value: %lf", mCaptureFps); return BAD_VALUE; @@ -1811,6 +1811,7 @@ status_t StagefrightRecorder::setupVideoEncoder( preferBFrames = false; tsLayers = 2; // use at least two layers as resulting video will likely be sped up } else if (mCaptureFps > maxPlaybackFps) { // slow-mo + format->setInt32("high-frame-rate", 1); maxPlaybackFps = mCaptureFps; // assume video will be played back at full capture speed preferBFrames = false; } diff --git a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp index c1c4b55400..8a81ef118b 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp @@ -2350,9 +2350,6 @@ void NuPlayer::performDecoderFlush(FlushCommand audio, FlushCommand video) { void NuPlayer::performReset() { ALOGV("performReset"); - CHECK(mAudioDecoder == NULL); - CHECK(mVideoDecoder == NULL); - updatePlaybackTimer(true /* stopping */, "performReset"); updateRebufferingTimer(true /* stopping */, true /* exiting */); diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp index c30f048c2d..7e8fe45121 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp @@ -922,6 +922,11 @@ size_t NuPlayer::Renderer::fillAudioBuffer(void *buffer, size_t size) { firstEntry = false; int64_t mediaTimeUs; CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); + if (mediaTimeUs < 0) { + ALOGD("fillAudioBuffer: reset negative media time %.2f secs to zero", + mediaTimeUs / 1E6); + mediaTimeUs = 0; + } ALOGV("fillAudioBuffer: rendering audio at media time %.2f secs", mediaTimeUs / 1E6); setAudioFirstAnchorTimeIfNeeded_l(mediaTimeUs); } diff --git a/media/libstagefright/ACodec.cpp b/media/libstagefright/ACodec.cpp index 8e480bff14..4a3b32f3ee 100644 --- a/media/libstagefright/ACodec.cpp +++ b/media/libstagefright/ACodec.cpp @@ -279,6 +279,13 @@ protected: void postFillThisBuffer(BufferInfo *info); + void maybePostExtraOutputMetadataBufferRequest() { + if (!mPendingExtraOutputMetadataBufferRequest) { + (new AMessage(kWhatSubmitExtraOutputMetadataBuffer, mCodec))->post(); + mPendingExtraOutputMetadataBufferRequest = true; + } + } + private: // Handles an OMX message. Returns true iff message was handled. bool onOMXMessage(const sp<AMessage> &msg); @@ -302,6 +309,8 @@ private: void getMoreInputDataIfPossible(); + bool mPendingExtraOutputMetadataBufferRequest; + DISALLOW_EVIL_CONSTRUCTORS(BaseState); }; @@ -555,6 +564,7 @@ ACodec::ACodec() mShutdownInProgress(false), mExplicitShutdown(false), mIsLegacyVP9Decoder(false), + mIsLowLatency(false), mEncoderDelay(0), mEncoderPadding(0), mRotationDegrees(0), @@ -2237,6 +2247,12 @@ status_t ACodec::configureCodec( } err = setupG711Codec(encoder, sampleRate, numChannels); } + } else if (!strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_OPUS)) { + int32_t numChannels = 1, sampleRate = 48000; + if (msg->findInt32("channel-count", &numChannels) && + msg->findInt32("sample-rate", &sampleRate)) { + err = setupOpusCodec(encoder, sampleRate, numChannels); + } } else if (!strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_FLAC)) { // numChannels needs to be set to properly communicate PCM values. int32_t numChannels = 2, sampleRate = 44100, compressionLevel = -1; @@ -2409,6 +2425,7 @@ status_t ACodec::setLowLatency(int32_t lowLatency) { if (err != OK) { ALOGE("decoder can not set low-latency to %d (err %d)", lowLatency, err); } + mIsLowLatency = (lowLatency && err == OK); return err; } @@ -3110,6 +3127,26 @@ status_t ACodec::setupG711Codec(bool encoder, int32_t sampleRate, int32_t numCha kPortIndexInput, sampleRate, numChannels); } +status_t ACodec::setupOpusCodec(bool encoder, int32_t sampleRate, int32_t numChannels) { + if (encoder) { + return INVALID_OPERATION; + } + OMX_AUDIO_PARAM_ANDROID_OPUSTYPE def; + InitOMXParams(&def); + def.nPortIndex = kPortIndexInput; + status_t err = mOMXNode->getParameter( + (OMX_INDEXTYPE)OMX_IndexParamAudioAndroidOpus, &def, sizeof(def)); + if (err != OK) { + ALOGE("setupOpusCodec(): Error %d getting OMX_IndexParamAudioAndroidOpus parameter", err); + return err; + } + def.nSampleRate = sampleRate; + def.nChannels = numChannels; + err = mOMXNode->setParameter( + (OMX_INDEXTYPE)OMX_IndexParamAudioAndroidOpus, &def, sizeof(def)); + return err; +} + status_t ACodec::setupFlacCodec( bool encoder, int32_t numChannels, int32_t sampleRate, int32_t compressionLevel, AudioEncoding encoding) { @@ -5294,21 +5331,20 @@ status_t ACodec::getPortFormat(OMX_U32 portIndex, sp<AMessage> ¬ify) { err = mOMXNode->getParameter( (OMX_INDEXTYPE)OMX_IndexParamAudioAndroidAacDrcPresentation, &presentation, sizeof(presentation)); - if (err != OK) { - return err; + if (err == OK) { + notify->setInt32("aac-encoded-target-level", + presentation.nEncodedTargetLevel); + notify->setInt32("aac-drc-cut-level", presentation.nDrcCut); + notify->setInt32("aac-drc-boost-level", presentation.nDrcBoost); + notify->setInt32("aac-drc-heavy-compression", + presentation.nHeavyCompression); + notify->setInt32("aac-target-ref-level", + presentation.nTargetReferenceLevel); + notify->setInt32("aac-drc-effect-type", presentation.nDrcEffectType); + notify->setInt32("aac-drc-album-mode", presentation.nDrcAlbumMode); + notify->setInt32("aac-drc-output-loudness", + presentation.nDrcOutputLoudness); } - notify->setInt32("aac-encoded-target-level", - presentation.nEncodedTargetLevel); - notify->setInt32("aac-drc-cut-level", presentation.nDrcCut); - notify->setInt32("aac-drc-boost-level", presentation.nDrcBoost); - notify->setInt32("aac-drc-heavy-compression", - presentation.nHeavyCompression); - notify->setInt32("aac-target-ref-level", - presentation.nTargetReferenceLevel); - notify->setInt32("aac-drc-effect-type", presentation.nDrcEffectType); - notify->setInt32("aac-drc-album-mode", presentation.nDrcAlbumMode); - notify->setInt32("aac-drc-output-loudness", - presentation.nDrcOutputLoudness); } } break; @@ -5750,7 +5786,8 @@ status_t ACodec::requestIDRFrame() { ACodec::BaseState::BaseState(ACodec *codec, const sp<AState> &parentState) : AState(parentState), - mCodec(codec) { + mCodec(codec), + mPendingExtraOutputMetadataBufferRequest(false) { } ACodec::BaseState::PortMode ACodec::BaseState::getPortMode( @@ -5851,6 +5888,21 @@ bool ACodec::BaseState::onMessageReceived(const sp<AMessage> &msg) { break; } + case kWhatSubmitExtraOutputMetadataBuffer: { + mPendingExtraOutputMetadataBufferRequest = false; + if (getPortMode(kPortIndexOutput) == RESUBMIT_BUFFERS && mCodec->mIsLowLatency) { + // Decoders often need more than one output buffer to be + // submitted before processing a single input buffer. + // For low latency codecs, we don't want to wait for more input + // to be queued to get those output buffers submitted. + if (mCodec->submitOutputMetadataBuffer() == OK + && mCodec->mMetadataBuffersToSubmit > 0) { + maybePostExtraOutputMetadataBufferRequest(); + } + } + break; + } + default: return false; } @@ -6207,7 +6259,12 @@ void ACodec::BaseState::onInputBufferFilled(const sp<AMessage> &msg) { (outputMode == FREE_BUFFERS ? "FREE" : outputMode == KEEP_BUFFERS ? "KEEP" : "RESUBMIT")); if (outputMode == RESUBMIT_BUFFERS) { - mCodec->submitOutputMetadataBuffer(); + status_t err = mCodec->submitOutputMetadataBuffer(); + if (mCodec->mIsLowLatency + && err == OK + && mCodec->mMetadataBuffersToSubmit > 0) { + maybePostExtraOutputMetadataBufferRequest(); + } } } info->checkReadFence("onInputBufferFilled"); @@ -7352,6 +7409,9 @@ void ACodec::ExecutingState::submitOutputMetaBuffers() { break; } } + if (mCodec->mIsLowLatency) { + maybePostExtraOutputMetadataBufferRequest(); + } // *** NOTE: THE FOLLOWING WORKAROUND WILL BE REMOVED *** mCodec->signalSubmitOutputMetadataBufferIfEOS_workaround(); @@ -7995,6 +8055,7 @@ status_t ACodec::setVendorParameters(const sp<AMessage> ¶ms) { // don't bother component if we don't have vendor extensions as they may not have implemented // the android vendor extension support, which will lead to unnecessary OMX failure logs. if (vendorKeys.empty()) { + mVendorExtensionsStatus = kExtensionsNone; return OK; } @@ -8286,13 +8347,34 @@ bool ACodec::OutputPortSettingsChangedState::onMessageReceived( FALLTHROUGH_INTENDED; } case kWhatResume: + { + ALOGV("[%s] Deferring resume", mCodec->mComponentName.c_str()); + + mCodec->deferMessage(msg); + handled = true; + break; + } + case kWhatSetParameters: { - if (msg->what() == kWhatResume) { - ALOGV("[%s] Deferring resume", mCodec->mComponentName.c_str()); + sp<AMessage> params; + CHECK(msg->findMessage("params", ¶ms)); + + sp<ABuffer> hdr10PlusInfo; + if (params->findBuffer("hdr10-plus-info", &hdr10PlusInfo)) { + if (hdr10PlusInfo != nullptr && hdr10PlusInfo->size() > 0) { + (void)mCodec->setHdr10PlusInfo(hdr10PlusInfo); + } + params->removeEntryAt(params->findEntryByName("hdr10-plus-info")); + + if (params->countEntries() == 0) { + msg->removeEntryAt(msg->findEntryByName("params")); + } } - mCodec->deferMessage(msg); + if (msg->countEntries() > 0) { + mCodec->deferMessage(msg); + } handled = true; break; } @@ -8407,6 +8489,15 @@ bool ACodec::OutputPortSettingsChangedState::onOMXEvent( return false; } + case OMX_EventConfigUpdate: + { + CHECK_EQ(data1, (OMX_U32)kPortIndexOutput); + + mCodec->onConfigUpdate((OMX_INDEXTYPE)data2); + + return true; + } + default: return BaseState::onOMXEvent(event, data1, data2); } diff --git a/media/libstagefright/ANetworkSession.cpp b/media/libstagefright/ANetworkSession.cpp new file mode 100644 index 0000000000..e3b77ec99a --- /dev/null +++ b/media/libstagefright/ANetworkSession.cpp @@ -0,0 +1,1400 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "NetworkSession" +#include <utils/Log.h> + +#include <arpa/inet.h> +#include <fcntl.h> +#include <linux/tcp.h> +#include <net/if.h> +#include <netdb.h> +#include <netinet/in.h> +#include <sys/ioctl.h> +#include <sys/socket.h> + +#include <media/stagefright/ANetworkSession.h> +#include <media/stagefright/ParsedMessage.h> +#include <media/stagefright/foundation/ABuffer.h> +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> +#include <media/stagefright/foundation/ByteUtils.h> +#include <media/stagefright/foundation/hexdump.h> + +namespace android { + +static const size_t kMaxUDPSize = 1500; +static const int32_t kMaxUDPRetries = 200; + +struct ANetworkSession::NetworkThread : public Thread { + explicit NetworkThread(ANetworkSession *session); + +protected: + virtual ~NetworkThread(); + +private: + ANetworkSession *mSession; + + virtual bool threadLoop(); + + DISALLOW_EVIL_CONSTRUCTORS(NetworkThread); +}; + +struct ANetworkSession::Session : public RefBase { + enum Mode { + MODE_RTSP, + MODE_DATAGRAM, + MODE_WEBSOCKET, + }; + + enum State { + CONNECTING, + CONNECTED, + LISTENING_RTSP, + LISTENING_TCP_DGRAMS, + DATAGRAM, + }; + + Session(int32_t sessionID, + State state, + int s, + const sp<AMessage> ¬ify); + + int32_t sessionID() const; + int socket() const; + sp<AMessage> getNotificationMessage() const; + + bool isRTSPServer() const; + bool isTCPDatagramServer() const; + + bool wantsToRead(); + bool wantsToWrite(); + + status_t readMore(); + status_t writeMore(); + + status_t sendRequest( + const void *data, ssize_t size, bool timeValid, int64_t timeUs); + + void setMode(Mode mode); + + status_t switchToWebSocketMode(); + +protected: + virtual ~Session(); + +private: + enum { + FRAGMENT_FLAG_TIME_VALID = 1, + }; + struct Fragment { + uint32_t mFlags; + int64_t mTimeUs; + sp<ABuffer> mBuffer; + }; + + int32_t mSessionID; + State mState; + Mode mMode; + int mSocket; + sp<AMessage> mNotify; + bool mSawReceiveFailure, mSawSendFailure; + int32_t mUDPRetries; + + List<Fragment> mOutFragments; + + AString mInBuffer; + + int64_t mLastStallReportUs; + + void notifyError(bool send, status_t err, const char *detail); + void notify(NotificationReason reason); + + void dumpFragmentStats(const Fragment &frag); + + DISALLOW_EVIL_CONSTRUCTORS(Session); +}; +//////////////////////////////////////////////////////////////////////////////// + +ANetworkSession::NetworkThread::NetworkThread(ANetworkSession *session) + : mSession(session) { +} + +ANetworkSession::NetworkThread::~NetworkThread() { +} + +bool ANetworkSession::NetworkThread::threadLoop() { + mSession->threadLoop(); + + return true; +} + +//////////////////////////////////////////////////////////////////////////////// + +ANetworkSession::Session::Session( + int32_t sessionID, + State state, + int s, + const sp<AMessage> ¬ify) + : mSessionID(sessionID), + mState(state), + mMode(MODE_DATAGRAM), + mSocket(s), + mNotify(notify), + mSawReceiveFailure(false), + mSawSendFailure(false), + mUDPRetries(kMaxUDPRetries), + mLastStallReportUs(-1ll) { + if (mState == CONNECTED) { + struct sockaddr_in localAddr; + socklen_t localAddrLen = sizeof(localAddr); + + int res = getsockname( + mSocket, (struct sockaddr *)&localAddr, &localAddrLen); + CHECK_GE(res, 0); + + struct sockaddr_in remoteAddr; + socklen_t remoteAddrLen = sizeof(remoteAddr); + + res = getpeername( + mSocket, (struct sockaddr *)&remoteAddr, &remoteAddrLen); + CHECK_GE(res, 0); + + in_addr_t addr = ntohl(localAddr.sin_addr.s_addr); + AString localAddrString = AStringPrintf( + "%d.%d.%d.%d", + (addr >> 24), + (addr >> 16) & 0xff, + (addr >> 8) & 0xff, + addr & 0xff); + + addr = ntohl(remoteAddr.sin_addr.s_addr); + AString remoteAddrString = AStringPrintf( + "%d.%d.%d.%d", + (addr >> 24), + (addr >> 16) & 0xff, + (addr >> 8) & 0xff, + addr & 0xff); + + sp<AMessage> msg = mNotify->dup(); + msg->setInt32("sessionID", mSessionID); + msg->setInt32("reason", kWhatClientConnected); + msg->setString("server-ip", localAddrString.c_str()); + msg->setInt32("server-port", ntohs(localAddr.sin_port)); + msg->setString("client-ip", remoteAddrString.c_str()); + msg->setInt32("client-port", ntohs(remoteAddr.sin_port)); + msg->post(); + } +} + +ANetworkSession::Session::~Session() { + ALOGV("Session %d gone", mSessionID); + + close(mSocket); + mSocket = -1; +} + +int32_t ANetworkSession::Session::sessionID() const { + return mSessionID; +} + +int ANetworkSession::Session::socket() const { + return mSocket; +} + +void ANetworkSession::Session::setMode(Mode mode) { + mMode = mode; +} + +status_t ANetworkSession::Session::switchToWebSocketMode() { + if (mState != CONNECTED || mMode != MODE_RTSP) { + return INVALID_OPERATION; + } + + mMode = MODE_WEBSOCKET; + + return OK; +} + +sp<AMessage> ANetworkSession::Session::getNotificationMessage() const { + return mNotify; +} + +bool ANetworkSession::Session::isRTSPServer() const { + return mState == LISTENING_RTSP; +} + +bool ANetworkSession::Session::isTCPDatagramServer() const { + return mState == LISTENING_TCP_DGRAMS; +} + +bool ANetworkSession::Session::wantsToRead() { + return !mSawReceiveFailure && mState != CONNECTING; +} + +bool ANetworkSession::Session::wantsToWrite() { + return !mSawSendFailure + && (mState == CONNECTING + || (mState == CONNECTED && !mOutFragments.empty()) + || (mState == DATAGRAM && !mOutFragments.empty())); +} + +status_t ANetworkSession::Session::readMore() { + if (mState == DATAGRAM) { + CHECK_EQ(mMode, MODE_DATAGRAM); + + status_t err; + do { + sp<ABuffer> buf = new ABuffer(kMaxUDPSize); + + struct sockaddr_in remoteAddr; + socklen_t remoteAddrLen = sizeof(remoteAddr); + + ssize_t n; + do { + n = recvfrom( + mSocket, buf->data(), buf->capacity(), 0, + (struct sockaddr *)&remoteAddr, &remoteAddrLen); + } while (n < 0 && errno == EINTR); + + err = OK; + if (n < 0) { + err = -errno; + } else if (n == 0) { + err = -ECONNRESET; + } else { + buf->setRange(0, n); + + int64_t nowUs = ALooper::GetNowUs(); + buf->meta()->setInt64("arrivalTimeUs", nowUs); + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("sessionID", mSessionID); + notify->setInt32("reason", kWhatDatagram); + + uint32_t ip = ntohl(remoteAddr.sin_addr.s_addr); + notify->setString( + "fromAddr", + AStringPrintf( + "%u.%u.%u.%u", + ip >> 24, + (ip >> 16) & 0xff, + (ip >> 8) & 0xff, + ip & 0xff).c_str()); + + notify->setInt32("fromPort", ntohs(remoteAddr.sin_port)); + + notify->setBuffer("data", buf); + notify->post(); + } + } while (err == OK); + + if (err == -EAGAIN) { + err = OK; + } + + if (err != OK) { + if (!mUDPRetries) { + notifyError(false /* send */, err, "Recvfrom failed."); + mSawReceiveFailure = true; + } else { + mUDPRetries--; + ALOGE("Recvfrom failed, %d/%d retries left", + mUDPRetries, kMaxUDPRetries); + err = OK; + } + } else { + mUDPRetries = kMaxUDPRetries; + } + + return err; + } + + char tmp[512]; + ssize_t n; + do { + n = recv(mSocket, tmp, sizeof(tmp), 0); + } while (n < 0 && errno == EINTR); + + status_t err = OK; + + if (n > 0) { + mInBuffer.append(tmp, n); + +#if 0 + ALOGI("in:"); + hexdump(tmp, n); +#endif + } else if (n < 0) { + err = -errno; + } else { + err = -ECONNRESET; + } + + if (mMode == MODE_DATAGRAM) { + // TCP stream carrying 16-bit length-prefixed datagrams. + + while (mInBuffer.size() >= 2) { + size_t packetSize = U16_AT((const uint8_t *)mInBuffer.c_str()); + + if (mInBuffer.size() < packetSize + 2) { + break; + } + + sp<ABuffer> packet = new ABuffer(packetSize); + memcpy(packet->data(), mInBuffer.c_str() + 2, packetSize); + + int64_t nowUs = ALooper::GetNowUs(); + packet->meta()->setInt64("arrivalTimeUs", nowUs); + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("sessionID", mSessionID); + notify->setInt32("reason", kWhatDatagram); + notify->setBuffer("data", packet); + notify->post(); + + mInBuffer.erase(0, packetSize + 2); + } + } else if (mMode == MODE_RTSP) { + for (;;) { + size_t length; + + if (mInBuffer.size() > 0 && mInBuffer.c_str()[0] == '$') { + if (mInBuffer.size() < 4) { + break; + } + + length = U16_AT((const uint8_t *)mInBuffer.c_str() + 2); + + if (mInBuffer.size() < 4 + length) { + break; + } + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("sessionID", mSessionID); + notify->setInt32("reason", kWhatBinaryData); + notify->setInt32("channel", mInBuffer.c_str()[1]); + + sp<ABuffer> data = new ABuffer(length); + memcpy(data->data(), mInBuffer.c_str() + 4, length); + + int64_t nowUs = ALooper::GetNowUs(); + data->meta()->setInt64("arrivalTimeUs", nowUs); + + notify->setBuffer("data", data); + notify->post(); + + mInBuffer.erase(0, 4 + length); + continue; + } + + sp<ParsedMessage> msg = + ParsedMessage::Parse( + mInBuffer.c_str(), mInBuffer.size(), err != OK, &length); + + if (msg == NULL) { + break; + } + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("sessionID", mSessionID); + notify->setInt32("reason", kWhatData); + notify->setObject("data", msg); + notify->post(); + +#if 1 + // XXX The (old) dongle sends the wrong content length header on a + // SET_PARAMETER request that signals a "wfd_idr_request". + // (17 instead of 19). + const char *content = msg->getContent(); + if (content + && !memcmp(content, "wfd_idr_request\r\n", 17) + && length >= 19 + && mInBuffer.c_str()[length] == '\r' + && mInBuffer.c_str()[length + 1] == '\n') { + length += 2; + } +#endif + + mInBuffer.erase(0, length); + + if (err != OK) { + break; + } + } + } else { + CHECK_EQ(mMode, MODE_WEBSOCKET); + + const uint8_t *data = (const uint8_t *)mInBuffer.c_str(); + // hexdump(data, mInBuffer.size()); + + while (mInBuffer.size() >= 2) { + size_t offset = 2; + + uint64_t payloadLen = data[1] & 0x7f; + if (payloadLen == 126) { + if (offset + 2 > mInBuffer.size()) { + break; + } + + payloadLen = U16_AT(&data[offset]); + offset += 2; + } else if (payloadLen == 127) { + if (offset + 8 > mInBuffer.size()) { + break; + } + + payloadLen = U64_AT(&data[offset]); + offset += 8; + } + + uint32_t mask = 0; + if (data[1] & 0x80) { + // MASK==1 + if (offset + 4 > mInBuffer.size()) { + break; + } + + mask = U32_AT(&data[offset]); + offset += 4; + } + + if (payloadLen > mInBuffer.size() || offset > mInBuffer.size() - payloadLen) { + break; + } + + // We have the full message. + + sp<ABuffer> packet = new ABuffer(payloadLen); + memcpy(packet->data(), &data[offset], payloadLen); + + if (mask != 0) { + for (size_t i = 0; i < payloadLen; ++i) { + packet->data()[i] = + data[offset + i] + ^ ((mask >> (8 * (3 - (i % 4)))) & 0xff); + } + } + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("sessionID", mSessionID); + notify->setInt32("reason", kWhatWebSocketMessage); + notify->setBuffer("data", packet); + notify->setInt32("headerByte", data[0]); + notify->post(); + + mInBuffer.erase(0, offset + payloadLen); + } + } + + if (err != OK) { + notifyError(false /* send */, err, "Recv failed."); + mSawReceiveFailure = true; + } + + return err; +} + +void ANetworkSession::Session::dumpFragmentStats(const Fragment & /* frag */) { +#if 0 + int64_t nowUs = ALooper::GetNowUs(); + int64_t delayMs = (nowUs - frag.mTimeUs) / 1000ll; + + static const int64_t kMinDelayMs = 0; + static const int64_t kMaxDelayMs = 300; + + const char *kPattern = "########################################"; + size_t kPatternSize = strlen(kPattern); + + int n = (kPatternSize * (delayMs - kMinDelayMs)) + / (kMaxDelayMs - kMinDelayMs); + + if (n < 0) { + n = 0; + } else if ((size_t)n > kPatternSize) { + n = kPatternSize; + } + + ALOGI("[%lld]: (%4lld ms) %s\n", + frag.mTimeUs / 1000, + delayMs, + kPattern + kPatternSize - n); +#endif +} + +status_t ANetworkSession::Session::writeMore() { + if (mState == DATAGRAM) { + CHECK(!mOutFragments.empty()); + + status_t err; + do { + const Fragment &frag = *mOutFragments.begin(); + const sp<ABuffer> &datagram = frag.mBuffer; + + int n; + do { + n = send(mSocket, datagram->data(), datagram->size(), 0); + } while (n < 0 && errno == EINTR); + + err = OK; + + if (n > 0) { + if (frag.mFlags & FRAGMENT_FLAG_TIME_VALID) { + dumpFragmentStats(frag); + } + + mOutFragments.erase(mOutFragments.begin()); + } else if (n < 0) { + err = -errno; + } else if (n == 0) { + err = -ECONNRESET; + } + } while (err == OK && !mOutFragments.empty()); + + if (err == -EAGAIN) { + if (!mOutFragments.empty()) { + ALOGI("%zu datagrams remain queued.", mOutFragments.size()); + } + err = OK; + } + + if (err != OK) { + if (!mUDPRetries) { + notifyError(true /* send */, err, "Send datagram failed."); + mSawSendFailure = true; + } else { + mUDPRetries--; + ALOGE("Send datagram failed, %d/%d retries left", + mUDPRetries, kMaxUDPRetries); + err = OK; + } + } else { + mUDPRetries = kMaxUDPRetries; + } + + return err; + } + + if (mState == CONNECTING) { + int err; + socklen_t optionLen = sizeof(err); + CHECK_EQ(getsockopt(mSocket, SOL_SOCKET, SO_ERROR, &err, &optionLen), 0); + CHECK_EQ(optionLen, (socklen_t)sizeof(err)); + + if (err != 0) { + notifyError(kWhatError, -err, "Connection failed"); + mSawSendFailure = true; + + return -err; + } + + mState = CONNECTED; + notify(kWhatConnected); + + return OK; + } + + CHECK_EQ(mState, CONNECTED); + CHECK(!mOutFragments.empty()); + + ssize_t n = -1; + while (!mOutFragments.empty()) { + const Fragment &frag = *mOutFragments.begin(); + + do { + n = send(mSocket, frag.mBuffer->data(), frag.mBuffer->size(), 0); + } while (n < 0 && errno == EINTR); + + if (n <= 0) { + break; + } + + frag.mBuffer->setRange( + frag.mBuffer->offset() + n, frag.mBuffer->size() - n); + + if (frag.mBuffer->size() > 0) { + break; + } + + if (frag.mFlags & FRAGMENT_FLAG_TIME_VALID) { + dumpFragmentStats(frag); + } + + mOutFragments.erase(mOutFragments.begin()); + } + + status_t err = OK; + + if (n < 0) { + err = -errno; + } else if (n == 0) { + err = -ECONNRESET; + } + + if (err != OK) { + notifyError(true /* send */, err, "Send failed."); + mSawSendFailure = true; + } + +#if 0 + int numBytesQueued; + int res = ioctl(mSocket, SIOCOUTQ, &numBytesQueued); + if (res == 0 && numBytesQueued > 50 * 1024) { + if (numBytesQueued > 409600) { + ALOGW("!!! numBytesQueued = %d", numBytesQueued); + } + + int64_t nowUs = ALooper::GetNowUs(); + + if (mLastStallReportUs < 0ll + || nowUs > mLastStallReportUs + 100000ll) { + sp<AMessage> msg = mNotify->dup(); + msg->setInt32("sessionID", mSessionID); + msg->setInt32("reason", kWhatNetworkStall); + msg->setSize("numBytesQueued", numBytesQueued); + msg->post(); + + mLastStallReportUs = nowUs; + } + } +#endif + + return err; +} + +status_t ANetworkSession::Session::sendRequest( + const void *data, ssize_t size, bool timeValid, int64_t timeUs) { + CHECK(mState == CONNECTED || mState == DATAGRAM); + + if (size < 0) { + size = strlen((const char *)data); + } + + if (size == 0) { + return OK; + } + + sp<ABuffer> buffer; + + if (mState == CONNECTED && mMode == MODE_DATAGRAM) { + CHECK_LE(size, 65535); + + buffer = new ABuffer(size + 2); + buffer->data()[0] = size >> 8; + buffer->data()[1] = size & 0xff; + memcpy(buffer->data() + 2, data, size); + } else if (mState == CONNECTED && mMode == MODE_WEBSOCKET) { + static const bool kUseMask = false; // Chromium doesn't like it. + + size_t numHeaderBytes = 2 + (kUseMask ? 4 : 0); + if (size > 65535) { + numHeaderBytes += 8; + } else if (size > 125) { + numHeaderBytes += 2; + } + + buffer = new ABuffer(numHeaderBytes + size); + buffer->data()[0] = 0x81; // FIN==1 | opcode=1 (text) + buffer->data()[1] = kUseMask ? 0x80 : 0x00; + + if (size > 65535) { + buffer->data()[1] |= 127; + buffer->data()[2] = 0x00; + buffer->data()[3] = 0x00; + buffer->data()[4] = 0x00; + buffer->data()[5] = 0x00; + buffer->data()[6] = (size >> 24) & 0xff; + buffer->data()[7] = (size >> 16) & 0xff; + buffer->data()[8] = (size >> 8) & 0xff; + buffer->data()[9] = size & 0xff; + } else if (size > 125) { + buffer->data()[1] |= 126; + buffer->data()[2] = (size >> 8) & 0xff; + buffer->data()[3] = size & 0xff; + } else { + buffer->data()[1] |= size; + } + + if (kUseMask) { + uint32_t mask = rand(); + + buffer->data()[numHeaderBytes - 4] = (mask >> 24) & 0xff; + buffer->data()[numHeaderBytes - 3] = (mask >> 16) & 0xff; + buffer->data()[numHeaderBytes - 2] = (mask >> 8) & 0xff; + buffer->data()[numHeaderBytes - 1] = mask & 0xff; + + for (size_t i = 0; i < (size_t)size; ++i) { + buffer->data()[numHeaderBytes + i] = + ((const uint8_t *)data)[i] + ^ ((mask >> (8 * (3 - (i % 4)))) & 0xff); + } + } else { + memcpy(buffer->data() + numHeaderBytes, data, size); + } + } else { + buffer = new ABuffer(size); + memcpy(buffer->data(), data, size); + } + + Fragment frag; + + frag.mFlags = 0; + if (timeValid) { + frag.mFlags = FRAGMENT_FLAG_TIME_VALID; + frag.mTimeUs = timeUs; + } + + frag.mBuffer = buffer; + + mOutFragments.push_back(frag); + + return OK; +} + +void ANetworkSession::Session::notifyError( + bool send, status_t err, const char *detail) { + sp<AMessage> msg = mNotify->dup(); + msg->setInt32("sessionID", mSessionID); + msg->setInt32("reason", kWhatError); + msg->setInt32("send", send); + msg->setInt32("err", err); + msg->setString("detail", detail); + msg->post(); +} + +void ANetworkSession::Session::notify(NotificationReason reason) { + sp<AMessage> msg = mNotify->dup(); + msg->setInt32("sessionID", mSessionID); + msg->setInt32("reason", reason); + msg->post(); +} + +//////////////////////////////////////////////////////////////////////////////// + +ANetworkSession::ANetworkSession() + : mNextSessionID(1) { + mPipeFd[0] = mPipeFd[1] = -1; +} + +ANetworkSession::~ANetworkSession() { + stop(); +} + +status_t ANetworkSession::start() { + if (mThread != NULL) { + return INVALID_OPERATION; + } + + int res = pipe(mPipeFd); + if (res != 0) { + mPipeFd[0] = mPipeFd[1] = -1; + return -errno; + } + + mThread = new NetworkThread(this); + + status_t err = mThread->run("ANetworkSession", ANDROID_PRIORITY_AUDIO); + + if (err != OK) { + mThread.clear(); + + close(mPipeFd[0]); + close(mPipeFd[1]); + mPipeFd[0] = mPipeFd[1] = -1; + + return err; + } + + return OK; +} + +status_t ANetworkSession::stop() { + if (mThread == NULL) { + return INVALID_OPERATION; + } + + mThread->requestExit(); + interrupt(); + mThread->requestExitAndWait(); + + mThread.clear(); + + close(mPipeFd[0]); + close(mPipeFd[1]); + mPipeFd[0] = mPipeFd[1] = -1; + + return OK; +} + +status_t ANetworkSession::createRTSPClient( + const char *host, unsigned port, const sp<AMessage> ¬ify, + int32_t *sessionID) { + return createClientOrServer( + kModeCreateRTSPClient, + NULL /* addr */, + 0 /* port */, + host, + port, + notify, + sessionID); +} + +status_t ANetworkSession::createRTSPServer( + const struct in_addr &addr, unsigned port, + const sp<AMessage> ¬ify, int32_t *sessionID) { + return createClientOrServer( + kModeCreateRTSPServer, + &addr, + port, + NULL /* remoteHost */, + 0 /* remotePort */, + notify, + sessionID); +} + +status_t ANetworkSession::createUDPSession( + unsigned localPort, const sp<AMessage> ¬ify, int32_t *sessionID) { + return createUDPSession(localPort, NULL, 0, notify, sessionID); +} + +status_t ANetworkSession::createUDPSession( + unsigned localPort, + const char *remoteHost, + unsigned remotePort, + const sp<AMessage> ¬ify, + int32_t *sessionID) { + return createClientOrServer( + kModeCreateUDPSession, + NULL /* addr */, + localPort, + remoteHost, + remotePort, + notify, + sessionID); +} + +status_t ANetworkSession::createTCPDatagramSession( + const struct in_addr &addr, unsigned port, + const sp<AMessage> ¬ify, int32_t *sessionID) { + return createClientOrServer( + kModeCreateTCPDatagramSessionPassive, + &addr, + port, + NULL /* remoteHost */, + 0 /* remotePort */, + notify, + sessionID); +} + +status_t ANetworkSession::createTCPDatagramSession( + unsigned localPort, + const char *remoteHost, + unsigned remotePort, + const sp<AMessage> ¬ify, + int32_t *sessionID) { + return createClientOrServer( + kModeCreateTCPDatagramSessionActive, + NULL /* addr */, + localPort, + remoteHost, + remotePort, + notify, + sessionID); +} + +status_t ANetworkSession::destroySession(int32_t sessionID) { + Mutex::Autolock autoLock(mLock); + + ssize_t index = mSessions.indexOfKey(sessionID); + + if (index < 0) { + return -ENOENT; + } + + mSessions.removeItemsAt(index); + + interrupt(); + + return OK; +} + +// static +status_t ANetworkSession::MakeSocketNonBlocking(int s) { + int flags = fcntl(s, F_GETFL, 0); + if (flags < 0) { + flags = 0; + } + + int res = fcntl(s, F_SETFL, flags | O_NONBLOCK); + if (res < 0) { + return -errno; + } + + return OK; +} + +status_t ANetworkSession::createClientOrServer( + Mode mode, + const struct in_addr *localAddr, + unsigned port, + const char *remoteHost, + unsigned remotePort, + const sp<AMessage> ¬ify, + int32_t *sessionID) { + Mutex::Autolock autoLock(mLock); + + *sessionID = 0; + status_t err = OK; + int s, res; + sp<Session> session; + + s = socket( + AF_INET, + (mode == kModeCreateUDPSession) ? SOCK_DGRAM : SOCK_STREAM, + 0); + + if (s < 0) { + err = -errno; + goto bail; + } + + if (mode == kModeCreateRTSPServer + || mode == kModeCreateTCPDatagramSessionPassive) { + const int yes = 1; + res = setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)); + + if (res < 0) { + err = -errno; + goto bail2; + } + } + + if (mode == kModeCreateUDPSession) { + int size = 256 * 1024; + + res = setsockopt(s, SOL_SOCKET, SO_RCVBUF, &size, sizeof(size)); + + if (res < 0) { + err = -errno; + goto bail2; + } + + res = setsockopt(s, SOL_SOCKET, SO_SNDBUF, &size, sizeof(size)); + + if (res < 0) { + err = -errno; + goto bail2; + } + } else if (mode == kModeCreateTCPDatagramSessionActive) { + int flag = 1; + res = setsockopt(s, IPPROTO_TCP, TCP_NODELAY, &flag, sizeof(flag)); + + if (res < 0) { + err = -errno; + goto bail2; + } + + int tos = 224; // VOICE + res = setsockopt(s, IPPROTO_IP, IP_TOS, &tos, sizeof(tos)); + + if (res < 0) { + err = -errno; + goto bail2; + } + } + + err = MakeSocketNonBlocking(s); + + if (err != OK) { + goto bail2; + } + + struct sockaddr_in addr; + memset(addr.sin_zero, 0, sizeof(addr.sin_zero)); + addr.sin_family = AF_INET; + + if (mode == kModeCreateRTSPClient + || mode == kModeCreateTCPDatagramSessionActive) { + struct hostent *ent= gethostbyname(remoteHost); + if (ent == NULL) { + err = -h_errno; + goto bail2; + } + + addr.sin_addr.s_addr = *(in_addr_t *)ent->h_addr; + addr.sin_port = htons(remotePort); + } else if (localAddr != NULL) { + addr.sin_addr = *localAddr; + addr.sin_port = htons(port); + } else { + addr.sin_addr.s_addr = htonl(INADDR_ANY); + addr.sin_port = htons(port); + } + + if (mode == kModeCreateRTSPClient + || mode == kModeCreateTCPDatagramSessionActive) { + in_addr_t x = ntohl(addr.sin_addr.s_addr); + ALOGI("connecting socket %d to %d.%d.%d.%d:%d", + s, + (x >> 24), + (x >> 16) & 0xff, + (x >> 8) & 0xff, + x & 0xff, + ntohs(addr.sin_port)); + + res = connect(s, (const struct sockaddr *)&addr, sizeof(addr)); + + CHECK_LT(res, 0); + if (errno == EINPROGRESS) { + res = 0; + } + } else { + res = bind(s, (const struct sockaddr *)&addr, sizeof(addr)); + + if (res == 0) { + if (mode == kModeCreateRTSPServer + || mode == kModeCreateTCPDatagramSessionPassive) { + res = listen(s, 4); + } else { + CHECK_EQ(mode, kModeCreateUDPSession); + + if (remoteHost != NULL) { + struct sockaddr_in remoteAddr; + memset(remoteAddr.sin_zero, 0, sizeof(remoteAddr.sin_zero)); + remoteAddr.sin_family = AF_INET; + remoteAddr.sin_port = htons(remotePort); + + struct hostent *ent= gethostbyname(remoteHost); + if (ent == NULL) { + err = -h_errno; + goto bail2; + } + + remoteAddr.sin_addr.s_addr = *(in_addr_t *)ent->h_addr; + + res = connect( + s, + (const struct sockaddr *)&remoteAddr, + sizeof(remoteAddr)); + } + } + } + } + + if (res < 0) { + err = -errno; + goto bail2; + } + + Session::State state; + switch (mode) { + case kModeCreateRTSPClient: + state = Session::CONNECTING; + break; + + case kModeCreateTCPDatagramSessionActive: + state = Session::CONNECTING; + break; + + case kModeCreateTCPDatagramSessionPassive: + state = Session::LISTENING_TCP_DGRAMS; + break; + + case kModeCreateRTSPServer: + state = Session::LISTENING_RTSP; + break; + + default: + CHECK_EQ(mode, kModeCreateUDPSession); + state = Session::DATAGRAM; + break; + } + + session = new Session( + mNextSessionID++, + state, + s, + notify); + + if (mode == kModeCreateTCPDatagramSessionActive) { + session->setMode(Session::MODE_DATAGRAM); + } else if (mode == kModeCreateRTSPClient) { + session->setMode(Session::MODE_RTSP); + } + + mSessions.add(session->sessionID(), session); + + interrupt(); + + *sessionID = session->sessionID(); + + goto bail; + +bail2: + close(s); + s = -1; + +bail: + return err; +} + +status_t ANetworkSession::connectUDPSession( + int32_t sessionID, const char *remoteHost, unsigned remotePort) { + Mutex::Autolock autoLock(mLock); + + ssize_t index = mSessions.indexOfKey(sessionID); + + if (index < 0) { + return -ENOENT; + } + + const sp<Session> session = mSessions.valueAt(index); + int s = session->socket(); + + struct sockaddr_in remoteAddr; + memset(remoteAddr.sin_zero, 0, sizeof(remoteAddr.sin_zero)); + remoteAddr.sin_family = AF_INET; + remoteAddr.sin_port = htons(remotePort); + + status_t err = OK; + struct hostent *ent = gethostbyname(remoteHost); + if (ent == NULL) { + err = -h_errno; + } else { + remoteAddr.sin_addr.s_addr = *(in_addr_t *)ent->h_addr; + + int res = connect( + s, + (const struct sockaddr *)&remoteAddr, + sizeof(remoteAddr)); + + if (res < 0) { + err = -errno; + } + } + + return err; +} + +status_t ANetworkSession::sendRequest( + int32_t sessionID, const void *data, ssize_t size, + bool timeValid, int64_t timeUs) { + Mutex::Autolock autoLock(mLock); + + ssize_t index = mSessions.indexOfKey(sessionID); + + if (index < 0) { + return -ENOENT; + } + + const sp<Session> session = mSessions.valueAt(index); + + status_t err = session->sendRequest(data, size, timeValid, timeUs); + + interrupt(); + + return err; +} + +status_t ANetworkSession::switchToWebSocketMode(int32_t sessionID) { + Mutex::Autolock autoLock(mLock); + + ssize_t index = mSessions.indexOfKey(sessionID); + + if (index < 0) { + return -ENOENT; + } + + const sp<Session> session = mSessions.valueAt(index); + return session->switchToWebSocketMode(); +} + +void ANetworkSession::interrupt() { + static const char dummy = 0; + + ssize_t n; + do { + n = write(mPipeFd[1], &dummy, 1); + } while (n < 0 && errno == EINTR); + + if (n < 0) { + ALOGW("Error writing to pipe (%s)", strerror(errno)); + } +} + +void ANetworkSession::threadLoop() { + fd_set rs, ws; + FD_ZERO(&rs); + FD_ZERO(&ws); + + FD_SET(mPipeFd[0], &rs); + int maxFd = mPipeFd[0]; + + { + Mutex::Autolock autoLock(mLock); + + for (size_t i = 0; i < mSessions.size(); ++i) { + const sp<Session> &session = mSessions.valueAt(i); + + int s = session->socket(); + + if (s < 0) { + continue; + } + + if (session->wantsToRead()) { + FD_SET(s, &rs); + if (s > maxFd) { + maxFd = s; + } + } + + if (session->wantsToWrite()) { + FD_SET(s, &ws); + if (s > maxFd) { + maxFd = s; + } + } + } + } + + int res = select(maxFd + 1, &rs, &ws, NULL, NULL /* tv */); + + if (res == 0) { + return; + } + + if (res < 0) { + if (errno == EINTR) { + return; + } + + ALOGE("select failed w/ error %d (%s)", errno, strerror(errno)); + return; + } + + if (FD_ISSET(mPipeFd[0], &rs)) { + char c; + ssize_t n; + do { + n = read(mPipeFd[0], &c, 1); + } while (n < 0 && errno == EINTR); + + if (n < 0) { + ALOGW("Error reading from pipe (%s)", strerror(errno)); + } + + --res; + } + + { + Mutex::Autolock autoLock(mLock); + + List<sp<Session> > sessionsToAdd; + + for (size_t i = mSessions.size(); res > 0 && i > 0;) { + i--; + const sp<Session> &session = mSessions.valueAt(i); + + int s = session->socket(); + + if (s < 0) { + continue; + } + + if (FD_ISSET(s, &rs) || FD_ISSET(s, &ws)) { + --res; + } + + if (FD_ISSET(s, &rs)) { + if (session->isRTSPServer() || session->isTCPDatagramServer()) { + struct sockaddr_in remoteAddr; + socklen_t remoteAddrLen = sizeof(remoteAddr); + + int clientSocket = accept( + s, (struct sockaddr *)&remoteAddr, &remoteAddrLen); + + if (clientSocket >= 0) { + status_t err = MakeSocketNonBlocking(clientSocket); + + if (err != OK) { + ALOGE("Unable to make client socket non blocking, " + "failed w/ error %d (%s)", + err, strerror(-err)); + + close(clientSocket); + clientSocket = -1; + } else { + in_addr_t addr = ntohl(remoteAddr.sin_addr.s_addr); + + ALOGI("incoming connection from %d.%d.%d.%d:%d " + "(socket %d)", + (addr >> 24), + (addr >> 16) & 0xff, + (addr >> 8) & 0xff, + addr & 0xff, + ntohs(remoteAddr.sin_port), + clientSocket); + + sp<Session> clientSession = + new Session( + mNextSessionID++, + Session::CONNECTED, + clientSocket, + session->getNotificationMessage()); + + clientSession->setMode( + session->isRTSPServer() + ? Session::MODE_RTSP + : Session::MODE_DATAGRAM); + + sessionsToAdd.push_back(clientSession); + } + } else { + ALOGE("accept returned error %d (%s)", + errno, strerror(errno)); + } + } else { + status_t err = session->readMore(); + if (err != OK) { + ALOGE("readMore on socket %d failed w/ error %d (%s)", + s, err, strerror(-err)); + } + } + } + + if (FD_ISSET(s, &ws)) { + status_t err = session->writeMore(); + if (err != OK) { + ALOGE("writeMore on socket %d failed w/ error %d (%s)", + s, err, strerror(-err)); + } + } + } + + while (!sessionsToAdd.empty()) { + sp<Session> session = *sessionsToAdd.begin(); + sessionsToAdd.erase(sessionsToAdd.begin()); + + mSessions.add(session->sessionID(), session); + + ALOGI("added clientSession %d", session->sessionID()); + } + } +} + +} // namespace android diff --git a/media/libstagefright/Android.bp b/media/libstagefright/Android.bp index 3bccb7be7a..d4bc8adb1d 100644 --- a/media/libstagefright/Android.bp +++ b/media/libstagefright/Android.bp @@ -208,6 +208,7 @@ cc_library { "ACodecBufferChannel.cpp", "AHierarchicalStateMachine.cpp", "AMRWriter.cpp", + "ANetworkSession.cpp", "AudioSource.cpp", "BufferImpl.cpp", "CallbackDataSource.cpp", @@ -237,10 +238,12 @@ cc_library { "OggWriter.cpp", "OMXClient.cpp", "OmxInfoBuilder.cpp", + "ParsedMessage.cpp", "RemoteMediaExtractor.cpp", "RemoteMediaSource.cpp", "SimpleDecodingSource.cpp", "StagefrightMediaScanner.cpp", + "SurfaceMediaSource.cpp", "SurfaceUtils.cpp", "ThrottledSource.cpp", "Utils.cpp", diff --git a/media/libstagefright/CameraSource.cpp b/media/libstagefright/CameraSource.cpp index 9b3f4200d5..be3fde8a0c 100644 --- a/media/libstagefright/CameraSource.cpp +++ b/media/libstagefright/CameraSource.cpp @@ -162,6 +162,10 @@ static int32_t getColorFormat(const char* colorFormat) { return OMX_COLOR_FormatAndroidOpaque; } + if (!strcmp(colorFormat, "YVU420SemiPlanar")) { + return OMX_QCOM_COLOR_FormatYVU420SemiPlanar; + } + ALOGE("Uknown color format (%s), please add it to " "CameraSource::getColorFormat", colorFormat); @@ -338,6 +342,12 @@ status_t CameraSource::isCameraColorFormatSupported( return OK; } +static int32_t getHighSpeedFrameRate(const CameraParameters& params) { + const char* hsr = params.get("video-hsr"); + int32_t rate = (hsr != NULL && strncmp(hsr, "off", 3)) ? strtol(hsr, NULL, 10) : 0; + return std::min(rate, 240); +} + /* * Configure the camera to use the requested video size * (width and height) and/or frame rate. If both width and @@ -385,11 +395,15 @@ status_t CameraSource::configureCamera( } if (frameRate != -1) { - CHECK(frameRate > 0 && frameRate <= 120); + CHECK(frameRate > 0 && frameRate <= 240); const char* supportedFrameRates = params->get(CameraParameters::KEY_SUPPORTED_PREVIEW_FRAME_RATES); CHECK(supportedFrameRates != NULL); ALOGV("Supported frame rates: %s", supportedFrameRates); + if (getHighSpeedFrameRate(*params)) { + ALOGI("Use default 30fps for HighSpeed %dfps", frameRate); + frameRate = 30; + } char buf[4]; snprintf(buf, 4, "%d", frameRate); if (strstr(supportedFrameRates, buf) == NULL) { @@ -491,6 +505,8 @@ status_t CameraSource::checkFrameRate( ALOGE("Failed to retrieve preview frame rate (%d)", frameRateActual); return UNKNOWN_ERROR; } + int32_t highSpeedRate = getHighSpeedFrameRate(params); + frameRateActual = highSpeedRate ? highSpeedRate : frameRateActual; // Check the actual video frame rate against the target/requested // video frame rate. diff --git a/media/libstagefright/CameraSourceTimeLapse.cpp b/media/libstagefright/CameraSourceTimeLapse.cpp index e0a6eb3a57..a00a1786ae 100644 --- a/media/libstagefright/CameraSourceTimeLapse.cpp +++ b/media/libstagefright/CameraSourceTimeLapse.cpp @@ -298,7 +298,8 @@ bool CameraSourceTimeLapse::skipFrameAndModifyTimeStamp(int64_t *timestampUs) { // The first 2 output frames from the encoder are: decoder specific info and // the compressed video frame data for the first input video frame. if (mNumFramesEncoded >= 1 && *timestampUs < - (mLastTimeLapseFrameRealTimestampUs + mTimeBetweenFrameCaptureUs)) { + (mLastTimeLapseFrameRealTimestampUs + mTimeBetweenFrameCaptureUs) && + (mTimeBetweenFrameCaptureUs > mTimeBetweenTimeLapseVideoFramesUs + 1)) { // Skip all frames from last encoded frame until // sufficient time (mTimeBetweenFrameCaptureUs) has passed. // Tell the camera to release its recording frame and return. @@ -313,6 +314,12 @@ bool CameraSourceTimeLapse::skipFrameAndModifyTimeStamp(int64_t *timestampUs) { mLastTimeLapseFrameRealTimestampUs = *timestampUs; *timestampUs = mLastFrameTimestampUs + mTimeBetweenTimeLapseVideoFramesUs; + // Update start-time once the captured-time reaches the expected start-time. + // Not doing so will result in CameraSource always dropping frames since + // updated-timestamp will never intersect start-timestamp + if ((mNumFramesReceived == 0 && mLastTimeLapseFrameRealTimestampUs >= mStartTimeUs)) { + mStartTimeUs = *timestampUs; + } return false; } return false; diff --git a/media/libstagefright/ParsedMessage.cpp b/media/libstagefright/ParsedMessage.cpp new file mode 100644 index 0000000000..4bfc454338 --- /dev/null +++ b/media/libstagefright/ParsedMessage.cpp @@ -0,0 +1,301 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <ctype.h> +#include <media/stagefright/ParsedMessage.h> +#include <media/stagefright/foundation/ABuffer.h> +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/hexdump.h> + +namespace android { + +// static +sp<ParsedMessage> ParsedMessage::Parse( + const char *data, size_t size, bool noMoreData, size_t *length) { + sp<ParsedMessage> msg = new ParsedMessage; + ssize_t res = msg->parse(data, size, noMoreData); + + if (res < 0) { + *length = 0; + return NULL; + } + + *length = res; + return msg; +} + +ParsedMessage::ParsedMessage() { +} + +ParsedMessage::~ParsedMessage() { +} + +bool ParsedMessage::findString(const char *name, AString *value) const { + AString key = name; + key.tolower(); + + ssize_t index = mDict.indexOfKey(key); + + if (index < 0) { + value->clear(); + + return false; + } + + *value = mDict.valueAt(index); + return true; +} + +bool ParsedMessage::findInt32(const char *name, int32_t *value) const { + AString stringValue; + + if (!findString(name, &stringValue)) { + return false; + } + + char *end; + *value = strtol(stringValue.c_str(), &end, 10); + + if (end == stringValue.c_str() || *end != '\0') { + *value = 0; + return false; + } + + return true; +} + +const char *ParsedMessage::getContent() const { + return mContent.c_str(); +} + +ssize_t ParsedMessage::parse(const char *data, size_t size, bool noMoreData) { + if (size == 0) { + return -1; + } + + ssize_t lastDictIndex = -1; + + size_t offset = 0; + bool headersComplete = false; + while (offset < size) { + size_t lineEndOffset = offset; + while (lineEndOffset + 1 < size + && (data[lineEndOffset] != '\r' + || data[lineEndOffset + 1] != '\n')) { + ++lineEndOffset; + } + + if (lineEndOffset + 1 >= size) { + return -1; + } + + AString line(&data[offset], lineEndOffset - offset); + + if (offset == 0) { + // Special handling for the request/status line. + + mDict.add(AString("_"), line); + offset = lineEndOffset + 2; + + continue; + } + + if (lineEndOffset == offset) { + // An empty line separates headers from body. + headersComplete = true; + offset += 2; + break; + } + + if (line.c_str()[0] == ' ' || line.c_str()[0] == '\t') { + // Support for folded header values. + + if (lastDictIndex >= 0) { + // Otherwise it's malformed since the first header line + // cannot continue anything... + + AString &value = mDict.editValueAt(lastDictIndex); + value.append(line); + } + + offset = lineEndOffset + 2; + continue; + } + + ssize_t colonPos = line.find(":"); + if (colonPos >= 0) { + AString key(line, 0, colonPos); + key.trim(); + key.tolower(); + + line.erase(0, colonPos + 1); + + lastDictIndex = mDict.add(key, line); + } + + offset = lineEndOffset + 2; + } + + if (!headersComplete && (!noMoreData || offset == 0)) { + // We either saw the empty line separating headers from body + // or we saw at least the status line and know that no more data + // is going to follow. + return -1; + } + + for (size_t i = 0; i < mDict.size(); ++i) { + mDict.editValueAt(i).trim(); + } + + int32_t contentLength; + if (!findInt32("content-length", &contentLength) || contentLength < 0) { + contentLength = 0; + } + + size_t totalLength = offset + contentLength; + + if (size < totalLength) { + return -1; + } + + mContent.setTo(&data[offset], contentLength); + + return totalLength; +} + +bool ParsedMessage::getRequestField(size_t index, AString *field) const { + AString line; + CHECK(findString("_", &line)); + + size_t prevOffset = 0; + size_t offset = 0; + for (size_t i = 0; i <= index; ++i) { + if (offset >= line.size()) { + return false; + } + + ssize_t spacePos = line.find(" ", offset); + + if (spacePos < 0) { + spacePos = line.size(); + } + + prevOffset = offset; + offset = spacePos + 1; + } + + field->setTo(line, prevOffset, offset - prevOffset - 1); + + return true; +} + +bool ParsedMessage::getStatusCode(int32_t *statusCode) const { + AString statusCodeString; + if (!getRequestField(1, &statusCodeString)) { + *statusCode = 0; + return false; + } + + char *end; + *statusCode = strtol(statusCodeString.c_str(), &end, 10); + + if (*end != '\0' || end == statusCodeString.c_str() + || (*statusCode) < 100 || (*statusCode) > 999) { + *statusCode = 0; + return false; + } + + return true; +} + +AString ParsedMessage::debugString() const { + AString line; + CHECK(findString("_", &line)); + + line.append("\n"); + + for (size_t i = 0; i < mDict.size(); ++i) { + const AString &key = mDict.keyAt(i); + const AString &value = mDict.valueAt(i); + + if (key == AString("_")) { + continue; + } + + line.append(key); + line.append(": "); + line.append(value); + line.append("\n"); + } + + line.append("\n"); + line.append(mContent); + + return line; +} + +// static +bool ParsedMessage::GetAttribute( + const char *s, const char *key, AString *value) { + value->clear(); + + size_t keyLen = strlen(key); + + for (;;) { + while (isspace(*s)) { + ++s; + } + + const char *colonPos = strchr(s, ';'); + + size_t len = + (colonPos == NULL) ? strlen(s) : colonPos - s; + + if (len >= keyLen + 1 && s[keyLen] == '=' && !strncmp(s, key, keyLen)) { + value->setTo(&s[keyLen + 1], len - keyLen - 1); + return true; + } + + if (colonPos == NULL) { + return false; + } + + s = colonPos + 1; + } +} + +// static +bool ParsedMessage::GetInt32Attribute( + const char *s, const char *key, int32_t *value) { + AString stringValue; + if (!GetAttribute(s, key, &stringValue)) { + *value = 0; + return false; + } + + char *end; + *value = strtol(stringValue.c_str(), &end, 10); + + if (end == stringValue.c_str() || *end != '\0') { + *value = 0; + return false; + } + + return true; +} + +} // namespace android + diff --git a/media/libstagefright/SurfaceMediaSource.cpp b/media/libstagefright/SurfaceMediaSource.cpp new file mode 100644 index 0000000000..d7370551e0 --- /dev/null +++ b/media/libstagefright/SurfaceMediaSource.cpp @@ -0,0 +1,485 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +//#define LOG_NDEBUG 0 +#define LOG_TAG "SurfaceMediaSource" + +#include <inttypes.h> + +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/SurfaceMediaSource.h> +#include <media/stagefright/MediaDefs.h> +#include <media/stagefright/MetaData.h> +#include <OMX_IVCommon.h> +#include <media/hardware/HardwareAPI.h> +#include <media/hardware/MetadataBufferType.h> + +#include <ui/GraphicBuffer.h> +#include <gui/BufferItem.h> +#include <gui/ISurfaceComposer.h> +#include <OMX_Component.h> + +#include <utils/Log.h> +#include <utils/String8.h> + +#include <private/gui/ComposerService.h> + +namespace android { + +SurfaceMediaSource::SurfaceMediaSource(uint32_t bufferWidth, uint32_t bufferHeight) : + mWidth(bufferWidth), + mHeight(bufferHeight), + mCurrentSlot(BufferQueue::INVALID_BUFFER_SLOT), + mNumPendingBuffers(0), + mCurrentTimestamp(0), + mFrameRate(30), + mStarted(false), + mNumFramesReceived(0), + mNumFramesEncoded(0), + mFirstFrameTimestamp(0), + mMaxAcquiredBufferCount(4), // XXX double-check the default + mUseAbsoluteTimestamps(false) { + ALOGV("SurfaceMediaSource"); + + if (bufferWidth == 0 || bufferHeight == 0) { + ALOGE("Invalid dimensions %dx%d", bufferWidth, bufferHeight); + } + + BufferQueue::createBufferQueue(&mProducer, &mConsumer); + mConsumer->setDefaultBufferSize(bufferWidth, bufferHeight); + mConsumer->setConsumerUsageBits(GRALLOC_USAGE_HW_VIDEO_ENCODER | + GRALLOC_USAGE_HW_TEXTURE); + + sp<ISurfaceComposer> composer(ComposerService::getComposerService()); + + // Note that we can't create an sp<...>(this) in a ctor that will not keep a + // reference once the ctor ends, as that would cause the refcount of 'this' + // dropping to 0 at the end of the ctor. Since all we need is a wp<...> + // that's what we create. + wp<ConsumerListener> listener = static_cast<ConsumerListener*>(this); + sp<BufferQueue::ProxyConsumerListener> proxy = new BufferQueue::ProxyConsumerListener(listener); + + status_t err = mConsumer->consumerConnect(proxy, false); + if (err != NO_ERROR) { + ALOGE("SurfaceMediaSource: error connecting to BufferQueue: %s (%d)", + strerror(-err), err); + } +} + +SurfaceMediaSource::~SurfaceMediaSource() { + ALOGV("~SurfaceMediaSource"); + CHECK(!mStarted); +} + +nsecs_t SurfaceMediaSource::getTimestamp() { + ALOGV("getTimestamp"); + Mutex::Autolock lock(mMutex); + return mCurrentTimestamp; +} + +void SurfaceMediaSource::setFrameAvailableListener( + const sp<FrameAvailableListener>& listener) { + ALOGV("setFrameAvailableListener"); + Mutex::Autolock lock(mMutex); + mFrameAvailableListener = listener; +} + +void SurfaceMediaSource::dumpState(String8& result) const +{ + char buffer[1024]; + dumpState(result, "", buffer, 1024); +} + +void SurfaceMediaSource::dumpState( + String8& result, + const char* /* prefix */, + char* buffer, + size_t /* SIZE */) const +{ + Mutex::Autolock lock(mMutex); + + result.append(buffer); + mConsumer->dumpState(result, ""); +} + +status_t SurfaceMediaSource::setFrameRate(int32_t fps) +{ + ALOGV("setFrameRate"); + Mutex::Autolock lock(mMutex); + const int MAX_FRAME_RATE = 60; + if (fps < 0 || fps > MAX_FRAME_RATE) { + return BAD_VALUE; + } + mFrameRate = fps; + return OK; +} + +MetadataBufferType SurfaceMediaSource::metaDataStoredInVideoBuffers() const { + ALOGV("isMetaDataStoredInVideoBuffers"); + return kMetadataBufferTypeANWBuffer; +} + +int32_t SurfaceMediaSource::getFrameRate( ) const { + ALOGV("getFrameRate"); + Mutex::Autolock lock(mMutex); + return mFrameRate; +} + +status_t SurfaceMediaSource::start(MetaData *params) +{ + ALOGV("start"); + + Mutex::Autolock lock(mMutex); + + CHECK(!mStarted); + + mStartTimeNs = 0; + int64_t startTimeUs; + int32_t bufferCount = 0; + if (params) { + if (params->findInt64(kKeyTime, &startTimeUs)) { + mStartTimeNs = startTimeUs * 1000; + } + + if (!params->findInt32(kKeyNumBuffers, &bufferCount)) { + ALOGE("Failed to find the advertised buffer count"); + return UNKNOWN_ERROR; + } + + if (bufferCount <= 1) { + ALOGE("bufferCount %d is too small", bufferCount); + return BAD_VALUE; + } + + mMaxAcquiredBufferCount = bufferCount; + } + + CHECK_GT(mMaxAcquiredBufferCount, 1u); + + status_t err = + mConsumer->setMaxAcquiredBufferCount(mMaxAcquiredBufferCount); + + if (err != OK) { + return err; + } + + mNumPendingBuffers = 0; + mStarted = true; + + return OK; +} + +status_t SurfaceMediaSource::setMaxAcquiredBufferCount(size_t count) { + ALOGV("setMaxAcquiredBufferCount(%zu)", count); + Mutex::Autolock lock(mMutex); + + CHECK_GT(count, 1u); + mMaxAcquiredBufferCount = count; + + return OK; +} + +status_t SurfaceMediaSource::setUseAbsoluteTimestamps() { + ALOGV("setUseAbsoluteTimestamps"); + Mutex::Autolock lock(mMutex); + mUseAbsoluteTimestamps = true; + + return OK; +} + +status_t SurfaceMediaSource::stop() +{ + ALOGV("stop"); + Mutex::Autolock lock(mMutex); + + if (!mStarted) { + return OK; + } + + mStarted = false; + mFrameAvailableCondition.signal(); + + while (mNumPendingBuffers > 0) { + ALOGI("Still waiting for %zu buffers to be returned.", + mNumPendingBuffers); + +#if DEBUG_PENDING_BUFFERS + for (size_t i = 0; i < mPendingBuffers.size(); ++i) { + ALOGI("%zu: %p", i, mPendingBuffers.itemAt(i)); + } +#endif + + mMediaBuffersAvailableCondition.wait(mMutex); + } + + mMediaBuffersAvailableCondition.signal(); + + return mConsumer->consumerDisconnect(); +} + +sp<MetaData> SurfaceMediaSource::getFormat() +{ + ALOGV("getFormat"); + + Mutex::Autolock lock(mMutex); + sp<MetaData> meta = new MetaData; + + meta->setInt32(kKeyWidth, mWidth); + meta->setInt32(kKeyHeight, mHeight); + // The encoder format is set as an opaque colorformat + // The encoder will later find out the actual colorformat + // from the GL Frames itself. + meta->setInt32(kKeyColorFormat, OMX_COLOR_FormatAndroidOpaque); + meta->setInt32(kKeyStride, mWidth); + meta->setInt32(kKeySliceHeight, mHeight); + meta->setInt32(kKeyFrameRate, mFrameRate); + meta->setCString(kKeyMIMEType, MEDIA_MIMETYPE_VIDEO_RAW); + return meta; +} + +// Pass the data to the MediaBuffer. Pass in only the metadata +// Note: Call only when you have the lock +void SurfaceMediaSource::passMetadataBuffer_l(MediaBufferBase **buffer, + ANativeWindowBuffer *bufferHandle) const { + *buffer = new MediaBuffer(sizeof(VideoNativeMetadata)); + VideoNativeMetadata *data = (VideoNativeMetadata *)(*buffer)->data(); + if (data == NULL) { + ALOGE("Cannot allocate memory for metadata buffer!"); + return; + } + data->eType = metaDataStoredInVideoBuffers(); + data->pBuffer = bufferHandle; + data->nFenceFd = -1; + ALOGV("handle = %p, offset = %zu, length = %zu", + bufferHandle, (*buffer)->range_length(), (*buffer)->range_offset()); +} + +status_t SurfaceMediaSource::read( + MediaBufferBase **buffer, const ReadOptions * /* options */) { + ALOGV("read"); + Mutex::Autolock lock(mMutex); + + *buffer = NULL; + + while (mStarted && mNumPendingBuffers == mMaxAcquiredBufferCount) { + mMediaBuffersAvailableCondition.wait(mMutex); + } + + // Update the current buffer info + // TODO: mCurrentSlot can be made a bufferstate since there + // can be more than one "current" slots. + + BufferItem item; + // If the recording has started and the queue is empty, then just + // wait here till the frames come in from the client side + while (mStarted) { + + status_t err = mConsumer->acquireBuffer(&item, 0); + if (err == BufferQueue::NO_BUFFER_AVAILABLE) { + // wait for a buffer to be queued + mFrameAvailableCondition.wait(mMutex); + } else if (err == OK) { + err = item.mFence->waitForever("SurfaceMediaSource::read"); + if (err) { + ALOGW("read: failed to wait for buffer fence: %d", err); + } + + // First time seeing the buffer? Added it to the SMS slot + if (item.mGraphicBuffer != NULL) { + mSlots[item.mSlot].mGraphicBuffer = item.mGraphicBuffer; + } + mSlots[item.mSlot].mFrameNumber = item.mFrameNumber; + + // check for the timing of this buffer + if (mNumFramesReceived == 0 && !mUseAbsoluteTimestamps) { + mFirstFrameTimestamp = item.mTimestamp; + // Initial delay + if (mStartTimeNs > 0) { + if (item.mTimestamp < mStartTimeNs) { + // This frame predates start of record, discard + mConsumer->releaseBuffer( + item.mSlot, item.mFrameNumber, EGL_NO_DISPLAY, + EGL_NO_SYNC_KHR, Fence::NO_FENCE); + continue; + } + mStartTimeNs = item.mTimestamp - mStartTimeNs; + } + } + item.mTimestamp = mStartTimeNs + (item.mTimestamp - mFirstFrameTimestamp); + + mNumFramesReceived++; + + break; + } else { + ALOGE("read: acquire failed with error code %d", err); + return ERROR_END_OF_STREAM; + } + + } + + // If the loop was exited as a result of stopping the recording, + // it is OK + if (!mStarted) { + ALOGV("Read: SurfaceMediaSource is stopped. Returning ERROR_END_OF_STREAM."); + return ERROR_END_OF_STREAM; + } + + mCurrentSlot = item.mSlot; + + // First time seeing the buffer? Added it to the SMS slot + if (item.mGraphicBuffer != NULL) { + mSlots[item.mSlot].mGraphicBuffer = item.mGraphicBuffer; + } + mSlots[item.mSlot].mFrameNumber = item.mFrameNumber; + + mCurrentBuffers.push_back(mSlots[mCurrentSlot].mGraphicBuffer); + int64_t prevTimeStamp = mCurrentTimestamp; + mCurrentTimestamp = item.mTimestamp; + + mNumFramesEncoded++; + // Pass the data to the MediaBuffer. Pass in only the metadata + + passMetadataBuffer_l(buffer, mSlots[mCurrentSlot].mGraphicBuffer->getNativeBuffer()); + + (*buffer)->setObserver(this); + (*buffer)->add_ref(); + (*buffer)->meta_data().setInt64(kKeyTime, mCurrentTimestamp / 1000); + ALOGV("Frames encoded = %d, timestamp = %" PRId64 ", time diff = %" PRId64, + mNumFramesEncoded, mCurrentTimestamp / 1000, + mCurrentTimestamp / 1000 - prevTimeStamp / 1000); + + ++mNumPendingBuffers; + +#if DEBUG_PENDING_BUFFERS + mPendingBuffers.push_back(*buffer); +#endif + + ALOGV("returning mbuf %p", *buffer); + + return OK; +} + +static buffer_handle_t getMediaBufferHandle(MediaBufferBase *buffer) { + // need to convert to char* for pointer arithmetic and then + // copy the byte stream into our handle + buffer_handle_t bufferHandle; + VideoNativeMetadata *data = (VideoNativeMetadata *)buffer->data(); + ANativeWindowBuffer *anwbuffer = (ANativeWindowBuffer *)data->pBuffer; + bufferHandle = anwbuffer->handle; + return bufferHandle; +} + +void SurfaceMediaSource::signalBufferReturned(MediaBufferBase *buffer) { + ALOGV("signalBufferReturned"); + + bool foundBuffer = false; + + Mutex::Autolock lock(mMutex); + + buffer_handle_t bufferHandle = getMediaBufferHandle(buffer); + ANativeWindowBuffer* curNativeHandle = NULL; + + for (size_t i = 0; i < mCurrentBuffers.size(); i++) { + curNativeHandle = mCurrentBuffers[i]->getNativeBuffer(); + if ((mCurrentBuffers[i]->handle == bufferHandle) || + ((buffer_handle_t)curNativeHandle == bufferHandle)) { + mCurrentBuffers.removeAt(i); + foundBuffer = true; + break; + } + } + + if (!foundBuffer) { + ALOGW("returned buffer was not found in the current buffer list"); + } + + for (int id = 0; id < BufferQueue::NUM_BUFFER_SLOTS; id++) { + if (mSlots[id].mGraphicBuffer == NULL) { + continue; + } + + curNativeHandle = mSlots[id].mGraphicBuffer->getNativeBuffer(); + + if ((bufferHandle == mSlots[id].mGraphicBuffer->handle) || + (bufferHandle == (buffer_handle_t)curNativeHandle)) { + ALOGV("Slot %d returned, matches handle = %p", id, + mSlots[id].mGraphicBuffer->handle); + + mConsumer->releaseBuffer(id, mSlots[id].mFrameNumber, + EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, + Fence::NO_FENCE); + + buffer->setObserver(0); + buffer->release(); + + foundBuffer = true; + break; + } + } + + if (!foundBuffer) { + CHECK(!"signalBufferReturned: bogus buffer"); + } + +#if DEBUG_PENDING_BUFFERS + for (size_t i = 0; i < mPendingBuffers.size(); ++i) { + if (mPendingBuffers.itemAt(i) == buffer) { + mPendingBuffers.removeAt(i); + break; + } + } +#endif + + --mNumPendingBuffers; + mMediaBuffersAvailableCondition.broadcast(); +} + +// Part of the BufferQueue::ConsumerListener +void SurfaceMediaSource::onFrameAvailable(const BufferItem& /* item */) { + ALOGV("onFrameAvailable"); + + sp<FrameAvailableListener> listener; + { // scope for the lock + Mutex::Autolock lock(mMutex); + mFrameAvailableCondition.broadcast(); + listener = mFrameAvailableListener; + } + + if (listener != NULL) { + ALOGV("actually calling onFrameAvailable"); + listener->onFrameAvailable(); + } +} + +// SurfaceMediaSource hijacks this event to assume +// the prodcuer is disconnecting from the BufferQueue +// and that it should stop the recording +void SurfaceMediaSource::onBuffersReleased() { + ALOGV("onBuffersReleased"); + + Mutex::Autolock lock(mMutex); + + mFrameAvailableCondition.signal(); + + for (int i = 0; i < BufferQueue::NUM_BUFFER_SLOTS; i++) { + mSlots[i].mGraphicBuffer = 0; + } +} + +void SurfaceMediaSource::onSidebandStreamChanged() { + ALOG_ASSERT(false, "SurfaceMediaSource can't consume sideband streams"); +} + +} // end of namespace android diff --git a/media/libstagefright/SurfaceUtils.cpp b/media/libstagefright/SurfaceUtils.cpp index c284ef7d06..1b3816d3b5 100644 --- a/media/libstagefright/SurfaceUtils.cpp +++ b/media/libstagefright/SurfaceUtils.cpp @@ -111,8 +111,9 @@ status_t setNativeWindowSizeFormatAndUsage( } } - int finalUsage = usage | consumerUsage; - ALOGV("gralloc usage: %#x(producer) + %#x(consumer) = %#x", usage, consumerUsage, finalUsage); + uint64_t finalUsage = (usage | consumerUsage) & 0xffffffffLL; + ALOGV("gralloc usage: %#x(producer) + %#x(consumer) = %#" PRIx64, + usage, consumerUsage, finalUsage); err = native_window_set_usage(nativeWindow, finalUsage); if (err != NO_ERROR) { ALOGE("native_window_set_usage failed: %s (%d)", strerror(-err), -err); @@ -126,7 +127,7 @@ status_t setNativeWindowSizeFormatAndUsage( return err; } - ALOGD("set up nativeWindow %p for %dx%d, color %#x, rotation %d, usage %#x", + ALOGD("set up nativeWindow %p for %dx%d, color %#x, rotation %d, usage %#" PRIx64, nativeWindow, width, height, format, rotation, finalUsage); return NO_ERROR; } diff --git a/media/libstagefright/bqhelper/Android.bp b/media/libstagefright/bqhelper/Android.bp index 8698d33f9b..9e2a339391 100644 --- a/media/libstagefright/bqhelper/Android.bp +++ b/media/libstagefright/bqhelper/Android.bp @@ -1,5 +1,6 @@ cc_defaults { name: "libstagefright_bufferqueue-defaults", + defaults: ["stagefright_qcom_legacy_defaults"], double_loadable: true, srcs: [ diff --git a/media/libstagefright/bqhelper/GraphicBufferSource.cpp b/media/libstagefright/bqhelper/GraphicBufferSource.cpp index cff14ac185..d031e91f5a 100644 --- a/media/libstagefright/bqhelper/GraphicBufferSource.cpp +++ b/media/libstagefright/bqhelper/GraphicBufferSource.cpp @@ -891,11 +891,13 @@ status_t GraphicBufferSource::submitBuffer_l(const VideoBuffer &item) { return UNKNOWN_ERROR; } +#ifndef QCOM_BSP_LEGACY if ((android_dataspace)item.mDataspace != mLastDataspace) { onDataspaceChanged_l( item.mDataspace, (android_pixel_format)item.mBuffer->getGraphicBuffer()->format); } +#endif std::shared_ptr<AcquiredBuffer> buffer = item.mBuffer; // use a GraphicBuffer for now as component is using GraphicBuffers to hold references diff --git a/media/libstagefright/codecs/opus/dec/SoftOpus.cpp b/media/libstagefright/codecs/opus/dec/SoftOpus.cpp index 4f61aa8be2..5bb1879da1 100644 --- a/media/libstagefright/codecs/opus/dec/SoftOpus.cpp +++ b/media/libstagefright/codecs/opus/dec/SoftOpus.cpp @@ -58,6 +58,8 @@ SoftOpus::SoftOpus( mInputBufferCount(0), mDecoder(NULL), mHeader(NULL), + mNumChannels(1), + mSamplingRate(kRate), mCodecDelay(0), mSeekPreRoll(0), mAnchorTimeUs(0), @@ -169,11 +171,11 @@ OMX_ERRORTYPE SoftOpus::internalGetParameter( } opusParams->nAudioBandWidth = 0; - opusParams->nSampleRate = kRate; + opusParams->nSampleRate = mSamplingRate; opusParams->nBitRate = 0; if (!isConfigured()) { - opusParams->nChannels = 1; + opusParams->nChannels = mNumChannels; } else { opusParams->nChannels = mHeader->channels; } @@ -274,7 +276,8 @@ OMX_ERRORTYPE SoftOpus::internalSetParameter( if (opusParams->nPortIndex != 0) { return OMX_ErrorUndefined; } - + mNumChannels = opusParams->nChannels; + mSamplingRate = opusParams->nSampleRate; return OMX_ErrorNone; } @@ -496,6 +499,8 @@ void SoftOpus::onQueueFilled(OMX_U32 /* portIndex */) { *(reinterpret_cast<int64_t*>(inHeader->pBuffer + inHeader->nOffset)), kRate); + mSamplingRate = kRate; + mNumChannels = mHeader->channels; notify(OMX_EventPortSettingsChanged, 1, 0, NULL); mOutputPortSettingsChange = AWAITING_DISABLED; } diff --git a/media/libstagefright/codecs/opus/dec/SoftOpus.h b/media/libstagefright/codecs/opus/dec/SoftOpus.h index 91cafa14c7..00058c8212 100644 --- a/media/libstagefright/codecs/opus/dec/SoftOpus.h +++ b/media/libstagefright/codecs/opus/dec/SoftOpus.h @@ -70,6 +70,8 @@ private: OpusMSDecoder *mDecoder; OpusHeader *mHeader; + int32_t mNumChannels; + int32_t mSamplingRate; int64_t mCodecDelay; int64_t mSeekPreRoll; int64_t mSamplesToDiscard; diff --git a/media/libstagefright/include/media/stagefright/ACodec.h b/media/libstagefright/include/media/stagefright/ACodec.h index 83e92b9f43..8ef92783ec 100644 --- a/media/libstagefright/include/media/stagefright/ACodec.h +++ b/media/libstagefright/include/media/stagefright/ACodec.h @@ -147,6 +147,7 @@ private: kWhatReleaseCodecInstance = 'relC', kWhatForceStateTransition = 'fstt', kWhatCheckIfStuck = 'Cstk', + kWhatSubmitExtraOutputMetadataBuffer = 'sbxo', }; enum { @@ -272,6 +273,7 @@ private: bool mShutdownInProgress; bool mExplicitShutdown; bool mIsLegacyVP9Decoder; + bool mIsLowLatency; // If "mKeepComponentAllocated" we only transition back to Loaded state // and do not release the component instance. @@ -499,6 +501,7 @@ private: status_t setupAMRCodec(bool encoder, bool isWAMR, int32_t bitRate); status_t setupG711Codec(bool encoder, int32_t sampleRate, int32_t numChannels); + status_t setupOpusCodec(bool encoder, int32_t sampleRate, int32_t numChannels); status_t setupFlacCodec( bool encoder, int32_t numChannels, int32_t sampleRate, int32_t compressionLevel, AudioEncoding encoding); diff --git a/media/libstagefright/include/media/stagefright/ANetworkSession.h b/media/libstagefright/include/media/stagefright/ANetworkSession.h new file mode 100644 index 0000000000..fd3ebaaa28 --- /dev/null +++ b/media/libstagefright/include/media/stagefright/ANetworkSession.h @@ -0,0 +1,135 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef A_NETWORK_SESSION_H_ + +#define A_NETWORK_SESSION_H_ + +#include <media/stagefright/foundation/ABase.h> +#include <utils/KeyedVector.h> +#include <utils/RefBase.h> +#include <utils/Thread.h> + +#include <netinet/in.h> + +namespace android { + +struct AMessage; + +// Helper class to manage a number of live sockets (datagram and stream-based) +// on a single thread. Clients are notified about activity through AMessages. +struct ANetworkSession : public RefBase { + ANetworkSession(); + + status_t start(); + status_t stop(); + + status_t createRTSPClient( + const char *host, unsigned port, const sp<AMessage> ¬ify, + int32_t *sessionID); + + status_t createRTSPServer( + const struct in_addr &addr, unsigned port, + const sp<AMessage> ¬ify, int32_t *sessionID); + + status_t createUDPSession( + unsigned localPort, const sp<AMessage> ¬ify, int32_t *sessionID); + + status_t createUDPSession( + unsigned localPort, + const char *remoteHost, + unsigned remotePort, + const sp<AMessage> ¬ify, + int32_t *sessionID); + + status_t connectUDPSession( + int32_t sessionID, const char *remoteHost, unsigned remotePort); + + // passive + status_t createTCPDatagramSession( + const struct in_addr &addr, unsigned port, + const sp<AMessage> ¬ify, int32_t *sessionID); + + // active + status_t createTCPDatagramSession( + unsigned localPort, + const char *remoteHost, + unsigned remotePort, + const sp<AMessage> ¬ify, + int32_t *sessionID); + + status_t destroySession(int32_t sessionID); + + status_t sendRequest( + int32_t sessionID, const void *data, ssize_t size = -1, + bool timeValid = false, int64_t timeUs = -1ll); + + status_t switchToWebSocketMode(int32_t sessionID); + + enum NotificationReason { + kWhatError, + kWhatConnected, + kWhatClientConnected, + kWhatData, + kWhatDatagram, + kWhatBinaryData, + kWhatWebSocketMessage, + kWhatNetworkStall, + }; + +protected: + virtual ~ANetworkSession(); + +private: + struct NetworkThread; + struct Session; + + Mutex mLock; + sp<Thread> mThread; + + int32_t mNextSessionID; + + int mPipeFd[2]; + + KeyedVector<int32_t, sp<Session> > mSessions; + + enum Mode { + kModeCreateUDPSession, + kModeCreateTCPDatagramSessionPassive, + kModeCreateTCPDatagramSessionActive, + kModeCreateRTSPServer, + kModeCreateRTSPClient, + }; + status_t createClientOrServer( + Mode mode, + const struct in_addr *addr, + unsigned port, + const char *remoteHost, + unsigned remotePort, + const sp<AMessage> ¬ify, + int32_t *sessionID); + + void threadLoop(); + void interrupt(); + + static status_t MakeSocketNonBlocking(int s); + + DISALLOW_EVIL_CONSTRUCTORS(ANetworkSession); +}; + +} // namespace android + +#endif // A_NETWORK_SESSION_H_ diff --git a/media/libstagefright/include/media/stagefright/ParsedMessage.h b/media/libstagefright/include/media/stagefright/ParsedMessage.h new file mode 100644 index 0000000000..9d43a93319 --- /dev/null +++ b/media/libstagefright/include/media/stagefright/ParsedMessage.h @@ -0,0 +1,60 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <media/stagefright/foundation/ABase.h> +#include <media/stagefright/foundation/AString.h> +#include <utils/KeyedVector.h> +#include <utils/RefBase.h> + +namespace android { + +// Encapsulates an "HTTP/RTSP style" response, i.e. a status line, +// key/value pairs making up the headers and an optional body/content. +struct ParsedMessage : public RefBase { + static sp<ParsedMessage> Parse( + const char *data, size_t size, bool noMoreData, size_t *length); + + bool findString(const char *name, AString *value) const; + bool findInt32(const char *name, int32_t *value) const; + + const char *getContent() const; + + bool getRequestField(size_t index, AString *field) const; + bool getStatusCode(int32_t *statusCode) const; + + AString debugString() const; + + static bool GetAttribute(const char *s, const char *key, AString *value); + + static bool GetInt32Attribute( + const char *s, const char *key, int32_t *value); + + +protected: + virtual ~ParsedMessage(); + +private: + KeyedVector<AString, AString> mDict; + AString mContent; + + ParsedMessage(); + + ssize_t parse(const char *data, size_t size, bool noMoreData); + + DISALLOW_EVIL_CONSTRUCTORS(ParsedMessage); +}; + +} // namespace android diff --git a/media/libstagefright/include/media/stagefright/SurfaceMediaSource.h b/media/libstagefright/include/media/stagefright/SurfaceMediaSource.h new file mode 100644 index 0000000000..c67defe30b --- /dev/null +++ b/media/libstagefright/include/media/stagefright/SurfaceMediaSource.h @@ -0,0 +1,248 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ANDROID_GUI_SURFACEMEDIASOURCE_H +#define ANDROID_GUI_SURFACEMEDIASOURCE_H + +#include <gui/IGraphicBufferProducer.h> +#include <gui/BufferQueue.h> + +#include <utils/threads.h> +#include <utils/Vector.h> +#include <media/stagefright/MediaSource.h> +#include <media/stagefright/MediaBuffer.h> + +#include <media/hardware/MetadataBufferType.h> + +#include <media/stagefright/foundation/ABase.h> + +namespace android { +// ---------------------------------------------------------------------------- + +class String8; +class GraphicBuffer; + +// ASSUMPTIONS +// 1. SurfaceMediaSource is initialized with width*height which +// can never change. However, deqeueue buffer does not currently +// enforce this as in BufferQueue, dequeue can be used by Surface +// which can modify the default width and heght. Also neither the width +// nor height can be 0. +// 2. setSynchronousMode is never used (basically no one should call +// setSynchronousMode(false) +// 3. setCrop, setTransform, setScalingMode should never be used +// 4. queueBuffer returns a filled buffer to the SurfaceMediaSource. In addition, a +// timestamp must be provided for the buffer. The timestamp is in +// nanoseconds, and must be monotonically increasing. Its other semantics +// (zero point, etc) are client-dependent and should be documented by the +// client. +// 5. Once disconnected, SurfaceMediaSource can be reused (can not +// connect again) +// 6. Stop is a hard stop, the last few frames held by the encoder +// may be dropped. It is possible to wait for the buffers to be +// returned (but not implemented) + +#define DEBUG_PENDING_BUFFERS 0 + +class SurfaceMediaSource : public MediaSource, + public MediaBufferObserver, + protected ConsumerListener { +public: + enum { MIN_UNDEQUEUED_BUFFERS = 4}; + + struct FrameAvailableListener : public virtual RefBase { + // onFrameAvailable() is called from queueBuffer() is the FIFO is + // empty. You can use SurfaceMediaSource::getQueuedCount() to + // figure out if there are more frames waiting. + // This is called without any lock held can be called concurrently by + // multiple threads. + virtual void onFrameAvailable() = 0; + }; + + SurfaceMediaSource(uint32_t bufferWidth, uint32_t bufferHeight); + + virtual ~SurfaceMediaSource(); + + // For the MediaSource interface for use by StageFrightRecorder: + virtual status_t start(MetaData *params = NULL); + virtual status_t stop(); + virtual status_t read(MediaBufferBase **buffer, + const ReadOptions *options = NULL); + virtual sp<MetaData> getFormat(); + + // Get / Set the frame rate used for encoding. Default fps = 30 + status_t setFrameRate(int32_t fps) ; + int32_t getFrameRate( ) const; + + // The call for the StageFrightRecorder to tell us that + // it is done using the MediaBuffer data so that its state + // can be set to FREE for dequeuing + virtual void signalBufferReturned(MediaBufferBase* buffer); + // end of MediaSource interface + + // getTimestamp retrieves the timestamp associated with the image + // set by the most recent call to read() + // + // The timestamp is in nanoseconds, and is monotonically increasing. Its + // other semantics (zero point, etc) are source-dependent and should be + // documented by the source. + int64_t getTimestamp(); + + // setFrameAvailableListener sets the listener object that will be notified + // when a new frame becomes available. + void setFrameAvailableListener(const sp<FrameAvailableListener>& listener); + + // dump our state in a String + void dumpState(String8& result) const; + void dumpState(String8& result, const char* prefix, char* buffer, + size_t SIZE) const; + + // metaDataStoredInVideoBuffers tells the encoder what kind of metadata + // is passed through the buffers. Currently, it is set to ANWBuffer + MetadataBufferType metaDataStoredInVideoBuffers() const; + + sp<IGraphicBufferProducer> getProducer() const { return mProducer; } + + // To be called before start() + status_t setMaxAcquiredBufferCount(size_t count); + + // To be called before start() + status_t setUseAbsoluteTimestamps(); + +protected: + + // Implementation of the BufferQueue::ConsumerListener interface. These + // calls are used to notify the Surface of asynchronous events in the + // BufferQueue. + virtual void onFrameAvailable(const BufferItem& item); + + // Used as a hook to BufferQueue::disconnect() + // This is called by the client side when it is done + // TODO: Currently, this also sets mStopped to true which + // is needed for unblocking the encoder which might be + // waiting to read more frames. So if on the client side, + // the same thread supplies the frames and also calls stop + // on the encoder, the client has to call disconnect before + // it calls stop. + // In the case of the camera, + // that need not be required since the thread supplying the + // frames is separate than the one calling stop. + virtual void onBuffersReleased(); + + // SurfaceMediaSource can't handle sideband streams, so this is not expected + // to ever be called. Does nothing. + virtual void onSidebandStreamChanged(); + + static bool isExternalFormat(uint32_t format); + +private: + // A BufferQueue, represented by these interfaces, is the exchange point + // between the producer and this consumer + sp<IGraphicBufferProducer> mProducer; + sp<IGraphicBufferConsumer> mConsumer; + + struct SlotData { + sp<GraphicBuffer> mGraphicBuffer; + uint64_t mFrameNumber; + }; + + // mSlots caches GraphicBuffers and frameNumbers from the buffer queue + SlotData mSlots[BufferQueue::NUM_BUFFER_SLOTS]; + + // The permenent width and height of SMS buffers + int mWidth; + int mHeight; + + // mCurrentSlot is the buffer slot index of the buffer that is currently + // being used by buffer consumer + // (e.g. StageFrightRecorder in the case of SurfaceMediaSource or GLTexture + // in the case of Surface). + // It is initialized to INVALID_BUFFER_SLOT, + // indicating that no buffer slot is currently bound to the texture. Note, + // however, that a value of INVALID_BUFFER_SLOT does not necessarily mean + // that no buffer is bound to the texture. A call to setBufferCount will + // reset mCurrentTexture to INVALID_BUFFER_SLOT. + int mCurrentSlot; + + // mCurrentBuffers is a list of the graphic buffers that are being used by + // buffer consumer (i.e. the video encoder). It's possible that these + // buffers are not associated with any buffer slots, so we must track them + // separately. Buffers are added to this list in read, and removed from + // this list in signalBufferReturned + Vector<sp<GraphicBuffer> > mCurrentBuffers; + + size_t mNumPendingBuffers; + +#if DEBUG_PENDING_BUFFERS + Vector<MediaBufferBase *> mPendingBuffers; +#endif + + // mCurrentTimestamp is the timestamp for the current texture. It + // gets set to mLastQueuedTimestamp each time updateTexImage is called. + int64_t mCurrentTimestamp; + + // mFrameAvailableListener is the listener object that will be called when a + // new frame becomes available. If it is not NULL it will be called from + // queueBuffer. + sp<FrameAvailableListener> mFrameAvailableListener; + + // mMutex is the mutex used to prevent concurrent access to the member + // variables of SurfaceMediaSource objects. It must be locked whenever the + // member variables are accessed. + mutable Mutex mMutex; + + ////////////////////////// For MediaSource + // Set to a default of 30 fps if not specified by the client side + int32_t mFrameRate; + + // mStarted is a flag to check if the recording is going on + bool mStarted; + + // mNumFramesReceived indicates the number of frames recieved from + // the client side + int mNumFramesReceived; + // mNumFramesEncoded indicates the number of frames passed on to the + // encoder + int mNumFramesEncoded; + + // mFirstFrameTimestamp is the timestamp of the first received frame. + // It is used to offset the output timestamps so recording starts at time 0. + int64_t mFirstFrameTimestamp; + // mStartTimeNs is the start time passed into the source at start, used to + // offset timestamps. + int64_t mStartTimeNs; + + size_t mMaxAcquiredBufferCount; + + bool mUseAbsoluteTimestamps; + + // mFrameAvailableCondition condition used to indicate whether there + // is a frame available for dequeuing + Condition mFrameAvailableCondition; + + Condition mMediaBuffersAvailableCondition; + + // Allocate and return a new MediaBuffer and pass the ANW buffer as metadata into it. + void passMetadataBuffer_l(MediaBufferBase **buffer, ANativeWindowBuffer *bufferHandle) const; + + // Avoid copying and equating and default constructor + DISALLOW_EVIL_CONSTRUCTORS(SurfaceMediaSource); +}; + +// ---------------------------------------------------------------------------- +}; // namespace android + +#endif // ANDROID_GUI_SURFACEMEDIASOURCE_H diff --git a/media/libstagefright/omx/1.0/OmxStore.cpp b/media/libstagefright/omx/1.0/OmxStore.cpp index 67f478ead0..b5c116656f 100644 --- a/media/libstagefright/omx/1.0/OmxStore.cpp +++ b/media/libstagefright/omx/1.0/OmxStore.cpp @@ -54,6 +54,24 @@ OmxStore::OmxStore( }); } + if (!nodes.empty()) { + auto anyNode = nodes.cbegin(); + std::string::const_iterator first = anyNode->cbegin(); + std::string::const_iterator last = anyNode->cend(); + for (const std::string &name : nodes) { + std::string::const_iterator it1 = first; + for (std::string::const_iterator it2 = name.cbegin(); + it1 != last && it2 != name.cend() && tolower(*it1) == tolower(*it2); + ++it1, ++it2) { + } + last = it1; + } + mPrefix = std::string(first, last); + LOG(INFO) << "omx common prefix: '" << mPrefix.c_str() << "'"; + } else { + LOG(INFO) << "omx common prefix: no nodes"; + } + MediaCodecsXmlParser parser; parser.parseXmlFilesInSearchDirs(xmlNames, searchDirs); if (profilingResultsXmlPath != nullptr) { @@ -112,8 +130,6 @@ OmxStore::OmxStore( mRoleList[i] = std::move(role); ++i; } - - mPrefix = parser.getCommonPrefix(); } OmxStore::~OmxStore() { diff --git a/media/libstagefright/omx/Android.bp b/media/libstagefright/omx/Android.bp index 78b4f192d0..a049fc57ec 100644 --- a/media/libstagefright/omx/Android.bp +++ b/media/libstagefright/omx/Android.bp @@ -1,5 +1,6 @@ cc_library_shared { name: "libstagefright_omx", + defaults: ["stagefright_qcom_legacy_defaults"], vendor_available: true, vndk: { enabled: true, diff --git a/media/libstagefright/omx/OMXNodeInstance.cpp b/media/libstagefright/omx/OMXNodeInstance.cpp index ac42373bf0..d996e1a085 100644 --- a/media/libstagefright/omx/OMXNodeInstance.cpp +++ b/media/libstagefright/omx/OMXNodeInstance.cpp @@ -453,7 +453,11 @@ OMXNodeInstance::OMXNodeInstance( mGraphicBufferEnabled[0] = false; mGraphicBufferEnabled[1] = false; mIsSecure = AString(name).endsWith(".secure"); +#ifdef QCOM_BSP_LEGACY + mLegacyAdaptiveExperiment = true; +#else mLegacyAdaptiveExperiment = ADebug::isExperimentEnabled("legacy-adaptive"); +#endif } OMXNodeInstance::~OMXNodeInstance() { @@ -580,6 +584,10 @@ status_t OMXNodeInstance::freeNode() { break; } + if (mActiveBuffers.size() > 0) { + freeActiveBuffers(); + } + Mutex::Autolock _l(mLock); status_t err = mOwner->freeNode(this); diff --git a/media/libstagefright/tests/Android.bp b/media/libstagefright/tests/Android.bp index a7f94c1365..62be2fbf98 100644 --- a/media/libstagefright/tests/Android.bp +++ b/media/libstagefright/tests/Android.bp @@ -1,6 +1,45 @@ // Build the unit tests. cc_test { + name: "SurfaceMediaSource_test", + + srcs: [ + "SurfaceMediaSource_test.cpp", + "DummyRecorder.cpp", + ], + + shared_libs: [ + "libEGL", + "libGLESv2", + "libbinder", + "libcutils", + "libgui", + "libmedia", + "libstagefright", + "libstagefright_foundation", + "libstagefright_omx", + "libsync", + "libui", + "libutils", + "liblog", + ], + + include_dirs: [ + "frameworks/av/media/libstagefright", + "frameworks/av/media/libstagefright/include", + "frameworks/native/include/media/openmax", + "frameworks/native/include/media/hardware", + ], + + cflags: [ + "-Werror", + "-Wall", + ], + + compile_multilib: "32", +} + +cc_test { name: "MediaCodecListOverrides_test", srcs: ["MediaCodecListOverrides_test.cpp"], diff --git a/media/libstagefright/tests/DummyRecorder.cpp b/media/libstagefright/tests/DummyRecorder.cpp new file mode 100644 index 0000000000..596b7e6fd7 --- /dev/null +++ b/media/libstagefright/tests/DummyRecorder.cpp @@ -0,0 +1,91 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "DummyRecorder" +// #define LOG_NDEBUG 0 + +#include <media/stagefright/MediaSource.h> +#include <media/stagefright/MediaBuffer.h> +#include "DummyRecorder.h" + +#include <utils/Log.h> + +namespace android { + +// static +void *DummyRecorder::threadWrapper(void *pthis) { + ALOGV("ThreadWrapper: %p", pthis); + DummyRecorder *writer = static_cast<DummyRecorder *>(pthis); + writer->readFromSource(); + return NULL; +} + + +status_t DummyRecorder::start() { + ALOGV("Start"); + mStarted = true; + + mSource->start(); + + pthread_attr_t attr; + pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); + int err = pthread_create(&mThread, &attr, threadWrapper, this); + pthread_attr_destroy(&attr); + + if (err) { + ALOGE("Error creating thread!"); + return -ENODEV; + } + return OK; +} + + +status_t DummyRecorder::stop() { + ALOGV("Stop"); + mStarted = false; + + mSource->stop(); + void *dummy; + pthread_join(mThread, &dummy); + status_t err = static_cast<status_t>(reinterpret_cast<uintptr_t>(dummy)); + + ALOGV("Ending the reading thread"); + return err; +} + +// pretend to read the source buffers +void DummyRecorder::readFromSource() { + ALOGV("ReadFromSource"); + if (!mStarted) { + return; + } + + status_t err = OK; + MediaBufferBase *buffer; + ALOGV("A fake writer accessing the frames"); + while (mStarted && (err = mSource->read(&buffer)) == OK){ + // if not getting a valid buffer from source, then exit + if (buffer == NULL) { + return; + } + buffer->release(); + buffer = NULL; + } +} + + +} // end of namespace android diff --git a/media/libstagefright/tests/DummyRecorder.h b/media/libstagefright/tests/DummyRecorder.h new file mode 100644 index 0000000000..075977784f --- /dev/null +++ b/media/libstagefright/tests/DummyRecorder.h @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef DUMMY_RECORDER_H_ +#define DUMMY_RECORDER_H_ + +#include <pthread.h> +#include <utils/String8.h> +#include <media/stagefright/foundation/ABase.h> + + +namespace android { + +struct MediaSource; +class MediaBuffer; + +class DummyRecorder { + public: + // The media source from which this will receive frames + sp<MediaSource> mSource; + bool mStarted; + pthread_t mThread; + + status_t start(); + status_t stop(); + + // actual entry point for the thread + void readFromSource(); + + // static function to wrap the actual thread entry point + static void *threadWrapper(void *pthis); + + explicit DummyRecorder(const sp<MediaSource> &source) : mSource(source) + , mStarted(false) {} + ~DummyRecorder( ) {} + + private: + + DISALLOW_EVIL_CONSTRUCTORS(DummyRecorder); +}; + +} // end of namespace android +#endif + + diff --git a/media/libstagefright/tests/SurfaceMediaSource_test.cpp b/media/libstagefright/tests/SurfaceMediaSource_test.cpp new file mode 100644 index 0000000000..1b1c3b8cdb --- /dev/null +++ b/media/libstagefright/tests/SurfaceMediaSource_test.cpp @@ -0,0 +1,944 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "SurfaceMediaSource_test" + +#include <gtest/gtest.h> +#include <utils/String8.h> +#include <utils/String16.h> +#include <utils/Errors.h> +#include <fcntl.h> +#include <unistd.h> + +#include <GLES2/gl2.h> + +#include <media/stagefright/SurfaceMediaSource.h> +#include <media/mediarecorder.h> + +#include <ui/GraphicBuffer.h> +#include <gui/Surface.h> +#include <gui/ISurfaceComposer.h> +#include <gui/Surface.h> +#include <gui/SurfaceComposerClient.h> + +#include <binder/ProcessState.h> + +#include <media/stagefright/foundation/ADebug.h> +#include <OMX_Component.h> + +#include "DummyRecorder.h" + + +namespace android { + +class GLTest : public ::testing::Test { +protected: + + GLTest(): + mEglDisplay(EGL_NO_DISPLAY), + mEglSurface(EGL_NO_SURFACE), + mEglContext(EGL_NO_CONTEXT) { + } + + virtual void SetUp() { + ALOGV("GLTest::SetUp()"); + mEglDisplay = eglGetDisplay(EGL_DEFAULT_DISPLAY); + ASSERT_EQ(EGL_SUCCESS, eglGetError()); + ASSERT_NE(EGL_NO_DISPLAY, mEglDisplay); + + EGLint majorVersion; + EGLint minorVersion; + EXPECT_TRUE(eglInitialize(mEglDisplay, &majorVersion, &minorVersion)); + ASSERT_EQ(EGL_SUCCESS, eglGetError()); + RecordProperty("EglVersionMajor", majorVersion); + RecordProperty("EglVersionMajor", minorVersion); + + EGLint numConfigs = 0; + EXPECT_TRUE(eglChooseConfig(mEglDisplay, getConfigAttribs(), &mGlConfig, + 1, &numConfigs)); + ASSERT_EQ(EGL_SUCCESS, eglGetError()); + + char* displaySecsEnv = getenv("GLTEST_DISPLAY_SECS"); + if (displaySecsEnv != NULL) { + mDisplaySecs = atoi(displaySecsEnv); + if (mDisplaySecs < 0) { + mDisplaySecs = 0; + } + } else { + mDisplaySecs = 0; + } + + if (mDisplaySecs > 0) { + mComposerClient = new SurfaceComposerClient; + ASSERT_EQ(NO_ERROR, mComposerClient->initCheck()); + + mSurfaceControl = mComposerClient->createSurface( + String8("Test Surface"), + getSurfaceWidth(), getSurfaceHeight(), + PIXEL_FORMAT_RGB_888, 0); + + ASSERT_TRUE(mSurfaceControl != NULL); + ASSERT_TRUE(mSurfaceControl->isValid()); + + SurfaceComposerClient::Transaction{} + .setLayer(mSurfaceControl, 0x7FFFFFFF) + .show(mSurfaceControl) + .apply(); + + sp<ANativeWindow> window = mSurfaceControl->getSurface(); + mEglSurface = eglCreateWindowSurface(mEglDisplay, mGlConfig, + window.get(), NULL); + } else { + ALOGV("No actual display. Choosing EGLSurface based on SurfaceMediaSource"); + sp<IGraphicBufferProducer> sms = (new SurfaceMediaSource( + getSurfaceWidth(), getSurfaceHeight()))->getProducer(); + sp<Surface> stc = new Surface(sms); + sp<ANativeWindow> window = stc; + + mEglSurface = eglCreateWindowSurface(mEglDisplay, mGlConfig, + window.get(), NULL); + } + ASSERT_EQ(EGL_SUCCESS, eglGetError()); + ASSERT_NE(EGL_NO_SURFACE, mEglSurface); + + mEglContext = eglCreateContext(mEglDisplay, mGlConfig, EGL_NO_CONTEXT, + getContextAttribs()); + ASSERT_EQ(EGL_SUCCESS, eglGetError()); + ASSERT_NE(EGL_NO_CONTEXT, mEglContext); + + EXPECT_TRUE(eglMakeCurrent(mEglDisplay, mEglSurface, mEglSurface, + mEglContext)); + ASSERT_EQ(EGL_SUCCESS, eglGetError()); + + EGLint w, h; + EXPECT_TRUE(eglQuerySurface(mEglDisplay, mEglSurface, EGL_WIDTH, &w)); + ASSERT_EQ(EGL_SUCCESS, eglGetError()); + EXPECT_TRUE(eglQuerySurface(mEglDisplay, mEglSurface, EGL_HEIGHT, &h)); + ASSERT_EQ(EGL_SUCCESS, eglGetError()); + RecordProperty("EglSurfaceWidth", w); + RecordProperty("EglSurfaceHeight", h); + + glViewport(0, 0, w, h); + ASSERT_EQ(GLenum(GL_NO_ERROR), glGetError()); + } + + virtual void TearDown() { + // Display the result + if (mDisplaySecs > 0 && mEglSurface != EGL_NO_SURFACE) { + eglSwapBuffers(mEglDisplay, mEglSurface); + sleep(mDisplaySecs); + } + + if (mComposerClient != NULL) { + mComposerClient->dispose(); + } + if (mEglContext != EGL_NO_CONTEXT) { + eglDestroyContext(mEglDisplay, mEglContext); + } + if (mEglSurface != EGL_NO_SURFACE) { + eglDestroySurface(mEglDisplay, mEglSurface); + } + if (mEglDisplay != EGL_NO_DISPLAY) { + eglTerminate(mEglDisplay); + } + ASSERT_EQ(EGL_SUCCESS, eglGetError()); + } + + virtual EGLint const* getConfigAttribs() { + ALOGV("GLTest getConfigAttribs"); + static EGLint sDefaultConfigAttribs[] = { + EGL_SURFACE_TYPE, EGL_PBUFFER_BIT, + EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT, + EGL_RED_SIZE, 8, + EGL_GREEN_SIZE, 8, + EGL_BLUE_SIZE, 8, + EGL_ALPHA_SIZE, 8, + EGL_DEPTH_SIZE, 16, + EGL_STENCIL_SIZE, 8, + EGL_NONE }; + + return sDefaultConfigAttribs; + } + + virtual EGLint const* getContextAttribs() { + static EGLint sDefaultContextAttribs[] = { + EGL_CONTEXT_CLIENT_VERSION, 2, + EGL_NONE }; + + return sDefaultContextAttribs; + } + + virtual EGLint getSurfaceWidth() { + return 512; + } + + virtual EGLint getSurfaceHeight() { + return 512; + } + + void loadShader(GLenum shaderType, const char* pSource, GLuint* outShader) { + GLuint shader = glCreateShader(shaderType); + ASSERT_EQ(GLenum(GL_NO_ERROR), glGetError()); + if (shader) { + glShaderSource(shader, 1, &pSource, NULL); + ASSERT_EQ(GLenum(GL_NO_ERROR), glGetError()); + glCompileShader(shader); + ASSERT_EQ(GLenum(GL_NO_ERROR), glGetError()); + GLint compiled = 0; + glGetShaderiv(shader, GL_COMPILE_STATUS, &compiled); + ASSERT_EQ(GLenum(GL_NO_ERROR), glGetError()); + if (!compiled) { + GLint infoLen = 0; + glGetShaderiv(shader, GL_INFO_LOG_LENGTH, &infoLen); + ASSERT_EQ(GLenum(GL_NO_ERROR), glGetError()); + if (infoLen) { + char* buf = (char*) malloc(infoLen); + if (buf) { + glGetShaderInfoLog(shader, infoLen, NULL, buf); + printf("Shader compile log:\n%s\n", buf); + free(buf); + FAIL(); + } + } else { + char* buf = (char*) malloc(0x1000); + if (buf) { + glGetShaderInfoLog(shader, 0x1000, NULL, buf); + printf("Shader compile log:\n%s\n", buf); + free(buf); + FAIL(); + } + } + glDeleteShader(shader); + shader = 0; + } + } + ASSERT_TRUE(shader != 0); + *outShader = shader; + } + + void createProgram(const char* pVertexSource, const char* pFragmentSource, + GLuint* outPgm) { + GLuint vertexShader, fragmentShader; + { + SCOPED_TRACE("compiling vertex shader"); + loadShader(GL_VERTEX_SHADER, pVertexSource, &vertexShader); + if (HasFatalFailure()) { + return; + } + } + { + SCOPED_TRACE("compiling fragment shader"); + loadShader(GL_FRAGMENT_SHADER, pFragmentSource, &fragmentShader); + if (HasFatalFailure()) { + return; + } + } + + GLuint program = glCreateProgram(); + ASSERT_EQ(GLenum(GL_NO_ERROR), glGetError()); + if (program) { + glAttachShader(program, vertexShader); + ASSERT_EQ(GLenum(GL_NO_ERROR), glGetError()); + glAttachShader(program, fragmentShader); + ASSERT_EQ(GLenum(GL_NO_ERROR), glGetError()); + glLinkProgram(program); + GLint linkStatus = GL_FALSE; + glGetProgramiv(program, GL_LINK_STATUS, &linkStatus); + if (linkStatus != GL_TRUE) { + GLint bufLength = 0; + glGetProgramiv(program, GL_INFO_LOG_LENGTH, &bufLength); + if (bufLength) { + char* buf = (char*) malloc(bufLength); + if (buf) { + glGetProgramInfoLog(program, bufLength, NULL, buf); + printf("Program link log:\n%s\n", buf); + free(buf); + FAIL(); + } + } + glDeleteProgram(program); + program = 0; + } + } + glDeleteShader(vertexShader); + glDeleteShader(fragmentShader); + ASSERT_TRUE(program != 0); + *outPgm = program; + } + + static int abs(int value) { + return value > 0 ? value : -value; + } + + ::testing::AssertionResult checkPixel(int x, int y, int r, + int g, int b, int a, int tolerance=2) { + GLubyte pixel[4]; + String8 msg; + glReadPixels(x, y, 1, 1, GL_RGBA, GL_UNSIGNED_BYTE, pixel); + GLenum err = glGetError(); + if (err != GL_NO_ERROR) { + msg += String8::format("error reading pixel: %#x", err); + while ((err = glGetError()) != GL_NO_ERROR) { + msg += String8::format(", %#x", err); + } + fprintf(stderr, "pixel check failure: %s\n", msg.string()); + return ::testing::AssertionFailure( + ::testing::Message(msg.string())); + } + if (r >= 0 && abs(r - int(pixel[0])) > tolerance) { + msg += String8::format("r(%d isn't %d)", pixel[0], r); + } + if (g >= 0 && abs(g - int(pixel[1])) > tolerance) { + if (!msg.isEmpty()) { + msg += " "; + } + msg += String8::format("g(%d isn't %d)", pixel[1], g); + } + if (b >= 0 && abs(b - int(pixel[2])) > tolerance) { + if (!msg.isEmpty()) { + msg += " "; + } + msg += String8::format("b(%d isn't %d)", pixel[2], b); + } + if (a >= 0 && abs(a - int(pixel[3])) > tolerance) { + if (!msg.isEmpty()) { + msg += " "; + } + msg += String8::format("a(%d isn't %d)", pixel[3], a); + } + if (!msg.isEmpty()) { + fprintf(stderr, "pixel check failure: %s\n", msg.string()); + return ::testing::AssertionFailure( + ::testing::Message(msg.string())); + } else { + return ::testing::AssertionSuccess(); + } + } + + int mDisplaySecs; + sp<SurfaceComposerClient> mComposerClient; + sp<SurfaceControl> mSurfaceControl; + + EGLDisplay mEglDisplay; + EGLSurface mEglSurface; + EGLContext mEglContext; + EGLConfig mGlConfig; +}; + +/////////////////////////////////////////////////////////////////////// +// Class for the NON-GL tests +/////////////////////////////////////////////////////////////////////// +class SurfaceMediaSourceTest : public ::testing::Test { +public: + + SurfaceMediaSourceTest( ): mYuvTexWidth(176), mYuvTexHeight(144) { } + void oneBufferPass(int width, int height ); + void oneBufferPassNoFill(int width, int height ); + static void fillYV12Buffer(uint8_t* buf, int w, int h, int stride) ; + static void fillYV12BufferRect(uint8_t* buf, int w, int h, + int stride, const android_native_rect_t& rect) ; +protected: + + virtual void SetUp() { + android::ProcessState::self()->startThreadPool(); + mSMS = new SurfaceMediaSource(mYuvTexWidth, mYuvTexHeight); + mSTC = new Surface(mSMS->getProducer()); + mANW = mSTC; + } + + virtual void TearDown() { + mSMS.clear(); + mSTC.clear(); + mANW.clear(); + } + + const int mYuvTexWidth; + const int mYuvTexHeight; + + sp<SurfaceMediaSource> mSMS; + sp<Surface> mSTC; + sp<ANativeWindow> mANW; +}; + +/////////////////////////////////////////////////////////////////////// +// Class for the GL tests +/////////////////////////////////////////////////////////////////////// +class SurfaceMediaSourceGLTest : public GLTest { +public: + + SurfaceMediaSourceGLTest( ): mYuvTexWidth(176), mYuvTexHeight(144) { } + virtual EGLint const* getConfigAttribs(); + void oneBufferPassGL(int num = 0); + static sp<MediaRecorder> setUpMediaRecorder(int fileDescriptor, int videoSource, + int outputFormat, int videoEncoder, int width, int height, int fps); +protected: + + virtual void SetUp() { + ALOGV("SMS-GLTest::SetUp()"); + android::ProcessState::self()->startThreadPool(); + mSMS = new SurfaceMediaSource(mYuvTexWidth, mYuvTexHeight); + mSTC = new Surface(mSMS->getProducer()); + mANW = mSTC; + + // Doing the setup related to the GL Side + GLTest::SetUp(); + } + + virtual void TearDown() { + mSMS.clear(); + mSTC.clear(); + mANW.clear(); + GLTest::TearDown(); + } + + void setUpEGLSurfaceFromMediaRecorder(sp<MediaRecorder>& mr); + + const int mYuvTexWidth; + const int mYuvTexHeight; + + sp<SurfaceMediaSource> mSMS; + sp<Surface> mSTC; + sp<ANativeWindow> mANW; +}; + +///////////////////////////////////////////////////////////////////// +// Methods in SurfaceMediaSourceGLTest +///////////////////////////////////////////////////////////////////// +EGLint const* SurfaceMediaSourceGLTest::getConfigAttribs() { + ALOGV("SurfaceMediaSourceGLTest getConfigAttribs"); + static EGLint sDefaultConfigAttribs[] = { + EGL_SURFACE_TYPE, EGL_WINDOW_BIT, + EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT, + EGL_RED_SIZE, 8, + EGL_GREEN_SIZE, 8, + EGL_BLUE_SIZE, 8, + EGL_RECORDABLE_ANDROID, EGL_TRUE, + EGL_NONE }; + + return sDefaultConfigAttribs; +} + +// One pass of dequeuing and queuing a GLBuffer +void SurfaceMediaSourceGLTest::oneBufferPassGL(int num) { + int d = num % 50; + float f = 0.2f; // 0.1f * d; + + glClearColor(0, 0.3, 0, 0.6); + glClear(GL_COLOR_BUFFER_BIT); + + glEnable(GL_SCISSOR_TEST); + glScissor(4 + d, 4 + d, 4, 4); + glClearColor(1.0 - f, f, f, 1.0); + glClear(GL_COLOR_BUFFER_BIT); + + glScissor(24 + d, 48 + d, 4, 4); + glClearColor(f, 1.0 - f, f, 1.0); + glClear(GL_COLOR_BUFFER_BIT); + + glScissor(37 + d, 17 + d, 4, 4); + glClearColor(f, f, 1.0 - f, 1.0); + glClear(GL_COLOR_BUFFER_BIT); + + // The following call dequeues and queues the buffer + eglSwapBuffers(mEglDisplay, mEglSurface); + ASSERT_EQ(EGL_SUCCESS, eglGetError()); + glDisable(GL_SCISSOR_TEST); +} + +// Set up the MediaRecorder which runs in the same process as mediaserver +sp<MediaRecorder> SurfaceMediaSourceGLTest::setUpMediaRecorder(int fd, int videoSource, + int outputFormat, int videoEncoder, int width, int height, int fps) { + sp<MediaRecorder> mr = new MediaRecorder(String16()); + mr->setVideoSource(videoSource); + mr->setOutputFormat(outputFormat); + mr->setVideoEncoder(videoEncoder); + mr->setOutputFile(fd); + mr->setVideoSize(width, height); + mr->setVideoFrameRate(fps); + mr->prepare(); + ALOGV("Starting MediaRecorder..."); + CHECK_EQ((status_t)OK, mr->start()); + return mr; +} + +// query the mediarecorder for a surfacemeidasource and create an egl surface with that +void SurfaceMediaSourceGLTest::setUpEGLSurfaceFromMediaRecorder(sp<MediaRecorder>& mr) { + sp<IGraphicBufferProducer> iST = mr->querySurfaceMediaSourceFromMediaServer(); + mSTC = new Surface(iST); + mANW = mSTC; + + if (mEglSurface != EGL_NO_SURFACE) { + EXPECT_TRUE(eglDestroySurface(mEglDisplay, mEglSurface)); + mEglSurface = EGL_NO_SURFACE; + } + mEglSurface = eglCreateWindowSurface(mEglDisplay, mGlConfig, + mANW.get(), NULL); + ASSERT_EQ(EGL_SUCCESS, eglGetError()); + ASSERT_NE(EGL_NO_SURFACE, mEglSurface) ; + + EXPECT_TRUE(eglMakeCurrent(mEglDisplay, mEglSurface, mEglSurface, + mEglContext)); + ASSERT_EQ(EGL_SUCCESS, eglGetError()); +} + + +///////////////////////////////////////////////////////////////////// +// Methods in SurfaceMediaSourceTest +///////////////////////////////////////////////////////////////////// + +// One pass of dequeuing and queuing the buffer. Fill it in with +// cpu YV12 buffer +void SurfaceMediaSourceTest::oneBufferPass(int width, int height ) { + ANativeWindowBuffer* anb; + ASSERT_EQ(NO_ERROR, native_window_dequeue_buffer_and_wait(mANW.get(), &anb)); + ASSERT_TRUE(anb != NULL); + + + // Fill the buffer with the a checkerboard pattern + uint8_t* img = NULL; + sp<GraphicBuffer> buf(GraphicBuffer::from(anb)); + buf->lock(GRALLOC_USAGE_SW_WRITE_OFTEN, (void**)(&img)); + SurfaceMediaSourceTest::fillYV12Buffer(img, width, height, buf->getStride()); + buf->unlock(); + + ASSERT_EQ(NO_ERROR, mANW->queueBuffer(mANW.get(), buf->getNativeBuffer(), + -1)); +} + +// Dequeuing and queuing the buffer without really filling it in. +void SurfaceMediaSourceTest::oneBufferPassNoFill( + int /* width */, int /* height */) { + ANativeWindowBuffer* anb; + ASSERT_EQ(NO_ERROR, native_window_dequeue_buffer_and_wait(mANW.get(), &anb)); + ASSERT_TRUE(anb != NULL); + + // We do not fill the buffer in. Just queue it back. + sp<GraphicBuffer> buf(GraphicBuffer::from(anb)); + ASSERT_EQ(NO_ERROR, mANW->queueBuffer(mANW.get(), buf->getNativeBuffer(), + -1)); +} + +// Fill a YV12 buffer with a multi-colored checkerboard pattern +void SurfaceMediaSourceTest::fillYV12Buffer(uint8_t* buf, int w, int h, int stride) { + const int blockWidth = w > 16 ? w / 16 : 1; + const int blockHeight = h > 16 ? h / 16 : 1; + const int yuvTexOffsetY = 0; + int yuvTexStrideY = stride; + int yuvTexOffsetV = yuvTexStrideY * h; + int yuvTexStrideV = (yuvTexStrideY/2 + 0xf) & ~0xf; + int yuvTexOffsetU = yuvTexOffsetV + yuvTexStrideV * h/2; + int yuvTexStrideU = yuvTexStrideV; + for (int x = 0; x < w; x++) { + for (int y = 0; y < h; y++) { + int parityX = (x / blockWidth) & 1; + int parityY = (y / blockHeight) & 1; + unsigned char intensity = (parityX ^ parityY) ? 63 : 191; + buf[yuvTexOffsetY + (y * yuvTexStrideY) + x] = intensity; + if (x < w / 2 && y < h / 2) { + buf[yuvTexOffsetU + (y * yuvTexStrideU) + x] = intensity; + if (x * 2 < w / 2 && y * 2 < h / 2) { + buf[yuvTexOffsetV + (y*2 * yuvTexStrideV) + x*2 + 0] = + buf[yuvTexOffsetV + (y*2 * yuvTexStrideV) + x*2 + 1] = + buf[yuvTexOffsetV + ((y*2+1) * yuvTexStrideV) + x*2 + 0] = + buf[yuvTexOffsetV + ((y*2+1) * yuvTexStrideV) + x*2 + 1] = + intensity; + } + } + } + } +} + +// Fill a YV12 buffer with red outside a given rectangle and green inside it. +void SurfaceMediaSourceTest::fillYV12BufferRect(uint8_t* buf, int w, + int h, int stride, const android_native_rect_t& rect) { + const int yuvTexOffsetY = 0; + int yuvTexStrideY = stride; + int yuvTexOffsetV = yuvTexStrideY * h; + int yuvTexStrideV = (yuvTexStrideY/2 + 0xf) & ~0xf; + int yuvTexOffsetU = yuvTexOffsetV + yuvTexStrideV * h/2; + int yuvTexStrideU = yuvTexStrideV; + for (int x = 0; x < w; x++) { + for (int y = 0; y < h; y++) { + bool inside = rect.left <= x && x < rect.right && + rect.top <= y && y < rect.bottom; + buf[yuvTexOffsetY + (y * yuvTexStrideY) + x] = inside ? 240 : 64; + if (x < w / 2 && y < h / 2) { + bool inside = rect.left <= 2*x && 2*x < rect.right && + rect.top <= 2*y && 2*y < rect.bottom; + buf[yuvTexOffsetU + (y * yuvTexStrideU) + x] = 16; + buf[yuvTexOffsetV + (y * yuvTexStrideV) + x] = + inside ? 16 : 255; + } + } + } +} ///////// End of class SurfaceMediaSourceTest + +/////////////////////////////////////////////////////////////////// +// Class to imitate the recording ///////////////////////////// +// //////////////////////////////////////////////////////////////// +struct SimpleDummyRecorder { + sp<MediaSource> mSource; + + explicit SimpleDummyRecorder + (const sp<MediaSource> &source): mSource(source) {} + + status_t start() { return mSource->start();} + status_t stop() { return mSource->stop();} + + // fakes reading from a media source + status_t readFromSource() { + MediaBufferBase *buffer; + status_t err = mSource->read(&buffer); + if (err != OK) { + return err; + } + buffer->release(); + buffer = NULL; + return OK; + } +}; +/////////////////////////////////////////////////////////////////// +// TESTS +// SurfaceMediaSourceTest class contains tests that fill the buffers +// using the cpu calls +// SurfaceMediaSourceGLTest class contains tests that fill the buffers +// using the GL calls. +// TODO: None of the tests actually verify the encoded images.. so at this point, +// these are mostly functionality tests + visual inspection +////////////////////////////////////////////////////////////////////// + +// Just pass one buffer from the native_window to the SurfaceMediaSource +// Dummy Encoder +static int testId = 1; +TEST_F(SurfaceMediaSourceTest, DISABLED_DummyEncodingFromCpuFilledYV12BufferNpotOneBufferPass) { + ALOGV("Test # %d", testId++); + ALOGV("Testing OneBufferPass ******************************"); + + ASSERT_EQ(NO_ERROR, native_window_set_buffers_format(mANW.get(), + HAL_PIXEL_FORMAT_YV12)); + oneBufferPass(mYuvTexWidth, mYuvTexHeight); +} + +// Pass the buffer with the wrong height and weight and should not be accepted +// Dummy Encoder +TEST_F(SurfaceMediaSourceTest, DISABLED_DummyEncodingFromCpuFilledYV12BufferNpotWrongSizeBufferPass) { + ALOGV("Test # %d", testId++); + ALOGV("Testing Wrong size BufferPass ******************************"); + + // setting the client side buffer size different than the server size + ASSERT_EQ(NO_ERROR, native_window_set_buffers_dimensions(mANW.get(), + 10, 10)); + ASSERT_EQ(NO_ERROR, native_window_set_buffers_format(mANW.get(), + HAL_PIXEL_FORMAT_YV12)); + + ANativeWindowBuffer* anb; + + // Note: make sure we get an ERROR back when dequeuing! + ASSERT_NE(NO_ERROR, native_window_dequeue_buffer_and_wait(mANW.get(), &anb)); +} + +// pass multiple buffers from the native_window the SurfaceMediaSource +// Dummy Encoder +TEST_F(SurfaceMediaSourceTest, DISABLED_DummyEncodingFromCpuFilledYV12BufferNpotMultiBufferPass) { + ALOGV("Test # %d", testId++); + ALOGV("Testing MultiBufferPass, Dummy Recorder *********************"); + ASSERT_EQ(NO_ERROR, native_window_set_buffers_format(mANW.get(), + HAL_PIXEL_FORMAT_YV12)); + + SimpleDummyRecorder writer(mSMS); + writer.start(); + + int32_t nFramesCount = 0; + while (nFramesCount < 300) { + oneBufferPass(mYuvTexWidth, mYuvTexHeight); + + ASSERT_EQ(NO_ERROR, writer.readFromSource()); + + nFramesCount++; + } + writer.stop(); +} + +// Delayed pass of multiple buffers from the native_window the SurfaceMediaSource +// Dummy Encoder +TEST_F(SurfaceMediaSourceTest, DummyLagEncodingFromCpuFilledYV12BufferNpotMultiBufferPass) { + ALOGV("Test # %d", testId++); + ALOGV("Testing MultiBufferPass, Dummy Recorder Lagging **************"); + + ASSERT_EQ(NO_ERROR, native_window_set_buffers_format(mANW.get(), + HAL_PIXEL_FORMAT_YV12)); + + SimpleDummyRecorder writer(mSMS); + writer.start(); + + int32_t nFramesCount = 1; + const int FRAMES_LAG = SurfaceMediaSource::MIN_UNDEQUEUED_BUFFERS; + + while (nFramesCount <= 300) { + ALOGV("Frame: %d", nFramesCount); + oneBufferPass(mYuvTexWidth, mYuvTexHeight); + // Forcing the writer to lag behind a few frames + if (nFramesCount > FRAMES_LAG) { + ASSERT_EQ(NO_ERROR, writer.readFromSource()); + } + nFramesCount++; + } + writer.stop(); +} + +// pass multiple buffers from the native_window the SurfaceMediaSource +// A dummy writer (MULTITHREADED) is used to simulate actual MPEG4Writer +TEST_F(SurfaceMediaSourceTest, DummyThreadedEncodingFromCpuFilledYV12BufferNpotMultiBufferPass) { + ALOGV("Test # %d", testId++); + ALOGV("Testing MultiBufferPass, Dummy Recorder Multi-Threaded **********"); + ASSERT_EQ(NO_ERROR, native_window_set_buffers_format(mANW.get(), + HAL_PIXEL_FORMAT_YV12)); + + DummyRecorder writer(mSMS); + writer.start(); + + int32_t nFramesCount = 0; + while (nFramesCount <= 300) { + ALOGV("Frame: %d", nFramesCount); + oneBufferPass(mYuvTexWidth, mYuvTexHeight); + + nFramesCount++; + } + writer.stop(); +} + +// Test to examine actual encoding using mediarecorder +// We use the mediaserver to create a mediarecorder and send +// it back to us. So SurfaceMediaSource lives in the same process +// as the mediaserver. +// Very close to the actual camera, except that the +// buffers are filled and queueud by the CPU instead of GL. +TEST_F(SurfaceMediaSourceTest, DISABLED_EncodingFromCpuYV12BufferNpotWriteMediaServer) { + ALOGV("Test # %d", testId++); + ALOGV("************** Testing the whole pipeline with actual MediaRecorder ***********"); + ALOGV("************** SurfaceMediaSource is same process as mediaserver ***********"); + + const char *fileName = "/sdcard/outputSurfEncMSource.mp4"; + int fd = open(fileName, O_RDWR | O_CREAT, 0744); + if (fd < 0) { + ALOGE("ERROR: Could not open the the file %s, fd = %d !!", fileName, fd); + } + CHECK(fd >= 0); + + sp<MediaRecorder> mr = SurfaceMediaSourceGLTest::setUpMediaRecorder(fd, + VIDEO_SOURCE_SURFACE, OUTPUT_FORMAT_MPEG_4, VIDEO_ENCODER_H264, + mYuvTexWidth, mYuvTexHeight, 30); + // get the reference to the surfacemediasource living in + // mediaserver that is created by stagefrightrecorder + sp<IGraphicBufferProducer> iST = mr->querySurfaceMediaSourceFromMediaServer(); + mSTC = new Surface(iST); + mANW = mSTC; + ASSERT_EQ(NO_ERROR, native_window_api_connect(mANW.get(), NATIVE_WINDOW_API_CPU)); + ASSERT_EQ(NO_ERROR, native_window_set_buffers_format(mANW.get(), + HAL_PIXEL_FORMAT_YV12)); + + int32_t nFramesCount = 0; + while (nFramesCount <= 300) { + oneBufferPassNoFill(mYuvTexWidth, mYuvTexHeight); + nFramesCount++; + ALOGV("framesCount = %d", nFramesCount); + } + + ASSERT_EQ(NO_ERROR, native_window_api_disconnect(mANW.get(), NATIVE_WINDOW_API_CPU)); + ALOGV("Stopping MediaRecorder..."); + CHECK_EQ((status_t)OK, mr->stop()); + mr.clear(); + close(fd); +} + +////////////////////////////////////////////////////////////////////// +// GL tests +///////////////////////////////////////////////////////////////////// + +// Test to examine whether we can choose the Recordable Android GLConfig +// DummyRecorder used- no real encoding here +TEST_F(SurfaceMediaSourceGLTest, ChooseAndroidRecordableEGLConfigDummyWriter) { + ALOGV("Test # %d", testId++); + ALOGV("Verify creating a surface w/ right config + dummy writer*********"); + + mSMS = new SurfaceMediaSource(mYuvTexWidth, mYuvTexHeight); + mSTC = new Surface(mSMS->getProducer()); + mANW = mSTC; + + DummyRecorder writer(mSMS); + writer.start(); + + if (mEglSurface != EGL_NO_SURFACE) { + EXPECT_TRUE(eglDestroySurface(mEglDisplay, mEglSurface)); + mEglSurface = EGL_NO_SURFACE; + } + + mEglSurface = eglCreateWindowSurface(mEglDisplay, mGlConfig, + mANW.get(), NULL); + ASSERT_EQ(EGL_SUCCESS, eglGetError()); + ASSERT_NE(EGL_NO_SURFACE, mEglSurface) ; + + EXPECT_TRUE(eglMakeCurrent(mEglDisplay, mEglSurface, mEglSurface, + mEglContext)); + ASSERT_EQ(EGL_SUCCESS, eglGetError()); + + int32_t nFramesCount = 0; + while (nFramesCount <= 300) { + oneBufferPassGL(); + nFramesCount++; + ALOGV("framesCount = %d", nFramesCount); + } + + EXPECT_TRUE(eglMakeCurrent(mEglDisplay, EGL_NO_SURFACE, EGL_NO_SURFACE, + EGL_NO_CONTEXT)); + ASSERT_EQ(EGL_SUCCESS, eglGetError()); + eglDestroySurface(mEglDisplay, mEglSurface); + mEglSurface = EGL_NO_SURFACE; + + writer.stop(); +} +// Test to examine whether we can render GL buffers in to the surface +// created with the native window handle +TEST_F(SurfaceMediaSourceGLTest, RenderingToRecordableEGLSurfaceWorks) { + ALOGV("Test # %d", testId++); + ALOGV("RenderingToRecordableEGLSurfaceWorks *********************"); + // Do the producer side of things + glClearColor(0.6, 0.6, 0.6, 0.6); + glClear(GL_COLOR_BUFFER_BIT); + + glEnable(GL_SCISSOR_TEST); + glScissor(4, 4, 4, 4); + glClearColor(1.0, 0.0, 0.0, 1.0); + glClear(GL_COLOR_BUFFER_BIT); + + glScissor(24, 48, 4, 4); + glClearColor(0.0, 1.0, 0.0, 1.0); + glClear(GL_COLOR_BUFFER_BIT); + + glScissor(37, 17, 4, 4); + glClearColor(0.0, 0.0, 1.0, 1.0); + glClear(GL_COLOR_BUFFER_BIT); + + EXPECT_TRUE(checkPixel( 0, 0, 153, 153, 153, 153)); + EXPECT_TRUE(checkPixel(63, 0, 153, 153, 153, 153)); + EXPECT_TRUE(checkPixel(63, 63, 153, 153, 153, 153)); + EXPECT_TRUE(checkPixel( 0, 63, 153, 153, 153, 153)); + + EXPECT_TRUE(checkPixel( 4, 7, 255, 0, 0, 255)); + EXPECT_TRUE(checkPixel(25, 51, 0, 255, 0, 255)); + EXPECT_TRUE(checkPixel(40, 19, 0, 0, 255, 255)); + EXPECT_TRUE(checkPixel(29, 51, 153, 153, 153, 153)); + EXPECT_TRUE(checkPixel( 5, 32, 153, 153, 153, 153)); + EXPECT_TRUE(checkPixel(13, 8, 153, 153, 153, 153)); + EXPECT_TRUE(checkPixel(46, 3, 153, 153, 153, 153)); + EXPECT_TRUE(checkPixel(30, 33, 153, 153, 153, 153)); + EXPECT_TRUE(checkPixel( 6, 52, 153, 153, 153, 153)); + EXPECT_TRUE(checkPixel(55, 33, 153, 153, 153, 153)); + EXPECT_TRUE(checkPixel(16, 29, 153, 153, 153, 153)); + EXPECT_TRUE(checkPixel( 1, 30, 153, 153, 153, 153)); + EXPECT_TRUE(checkPixel(41, 37, 153, 153, 153, 153)); + EXPECT_TRUE(checkPixel(46, 29, 153, 153, 153, 153)); + EXPECT_TRUE(checkPixel(15, 25, 153, 153, 153, 153)); + EXPECT_TRUE(checkPixel( 3, 52, 153, 153, 153, 153)); +} + +// Test to examine the actual encoding with GL buffers +// Actual encoder, Actual GL Buffers Filled SurfaceMediaSource +// The same pattern is rendered every frame +TEST_F(SurfaceMediaSourceGLTest, EncodingFromGLRgbaSameImageEachBufNpotWrite) { + ALOGV("Test # %d", testId++); + ALOGV("************** Testing the whole pipeline with actual Recorder ***********"); + ALOGV("************** GL Filling the buffers ***********"); + // Note: No need to set the colorformat for the buffers. The colorformat is + // in the GRAlloc buffers itself. + + const char *fileName = "/sdcard/outputSurfEncMSourceGL.mp4"; + int fd = open(fileName, O_RDWR | O_CREAT, 0744); + if (fd < 0) { + ALOGE("ERROR: Could not open the the file %s, fd = %d !!", fileName, fd); + } + CHECK(fd >= 0); + + sp<MediaRecorder> mr = setUpMediaRecorder(fd, VIDEO_SOURCE_SURFACE, + OUTPUT_FORMAT_MPEG_4, VIDEO_ENCODER_H264, mYuvTexWidth, mYuvTexHeight, 30); + + // get the reference to the surfacemediasource living in + // mediaserver that is created by stagefrightrecorder + setUpEGLSurfaceFromMediaRecorder(mr); + + int32_t nFramesCount = 0; + while (nFramesCount <= 300) { + oneBufferPassGL(); + nFramesCount++; + ALOGV("framesCount = %d", nFramesCount); + } + + EXPECT_TRUE(eglMakeCurrent(mEglDisplay, EGL_NO_SURFACE, EGL_NO_SURFACE, + EGL_NO_CONTEXT)); + ASSERT_EQ(EGL_SUCCESS, eglGetError()); + eglDestroySurface(mEglDisplay, mEglSurface); + mEglSurface = EGL_NO_SURFACE; + + ALOGV("Stopping MediaRecorder..."); + CHECK_EQ((status_t)OK, mr->stop()); + mr.clear(); + close(fd); +} + +// Test to examine the actual encoding from the GL Buffers +// Actual encoder, Actual GL Buffers Filled SurfaceMediaSource +// A different pattern is rendered every frame +TEST_F(SurfaceMediaSourceGLTest, EncodingFromGLRgbaDiffImageEachBufNpotWrite) { + ALOGV("Test # %d", testId++); + ALOGV("************** Testing the whole pipeline with actual Recorder ***********"); + ALOGV("************** Diff GL Filling the buffers ***********"); + // Note: No need to set the colorformat for the buffers. The colorformat is + // in the GRAlloc buffers itself. + + const char *fileName = "/sdcard/outputSurfEncMSourceGLDiff.mp4"; + int fd = open(fileName, O_RDWR | O_CREAT, 0744); + if (fd < 0) { + ALOGE("ERROR: Could not open the the file %s, fd = %d !!", fileName, fd); + } + CHECK(fd >= 0); + + sp<MediaRecorder> mr = setUpMediaRecorder(fd, VIDEO_SOURCE_SURFACE, + OUTPUT_FORMAT_MPEG_4, VIDEO_ENCODER_H264, mYuvTexWidth, mYuvTexHeight, 30); + + // get the reference to the surfacemediasource living in + // mediaserver that is created by stagefrightrecorder + setUpEGLSurfaceFromMediaRecorder(mr); + + int32_t nFramesCount = 0; + while (nFramesCount <= 300) { + oneBufferPassGL(nFramesCount); + nFramesCount++; + ALOGV("framesCount = %d", nFramesCount); + } + + EXPECT_TRUE(eglMakeCurrent(mEglDisplay, EGL_NO_SURFACE, EGL_NO_SURFACE, + EGL_NO_CONTEXT)); + ASSERT_EQ(EGL_SUCCESS, eglGetError()); + eglDestroySurface(mEglDisplay, mEglSurface); + mEglSurface = EGL_NO_SURFACE; + + ALOGV("Stopping MediaRecorder..."); + CHECK_EQ((status_t)OK, mr->stop()); + mr.clear(); + close(fd); +} +} // namespace android diff --git a/media/libstagefright/wifi-display/Android.bp b/media/libstagefright/wifi-display/Android.bp new file mode 100644 index 0000000000..2f1e74943f --- /dev/null +++ b/media/libstagefright/wifi-display/Android.bp @@ -0,0 +1,57 @@ +cc_library_shared { + name: "libstagefright_wfd", + + srcs: [ + "MediaSender.cpp", + "Parameters.cpp", + "rtp/RTPSender.cpp", + "source/Converter.cpp", + "source/MediaPuller.cpp", + "source/PlaybackSession.cpp", + "source/RepeaterSource.cpp", + "source/TSPacketizer.cpp", + "source/WifiDisplaySource.cpp", + "VideoFormats.cpp", + ], + + include_dirs: [ + "frameworks/av/media/libstagefright", + "frameworks/native/include/media/openmax", + "frameworks/native/include/media/hardware", + "frameworks/av/media/libstagefright/mpeg2ts", + ], + + shared_libs: [ + "libbinder", + "libcutils", + "liblog", + "libmedia", + "libmedia_omx", + "libstagefright", + "libstagefright_foundation", + "libui", + "libgui", + "libutils", + ], + + header_libs: [ + "libmediadrm_headers", + "libmediametrics_headers", + ], + + cflags: [ + "-Wno-multichar", + "-Werror", + "-Wall", + ], + + sanitize: { + misc_undefined: [ + "signed-integer-overflow", + ], + cfi: true, + diag: { + cfi: true, + }, + }, +} diff --git a/media/libstagefright/wifi-display/MediaSender.cpp b/media/libstagefright/wifi-display/MediaSender.cpp new file mode 100644 index 0000000000..6e743b5711 --- /dev/null +++ b/media/libstagefright/wifi-display/MediaSender.cpp @@ -0,0 +1,519 @@ +/* + * Copyright 2013, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "MediaSender" +#include <utils/Log.h> + +#include "MediaSender.h" + +#include "rtp/RTPSender.h" +#include "source/TSPacketizer.h" + +#include <media/IHDCP.h> +#include <media/stagefright/foundation/avc_utils.h> +#include <media/stagefright/foundation/ABuffer.h> +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> +#include <media/stagefright/ANetworkSession.h> +#include <media/stagefright/MediaBuffer.h> +#include <media/stagefright/MediaBufferBase.h> +#include <ui/GraphicBuffer.h> + +namespace android { + +MediaSender::MediaSender( + const sp<ANetworkSession> &netSession, + const sp<AMessage> ¬ify) + : mNetSession(netSession), + mNotify(notify), + mMode(MODE_UNDEFINED), + mGeneration(0), + mPrevTimeUs(-1ll), + mInitDoneCount(0), + mLogFile(NULL) { + // mLogFile = fopen("/data/misc/log.ts", "wb"); +} + +MediaSender::~MediaSender() { + if (mLogFile != NULL) { + fclose(mLogFile); + mLogFile = NULL; + } +} + +status_t MediaSender::setHDCP(const sp<IHDCP> &hdcp) { + if (mMode != MODE_UNDEFINED) { + return INVALID_OPERATION; + } + + mHDCP = hdcp; + + return OK; +} + +ssize_t MediaSender::addTrack(const sp<AMessage> &format, uint32_t flags) { + if (mMode != MODE_UNDEFINED) { + return INVALID_OPERATION; + } + + TrackInfo info; + info.mFormat = format; + info.mFlags = flags; + info.mPacketizerTrackIndex = -1; + + AString mime; + CHECK(format->findString("mime", &mime)); + info.mIsAudio = !strncasecmp("audio/", mime.c_str(), 6); + + size_t index = mTrackInfos.size(); + mTrackInfos.push_back(info); + + return index; +} + +status_t MediaSender::initAsync( + ssize_t trackIndex, + const char *remoteHost, + int32_t remoteRTPPort, + RTPSender::TransportMode rtpMode, + int32_t remoteRTCPPort, + RTPSender::TransportMode rtcpMode, + int32_t *localRTPPort) { + if (trackIndex < 0) { + if (mMode != MODE_UNDEFINED) { + return INVALID_OPERATION; + } + + uint32_t flags = 0; + if (mHDCP != NULL) { + // XXX Determine proper HDCP version. + flags |= TSPacketizer::EMIT_HDCP20_DESCRIPTOR; + } + mTSPacketizer = new TSPacketizer(flags); + + status_t err = OK; + for (size_t i = 0; i < mTrackInfos.size(); ++i) { + TrackInfo *info = &mTrackInfos.editItemAt(i); + + ssize_t packetizerTrackIndex = + mTSPacketizer->addTrack(info->mFormat); + + if (packetizerTrackIndex < 0) { + err = packetizerTrackIndex; + break; + } + + info->mPacketizerTrackIndex = packetizerTrackIndex; + } + + if (err == OK) { + sp<AMessage> notify = new AMessage(kWhatSenderNotify, this); + notify->setInt32("generation", mGeneration); + mTSSender = new RTPSender(mNetSession, notify); + looper()->registerHandler(mTSSender); + + err = mTSSender->initAsync( + remoteHost, + remoteRTPPort, + rtpMode, + remoteRTCPPort, + rtcpMode, + localRTPPort); + + if (err != OK) { + looper()->unregisterHandler(mTSSender->id()); + mTSSender.clear(); + } + } + + if (err != OK) { + for (size_t i = 0; i < mTrackInfos.size(); ++i) { + TrackInfo *info = &mTrackInfos.editItemAt(i); + info->mPacketizerTrackIndex = -1; + } + + mTSPacketizer.clear(); + return err; + } + + mMode = MODE_TRANSPORT_STREAM; + mInitDoneCount = 1; + + return OK; + } + + if (mMode == MODE_TRANSPORT_STREAM) { + return INVALID_OPERATION; + } + + if ((size_t)trackIndex >= mTrackInfos.size()) { + return -ERANGE; + } + + TrackInfo *info = &mTrackInfos.editItemAt(trackIndex); + + if (info->mSender != NULL) { + return INVALID_OPERATION; + } + + sp<AMessage> notify = new AMessage(kWhatSenderNotify, this); + notify->setInt32("generation", mGeneration); + notify->setSize("trackIndex", trackIndex); + + info->mSender = new RTPSender(mNetSession, notify); + looper()->registerHandler(info->mSender); + + status_t err = info->mSender->initAsync( + remoteHost, + remoteRTPPort, + rtpMode, + remoteRTCPPort, + rtcpMode, + localRTPPort); + + if (err != OK) { + looper()->unregisterHandler(info->mSender->id()); + info->mSender.clear(); + + return err; + } + + if (mMode == MODE_UNDEFINED) { + mInitDoneCount = mTrackInfos.size(); + } + + mMode = MODE_ELEMENTARY_STREAMS; + + return OK; +} + +status_t MediaSender::queueAccessUnit( + size_t trackIndex, const sp<ABuffer> &accessUnit) { + if (mMode == MODE_UNDEFINED) { + return INVALID_OPERATION; + } + + if (trackIndex >= mTrackInfos.size()) { + return -ERANGE; + } + + if (mMode == MODE_TRANSPORT_STREAM) { + TrackInfo *info = &mTrackInfos.editItemAt(trackIndex); + info->mAccessUnits.push_back(accessUnit); + + mTSPacketizer->extractCSDIfNecessary(info->mPacketizerTrackIndex); + + for (;;) { + ssize_t minTrackIndex = -1; + int64_t minTimeUs = -1ll; + + for (size_t i = 0; i < mTrackInfos.size(); ++i) { + const TrackInfo &info = mTrackInfos.itemAt(i); + + if (info.mAccessUnits.empty()) { + minTrackIndex = -1; + minTimeUs = -1ll; + break; + } + + int64_t timeUs; + const sp<ABuffer> &accessUnit = *info.mAccessUnits.begin(); + CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs)); + + if (minTrackIndex < 0 || timeUs < minTimeUs) { + minTrackIndex = i; + minTimeUs = timeUs; + } + } + + if (minTrackIndex < 0) { + return OK; + } + + TrackInfo *info = &mTrackInfos.editItemAt(minTrackIndex); + sp<ABuffer> accessUnit = *info->mAccessUnits.begin(); + info->mAccessUnits.erase(info->mAccessUnits.begin()); + + sp<ABuffer> tsPackets; + status_t err = packetizeAccessUnit( + minTrackIndex, accessUnit, &tsPackets); + + if (err == OK) { + if (mLogFile != NULL) { + fwrite(tsPackets->data(), 1, tsPackets->size(), mLogFile); + } + + int64_t timeUs; + CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs)); + tsPackets->meta()->setInt64("timeUs", timeUs); + + err = mTSSender->queueBuffer( + tsPackets, + 33 /* packetType */, + RTPSender::PACKETIZATION_TRANSPORT_STREAM); + } + + if (err != OK) { + return err; + } + } + } + + TrackInfo *info = &mTrackInfos.editItemAt(trackIndex); + + return info->mSender->queueBuffer( + accessUnit, + info->mIsAudio ? 96 : 97 /* packetType */, + info->mIsAudio + ? RTPSender::PACKETIZATION_AAC : RTPSender::PACKETIZATION_H264); +} + +void MediaSender::onMessageReceived(const sp<AMessage> &msg) { + switch (msg->what()) { + case kWhatSenderNotify: + { + int32_t generation; + CHECK(msg->findInt32("generation", &generation)); + if (generation != mGeneration) { + break; + } + + onSenderNotify(msg); + break; + } + + default: + TRESPASS(); + } +} + +void MediaSender::onSenderNotify(const sp<AMessage> &msg) { + int32_t what; + CHECK(msg->findInt32("what", &what)); + + switch (what) { + case RTPSender::kWhatInitDone: + { + --mInitDoneCount; + + int32_t err; + CHECK(msg->findInt32("err", &err)); + + if (err != OK) { + notifyInitDone(err); + ++mGeneration; + break; + } + + if (mInitDoneCount == 0) { + notifyInitDone(OK); + } + break; + } + + case RTPSender::kWhatError: + { + int32_t err; + CHECK(msg->findInt32("err", &err)); + + notifyError(err); + break; + } + + case kWhatNetworkStall: + { + size_t numBytesQueued; + CHECK(msg->findSize("numBytesQueued", &numBytesQueued)); + + notifyNetworkStall(numBytesQueued); + break; + } + + case kWhatInformSender: + { + int64_t avgLatencyUs; + CHECK(msg->findInt64("avgLatencyUs", &avgLatencyUs)); + + int64_t maxLatencyUs; + CHECK(msg->findInt64("maxLatencyUs", &maxLatencyUs)); + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatInformSender); + notify->setInt64("avgLatencyUs", avgLatencyUs); + notify->setInt64("maxLatencyUs", maxLatencyUs); + notify->post(); + break; + } + + default: + TRESPASS(); + } +} + +void MediaSender::notifyInitDone(status_t err) { + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatInitDone); + notify->setInt32("err", err); + notify->post(); +} + +void MediaSender::notifyError(status_t err) { + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatError); + notify->setInt32("err", err); + notify->post(); +} + +void MediaSender::notifyNetworkStall(size_t numBytesQueued) { + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatNetworkStall); + notify->setSize("numBytesQueued", numBytesQueued); + notify->post(); +} + +status_t MediaSender::packetizeAccessUnit( + size_t trackIndex, + sp<ABuffer> accessUnit, + sp<ABuffer> *tsPackets) { + const TrackInfo &info = mTrackInfos.itemAt(trackIndex); + + uint32_t flags = 0; + + bool isHDCPEncrypted = false; + uint64_t inputCTR; + uint8_t HDCP_private_data[16]; + + bool manuallyPrependSPSPPS = + !info.mIsAudio + && (info.mFlags & FLAG_MANUALLY_PREPEND_SPS_PPS) + && IsIDR(accessUnit->data(), accessUnit->size()); + + if (mHDCP != NULL && !info.mIsAudio) { + isHDCPEncrypted = true; + + if (manuallyPrependSPSPPS) { + accessUnit = mTSPacketizer->prependCSD( + info.mPacketizerTrackIndex, accessUnit); + } + + status_t err; + native_handle_t* handle; + if (accessUnit->meta()->findPointer("handle", (void**)&handle) + && handle != NULL) { + int32_t rangeLength, rangeOffset; + sp<AMessage> notify; + CHECK(accessUnit->meta()->findInt32("rangeOffset", &rangeOffset)); + CHECK(accessUnit->meta()->findInt32("rangeLength", &rangeLength)); + CHECK(accessUnit->meta()->findMessage("notify", ¬ify) + && notify != NULL); + CHECK_GE((int32_t)accessUnit->size(), rangeLength); + + sp<GraphicBuffer> grbuf(new GraphicBuffer( + rangeOffset + rangeLength /* width */, 1 /* height */, + HAL_PIXEL_FORMAT_Y8, 1 /* layerCount */, + GRALLOC_USAGE_HW_VIDEO_ENCODER, + rangeOffset + rangeLength /* stride */, handle, + false /* keepOwnership */)); + + err = mHDCP->encryptNative( + grbuf, rangeOffset, rangeLength, + trackIndex /* streamCTR */, + &inputCTR, + accessUnit->data()); + notify->post(); + } else { + err = mHDCP->encrypt( + accessUnit->data(), accessUnit->size(), + trackIndex /* streamCTR */, + &inputCTR, + accessUnit->data()); + } + + if (err != OK) { + ALOGE("Failed to HDCP-encrypt media data (err %d)", + err); + + return err; + } + + HDCP_private_data[0] = 0x00; + + HDCP_private_data[1] = + (((trackIndex >> 30) & 3) << 1) | 1; + + HDCP_private_data[2] = (trackIndex >> 22) & 0xff; + + HDCP_private_data[3] = + (((trackIndex >> 15) & 0x7f) << 1) | 1; + + HDCP_private_data[4] = (trackIndex >> 7) & 0xff; + + HDCP_private_data[5] = + ((trackIndex & 0x7f) << 1) | 1; + + HDCP_private_data[6] = 0x00; + + HDCP_private_data[7] = + (((inputCTR >> 60) & 0x0f) << 1) | 1; + + HDCP_private_data[8] = (inputCTR >> 52) & 0xff; + + HDCP_private_data[9] = + (((inputCTR >> 45) & 0x7f) << 1) | 1; + + HDCP_private_data[10] = (inputCTR >> 37) & 0xff; + + HDCP_private_data[11] = + (((inputCTR >> 30) & 0x7f) << 1) | 1; + + HDCP_private_data[12] = (inputCTR >> 22) & 0xff; + + HDCP_private_data[13] = + (((inputCTR >> 15) & 0x7f) << 1) | 1; + + HDCP_private_data[14] = (inputCTR >> 7) & 0xff; + + HDCP_private_data[15] = + ((inputCTR & 0x7f) << 1) | 1; + + flags |= TSPacketizer::IS_ENCRYPTED; + } else if (manuallyPrependSPSPPS) { + flags |= TSPacketizer::PREPEND_SPS_PPS_TO_IDR_FRAMES; + } + + int64_t timeUs = ALooper::GetNowUs(); + if (mPrevTimeUs < 0ll || mPrevTimeUs + 100000ll <= timeUs) { + flags |= TSPacketizer::EMIT_PCR; + flags |= TSPacketizer::EMIT_PAT_AND_PMT; + + mPrevTimeUs = timeUs; + } + + mTSPacketizer->packetize( + info.mPacketizerTrackIndex, + accessUnit, + tsPackets, + flags, + !isHDCPEncrypted ? NULL : HDCP_private_data, + !isHDCPEncrypted ? 0 : sizeof(HDCP_private_data), + info.mIsAudio ? 2 : 0 /* numStuffingBytes */); + + return OK; +} + +} // namespace android + diff --git a/media/libstagefright/wifi-display/MediaSender.h b/media/libstagefright/wifi-display/MediaSender.h new file mode 100644 index 0000000000..04538ea1f6 --- /dev/null +++ b/media/libstagefright/wifi-display/MediaSender.h @@ -0,0 +1,132 @@ +/* + * Copyright 2013, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MEDIA_SENDER_H_ + +#define MEDIA_SENDER_H_ + +#include "rtp/RTPSender.h" + +#include <media/stagefright/foundation/ABase.h> +#include <media/stagefright/foundation/AHandler.h> +#include <utils/Errors.h> +#include <utils/Vector.h> + +namespace android { + +struct ABuffer; +struct ANetworkSession; +struct AMessage; +struct IHDCP; +struct TSPacketizer; + +// This class facilitates sending of data from one or more media tracks +// through one or more RTP channels, either providing a 1:1 mapping from +// track to RTP channel or muxing all tracks into a single RTP channel and +// using transport stream encapsulation. +// Optionally the (video) data is encrypted using the provided hdcp object. +struct MediaSender : public AHandler { + enum { + kWhatInitDone, + kWhatError, + kWhatNetworkStall, + kWhatInformSender, + }; + + MediaSender( + const sp<ANetworkSession> &netSession, + const sp<AMessage> ¬ify); + + status_t setHDCP(const sp<IHDCP> &hdcp); + + enum FlagBits { + FLAG_MANUALLY_PREPEND_SPS_PPS = 1, + }; + ssize_t addTrack(const sp<AMessage> &format, uint32_t flags); + + // If trackIndex == -1, initialize for transport stream muxing. + status_t initAsync( + ssize_t trackIndex, + const char *remoteHost, + int32_t remoteRTPPort, + RTPSender::TransportMode rtpMode, + int32_t remoteRTCPPort, + RTPSender::TransportMode rtcpMode, + int32_t *localRTPPort); + + status_t queueAccessUnit( + size_t trackIndex, const sp<ABuffer> &accessUnit); + +protected: + virtual void onMessageReceived(const sp<AMessage> &msg); + virtual ~MediaSender(); + +private: + enum { + kWhatSenderNotify, + }; + + enum Mode { + MODE_UNDEFINED, + MODE_TRANSPORT_STREAM, + MODE_ELEMENTARY_STREAMS, + }; + + struct TrackInfo { + sp<AMessage> mFormat; + uint32_t mFlags; + sp<RTPSender> mSender; + List<sp<ABuffer> > mAccessUnits; + ssize_t mPacketizerTrackIndex; + bool mIsAudio; + }; + + sp<ANetworkSession> mNetSession; + sp<AMessage> mNotify; + + sp<IHDCP> mHDCP; + + Mode mMode; + int32_t mGeneration; + + Vector<TrackInfo> mTrackInfos; + + sp<TSPacketizer> mTSPacketizer; + sp<RTPSender> mTSSender; + int64_t mPrevTimeUs; + + size_t mInitDoneCount; + + FILE *mLogFile; + + void onSenderNotify(const sp<AMessage> &msg); + + void notifyInitDone(status_t err); + void notifyError(status_t err); + void notifyNetworkStall(size_t numBytesQueued); + + status_t packetizeAccessUnit( + size_t trackIndex, + sp<ABuffer> accessUnit, + sp<ABuffer> *tsPackets); + + DISALLOW_EVIL_CONSTRUCTORS(MediaSender); +}; + +} // namespace android + +#endif // MEDIA_SENDER_H_ + diff --git a/media/libstagefright/wifi-display/Parameters.cpp b/media/libstagefright/wifi-display/Parameters.cpp new file mode 100644 index 0000000000..d2a61ea464 --- /dev/null +++ b/media/libstagefright/wifi-display/Parameters.cpp @@ -0,0 +1,92 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Parameters.h" + +#include <media/stagefright/MediaErrors.h> + +namespace android { + +// static +sp<Parameters> Parameters::Parse(const char *data, size_t size) { + sp<Parameters> params = new Parameters; + status_t err = params->parse(data, size); + + if (err != OK) { + return NULL; + } + + return params; +} + +Parameters::Parameters() {} + +Parameters::~Parameters() {} + +status_t Parameters::parse(const char *data, size_t size) { + size_t i = 0; + while (i < size) { + size_t nameStart = i; + while (i < size && data[i] != ':') { + ++i; + } + + if (i == size || i == nameStart) { + return ERROR_MALFORMED; + } + + AString name(&data[nameStart], i - nameStart); + name.trim(); + name.tolower(); + + ++i; + + size_t valueStart = i; + + while (i + 1 < size && (data[i] != '\r' || data[i + 1] != '\n')) { + ++i; + } + + AString value(&data[valueStart], i - valueStart); + value.trim(); + + mDict.add(name, value); + + while (i + 1 < size && data[i] == '\r' && data[i + 1] == '\n') { + i += 2; + } + } + + return OK; +} + +bool Parameters::findParameter(const char *name, AString *value) const { + AString key = name; + key.tolower(); + + ssize_t index = mDict.indexOfKey(key); + + if (index < 0) { + value->clear(); + + return false; + } + + *value = mDict.valueAt(index); + return true; +} + +} // namespace android diff --git a/media/libstagefright/wifi-display/Parameters.h b/media/libstagefright/wifi-display/Parameters.h new file mode 100644 index 0000000000..a5e787e234 --- /dev/null +++ b/media/libstagefright/wifi-display/Parameters.h @@ -0,0 +1,41 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <media/stagefright/foundation/ABase.h> +#include <media/stagefright/foundation/AString.h> +#include <utils/KeyedVector.h> +#include <utils/RefBase.h> + +namespace android { + +struct Parameters : public RefBase { + static sp<Parameters> Parse(const char *data, size_t size); + + bool findParameter(const char *name, AString *value) const; + +protected: + virtual ~Parameters(); + +private: + KeyedVector<AString, AString> mDict; + + Parameters(); + status_t parse(const char *data, size_t size); + + DISALLOW_EVIL_CONSTRUCTORS(Parameters); +}; + +} // namespace android diff --git a/media/libstagefright/wifi-display/VideoFormats.cpp b/media/libstagefright/wifi-display/VideoFormats.cpp new file mode 100644 index 0000000000..dbc511caa7 --- /dev/null +++ b/media/libstagefright/wifi-display/VideoFormats.cpp @@ -0,0 +1,550 @@ +/* + * Copyright 2013, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "VideoFormats" +#include <utils/Log.h> + +#include "VideoFormats.h" + +#include <media/stagefright/foundation/ADebug.h> + +namespace android { + +// static +const VideoFormats::config_t VideoFormats::mResolutionTable[][32] = { + { + // CEA Resolutions + { 640, 480, 60, false, 0, 0}, + { 720, 480, 60, false, 0, 0}, + { 720, 480, 60, true, 0, 0}, + { 720, 576, 50, false, 0, 0}, + { 720, 576, 50, true, 0, 0}, + { 1280, 720, 30, false, 0, 0}, + { 1280, 720, 60, false, 0, 0}, + { 1920, 1080, 30, false, 0, 0}, + { 1920, 1080, 60, false, 0, 0}, + { 1920, 1080, 60, true, 0, 0}, + { 1280, 720, 25, false, 0, 0}, + { 1280, 720, 50, false, 0, 0}, + { 1920, 1080, 25, false, 0, 0}, + { 1920, 1080, 50, false, 0, 0}, + { 1920, 1080, 50, true, 0, 0}, + { 1280, 720, 24, false, 0, 0}, + { 1920, 1080, 24, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + }, + { + // VESA Resolutions + { 800, 600, 30, false, 0, 0}, + { 800, 600, 60, false, 0, 0}, + { 1024, 768, 30, false, 0, 0}, + { 1024, 768, 60, false, 0, 0}, + { 1152, 864, 30, false, 0, 0}, + { 1152, 864, 60, false, 0, 0}, + { 1280, 768, 30, false, 0, 0}, + { 1280, 768, 60, false, 0, 0}, + { 1280, 800, 30, false, 0, 0}, + { 1280, 800, 60, false, 0, 0}, + { 1360, 768, 30, false, 0, 0}, + { 1360, 768, 60, false, 0, 0}, + { 1366, 768, 30, false, 0, 0}, + { 1366, 768, 60, false, 0, 0}, + { 1280, 1024, 30, false, 0, 0}, + { 1280, 1024, 60, false, 0, 0}, + { 1400, 1050, 30, false, 0, 0}, + { 1400, 1050, 60, false, 0, 0}, + { 1440, 900, 30, false, 0, 0}, + { 1440, 900, 60, false, 0, 0}, + { 1600, 900, 30, false, 0, 0}, + { 1600, 900, 60, false, 0, 0}, + { 1600, 1200, 30, false, 0, 0}, + { 1600, 1200, 60, false, 0, 0}, + { 1680, 1024, 30, false, 0, 0}, + { 1680, 1024, 60, false, 0, 0}, + { 1680, 1050, 30, false, 0, 0}, + { 1680, 1050, 60, false, 0, 0}, + { 1920, 1200, 30, false, 0, 0}, + { 1920, 1200, 60, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + }, + { + // HH Resolutions + { 800, 480, 30, false, 0, 0}, + { 800, 480, 60, false, 0, 0}, + { 854, 480, 30, false, 0, 0}, + { 854, 480, 60, false, 0, 0}, + { 864, 480, 30, false, 0, 0}, + { 864, 480, 60, false, 0, 0}, + { 640, 360, 30, false, 0, 0}, + { 640, 360, 60, false, 0, 0}, + { 960, 540, 30, false, 0, 0}, + { 960, 540, 60, false, 0, 0}, + { 848, 480, 30, false, 0, 0}, + { 848, 480, 60, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + } +}; + +VideoFormats::VideoFormats() { + memcpy(mConfigs, mResolutionTable, sizeof(mConfigs)); + + for (size_t i = 0; i < kNumResolutionTypes; ++i) { + mResolutionEnabled[i] = 0; + } + + setNativeResolution(RESOLUTION_CEA, 0); // default to 640x480 p60 +} + +void VideoFormats::setNativeResolution(ResolutionType type, size_t index) { + CHECK_LT(type, kNumResolutionTypes); + CHECK(GetConfiguration(type, index, NULL, NULL, NULL, NULL)); + + mNativeType = type; + mNativeIndex = index; + + setResolutionEnabled(type, index); +} + +void VideoFormats::getNativeResolution( + ResolutionType *type, size_t *index) const { + *type = mNativeType; + *index = mNativeIndex; +} + +void VideoFormats::disableAll() { + for (size_t i = 0; i < kNumResolutionTypes; ++i) { + mResolutionEnabled[i] = 0; + for (size_t j = 0; j < 32; j++) { + mConfigs[i][j].profile = mConfigs[i][j].level = 0; + } + } +} + +void VideoFormats::enableAll() { + for (size_t i = 0; i < kNumResolutionTypes; ++i) { + mResolutionEnabled[i] = 0xffffffff; + for (size_t j = 0; j < 32; j++) { + mConfigs[i][j].profile = (1ul << PROFILE_CBP); + mConfigs[i][j].level = (1ul << LEVEL_31); + } + } +} + +void VideoFormats::enableResolutionUpto( + ResolutionType type, size_t index, + ProfileType profile, LevelType level) { + size_t width, height, fps, score; + bool interlaced; + if (!GetConfiguration(type, index, &width, &height, + &fps, &interlaced)) { + ALOGE("Maximum resolution not found!"); + return; + } + score = width * height * fps * (!interlaced + 1); + for (size_t i = 0; i < kNumResolutionTypes; ++i) { + for (size_t j = 0; j < 32; j++) { + if (GetConfiguration((ResolutionType)i, j, + &width, &height, &fps, &interlaced) + && score >= width * height * fps * (!interlaced + 1)) { + setResolutionEnabled((ResolutionType)i, j); + setProfileLevel((ResolutionType)i, j, profile, level); + } + } + } +} + +void VideoFormats::setResolutionEnabled( + ResolutionType type, size_t index, bool enabled) { + CHECK_LT(type, kNumResolutionTypes); + CHECK(GetConfiguration(type, index, NULL, NULL, NULL, NULL)); + + if (enabled) { + mResolutionEnabled[type] |= (1ul << index); + mConfigs[type][index].profile = (1ul << PROFILE_CBP); + mConfigs[type][index].level = (1ul << LEVEL_31); + } else { + mResolutionEnabled[type] &= ~(1ul << index); + mConfigs[type][index].profile = 0; + mConfigs[type][index].level = 0; + } +} + +void VideoFormats::setProfileLevel( + ResolutionType type, size_t index, + ProfileType profile, LevelType level) { + CHECK_LT(type, kNumResolutionTypes); + CHECK(GetConfiguration(type, index, NULL, NULL, NULL, NULL)); + + mConfigs[type][index].profile = (1ul << profile); + mConfigs[type][index].level = (1ul << level); +} + +void VideoFormats::getProfileLevel( + ResolutionType type, size_t index, + ProfileType *profile, LevelType *level) const{ + CHECK_LT(type, kNumResolutionTypes); + CHECK(GetConfiguration(type, index, NULL, NULL, NULL, NULL)); + + int i, bestProfile = -1, bestLevel = -1; + + for (i = 0; i < kNumProfileTypes; ++i) { + if (mConfigs[type][index].profile & (1ul << i)) { + bestProfile = i; + } + } + + for (i = 0; i < kNumLevelTypes; ++i) { + if (mConfigs[type][index].level & (1ul << i)) { + bestLevel = i; + } + } + + if (bestProfile == -1 || bestLevel == -1) { + ALOGE("Profile or level not set for resolution type %d, index %zu", + type, index); + bestProfile = PROFILE_CBP; + bestLevel = LEVEL_31; + } + + *profile = (ProfileType) bestProfile; + *level = (LevelType) bestLevel; +} + +bool VideoFormats::isResolutionEnabled( + ResolutionType type, size_t index) const { + CHECK_LT(type, kNumResolutionTypes); + CHECK(GetConfiguration(type, index, NULL, NULL, NULL, NULL)); + + return mResolutionEnabled[type] & (1ul << index); +} + +// static +bool VideoFormats::GetConfiguration( + ResolutionType type, + size_t index, + size_t *width, size_t *height, size_t *framesPerSecond, + bool *interlaced) { + CHECK_LT(type, kNumResolutionTypes); + + if (index >= 32) { + return false; + } + + const config_t *config = &mResolutionTable[type][index]; + + if (config->width == 0) { + return false; + } + + if (width) { + *width = config->width; + } + + if (height) { + *height = config->height; + } + + if (framesPerSecond) { + *framesPerSecond = config->framesPerSecond; + } + + if (interlaced) { + *interlaced = config->interlaced; + } + + return true; +} + +bool VideoFormats::parseH264Codec(const char *spec) { + unsigned profile, level, res[3]; + + if (sscanf( + spec, + "%02x %02x %08X %08X %08X", + &profile, + &level, + &res[0], + &res[1], + &res[2]) != 5) { + return false; + } + + for (size_t i = 0; i < kNumResolutionTypes; ++i) { + for (size_t j = 0; j < 32; ++j) { + if (res[i] & (1ul << j)){ + mResolutionEnabled[i] |= (1ul << j); + if (profile > mConfigs[i][j].profile) { + // prefer higher profile (even if level is lower) + mConfigs[i][j].profile = profile; + mConfigs[i][j].level = level; + } else if (profile == mConfigs[i][j].profile && + level > mConfigs[i][j].level) { + mConfigs[i][j].level = level; + } + } + } + } + + return true; +} + +// static +bool VideoFormats::GetProfileLevel( + ProfileType profile, LevelType level, unsigned *profileIdc, + unsigned *levelIdc, unsigned *constraintSet) { + CHECK_LT(profile, kNumProfileTypes); + CHECK_LT(level, kNumLevelTypes); + + static const unsigned kProfileIDC[kNumProfileTypes] = { + 66, // PROFILE_CBP + 100, // PROFILE_CHP + }; + + static const unsigned kLevelIDC[kNumLevelTypes] = { + 31, // LEVEL_31 + 32, // LEVEL_32 + 40, // LEVEL_40 + 41, // LEVEL_41 + 42, // LEVEL_42 + }; + + static const unsigned kConstraintSet[kNumProfileTypes] = { + 0xc0, // PROFILE_CBP + 0x0c, // PROFILE_CHP + }; + + if (profileIdc) { + *profileIdc = kProfileIDC[profile]; + } + + if (levelIdc) { + *levelIdc = kLevelIDC[level]; + } + + if (constraintSet) { + *constraintSet = kConstraintSet[profile]; + } + + return true; +} + +bool VideoFormats::parseFormatSpec(const char *spec) { + CHECK_EQ(kNumResolutionTypes, 3); + + disableAll(); + + unsigned native, dummy; + size_t size = strlen(spec); + size_t offset = 0; + + if (sscanf(spec, "%02x %02x ", &native, &dummy) != 2) { + return false; + } + + offset += 6; // skip native and preferred-display-mode-supported + CHECK_LE(offset + 58, size); + while (offset < size) { + parseH264Codec(spec + offset); + offset += 60; // skip H.264-codec + ", " + } + + mNativeIndex = native >> 3; + mNativeType = (ResolutionType)(native & 7); + + bool success; + if (mNativeType >= kNumResolutionTypes) { + success = false; + } else { + success = GetConfiguration( + mNativeType, mNativeIndex, NULL, NULL, NULL, NULL); + } + + if (!success) { + ALOGW("sink advertised an illegal native resolution, fortunately " + "this value is ignored for the time being..."); + } + + return true; +} + +AString VideoFormats::getFormatSpec(bool forM4Message) const { + CHECK_EQ(kNumResolutionTypes, 3); + + // wfd_video_formats: + // 1 byte "native" + // 1 byte "preferred-display-mode-supported" 0 or 1 + // one or more avc codec structures + // 1 byte profile + // 1 byte level + // 4 byte CEA mask + // 4 byte VESA mask + // 4 byte HH mask + // 1 byte latency + // 2 byte min-slice-slice + // 2 byte slice-enc-params + // 1 byte framerate-control-support + // max-hres (none or 2 byte) + // max-vres (none or 2 byte) + + return AStringPrintf( + "%02x 00 %02x %02x %08x %08x %08x 00 0000 0000 00 none none", + forM4Message ? 0x00 : ((mNativeIndex << 3) | mNativeType), + mConfigs[mNativeType][mNativeIndex].profile, + mConfigs[mNativeType][mNativeIndex].level, + mResolutionEnabled[0], + mResolutionEnabled[1], + mResolutionEnabled[2]); +} + +// static +bool VideoFormats::PickBestFormat( + const VideoFormats &sinkSupported, + const VideoFormats &sourceSupported, + ResolutionType *chosenType, + size_t *chosenIndex, + ProfileType *chosenProfile, + LevelType *chosenLevel) { +#if 0 + // Support for the native format is a great idea, the spec includes + // these features, but nobody supports it and the tests don't validate it. + + ResolutionType nativeType; + size_t nativeIndex; + sinkSupported.getNativeResolution(&nativeType, &nativeIndex); + if (sinkSupported.isResolutionEnabled(nativeType, nativeIndex)) { + if (sourceSupported.isResolutionEnabled(nativeType, nativeIndex)) { + ALOGI("Choosing sink's native resolution"); + *chosenType = nativeType; + *chosenIndex = nativeIndex; + return true; + } + } else { + ALOGW("Sink advertised native resolution that it doesn't " + "actually support... ignoring"); + } + + sourceSupported.getNativeResolution(&nativeType, &nativeIndex); + if (sourceSupported.isResolutionEnabled(nativeType, nativeIndex)) { + if (sinkSupported.isResolutionEnabled(nativeType, nativeIndex)) { + ALOGI("Choosing source's native resolution"); + *chosenType = nativeType; + *chosenIndex = nativeIndex; + return true; + } + } else { + ALOGW("Source advertised native resolution that it doesn't " + "actually support... ignoring"); + } +#endif + + bool first = true; + uint32_t bestScore = 0; + size_t bestType = 0; + size_t bestIndex = 0; + for (size_t i = 0; i < kNumResolutionTypes; ++i) { + for (size_t j = 0; j < 32; ++j) { + size_t width, height, framesPerSecond; + bool interlaced; + if (!GetConfiguration( + (ResolutionType)i, + j, + &width, &height, &framesPerSecond, &interlaced)) { + break; + } + + if (!sinkSupported.isResolutionEnabled((ResolutionType)i, j) + || !sourceSupported.isResolutionEnabled( + (ResolutionType)i, j)) { + continue; + } + + ALOGV("type %zu, index %zu, %zu x %zu %c%zu supported", + i, j, width, height, interlaced ? 'i' : 'p', framesPerSecond); + + uint32_t score = width * height * framesPerSecond; + if (!interlaced) { + score *= 2; + } + + if (first || score > bestScore) { + bestScore = score; + bestType = i; + bestIndex = j; + + first = false; + } + } + } + + if (first) { + return false; + } + + *chosenType = (ResolutionType)bestType; + *chosenIndex = bestIndex; + + // Pick the best profile/level supported by both sink and source. + ProfileType srcProfile, sinkProfile; + LevelType srcLevel, sinkLevel; + sourceSupported.getProfileLevel( + (ResolutionType)bestType, bestIndex, + &srcProfile, &srcLevel); + sinkSupported.getProfileLevel( + (ResolutionType)bestType, bestIndex, + &sinkProfile, &sinkLevel); + *chosenProfile = srcProfile < sinkProfile ? srcProfile : sinkProfile; + *chosenLevel = srcLevel < sinkLevel ? srcLevel : sinkLevel; + + return true; +} + +} // namespace android + diff --git a/media/libstagefright/wifi-display/VideoFormats.h b/media/libstagefright/wifi-display/VideoFormats.h new file mode 100644 index 0000000000..fd38fd192e --- /dev/null +++ b/media/libstagefright/wifi-display/VideoFormats.h @@ -0,0 +1,125 @@ +/* + * Copyright 2013, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef VIDEO_FORMATS_H_ + +#define VIDEO_FORMATS_H_ + +#include <media/stagefright/foundation/ABase.h> + +#include <stdint.h> + +namespace android { + +struct AString; + +// This class encapsulates that video resolution capabilities of a wfd source +// or sink as outlined in the wfd specs. Currently three sets of resolutions +// are specified, each of which supports up to 32 resolutions. +// In addition to its capabilities each sink/source also publishes its +// "native" resolution, presumably one that is preferred among all others +// because it wouldn't require any scaling and directly corresponds to the +// display capabilities/pixels. +struct VideoFormats { + VideoFormats(); + + struct config_t { + size_t width, height, framesPerSecond; + bool interlaced; + unsigned char profile, level; + }; + + enum ProfileType { + PROFILE_CBP = 0, + PROFILE_CHP, + kNumProfileTypes, + }; + + enum LevelType { + LEVEL_31 = 0, + LEVEL_32, + LEVEL_40, + LEVEL_41, + LEVEL_42, + kNumLevelTypes, + }; + + enum ResolutionType { + RESOLUTION_CEA, + RESOLUTION_VESA, + RESOLUTION_HH, + kNumResolutionTypes, + }; + + void setNativeResolution(ResolutionType type, size_t index); + void getNativeResolution(ResolutionType *type, size_t *index) const; + + void disableAll(); + void enableAll(); + void enableResolutionUpto( + ResolutionType type, size_t index, + ProfileType profile, LevelType level); + + void setResolutionEnabled( + ResolutionType type, size_t index, bool enabled = true); + + bool isResolutionEnabled(ResolutionType type, size_t index) const; + + void setProfileLevel( + ResolutionType type, size_t index, + ProfileType profile, LevelType level); + + void getProfileLevel( + ResolutionType type, size_t index, + ProfileType *profile, LevelType *level) const; + + static bool GetConfiguration( + ResolutionType type, size_t index, + size_t *width, size_t *height, size_t *framesPerSecond, + bool *interlaced); + + static bool GetProfileLevel( + ProfileType profile, LevelType level, + unsigned *profileIdc, unsigned *levelIdc, + unsigned *constraintSet); + + bool parseFormatSpec(const char *spec); + AString getFormatSpec(bool forM4Message = false) const; + + static bool PickBestFormat( + const VideoFormats &sinkSupported, + const VideoFormats &sourceSupported, + ResolutionType *chosenType, + size_t *chosenIndex, + ProfileType *chosenProfile, + LevelType *chosenLevel); + +private: + bool parseH264Codec(const char *spec); + ResolutionType mNativeType; + size_t mNativeIndex; + + uint32_t mResolutionEnabled[kNumResolutionTypes]; + static const config_t mResolutionTable[kNumResolutionTypes][32]; + config_t mConfigs[kNumResolutionTypes][32]; + + DISALLOW_EVIL_CONSTRUCTORS(VideoFormats); +}; + +} // namespace android + +#endif // VIDEO_FORMATS_H_ + diff --git a/media/libstagefright/wifi-display/rtp/RTPBase.h b/media/libstagefright/wifi-display/rtp/RTPBase.h new file mode 100644 index 0000000000..194f1ee13b --- /dev/null +++ b/media/libstagefright/wifi-display/rtp/RTPBase.h @@ -0,0 +1,49 @@ +/* + * Copyright 2013, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RTP_BASE_H_ + +#define RTP_BASE_H_ + +namespace android { + +struct RTPBase { + enum PacketizationMode { + PACKETIZATION_TRANSPORT_STREAM, + PACKETIZATION_H264, + PACKETIZATION_AAC, + PACKETIZATION_NONE, + }; + + enum TransportMode { + TRANSPORT_UNDEFINED, + TRANSPORT_NONE, + TRANSPORT_UDP, + TRANSPORT_TCP, + TRANSPORT_TCP_INTERLEAVED, + }; + + // Really UDP _payload_ size + const unsigned int kMaxUDPPacketSize = 1472; // 1472 good, 1473 bad on Android@Home + + static int32_t PickRandomRTPPort(); +}; + +} // namespace android + +#endif // RTP_BASE_H_ + + diff --git a/media/libstagefright/wifi-display/rtp/RTPSender.cpp b/media/libstagefright/wifi-display/rtp/RTPSender.cpp new file mode 100644 index 0000000000..ba01b740ea --- /dev/null +++ b/media/libstagefright/wifi-display/rtp/RTPSender.cpp @@ -0,0 +1,808 @@ +/* + * Copyright 2013, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "RTPSender" +#include <utils/Log.h> + +#include "RTPSender.h" + +#include <media/stagefright/foundation/avc_utils.h> +#include <media/stagefright/foundation/ABuffer.h> +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> +#include <media/stagefright/foundation/ByteUtils.h> +#include <media/stagefright/foundation/hexdump.h> +#include <media/stagefright/ANetworkSession.h> +#include <media/stagefright/MediaErrors.h> +#include <media/stagefright/Utils.h> + +namespace android { + +RTPSender::RTPSender( + const sp<ANetworkSession> &netSession, + const sp<AMessage> ¬ify) + : mNetSession(netSession), + mNotify(notify), + mRTPMode(TRANSPORT_UNDEFINED), + mRTCPMode(TRANSPORT_UNDEFINED), + mRTPSessionID(0), + mRTCPSessionID(0), + mRTPConnected(false), + mRTCPConnected(false), + mLastNTPTime(0), + mLastRTPTime(0), + mNumRTPSent(0), + mNumRTPOctetsSent(0), + mNumSRsSent(0), + mRTPSeqNo(0), + mHistorySize(0) { +} + +RTPSender::~RTPSender() { + if (mRTCPSessionID != 0) { + mNetSession->destroySession(mRTCPSessionID); + mRTCPSessionID = 0; + } + + if (mRTPSessionID != 0) { + mNetSession->destroySession(mRTPSessionID); + mRTPSessionID = 0; + } +} + +// static +int32_t RTPBase::PickRandomRTPPort() { + // Pick an even integer in range [1024, 65534) + + static const size_t kRange = (65534 - 1024) / 2; + + return (int32_t)(((float)(kRange + 1) * rand()) / RAND_MAX) * 2 + 1024; +} + +status_t RTPSender::initAsync( + const char *remoteHost, + int32_t remoteRTPPort, + TransportMode rtpMode, + int32_t remoteRTCPPort, + TransportMode rtcpMode, + int32_t *outLocalRTPPort) { + if (mRTPMode != TRANSPORT_UNDEFINED + || rtpMode == TRANSPORT_UNDEFINED + || rtpMode == TRANSPORT_NONE + || rtcpMode == TRANSPORT_UNDEFINED) { + return INVALID_OPERATION; + } + + CHECK_NE(rtpMode, TRANSPORT_TCP_INTERLEAVED); + CHECK_NE(rtcpMode, TRANSPORT_TCP_INTERLEAVED); + + if ((rtcpMode == TRANSPORT_NONE && remoteRTCPPort >= 0) + || (rtcpMode != TRANSPORT_NONE && remoteRTCPPort < 0)) { + return INVALID_OPERATION; + } + + sp<AMessage> rtpNotify = new AMessage(kWhatRTPNotify, this); + + sp<AMessage> rtcpNotify; + if (remoteRTCPPort >= 0) { + rtcpNotify = new AMessage(kWhatRTCPNotify, this); + } + + CHECK_EQ(mRTPSessionID, 0); + CHECK_EQ(mRTCPSessionID, 0); + + int32_t localRTPPort; + + for (;;) { + localRTPPort = PickRandomRTPPort(); + + status_t err; + if (rtpMode == TRANSPORT_UDP) { + err = mNetSession->createUDPSession( + localRTPPort, + remoteHost, + remoteRTPPort, + rtpNotify, + &mRTPSessionID); + } else { + CHECK_EQ(rtpMode, TRANSPORT_TCP); + err = mNetSession->createTCPDatagramSession( + localRTPPort, + remoteHost, + remoteRTPPort, + rtpNotify, + &mRTPSessionID); + } + + if (err != OK) { + continue; + } + + if (remoteRTCPPort < 0) { + break; + } + + if (rtcpMode == TRANSPORT_UDP) { + err = mNetSession->createUDPSession( + localRTPPort + 1, + remoteHost, + remoteRTCPPort, + rtcpNotify, + &mRTCPSessionID); + } else { + CHECK_EQ(rtcpMode, TRANSPORT_TCP); + err = mNetSession->createTCPDatagramSession( + localRTPPort + 1, + remoteHost, + remoteRTCPPort, + rtcpNotify, + &mRTCPSessionID); + } + + if (err == OK) { + break; + } + + mNetSession->destroySession(mRTPSessionID); + mRTPSessionID = 0; + } + + if (rtpMode == TRANSPORT_UDP) { + mRTPConnected = true; + } + + if (rtcpMode == TRANSPORT_UDP) { + mRTCPConnected = true; + } + + mRTPMode = rtpMode; + mRTCPMode = rtcpMode; + *outLocalRTPPort = localRTPPort; + + if (mRTPMode == TRANSPORT_UDP + && (mRTCPMode == TRANSPORT_UDP || mRTCPMode == TRANSPORT_NONE)) { + notifyInitDone(OK); + } + + return OK; +} + +status_t RTPSender::queueBuffer( + const sp<ABuffer> &buffer, uint8_t packetType, PacketizationMode mode) { + status_t err; + + switch (mode) { + case PACKETIZATION_NONE: + err = queueRawPacket(buffer, packetType); + break; + + case PACKETIZATION_TRANSPORT_STREAM: + err = queueTSPackets(buffer, packetType); + break; + + case PACKETIZATION_H264: + err = queueAVCBuffer(buffer, packetType); + break; + + default: + TRESPASS(); + } + + return err; +} + +status_t RTPSender::queueRawPacket( + const sp<ABuffer> &packet, uint8_t packetType) { + CHECK_LE(packet->size(), kMaxUDPPacketSize - 12); + + int64_t timeUs; + CHECK(packet->meta()->findInt64("timeUs", &timeUs)); + + sp<ABuffer> udpPacket = new ABuffer(12 + packet->size()); + + udpPacket->setInt32Data(mRTPSeqNo); + + uint8_t *rtp = udpPacket->data(); + rtp[0] = 0x80; + rtp[1] = packetType; + + rtp[2] = (mRTPSeqNo >> 8) & 0xff; + rtp[3] = mRTPSeqNo & 0xff; + ++mRTPSeqNo; + + uint32_t rtpTime = (timeUs * 9) / 100ll; + + rtp[4] = rtpTime >> 24; + rtp[5] = (rtpTime >> 16) & 0xff; + rtp[6] = (rtpTime >> 8) & 0xff; + rtp[7] = rtpTime & 0xff; + + rtp[8] = kSourceID >> 24; + rtp[9] = (kSourceID >> 16) & 0xff; + rtp[10] = (kSourceID >> 8) & 0xff; + rtp[11] = kSourceID & 0xff; + + memcpy(&rtp[12], packet->data(), packet->size()); + + return sendRTPPacket( + udpPacket, + true /* storeInHistory */, + true /* timeValid */, + ALooper::GetNowUs()); +} + +status_t RTPSender::queueTSPackets( + const sp<ABuffer> &tsPackets, uint8_t packetType) { + CHECK_EQ(0u, tsPackets->size() % 188); + + int64_t timeUs; + CHECK(tsPackets->meta()->findInt64("timeUs", &timeUs)); + + size_t srcOffset = 0; + while (srcOffset < tsPackets->size()) { + sp<ABuffer> udpPacket = + new ABuffer(12 + kMaxNumTSPacketsPerRTPPacket * 188); + + udpPacket->setInt32Data(mRTPSeqNo); + + uint8_t *rtp = udpPacket->data(); + rtp[0] = 0x80; + rtp[1] = packetType; + + rtp[2] = (mRTPSeqNo >> 8) & 0xff; + rtp[3] = mRTPSeqNo & 0xff; + ++mRTPSeqNo; + + int64_t nowUs = ALooper::GetNowUs(); + uint32_t rtpTime = (nowUs * 9) / 100ll; + + rtp[4] = rtpTime >> 24; + rtp[5] = (rtpTime >> 16) & 0xff; + rtp[6] = (rtpTime >> 8) & 0xff; + rtp[7] = rtpTime & 0xff; + + rtp[8] = kSourceID >> 24; + rtp[9] = (kSourceID >> 16) & 0xff; + rtp[10] = (kSourceID >> 8) & 0xff; + rtp[11] = kSourceID & 0xff; + + size_t numTSPackets = (tsPackets->size() - srcOffset) / 188; + if (numTSPackets > kMaxNumTSPacketsPerRTPPacket) { + numTSPackets = kMaxNumTSPacketsPerRTPPacket; + } + + memcpy(&rtp[12], tsPackets->data() + srcOffset, numTSPackets * 188); + + udpPacket->setRange(0, 12 + numTSPackets * 188); + + srcOffset += numTSPackets * 188; + bool isLastPacket = (srcOffset == tsPackets->size()); + + status_t err = sendRTPPacket( + udpPacket, + true /* storeInHistory */, + isLastPacket /* timeValid */, + timeUs); + + if (err != OK) { + return err; + } + } + + return OK; +} + +status_t RTPSender::queueAVCBuffer( + const sp<ABuffer> &accessUnit, uint8_t packetType) { + int64_t timeUs; + CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs)); + + uint32_t rtpTime = (timeUs * 9 / 100ll); + + List<sp<ABuffer> > packets; + + sp<ABuffer> out = new ABuffer(kMaxUDPPacketSize); + size_t outBytesUsed = 12; // Placeholder for RTP header. + + const uint8_t *data = accessUnit->data(); + size_t size = accessUnit->size(); + const uint8_t *nalStart; + size_t nalSize; + while (getNextNALUnit( + &data, &size, &nalStart, &nalSize, + true /* startCodeFollows */) == OK) { + size_t bytesNeeded = nalSize + 2; + if (outBytesUsed == 12) { + ++bytesNeeded; + } + + if (outBytesUsed + bytesNeeded > out->capacity()) { + bool emitSingleNALPacket = false; + + if (outBytesUsed == 12 + && outBytesUsed + nalSize <= out->capacity()) { + // We haven't emitted anything into the current packet yet and + // this NAL unit fits into a single-NAL-unit-packet while + // it wouldn't have fit as part of a STAP-A packet. + + memcpy(out->data() + outBytesUsed, nalStart, nalSize); + outBytesUsed += nalSize; + + emitSingleNALPacket = true; + } + + if (outBytesUsed > 12) { + out->setRange(0, outBytesUsed); + packets.push_back(out); + out = new ABuffer(kMaxUDPPacketSize); + outBytesUsed = 12; // Placeholder for RTP header + } + + if (emitSingleNALPacket) { + continue; + } + } + + if (outBytesUsed + bytesNeeded <= out->capacity()) { + uint8_t *dst = out->data() + outBytesUsed; + + if (outBytesUsed == 12) { + *dst++ = 24; // STAP-A header + } + + *dst++ = (nalSize >> 8) & 0xff; + *dst++ = nalSize & 0xff; + memcpy(dst, nalStart, nalSize); + + outBytesUsed += bytesNeeded; + continue; + } + + // This single NAL unit does not fit into a single RTP packet, + // we need to emit an FU-A. + + CHECK_EQ(outBytesUsed, 12u); + + uint8_t nalType = nalStart[0] & 0x1f; + uint8_t nri = (nalStart[0] >> 5) & 3; + + size_t srcOffset = 1; + while (srcOffset < nalSize) { + size_t copy = out->capacity() - outBytesUsed - 2; + if (copy > nalSize - srcOffset) { + copy = nalSize - srcOffset; + } + + uint8_t *dst = out->data() + outBytesUsed; + dst[0] = (nri << 5) | 28; + + dst[1] = nalType; + + if (srcOffset == 1) { + dst[1] |= 0x80; + } + + if (srcOffset + copy == nalSize) { + dst[1] |= 0x40; + } + + memcpy(&dst[2], nalStart + srcOffset, copy); + srcOffset += copy; + + out->setRange(0, outBytesUsed + copy + 2); + + packets.push_back(out); + out = new ABuffer(kMaxUDPPacketSize); + outBytesUsed = 12; // Placeholder for RTP header + } + } + + if (outBytesUsed > 12) { + out->setRange(0, outBytesUsed); + packets.push_back(out); + } + + while (!packets.empty()) { + sp<ABuffer> out = *packets.begin(); + packets.erase(packets.begin()); + + out->setInt32Data(mRTPSeqNo); + + bool last = packets.empty(); + + uint8_t *dst = out->data(); + + dst[0] = 0x80; + + dst[1] = packetType; + if (last) { + dst[1] |= 1 << 7; // M-bit + } + + dst[2] = (mRTPSeqNo >> 8) & 0xff; + dst[3] = mRTPSeqNo & 0xff; + ++mRTPSeqNo; + + dst[4] = rtpTime >> 24; + dst[5] = (rtpTime >> 16) & 0xff; + dst[6] = (rtpTime >> 8) & 0xff; + dst[7] = rtpTime & 0xff; + dst[8] = kSourceID >> 24; + dst[9] = (kSourceID >> 16) & 0xff; + dst[10] = (kSourceID >> 8) & 0xff; + dst[11] = kSourceID & 0xff; + + status_t err = sendRTPPacket(out, true /* storeInHistory */); + + if (err != OK) { + return err; + } + } + + return OK; +} + +status_t RTPSender::sendRTPPacket( + const sp<ABuffer> &buffer, bool storeInHistory, + bool timeValid, int64_t timeUs) { + CHECK(mRTPConnected); + + status_t err = mNetSession->sendRequest( + mRTPSessionID, buffer->data(), buffer->size(), + timeValid, timeUs); + + if (err != OK) { + return err; + } + + mLastNTPTime = GetNowNTP(); + mLastRTPTime = U32_AT(buffer->data() + 4); + + ++mNumRTPSent; + mNumRTPOctetsSent += buffer->size() - 12; + + if (storeInHistory) { + if (mHistorySize == kMaxHistorySize) { + mHistory.erase(mHistory.begin()); + } else { + ++mHistorySize; + } + mHistory.push_back(buffer); + } + + return OK; +} + +// static +uint64_t RTPSender::GetNowNTP() { + struct timeval tv; + gettimeofday(&tv, NULL /* timezone */); + + uint64_t nowUs = tv.tv_sec * 1000000ll + tv.tv_usec; + + nowUs += ((70ll * 365 + 17) * 24) * 60 * 60 * 1000000ll; + + uint64_t hi = nowUs / 1000000ll; + uint64_t lo = ((1ll << 32) * (nowUs % 1000000ll)) / 1000000ll; + + return (hi << 32) | lo; +} + +void RTPSender::onMessageReceived(const sp<AMessage> &msg) { + switch (msg->what()) { + case kWhatRTPNotify: + case kWhatRTCPNotify: + onNetNotify(msg->what() == kWhatRTPNotify, msg); + break; + + default: + TRESPASS(); + } +} + +void RTPSender::onNetNotify(bool isRTP, const sp<AMessage> &msg) { + int32_t reason; + CHECK(msg->findInt32("reason", &reason)); + + switch (reason) { + case ANetworkSession::kWhatError: + { + int32_t sessionID; + CHECK(msg->findInt32("sessionID", &sessionID)); + + int32_t err; + CHECK(msg->findInt32("err", &err)); + + int32_t errorOccuredDuringSend; + CHECK(msg->findInt32("send", &errorOccuredDuringSend)); + + AString detail; + CHECK(msg->findString("detail", &detail)); + + ALOGE("An error occurred during %s in session %d " + "(%d, '%s' (%s)).", + errorOccuredDuringSend ? "send" : "receive", + sessionID, + err, + detail.c_str(), + strerror(-err)); + + mNetSession->destroySession(sessionID); + + if (sessionID == mRTPSessionID) { + mRTPSessionID = 0; + } else if (sessionID == mRTCPSessionID) { + mRTCPSessionID = 0; + } + + if (!mRTPConnected + || (mRTPMode != TRANSPORT_NONE && !mRTCPConnected)) { + // We haven't completed initialization, attach the error + // to the notification instead. + notifyInitDone(err); + break; + } + + notifyError(err); + break; + } + + case ANetworkSession::kWhatDatagram: + { + sp<ABuffer> data; + CHECK(msg->findBuffer("data", &data)); + + if (isRTP) { + ALOGW("Huh? Received data on RTP connection..."); + } else { + onRTCPData(data); + } + break; + } + + case ANetworkSession::kWhatConnected: + { + int32_t sessionID; + CHECK(msg->findInt32("sessionID", &sessionID)); + + if (isRTP) { + CHECK_EQ(mRTPMode, TRANSPORT_TCP); + CHECK_EQ(sessionID, mRTPSessionID); + mRTPConnected = true; + } else { + CHECK_EQ(mRTCPMode, TRANSPORT_TCP); + CHECK_EQ(sessionID, mRTCPSessionID); + mRTCPConnected = true; + } + + if (mRTPConnected + && (mRTCPMode == TRANSPORT_NONE || mRTCPConnected)) { + notifyInitDone(OK); + } + break; + } + + case ANetworkSession::kWhatNetworkStall: + { + size_t numBytesQueued; + CHECK(msg->findSize("numBytesQueued", &numBytesQueued)); + + notifyNetworkStall(numBytesQueued); + break; + } + + default: + TRESPASS(); + } +} + +status_t RTPSender::onRTCPData(const sp<ABuffer> &buffer) { + const uint8_t *data = buffer->data(); + size_t size = buffer->size(); + + while (size > 0) { + if (size < 8) { + // Too short to be a valid RTCP header + return ERROR_MALFORMED; + } + + if ((data[0] >> 6) != 2) { + // Unsupported version. + return ERROR_UNSUPPORTED; + } + + if (data[0] & 0x20) { + // Padding present. + + size_t paddingLength = data[size - 1]; + + if (paddingLength + 12 > size) { + // If we removed this much padding we'd end up with something + // that's too short to be a valid RTP header. + return ERROR_MALFORMED; + } + + size -= paddingLength; + } + + size_t headerLength = 4 * (data[2] << 8 | data[3]) + 4; + + if (size < headerLength) { + // Only received a partial packet? + return ERROR_MALFORMED; + } + + switch (data[1]) { + case 200: + case 201: // RR + parseReceiverReport(data, headerLength); + break; + + case 202: // SDES + case 203: + break; + + case 204: // APP + parseAPP(data, headerLength); + break; + + case 205: // TSFB (transport layer specific feedback) + parseTSFB(data, headerLength); + break; + + case 206: // PSFB (payload specific feedback) + // hexdump(data, headerLength); + break; + + default: + { + ALOGW("Unknown RTCP packet type %u of size %zu", + (unsigned)data[1], headerLength); + break; + } + } + + data += headerLength; + size -= headerLength; + } + + return OK; +} + +status_t RTPSender::parseReceiverReport( + const uint8_t *data, size_t /* size */) { + float fractionLost = data[12] / 256.0f; + + ALOGI("lost %.2f %% of packets during report interval.", + 100.0f * fractionLost); + + return OK; +} + +status_t RTPSender::parseTSFB(const uint8_t *data, size_t size) { + if ((data[0] & 0x1f) != 1) { + return ERROR_UNSUPPORTED; // We only support NACK for now. + } + + uint32_t srcId = U32_AT(&data[8]); + if (srcId != kSourceID) { + return ERROR_MALFORMED; + } + + for (size_t i = 12; i < size; i += 4) { + uint16_t seqNo = U16_AT(&data[i]); + uint16_t blp = U16_AT(&data[i + 2]); + + List<sp<ABuffer> >::iterator it = mHistory.begin(); + bool foundSeqNo = false; + while (it != mHistory.end()) { + const sp<ABuffer> &buffer = *it; + + uint16_t bufferSeqNo = buffer->int32Data() & 0xffff; + + bool retransmit = false; + if (bufferSeqNo == seqNo) { + retransmit = true; + } else if (blp != 0) { + for (size_t i = 0; i < 16; ++i) { + if ((blp & (1 << i)) + && (bufferSeqNo == ((seqNo + i + 1) & 0xffff))) { + blp &= ~(1 << i); + retransmit = true; + } + } + } + + if (retransmit) { + ALOGV("retransmitting seqNo %d", bufferSeqNo); + + CHECK_EQ((status_t)OK, + sendRTPPacket(buffer, false /* storeInHistory */)); + + if (bufferSeqNo == seqNo) { + foundSeqNo = true; + } + + if (foundSeqNo && blp == 0) { + break; + } + } + + ++it; + } + + if (!foundSeqNo || blp != 0) { + ALOGI("Some sequence numbers were no longer available for " + "retransmission (seqNo = %d, foundSeqNo = %d, blp = 0x%04x)", + seqNo, foundSeqNo, blp); + + if (!mHistory.empty()) { + int32_t earliest = (*mHistory.begin())->int32Data() & 0xffff; + int32_t latest = (*--mHistory.end())->int32Data() & 0xffff; + + ALOGI("have seq numbers from %d - %d", earliest, latest); + } + } + } + + return OK; +} + +status_t RTPSender::parseAPP(const uint8_t *data, size_t size) { + static const size_t late_offset = 8; + static const char late_string[] = "late"; + static const size_t avgLatencyUs_offset = late_offset + sizeof(late_string) - 1; + static const size_t maxLatencyUs_offset = avgLatencyUs_offset + sizeof(int64_t); + + if ((size >= (maxLatencyUs_offset + sizeof(int64_t))) + && !memcmp(late_string, &data[late_offset], sizeof(late_string) - 1)) { + int64_t avgLatencyUs = (int64_t)U64_AT(&data[avgLatencyUs_offset]); + int64_t maxLatencyUs = (int64_t)U64_AT(&data[maxLatencyUs_offset]); + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatInformSender); + notify->setInt64("avgLatencyUs", avgLatencyUs); + notify->setInt64("maxLatencyUs", maxLatencyUs); + notify->post(); + } + + return OK; +} + +void RTPSender::notifyInitDone(status_t err) { + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatInitDone); + notify->setInt32("err", err); + notify->post(); +} + +void RTPSender::notifyError(status_t err) { + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatError); + notify->setInt32("err", err); + notify->post(); +} + +void RTPSender::notifyNetworkStall(size_t numBytesQueued) { + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatNetworkStall); + notify->setSize("numBytesQueued", numBytesQueued); + notify->post(); +} + +} // namespace android + diff --git a/media/libstagefright/wifi-display/rtp/RTPSender.h b/media/libstagefright/wifi-display/rtp/RTPSender.h new file mode 100644 index 0000000000..bedfd01a8a --- /dev/null +++ b/media/libstagefright/wifi-display/rtp/RTPSender.h @@ -0,0 +1,119 @@ +/* + * Copyright 2013, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RTP_SENDER_H_ + +#define RTP_SENDER_H_ + +#include "RTPBase.h" + +#include <media/stagefright/foundation/AHandler.h> + +namespace android { + +struct ABuffer; +struct ANetworkSession; + +// An object of this class facilitates sending of media data over an RTP +// channel. The channel is established over a UDP or TCP connection depending +// on which "TransportMode" was chosen. In addition different RTP packetization +// schemes are supported such as "Transport Stream Packets over RTP", +// or "AVC/H.264 encapsulation as specified in RFC 3984 (non-interleaved mode)" +struct RTPSender : public RTPBase, public AHandler { + enum { + kWhatInitDone, + kWhatError, + kWhatNetworkStall, + kWhatInformSender, + }; + RTPSender( + const sp<ANetworkSession> &netSession, + const sp<AMessage> ¬ify); + + status_t initAsync( + const char *remoteHost, + int32_t remoteRTPPort, + TransportMode rtpMode, + int32_t remoteRTCPPort, + TransportMode rtcpMode, + int32_t *outLocalRTPPort); + + status_t queueBuffer( + const sp<ABuffer> &buffer, + uint8_t packetType, + PacketizationMode mode); + +protected: + virtual ~RTPSender(); + virtual void onMessageReceived(const sp<AMessage> &msg); + +private: + enum { + kWhatRTPNotify, + kWhatRTCPNotify, + }; + + const unsigned int kMaxNumTSPacketsPerRTPPacket = (kMaxUDPPacketSize - 12) / 188; + const unsigned int kMaxHistorySize = 1024; + const unsigned int kSourceID = 0xdeadbeef; + + sp<ANetworkSession> mNetSession; + sp<AMessage> mNotify; + TransportMode mRTPMode; + TransportMode mRTCPMode; + int32_t mRTPSessionID; + int32_t mRTCPSessionID; + bool mRTPConnected; + bool mRTCPConnected; + + uint64_t mLastNTPTime; + uint32_t mLastRTPTime; + uint32_t mNumRTPSent; + uint32_t mNumRTPOctetsSent; + uint32_t mNumSRsSent; + + uint32_t mRTPSeqNo; + + List<sp<ABuffer> > mHistory; + size_t mHistorySize; + + static uint64_t GetNowNTP(); + + status_t queueRawPacket(const sp<ABuffer> &tsPackets, uint8_t packetType); + status_t queueTSPackets(const sp<ABuffer> &tsPackets, uint8_t packetType); + status_t queueAVCBuffer(const sp<ABuffer> &accessUnit, uint8_t packetType); + + status_t sendRTPPacket( + const sp<ABuffer> &packet, bool storeInHistory, + bool timeValid = false, int64_t timeUs = -1ll); + + void onNetNotify(bool isRTP, const sp<AMessage> &msg); + + status_t onRTCPData(const sp<ABuffer> &data); + status_t parseReceiverReport(const uint8_t *data, size_t size); + status_t parseTSFB(const uint8_t *data, size_t size); + status_t parseAPP(const uint8_t *data, size_t size); + + void notifyInitDone(status_t err); + void notifyError(status_t err); + void notifyNetworkStall(size_t numBytesQueued); + + DISALLOW_EVIL_CONSTRUCTORS(RTPSender); +}; + +} // namespace android + +#endif // RTP_SENDER_H_ diff --git a/media/libstagefright/wifi-display/source/Converter.cpp b/media/libstagefright/wifi-display/source/Converter.cpp new file mode 100644 index 0000000000..5502b063ab --- /dev/null +++ b/media/libstagefright/wifi-display/source/Converter.cpp @@ -0,0 +1,826 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "Converter" +#include <utils/Log.h> + +#include "Converter.h" + +#include "MediaPuller.h" + +#include <cutils/properties.h> +#include <gui/Surface.h> +#include <mediadrm/ICrypto.h> +#include <media/MediaCodecBuffer.h> +#include <media/stagefright/foundation/avc_utils.h> +#include <media/stagefright/foundation/ABuffer.h> +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> +#include <media/stagefright/MediaBufferBase.h> +#include <media/stagefright/MediaCodec.h> +#include <media/stagefright/MediaDefs.h> +#include <media/stagefright/MediaErrors.h> +#include <media/MediaBufferHolder.h> + +#include <arpa/inet.h> + +#include <OMX_Video.h> + +namespace android { + +Converter::Converter( + const sp<AMessage> ¬ify, + const sp<ALooper> &codecLooper, + const sp<AMessage> &outputFormat, + uint32_t flags) + : mNotify(notify), + mCodecLooper(codecLooper), + mOutputFormat(outputFormat), + mFlags(flags), + mIsVideo(false), + mIsH264(false), + mIsPCMAudio(false), + mNeedToManuallyPrependSPSPPS(false), + mDoMoreWorkPending(false) +#if ENABLE_SILENCE_DETECTION + ,mFirstSilentFrameUs(-1ll) + ,mInSilentMode(false) +#endif + ,mPrevVideoBitrate(-1) + ,mNumFramesToDrop(0) + ,mEncodingSuspended(false) + { + AString mime; + CHECK(mOutputFormat->findString("mime", &mime)); + + if (!strncasecmp("video/", mime.c_str(), 6)) { + mIsVideo = true; + + mIsH264 = !strcasecmp(mime.c_str(), MEDIA_MIMETYPE_VIDEO_AVC); + } else if (!strcasecmp(MEDIA_MIMETYPE_AUDIO_RAW, mime.c_str())) { + mIsPCMAudio = true; + } +} + +void Converter::releaseEncoder() { + if (mEncoder == NULL) { + return; + } + + mEncoder->release(); + mEncoder.clear(); + + mInputBufferQueue.clear(); + mEncoderInputBuffers.clear(); + mEncoderOutputBuffers.clear(); +} + +Converter::~Converter() { + CHECK(mEncoder == NULL); +} + +void Converter::shutdownAsync() { + ALOGV("shutdown"); + (new AMessage(kWhatShutdown, this))->post(); +} + +status_t Converter::init() { + status_t err = initEncoder(); + + if (err != OK) { + releaseEncoder(); + } + + return err; +} + +sp<IGraphicBufferProducer> Converter::getGraphicBufferProducer() { + CHECK(mFlags & FLAG_USE_SURFACE_INPUT); + return mGraphicBufferProducer; +} + +size_t Converter::getInputBufferCount() const { + return mEncoderInputBuffers.size(); +} + +sp<AMessage> Converter::getOutputFormat() const { + return mOutputFormat; +} + +bool Converter::needToManuallyPrependSPSPPS() const { + return mNeedToManuallyPrependSPSPPS; +} + +// static +int32_t Converter::GetInt32Property( + const char *propName, int32_t defaultValue) { + char val[PROPERTY_VALUE_MAX]; + if (property_get(propName, val, NULL)) { + char *end; + unsigned long x = strtoul(val, &end, 10); + + if (*end == '\0' && end > val && x > 0) { + return x; + } + } + + return defaultValue; +} + +status_t Converter::initEncoder() { + AString outputMIME; + CHECK(mOutputFormat->findString("mime", &outputMIME)); + + bool isAudio = !strncasecmp(outputMIME.c_str(), "audio/", 6); + + if (!mIsPCMAudio) { + mEncoder = MediaCodec::CreateByType( + mCodecLooper, outputMIME.c_str(), true /* encoder */); + + if (mEncoder == NULL) { + return ERROR_UNSUPPORTED; + } + } + + if (mIsPCMAudio) { + return OK; + } + + int32_t audioBitrate = GetInt32Property("media.wfd.audio-bitrate", 128000); + int32_t videoBitrate = GetInt32Property("media.wfd.video-bitrate", 5000000); + mPrevVideoBitrate = videoBitrate; + + ALOGI("using audio bitrate of %d bps, video bitrate of %d bps", + audioBitrate, videoBitrate); + + if (isAudio) { + mOutputFormat->setInt32("bitrate", audioBitrate); + } else { + mOutputFormat->setInt32("bitrate", videoBitrate); + mOutputFormat->setInt32("bitrate-mode", OMX_Video_ControlRateConstant); + mOutputFormat->setInt32("frame-rate", 30); + mOutputFormat->setInt32("i-frame-interval", 15); // Iframes every 15 secs + + // Configure encoder to use intra macroblock refresh mode + mOutputFormat->setInt32("intra-refresh-mode", OMX_VIDEO_IntraRefreshCyclic); + + int width, height, mbs; + if (!mOutputFormat->findInt32("width", &width) + || !mOutputFormat->findInt32("height", &height)) { + return ERROR_UNSUPPORTED; + } + + // Update macroblocks in a cyclic fashion with 10% of all MBs within + // frame gets updated at one time. It takes about 10 frames to + // completely update a whole video frame. If the frame rate is 30, + // it takes about 333 ms in the best case (if next frame is not an IDR) + // to recover from a lost/corrupted packet. + mbs = (((width + 15) / 16) * ((height + 15) / 16) * 10) / 100; + mOutputFormat->setInt32("intra-refresh-CIR-mbs", mbs); + } + + ALOGV("output format is '%s'", mOutputFormat->debugString(0).c_str()); + + mNeedToManuallyPrependSPSPPS = false; + + status_t err = NO_INIT; + + if (!isAudio) { + sp<AMessage> tmp = mOutputFormat->dup(); + tmp->setInt32("prepend-sps-pps-to-idr-frames", 1); + + err = mEncoder->configure( + tmp, + NULL /* nativeWindow */, + NULL /* crypto */, + MediaCodec::CONFIGURE_FLAG_ENCODE); + + if (err == OK) { + // Encoder supported prepending SPS/PPS, we don't need to emulate + // it. + mOutputFormat = tmp; + } else { + mNeedToManuallyPrependSPSPPS = true; + + ALOGI("We going to manually prepend SPS and PPS to IDR frames."); + } + } + + if (err != OK) { + // We'll get here for audio or if we failed to configure the encoder + // to automatically prepend SPS/PPS in the case of video. + + err = mEncoder->configure( + mOutputFormat, + NULL /* nativeWindow */, + NULL /* crypto */, + MediaCodec::CONFIGURE_FLAG_ENCODE); + } + + if (err != OK) { + return err; + } + + if (mFlags & FLAG_USE_SURFACE_INPUT) { + CHECK(mIsVideo); + + err = mEncoder->createInputSurface(&mGraphicBufferProducer); + + if (err != OK) { + return err; + } + } + + err = mEncoder->start(); + + if (err != OK) { + return err; + } + + err = mEncoder->getInputBuffers(&mEncoderInputBuffers); + + if (err != OK) { + return err; + } + + err = mEncoder->getOutputBuffers(&mEncoderOutputBuffers); + + if (err != OK) { + return err; + } + + if (mFlags & FLAG_USE_SURFACE_INPUT) { + scheduleDoMoreWork(); + } + + return OK; +} + +void Converter::notifyError(status_t err) { + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatError); + notify->setInt32("err", err); + notify->post(); +} + +// static +bool Converter::IsSilence(const sp<ABuffer> &accessUnit) { + const uint8_t *ptr = accessUnit->data(); + const uint8_t *end = ptr + accessUnit->size(); + while (ptr < end) { + if (*ptr != 0) { + return false; + } + ++ptr; + } + + return true; +} + +void Converter::onMessageReceived(const sp<AMessage> &msg) { + switch (msg->what()) { + case kWhatMediaPullerNotify: + { + int32_t what; + CHECK(msg->findInt32("what", &what)); + + if (!mIsPCMAudio && mEncoder == NULL) { + ALOGV("got msg '%s' after encoder shutdown.", + msg->debugString().c_str()); + + if (what == MediaPuller::kWhatAccessUnit) { + sp<ABuffer> accessUnit; + CHECK(msg->findBuffer("accessUnit", &accessUnit)); + + accessUnit->meta()->setObject("mediaBufferHolder", sp<MediaBufferHolder>(nullptr)); + } + break; + } + + if (what == MediaPuller::kWhatEOS) { + mInputBufferQueue.push_back(NULL); + + feedEncoderInputBuffers(); + + scheduleDoMoreWork(); + } else { + CHECK_EQ(what, MediaPuller::kWhatAccessUnit); + + sp<ABuffer> accessUnit; + CHECK(msg->findBuffer("accessUnit", &accessUnit)); + + if (mNumFramesToDrop > 0 || mEncodingSuspended) { + if (mNumFramesToDrop > 0) { + --mNumFramesToDrop; + ALOGI("dropping frame."); + } + + accessUnit->meta()->setObject("mediaBufferHolder", sp<MediaBufferHolder>(nullptr)); + break; + } + +#if 0 + MediaBuffer *mbuf = + (MediaBuffer *)(accessUnit->getMediaBufferBase()); + if (mbuf != NULL) { + ALOGI("queueing mbuf %p", mbuf); + mbuf->release(); + } +#endif + +#if ENABLE_SILENCE_DETECTION + if (!mIsVideo) { + if (IsSilence(accessUnit)) { + if (mInSilentMode) { + break; + } + + int64_t nowUs = ALooper::GetNowUs(); + + if (mFirstSilentFrameUs < 0ll) { + mFirstSilentFrameUs = nowUs; + } else if (nowUs >= mFirstSilentFrameUs + 10000000ll) { + mInSilentMode = true; + ALOGI("audio in silent mode now."); + break; + } + } else { + if (mInSilentMode) { + ALOGI("audio no longer in silent mode."); + } + mInSilentMode = false; + mFirstSilentFrameUs = -1ll; + } + } +#endif + + mInputBufferQueue.push_back(accessUnit); + + feedEncoderInputBuffers(); + + scheduleDoMoreWork(); + } + break; + } + + case kWhatEncoderActivity: + { +#if 0 + int64_t whenUs; + if (msg->findInt64("whenUs", &whenUs)) { + int64_t nowUs = ALooper::GetNowUs(); + ALOGI("[%s] kWhatEncoderActivity after %lld us", + mIsVideo ? "video" : "audio", nowUs - whenUs); + } +#endif + + mDoMoreWorkPending = false; + + if (mEncoder == NULL) { + break; + } + + status_t err = doMoreWork(); + + if (err != OK) { + notifyError(err); + } else { + scheduleDoMoreWork(); + } + break; + } + + case kWhatRequestIDRFrame: + { + if (mEncoder == NULL) { + break; + } + + if (mIsVideo) { + ALOGV("requesting IDR frame"); + mEncoder->requestIDRFrame(); + } + break; + } + + case kWhatShutdown: + { + ALOGI("shutting down %s encoder", mIsVideo ? "video" : "audio"); + + releaseEncoder(); + + AString mime; + CHECK(mOutputFormat->findString("mime", &mime)); + ALOGI("encoder (%s) shut down.", mime.c_str()); + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatShutdownCompleted); + notify->post(); + break; + } + + case kWhatDropAFrame: + { + ++mNumFramesToDrop; + break; + } + + case kWhatReleaseOutputBuffer: + { + if (mEncoder != NULL) { + size_t bufferIndex; + CHECK(msg->findInt32("bufferIndex", (int32_t*)&bufferIndex)); + CHECK(bufferIndex < mEncoderOutputBuffers.size()); + mEncoder->releaseOutputBuffer(bufferIndex); + } + break; + } + + case kWhatSuspendEncoding: + { + int32_t suspend; + CHECK(msg->findInt32("suspend", &suspend)); + + mEncodingSuspended = suspend; + + if (mFlags & FLAG_USE_SURFACE_INPUT) { + sp<AMessage> params = new AMessage; + params->setInt32("drop-input-frames",suspend); + mEncoder->setParameters(params); + } + break; + } + + default: + TRESPASS(); + } +} + +void Converter::scheduleDoMoreWork() { + if (mIsPCMAudio) { + // There's no encoder involved in this case. + return; + } + + if (mDoMoreWorkPending) { + return; + } + + mDoMoreWorkPending = true; + +#if 1 + if (mEncoderActivityNotify == NULL) { + mEncoderActivityNotify = new AMessage(kWhatEncoderActivity, this); + } + mEncoder->requestActivityNotification(mEncoderActivityNotify->dup()); +#else + sp<AMessage> notify = new AMessage(kWhatEncoderActivity, this); + notify->setInt64("whenUs", ALooper::GetNowUs()); + mEncoder->requestActivityNotification(notify); +#endif +} + +status_t Converter::feedRawAudioInputBuffers() { + // Split incoming PCM audio into buffers of 6 AUs of 80 audio frames each + // and add a 4 byte header according to the wifi display specs. + + while (!mInputBufferQueue.empty()) { + sp<ABuffer> buffer = *mInputBufferQueue.begin(); + mInputBufferQueue.erase(mInputBufferQueue.begin()); + + int16_t *ptr = (int16_t *)buffer->data(); + int16_t *stop = (int16_t *)(buffer->data() + buffer->size()); + while (ptr < stop) { + *ptr = htons(*ptr); + ++ptr; + } + + static const size_t kFrameSize = 2 * sizeof(int16_t); // stereo + static const size_t kFramesPerAU = 80; + static const size_t kNumAUsPerPESPacket = 6; + + if (mPartialAudioAU != NULL) { + size_t bytesMissingForFullAU = + kNumAUsPerPESPacket * kFramesPerAU * kFrameSize + - mPartialAudioAU->size() + 4; + + size_t copy = buffer->size(); + if(copy > bytesMissingForFullAU) { + copy = bytesMissingForFullAU; + } + + memcpy(mPartialAudioAU->data() + mPartialAudioAU->size(), + buffer->data(), + copy); + + mPartialAudioAU->setRange(0, mPartialAudioAU->size() + copy); + + buffer->setRange(buffer->offset() + copy, buffer->size() - copy); + + int64_t timeUs; + CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); + + int64_t copyUs = (int64_t)((copy / kFrameSize) * 1E6 / 48000.0); + timeUs += copyUs; + buffer->meta()->setInt64("timeUs", timeUs); + + if (bytesMissingForFullAU == copy) { + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatAccessUnit); + notify->setBuffer("accessUnit", mPartialAudioAU); + notify->post(); + + mPartialAudioAU.clear(); + } + } + + while (buffer->size() > 0) { + sp<ABuffer> partialAudioAU = + new ABuffer( + 4 + + kNumAUsPerPESPacket * kFrameSize * kFramesPerAU); + + uint8_t *ptr = partialAudioAU->data(); + ptr[0] = 0xa0; // 10100000b + ptr[1] = kNumAUsPerPESPacket; + ptr[2] = 0; // reserved, audio _emphasis_flag = 0 + + static const unsigned kQuantizationWordLength = 0; // 16-bit + static const unsigned kAudioSamplingFrequency = 2; // 48Khz + static const unsigned kNumberOfAudioChannels = 1; // stereo + + ptr[3] = (kQuantizationWordLength << 6) + | (kAudioSamplingFrequency << 3) + | kNumberOfAudioChannels; + + size_t copy = buffer->size(); + if (copy > partialAudioAU->size() - 4) { + copy = partialAudioAU->size() - 4; + } + + memcpy(&ptr[4], buffer->data(), copy); + + partialAudioAU->setRange(0, 4 + copy); + buffer->setRange(buffer->offset() + copy, buffer->size() - copy); + + int64_t timeUs; + CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); + + partialAudioAU->meta()->setInt64("timeUs", timeUs); + + int64_t copyUs = (int64_t)((copy / kFrameSize) * 1E6 / 48000.0); + timeUs += copyUs; + buffer->meta()->setInt64("timeUs", timeUs); + + if (copy == partialAudioAU->capacity() - 4) { + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatAccessUnit); + notify->setBuffer("accessUnit", partialAudioAU); + notify->post(); + + partialAudioAU.clear(); + continue; + } + + mPartialAudioAU = partialAudioAU; + } + } + + return OK; +} + +status_t Converter::feedEncoderInputBuffers() { + if (mIsPCMAudio) { + return feedRawAudioInputBuffers(); + } + + while (!mInputBufferQueue.empty() + && !mAvailEncoderInputIndices.empty()) { + sp<ABuffer> buffer = *mInputBufferQueue.begin(); + mInputBufferQueue.erase(mInputBufferQueue.begin()); + + size_t bufferIndex = *mAvailEncoderInputIndices.begin(); + mAvailEncoderInputIndices.erase(mAvailEncoderInputIndices.begin()); + + int64_t timeUs = 0ll; + uint32_t flags = 0; + + if (buffer != NULL) { + CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); + + memcpy(mEncoderInputBuffers.itemAt(bufferIndex)->data(), + buffer->data(), + buffer->size()); + + MediaBufferBase *mediaBuffer = NULL; + sp<RefBase> holder; + + if (buffer->meta()->findObject("mediaBufferHolder", &holder)) { + mediaBuffer = (holder != nullptr) ? + static_cast<MediaBufferHolder*>(holder.get())->mediaBuffer() : nullptr; + } + if (mediaBuffer != NULL) { + mEncoderInputBuffers.itemAt(bufferIndex)->meta()->setObject("mediaBufferHolder", new MediaBufferHolder(mediaBuffer)); + + buffer->meta()->setObject("mediaBufferHolder", sp<MediaBufferHolder>(nullptr)); + } + } else { + flags = MediaCodec::BUFFER_FLAG_EOS; + } + + status_t err = mEncoder->queueInputBuffer( + bufferIndex, 0, (buffer == NULL) ? 0 : buffer->size(), + timeUs, flags); + + if (err != OK) { + return err; + } + } + + return OK; +} + +sp<ABuffer> Converter::prependCSD(const sp<ABuffer> &accessUnit) const { + CHECK(mCSD0 != NULL); + + sp<ABuffer> dup = new ABuffer(accessUnit->size() + mCSD0->size()); + memcpy(dup->data(), mCSD0->data(), mCSD0->size()); + memcpy(dup->data() + mCSD0->size(), accessUnit->data(), accessUnit->size()); + + int64_t timeUs; + CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs)); + + dup->meta()->setInt64("timeUs", timeUs); + + return dup; +} + +status_t Converter::doMoreWork() { + status_t err; + + if (!(mFlags & FLAG_USE_SURFACE_INPUT)) { + for (;;) { + size_t bufferIndex; + err = mEncoder->dequeueInputBuffer(&bufferIndex); + + if (err != OK) { + break; + } + + mAvailEncoderInputIndices.push_back(bufferIndex); + } + + feedEncoderInputBuffers(); + } + + for (;;) { + size_t bufferIndex; + size_t offset; + size_t size; + int64_t timeUs; + uint32_t flags; + native_handle_t* handle = NULL; + err = mEncoder->dequeueOutputBuffer( + &bufferIndex, &offset, &size, &timeUs, &flags); + + if (err != OK) { + if (err == INFO_FORMAT_CHANGED) { + continue; + } else if (err == INFO_OUTPUT_BUFFERS_CHANGED) { + mEncoder->getOutputBuffers(&mEncoderOutputBuffers); + continue; + } + + if (err == -EAGAIN) { + err = OK; + } + break; + } + + if (flags & MediaCodec::BUFFER_FLAG_EOS) { + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatEOS); + notify->post(); + } else { +#if 0 + if (mIsVideo) { + int32_t videoBitrate = GetInt32Property( + "media.wfd.video-bitrate", 5000000); + + setVideoBitrate(videoBitrate); + } +#endif + + sp<ABuffer> buffer; + sp<MediaCodecBuffer> outbuf = mEncoderOutputBuffers.itemAt(bufferIndex); + + if (outbuf->meta()->findPointer("handle", (void**)&handle) && + handle != NULL) { + int32_t rangeLength, rangeOffset; + CHECK(outbuf->meta()->findInt32("rangeOffset", &rangeOffset)); + CHECK(outbuf->meta()->findInt32("rangeLength", &rangeLength)); + outbuf->meta()->setPointer("handle", NULL); + + // MediaSender will post the following message when HDCP + // is done, to release the output buffer back to encoder. + sp<AMessage> notify(new AMessage(kWhatReleaseOutputBuffer, this)); + notify->setInt32("bufferIndex", bufferIndex); + + buffer = new ABuffer( + rangeLength > (int32_t)size ? rangeLength : size); + buffer->meta()->setPointer("handle", handle); + buffer->meta()->setInt32("rangeOffset", rangeOffset); + buffer->meta()->setInt32("rangeLength", rangeLength); + buffer->meta()->setMessage("notify", notify); + } else { + buffer = new ABuffer(size); + } + + buffer->meta()->setInt64("timeUs", timeUs); + + ALOGV("[%s] time %lld us (%.2f secs)", + mIsVideo ? "video" : "audio", (long long)timeUs, timeUs / 1E6); + + memcpy(buffer->data(), outbuf->base() + offset, size); + + if (flags & MediaCodec::BUFFER_FLAG_CODECCONFIG) { + if (!handle) { + if (mIsH264) { + mCSD0 = buffer; + } + mOutputFormat->setBuffer("csd-0", buffer); + } + } else { + if (mNeedToManuallyPrependSPSPPS + && mIsH264 + && (mFlags & FLAG_PREPEND_CSD_IF_NECESSARY) + && IsIDR(buffer->data(), buffer->size())) { + buffer = prependCSD(buffer); + } + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatAccessUnit); + notify->setBuffer("accessUnit", buffer); + notify->post(); + } + } + + if (!handle) { + mEncoder->releaseOutputBuffer(bufferIndex); + } + + if (flags & MediaCodec::BUFFER_FLAG_EOS) { + break; + } + } + + return err; +} + +void Converter::requestIDRFrame() { + (new AMessage(kWhatRequestIDRFrame, this))->post(); +} + +void Converter::dropAFrame() { + // Unsupported in surface input mode. + CHECK(!(mFlags & FLAG_USE_SURFACE_INPUT)); + + (new AMessage(kWhatDropAFrame, this))->post(); +} + +void Converter::suspendEncoding(bool suspend) { + sp<AMessage> msg = new AMessage(kWhatSuspendEncoding, this); + msg->setInt32("suspend", suspend); + msg->post(); +} + +int32_t Converter::getVideoBitrate() const { + return mPrevVideoBitrate; +} + +void Converter::setVideoBitrate(int32_t bitRate) { + if (mIsVideo && mEncoder != NULL && bitRate != mPrevVideoBitrate) { + sp<AMessage> params = new AMessage; + params->setInt32("video-bitrate", bitRate); + + mEncoder->setParameters(params); + + mPrevVideoBitrate = bitRate; + } +} + +} // namespace android diff --git a/media/libstagefright/wifi-display/source/Converter.h b/media/libstagefright/wifi-display/source/Converter.h new file mode 100644 index 0000000000..ad95ab5ce9 --- /dev/null +++ b/media/libstagefright/wifi-display/source/Converter.h @@ -0,0 +1,157 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef CONVERTER_H_ + +#define CONVERTER_H_ + +#include <media/stagefright/foundation/AHandler.h> + +namespace android { + +struct ABuffer; +class IGraphicBufferProducer; +struct MediaCodec; +class MediaCodecBuffer; + +#define ENABLE_SILENCE_DETECTION 0 + +// Utility class that receives media access units and converts them into +// media access unit of a different format. +// Right now this'll convert raw video into H.264 and raw audio into AAC. +struct Converter : public AHandler { + enum { + kWhatAccessUnit, + kWhatEOS, + kWhatError, + kWhatShutdownCompleted, + }; + + enum FlagBits { + FLAG_USE_SURFACE_INPUT = 1, + FLAG_PREPEND_CSD_IF_NECESSARY = 2, + }; + Converter(const sp<AMessage> ¬ify, + const sp<ALooper> &codecLooper, + const sp<AMessage> &outputFormat, + uint32_t flags = 0); + + status_t init(); + + sp<IGraphicBufferProducer> getGraphicBufferProducer(); + + size_t getInputBufferCount() const; + + sp<AMessage> getOutputFormat() const; + bool needToManuallyPrependSPSPPS() const; + + void feedAccessUnit(const sp<ABuffer> &accessUnit); + void signalEOS(); + + void requestIDRFrame(); + + void dropAFrame(); + void suspendEncoding(bool suspend); + + void shutdownAsync(); + + int32_t getVideoBitrate() const; + void setVideoBitrate(int32_t bitrate); + + static int32_t GetInt32Property(const char *propName, int32_t defaultValue); + + enum { + // MUST not conflict with private enums below. + kWhatMediaPullerNotify = 'pulN', + }; + +protected: + virtual ~Converter(); + virtual void onMessageReceived(const sp<AMessage> &msg); + +private: + enum { + kWhatDoMoreWork, + kWhatRequestIDRFrame, + kWhatSuspendEncoding, + kWhatShutdown, + kWhatEncoderActivity, + kWhatDropAFrame, + kWhatReleaseOutputBuffer, + }; + + sp<AMessage> mNotify; + sp<ALooper> mCodecLooper; + sp<AMessage> mOutputFormat; + uint32_t mFlags; + bool mIsVideo; + bool mIsH264; + bool mIsPCMAudio; + bool mNeedToManuallyPrependSPSPPS; + + sp<MediaCodec> mEncoder; + sp<AMessage> mEncoderActivityNotify; + + sp<IGraphicBufferProducer> mGraphicBufferProducer; + + Vector<sp<MediaCodecBuffer> > mEncoderInputBuffers; + Vector<sp<MediaCodecBuffer> > mEncoderOutputBuffers; + + List<size_t> mAvailEncoderInputIndices; + + List<sp<ABuffer> > mInputBufferQueue; + + sp<ABuffer> mCSD0; + + bool mDoMoreWorkPending; + +#if ENABLE_SILENCE_DETECTION + int64_t mFirstSilentFrameUs; + bool mInSilentMode; +#endif + + sp<ABuffer> mPartialAudioAU; + + int32_t mPrevVideoBitrate; + + int32_t mNumFramesToDrop; + bool mEncodingSuspended; + + status_t initEncoder(); + void releaseEncoder(); + + status_t feedEncoderInputBuffers(); + + void scheduleDoMoreWork(); + status_t doMoreWork(); + + void notifyError(status_t err); + + // Packetizes raw PCM audio data available in mInputBufferQueue + // into a format suitable for transport stream inclusion and + // notifies the observer. + status_t feedRawAudioInputBuffers(); + + static bool IsSilence(const sp<ABuffer> &accessUnit); + + sp<ABuffer> prependCSD(const sp<ABuffer> &accessUnit) const; + + DISALLOW_EVIL_CONSTRUCTORS(Converter); +}; + +} // namespace android + +#endif // CONVERTER_H_ diff --git a/media/libstagefright/wifi-display/source/MediaPuller.cpp b/media/libstagefright/wifi-display/source/MediaPuller.cpp new file mode 100644 index 0000000000..5bf893d1a7 --- /dev/null +++ b/media/libstagefright/wifi-display/source/MediaPuller.cpp @@ -0,0 +1,227 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "MediaPuller" +#include <utils/Log.h> + +#include "MediaPuller.h" + +#include <media/stagefright/foundation/ABuffer.h> +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> +#include <media/stagefright/MediaBufferBase.h> +#include <media/stagefright/MediaSource.h> +#include <media/stagefright/MetaData.h> +#include <media/MediaBufferHolder.h> + +namespace android { + +MediaPuller::MediaPuller( + const sp<MediaSource> &source, const sp<AMessage> ¬ify) + : mSource(source), + mNotify(notify), + mPullGeneration(0), + mIsAudio(false), + mPaused(false) { + sp<MetaData> meta = source->getFormat(); + const char *mime; + CHECK(meta->findCString(kKeyMIMEType, &mime)); + + mIsAudio = !strncasecmp(mime, "audio/", 6); +} + +MediaPuller::~MediaPuller() { +} + +status_t MediaPuller::postSynchronouslyAndReturnError( + const sp<AMessage> &msg) { + sp<AMessage> response; + status_t err = msg->postAndAwaitResponse(&response); + + if (err != OK) { + return err; + } + + if (!response->findInt32("err", &err)) { + err = OK; + } + + return err; +} + +status_t MediaPuller::start() { + return postSynchronouslyAndReturnError(new AMessage(kWhatStart, this)); +} + +void MediaPuller::stopAsync(const sp<AMessage> ¬ify) { + sp<AMessage> msg = new AMessage(kWhatStop, this); + msg->setMessage("notify", notify); + msg->post(); +} + +void MediaPuller::pause() { + (new AMessage(kWhatPause, this))->post(); +} + +void MediaPuller::resume() { + (new AMessage(kWhatResume, this))->post(); +} + +void MediaPuller::onMessageReceived(const sp<AMessage> &msg) { + switch (msg->what()) { + case kWhatStart: + { + status_t err; + if (mIsAudio) { + // This atrocity causes AudioSource to deliver absolute + // systemTime() based timestamps (off by 1 us). + sp<MetaData> params = new MetaData; + params->setInt64(kKeyTime, 1ll); + err = mSource->start(params.get()); + } else { + err = mSource->start(); + if (err != OK) { + ALOGE("source failed to start w/ err %d", err); + } + } + + if (err == OK) { + schedulePull(); + } + + sp<AMessage> response = new AMessage; + response->setInt32("err", err); + + sp<AReplyToken> replyID; + CHECK(msg->senderAwaitsResponse(&replyID)); + response->postReply(replyID); + break; + } + + case kWhatStop: + { + sp<MetaData> meta = mSource->getFormat(); + const char *tmp; + CHECK(meta->findCString(kKeyMIMEType, &tmp)); + AString mime = tmp; + + ALOGI("MediaPuller(%s) stopping.", mime.c_str()); + mSource->stop(); + ALOGI("MediaPuller(%s) stopped.", mime.c_str()); + ++mPullGeneration; + + sp<AMessage> notify; + CHECK(msg->findMessage("notify", ¬ify)); + notify->post(); + break; + } + + case kWhatPull: + { + int32_t generation; + CHECK(msg->findInt32("generation", &generation)); + + if (generation != mPullGeneration) { + break; + } + + MediaBufferBase *mbuf; + status_t err = mSource->read(&mbuf); + + if (mPaused) { + if (err == OK) { + mbuf->release(); + mbuf = NULL; + } + + schedulePull(); + break; + } + + if (err != OK) { + if (err == ERROR_END_OF_STREAM) { + ALOGI("stream ended."); + } else { + ALOGE("error %d reading stream.", err); + } + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatEOS); + notify->post(); + } else { + int64_t timeUs; + CHECK(mbuf->meta_data().findInt64(kKeyTime, &timeUs)); + + sp<ABuffer> accessUnit = new ABuffer(mbuf->range_length()); + + memcpy(accessUnit->data(), + (const uint8_t *)mbuf->data() + mbuf->range_offset(), + mbuf->range_length()); + + accessUnit->meta()->setInt64("timeUs", timeUs); + + if (mIsAudio) { + mbuf->release(); + mbuf = NULL; + } else { + // video encoder will release MediaBufferBase when done + // with underlying data. + accessUnit->meta()->setObject("mediaBufferHolder", new MediaBufferHolder(mbuf)); + mbuf->release(); + mbuf = NULL; + } + + sp<AMessage> notify = mNotify->dup(); + + notify->setInt32("what", kWhatAccessUnit); + notify->setBuffer("accessUnit", accessUnit); + notify->post(); + + if (mbuf != NULL) { + ALOGV("posted mbuf %p", mbuf); + } + + schedulePull(); + } + break; + } + + case kWhatPause: + { + mPaused = true; + break; + } + + case kWhatResume: + { + mPaused = false; + break; + } + + default: + TRESPASS(); + } +} + +void MediaPuller::schedulePull() { + sp<AMessage> msg = new AMessage(kWhatPull, this); + msg->setInt32("generation", mPullGeneration); + msg->post(); +} + +} // namespace android + diff --git a/media/libstagefright/wifi-display/source/MediaPuller.h b/media/libstagefright/wifi-display/source/MediaPuller.h new file mode 100644 index 0000000000..1291bb3f9e --- /dev/null +++ b/media/libstagefright/wifi-display/source/MediaPuller.h @@ -0,0 +1,68 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MEDIA_PULLER_H_ + +#define MEDIA_PULLER_H_ + +#include <media/stagefright/foundation/AHandler.h> + +namespace android { + +struct MediaSource; + +struct MediaPuller : public AHandler { + enum { + kWhatEOS, + kWhatAccessUnit + }; + + MediaPuller(const sp<MediaSource> &source, const sp<AMessage> ¬ify); + + status_t start(); + void stopAsync(const sp<AMessage> ¬ify); + + void pause(); + void resume(); + +protected: + virtual void onMessageReceived(const sp<AMessage> &msg); + virtual ~MediaPuller(); + +private: + enum { + kWhatStart, + kWhatStop, + kWhatPull, + kWhatPause, + kWhatResume, + }; + + sp<MediaSource> mSource; + sp<AMessage> mNotify; + int32_t mPullGeneration; + bool mIsAudio; + bool mPaused; + + status_t postSynchronouslyAndReturnError(const sp<AMessage> &msg); + void schedulePull(); + + DISALLOW_EVIL_CONSTRUCTORS(MediaPuller); +}; + +} // namespace android + +#endif // MEDIA_PULLER_H_ diff --git a/media/libstagefright/wifi-display/source/PlaybackSession.cpp b/media/libstagefright/wifi-display/source/PlaybackSession.cpp new file mode 100644 index 0000000000..0240d2a552 --- /dev/null +++ b/media/libstagefright/wifi-display/source/PlaybackSession.cpp @@ -0,0 +1,1116 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "PlaybackSession" +#include <utils/Log.h> + +#include "PlaybackSession.h" + +#include "Converter.h" +#include "MediaPuller.h" +#include "RepeaterSource.h" +#include "WifiDisplaySource.h" + +#include <binder/IServiceManager.h> +#include <cutils/properties.h> +#include <media/IHDCP.h> +#include <media/IMediaHTTPService.h> +#include <media/stagefright/foundation/avc_utils.h> +#include <media/stagefright/foundation/ABitReader.h> +#include <media/stagefright/foundation/ABuffer.h> +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> +#include <media/stagefright/foundation/hexdump.h> +#include <media/stagefright/AudioSource.h> +#include <media/stagefright/MediaDefs.h> +#include <media/stagefright/MediaErrors.h> +#include <media/stagefright/MediaSource.h> +#include <media/stagefright/MetaData.h> +#include <media/stagefright/NuMediaExtractor.h> +#include <media/stagefright/SurfaceMediaSource.h> +#include <media/stagefright/Utils.h> +#include <media/IOMX.h> + +#include <OMX_IVCommon.h> + +namespace android { + +struct WifiDisplaySource::PlaybackSession::Track : public AHandler { + enum { + kWhatStopped, + }; + + Track(const sp<AMessage> ¬ify, + const sp<ALooper> &pullLooper, + const sp<ALooper> &codecLooper, + const sp<MediaPuller> &mediaPuller, + const sp<Converter> &converter); + + Track(const sp<AMessage> ¬ify, const sp<AMessage> &format); + + void setRepeaterSource(const sp<RepeaterSource> &source); + + sp<AMessage> getFormat(); + bool isAudio() const; + + const sp<Converter> &converter() const; + const sp<RepeaterSource> &repeaterSource() const; + + ssize_t mediaSenderTrackIndex() const; + void setMediaSenderTrackIndex(size_t index); + + status_t start(); + void stopAsync(); + + void pause(); + void resume(); + + void queueAccessUnit(const sp<ABuffer> &accessUnit); + sp<ABuffer> dequeueAccessUnit(); + + bool hasOutputBuffer(int64_t *timeUs) const; + void queueOutputBuffer(const sp<ABuffer> &accessUnit); + sp<ABuffer> dequeueOutputBuffer(); + +#if SUSPEND_VIDEO_IF_IDLE + bool isSuspended() const; +#endif + + size_t countQueuedOutputBuffers() const { + return mQueuedOutputBuffers.size(); + } + + void requestIDRFrame(); + +protected: + virtual void onMessageReceived(const sp<AMessage> &msg); + virtual ~Track(); + +private: + enum { + kWhatMediaPullerStopped, + }; + + sp<AMessage> mNotify; + sp<ALooper> mPullLooper; + sp<ALooper> mCodecLooper; + sp<MediaPuller> mMediaPuller; + sp<Converter> mConverter; + sp<AMessage> mFormat; + bool mStarted; + ssize_t mMediaSenderTrackIndex; + bool mIsAudio; + List<sp<ABuffer> > mQueuedAccessUnits; + sp<RepeaterSource> mRepeaterSource; + List<sp<ABuffer> > mQueuedOutputBuffers; + int64_t mLastOutputBufferQueuedTimeUs; + + static bool IsAudioFormat(const sp<AMessage> &format); + + DISALLOW_EVIL_CONSTRUCTORS(Track); +}; + +WifiDisplaySource::PlaybackSession::Track::Track( + const sp<AMessage> ¬ify, + const sp<ALooper> &pullLooper, + const sp<ALooper> &codecLooper, + const sp<MediaPuller> &mediaPuller, + const sp<Converter> &converter) + : mNotify(notify), + mPullLooper(pullLooper), + mCodecLooper(codecLooper), + mMediaPuller(mediaPuller), + mConverter(converter), + mStarted(false), + mIsAudio(IsAudioFormat(mConverter->getOutputFormat())), + mLastOutputBufferQueuedTimeUs(-1ll) { +} + +WifiDisplaySource::PlaybackSession::Track::Track( + const sp<AMessage> ¬ify, const sp<AMessage> &format) + : mNotify(notify), + mFormat(format), + mStarted(false), + mIsAudio(IsAudioFormat(format)), + mLastOutputBufferQueuedTimeUs(-1ll) { +} + +WifiDisplaySource::PlaybackSession::Track::~Track() { + CHECK(!mStarted); +} + +// static +bool WifiDisplaySource::PlaybackSession::Track::IsAudioFormat( + const sp<AMessage> &format) { + AString mime; + CHECK(format->findString("mime", &mime)); + + return !strncasecmp(mime.c_str(), "audio/", 6); +} + +sp<AMessage> WifiDisplaySource::PlaybackSession::Track::getFormat() { + return mFormat != NULL ? mFormat : mConverter->getOutputFormat(); +} + +bool WifiDisplaySource::PlaybackSession::Track::isAudio() const { + return mIsAudio; +} + +const sp<Converter> &WifiDisplaySource::PlaybackSession::Track::converter() const { + return mConverter; +} + +const sp<RepeaterSource> & +WifiDisplaySource::PlaybackSession::Track::repeaterSource() const { + return mRepeaterSource; +} + +ssize_t WifiDisplaySource::PlaybackSession::Track::mediaSenderTrackIndex() const { + CHECK_GE(mMediaSenderTrackIndex, 0); + return mMediaSenderTrackIndex; +} + +void WifiDisplaySource::PlaybackSession::Track::setMediaSenderTrackIndex( + size_t index) { + mMediaSenderTrackIndex = index; +} + +status_t WifiDisplaySource::PlaybackSession::Track::start() { + ALOGV("Track::start isAudio=%d", mIsAudio); + + CHECK(!mStarted); + + status_t err = OK; + + if (mMediaPuller != NULL) { + err = mMediaPuller->start(); + } + + if (err == OK) { + mStarted = true; + } + + return err; +} + +void WifiDisplaySource::PlaybackSession::Track::stopAsync() { + ALOGV("Track::stopAsync isAudio=%d", mIsAudio); + + if (mConverter != NULL) { + mConverter->shutdownAsync(); + } + + sp<AMessage> msg = new AMessage(kWhatMediaPullerStopped, this); + + if (mStarted && mMediaPuller != NULL) { + if (mRepeaterSource != NULL) { + // Let's unblock MediaPuller's MediaSource::read(). + mRepeaterSource->wakeUp(); + } + + mMediaPuller->stopAsync(msg); + } else { + mStarted = false; + msg->post(); + } +} + +void WifiDisplaySource::PlaybackSession::Track::pause() { + mMediaPuller->pause(); +} + +void WifiDisplaySource::PlaybackSession::Track::resume() { + mMediaPuller->resume(); +} + +void WifiDisplaySource::PlaybackSession::Track::onMessageReceived( + const sp<AMessage> &msg) { + switch (msg->what()) { + case kWhatMediaPullerStopped: + { + mConverter.clear(); + + mStarted = false; + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatStopped); + notify->post(); + + ALOGI("kWhatStopped %s posted", mIsAudio ? "audio" : "video"); + break; + } + + default: + TRESPASS(); + } +} + +void WifiDisplaySource::PlaybackSession::Track::queueAccessUnit( + const sp<ABuffer> &accessUnit) { + mQueuedAccessUnits.push_back(accessUnit); +} + +sp<ABuffer> WifiDisplaySource::PlaybackSession::Track::dequeueAccessUnit() { + if (mQueuedAccessUnits.empty()) { + return NULL; + } + + sp<ABuffer> accessUnit = *mQueuedAccessUnits.begin(); + CHECK(accessUnit != NULL); + + mQueuedAccessUnits.erase(mQueuedAccessUnits.begin()); + + return accessUnit; +} + +void WifiDisplaySource::PlaybackSession::Track::setRepeaterSource( + const sp<RepeaterSource> &source) { + mRepeaterSource = source; +} + +void WifiDisplaySource::PlaybackSession::Track::requestIDRFrame() { + if (mIsAudio) { + return; + } + + if (mRepeaterSource != NULL) { + mRepeaterSource->wakeUp(); + } + + mConverter->requestIDRFrame(); +} + +bool WifiDisplaySource::PlaybackSession::Track::hasOutputBuffer( + int64_t *timeUs) const { + *timeUs = 0ll; + + if (mQueuedOutputBuffers.empty()) { + return false; + } + + const sp<ABuffer> &outputBuffer = *mQueuedOutputBuffers.begin(); + + CHECK(outputBuffer->meta()->findInt64("timeUs", timeUs)); + + return true; +} + +void WifiDisplaySource::PlaybackSession::Track::queueOutputBuffer( + const sp<ABuffer> &accessUnit) { + mQueuedOutputBuffers.push_back(accessUnit); + mLastOutputBufferQueuedTimeUs = ALooper::GetNowUs(); +} + +sp<ABuffer> WifiDisplaySource::PlaybackSession::Track::dequeueOutputBuffer() { + CHECK(!mQueuedOutputBuffers.empty()); + + sp<ABuffer> outputBuffer = *mQueuedOutputBuffers.begin(); + mQueuedOutputBuffers.erase(mQueuedOutputBuffers.begin()); + + return outputBuffer; +} + +#if SUSPEND_VIDEO_IF_IDLE +bool WifiDisplaySource::PlaybackSession::Track::isSuspended() const { + if (!mQueuedOutputBuffers.empty()) { + return false; + } + + if (mLastOutputBufferQueuedTimeUs < 0ll) { + // We've never seen an output buffer queued, but tracks start + // out live, not suspended. + return false; + } + + // If we've not seen new output data for 60ms or more, we consider + // this track suspended for the time being. + return (ALooper::GetNowUs() - mLastOutputBufferQueuedTimeUs) > 60000ll; +} +#endif + +//////////////////////////////////////////////////////////////////////////////// + +WifiDisplaySource::PlaybackSession::PlaybackSession( + const String16 &opPackageName, + const sp<ANetworkSession> &netSession, + const sp<AMessage> ¬ify, + const in_addr &interfaceAddr, + const sp<IHDCP> &hdcp, + const char *path) + : mOpPackageName(opPackageName), + mNetSession(netSession), + mNotify(notify), + mInterfaceAddr(interfaceAddr), + mHDCP(hdcp), + mLocalRTPPort(-1), + mWeAreDead(false), + mPaused(false), + mLastLifesignUs(), + mVideoTrackIndex(-1), + mPrevTimeUs(-1ll), + mPullExtractorPending(false), + mPullExtractorGeneration(0), + mFirstSampleTimeRealUs(-1ll), + mFirstSampleTimeUs(-1ll) { + if (path != NULL) { + mMediaPath.setTo(path); + } +} + +status_t WifiDisplaySource::PlaybackSession::init( + const char *clientIP, + int32_t clientRtp, + RTPSender::TransportMode rtpMode, + int32_t clientRtcp, + RTPSender::TransportMode rtcpMode, + bool enableAudio, + bool usePCMAudio, + bool enableVideo, + VideoFormats::ResolutionType videoResolutionType, + size_t videoResolutionIndex, + VideoFormats::ProfileType videoProfileType, + VideoFormats::LevelType videoLevelType) { + sp<AMessage> notify = new AMessage(kWhatMediaSenderNotify, this); + mMediaSender = new MediaSender(mNetSession, notify); + looper()->registerHandler(mMediaSender); + + mMediaSender->setHDCP(mHDCP); + + status_t err = setupPacketizer( + enableAudio, + usePCMAudio, + enableVideo, + videoResolutionType, + videoResolutionIndex, + videoProfileType, + videoLevelType); + + if (err == OK) { + err = mMediaSender->initAsync( + -1 /* trackIndex */, + clientIP, + clientRtp, + rtpMode, + clientRtcp, + rtcpMode, + &mLocalRTPPort); + } + + if (err != OK) { + mLocalRTPPort = -1; + + looper()->unregisterHandler(mMediaSender->id()); + mMediaSender.clear(); + + return err; + } + + updateLiveness(); + + return OK; +} + +WifiDisplaySource::PlaybackSession::~PlaybackSession() { +} + +int32_t WifiDisplaySource::PlaybackSession::getRTPPort() const { + return mLocalRTPPort; +} + +int64_t WifiDisplaySource::PlaybackSession::getLastLifesignUs() const { + return mLastLifesignUs; +} + +void WifiDisplaySource::PlaybackSession::updateLiveness() { + mLastLifesignUs = ALooper::GetNowUs(); +} + +status_t WifiDisplaySource::PlaybackSession::play() { + updateLiveness(); + + (new AMessage(kWhatResume, this))->post(); + + return OK; +} + +status_t WifiDisplaySource::PlaybackSession::onMediaSenderInitialized() { + for (size_t i = 0; i < mTracks.size(); ++i) { + CHECK_EQ((status_t)OK, mTracks.editValueAt(i)->start()); + } + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatSessionEstablished); + notify->post(); + + return OK; +} + +status_t WifiDisplaySource::PlaybackSession::pause() { + updateLiveness(); + + (new AMessage(kWhatPause, this))->post(); + + return OK; +} + +void WifiDisplaySource::PlaybackSession::destroyAsync() { + ALOGI("destroyAsync"); + + for (size_t i = 0; i < mTracks.size(); ++i) { + mTracks.valueAt(i)->stopAsync(); + } +} + +void WifiDisplaySource::PlaybackSession::onMessageReceived( + const sp<AMessage> &msg) { + switch (msg->what()) { + case kWhatConverterNotify: + { + if (mWeAreDead) { + ALOGV("dropping msg '%s' because we're dead", + msg->debugString().c_str()); + + break; + } + + int32_t what; + CHECK(msg->findInt32("what", &what)); + + size_t trackIndex; + CHECK(msg->findSize("trackIndex", &trackIndex)); + + if (what == Converter::kWhatAccessUnit) { + sp<ABuffer> accessUnit; + CHECK(msg->findBuffer("accessUnit", &accessUnit)); + + const sp<Track> &track = mTracks.valueFor(trackIndex); + + status_t err = mMediaSender->queueAccessUnit( + track->mediaSenderTrackIndex(), + accessUnit); + + if (err != OK) { + notifySessionDead(); + } + break; + } else if (what == Converter::kWhatEOS) { + CHECK_EQ(what, Converter::kWhatEOS); + + ALOGI("output EOS on track %zu", trackIndex); + + ssize_t index = mTracks.indexOfKey(trackIndex); + CHECK_GE(index, 0); + + const sp<Converter> &converter = + mTracks.valueAt(index)->converter(); + looper()->unregisterHandler(converter->id()); + + mTracks.removeItemsAt(index); + + if (mTracks.isEmpty()) { + ALOGI("Reached EOS"); + } + } else if (what != Converter::kWhatShutdownCompleted) { + CHECK_EQ(what, Converter::kWhatError); + + status_t err; + CHECK(msg->findInt32("err", &err)); + + ALOGE("converter signaled error %d", err); + + notifySessionDead(); + } + break; + } + + case kWhatMediaSenderNotify: + { + int32_t what; + CHECK(msg->findInt32("what", &what)); + + if (what == MediaSender::kWhatInitDone) { + status_t err; + CHECK(msg->findInt32("err", &err)); + + if (err == OK) { + onMediaSenderInitialized(); + } else { + notifySessionDead(); + } + } else if (what == MediaSender::kWhatError) { + notifySessionDead(); + } else if (what == MediaSender::kWhatNetworkStall) { + size_t numBytesQueued; + CHECK(msg->findSize("numBytesQueued", &numBytesQueued)); + + if (mVideoTrackIndex >= 0) { + const sp<Track> &videoTrack = + mTracks.valueFor(mVideoTrackIndex); + + sp<Converter> converter = videoTrack->converter(); + if (converter != NULL) { + converter->dropAFrame(); + } + } + } else if (what == MediaSender::kWhatInformSender) { + onSinkFeedback(msg); + } else { + TRESPASS(); + } + break; + } + + case kWhatTrackNotify: + { + int32_t what; + CHECK(msg->findInt32("what", &what)); + + size_t trackIndex; + CHECK(msg->findSize("trackIndex", &trackIndex)); + + if (what == Track::kWhatStopped) { + ALOGI("Track %zu stopped", trackIndex); + + sp<Track> track = mTracks.valueFor(trackIndex); + looper()->unregisterHandler(track->id()); + mTracks.removeItem(trackIndex); + track.clear(); + + if (!mTracks.isEmpty()) { + ALOGI("not all tracks are stopped yet"); + break; + } + + looper()->unregisterHandler(mMediaSender->id()); + mMediaSender.clear(); + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatSessionDestroyed); + notify->post(); + } + break; + } + + case kWhatPause: + { + if (mExtractor != NULL) { + ++mPullExtractorGeneration; + mFirstSampleTimeRealUs = -1ll; + mFirstSampleTimeUs = -1ll; + } + + if (mPaused) { + break; + } + + for (size_t i = 0; i < mTracks.size(); ++i) { + mTracks.editValueAt(i)->pause(); + } + + mPaused = true; + break; + } + + case kWhatResume: + { + if (mExtractor != NULL) { + schedulePullExtractor(); + } + + if (!mPaused) { + break; + } + + for (size_t i = 0; i < mTracks.size(); ++i) { + mTracks.editValueAt(i)->resume(); + } + + mPaused = false; + break; + } + + case kWhatPullExtractorSample: + { + int32_t generation; + CHECK(msg->findInt32("generation", &generation)); + + if (generation != mPullExtractorGeneration) { + break; + } + + mPullExtractorPending = false; + + onPullExtractor(); + break; + } + + default: + TRESPASS(); + } +} + +void WifiDisplaySource::PlaybackSession::onSinkFeedback(const sp<AMessage> &msg) { + int64_t avgLatencyUs; + CHECK(msg->findInt64("avgLatencyUs", &avgLatencyUs)); + + int64_t maxLatencyUs; + CHECK(msg->findInt64("maxLatencyUs", &maxLatencyUs)); + + ALOGI("sink reports avg. latency of %lld ms (max %lld ms)", + avgLatencyUs / 1000ll, + maxLatencyUs / 1000ll); + + if (mVideoTrackIndex >= 0) { + const sp<Track> &videoTrack = mTracks.valueFor(mVideoTrackIndex); + sp<Converter> converter = videoTrack->converter(); + + if (converter != NULL) { + int32_t videoBitrate = + Converter::GetInt32Property("media.wfd.video-bitrate", -1); + + char val[PROPERTY_VALUE_MAX]; + if (videoBitrate < 0 + && property_get("media.wfd.video-bitrate", val, NULL) + && !strcasecmp("adaptive", val)) { + videoBitrate = converter->getVideoBitrate(); + + if (avgLatencyUs > 300000ll) { + videoBitrate *= 0.6; + } else if (avgLatencyUs < 100000ll) { + videoBitrate *= 1.1; + } + } + + if (videoBitrate > 0) { + if (videoBitrate < 500000) { + videoBitrate = 500000; + } else if (videoBitrate > 10000000) { + videoBitrate = 10000000; + } + + if (videoBitrate != converter->getVideoBitrate()) { + ALOGI("setting video bitrate to %d bps", videoBitrate); + + converter->setVideoBitrate(videoBitrate); + } + } + } + + sp<RepeaterSource> repeaterSource = videoTrack->repeaterSource(); + if (repeaterSource != NULL) { + double rateHz = + Converter::GetInt32Property( + "media.wfd.video-framerate", -1); + + char val[PROPERTY_VALUE_MAX]; + if (rateHz < 0.0 + && property_get("media.wfd.video-framerate", val, NULL) + && !strcasecmp("adaptive", val)) { + rateHz = repeaterSource->getFrameRate(); + + if (avgLatencyUs > 300000ll) { + rateHz *= 0.9; + } else if (avgLatencyUs < 200000ll) { + rateHz *= 1.1; + } + } + + if (rateHz > 0) { + if (rateHz < 5.0) { + rateHz = 5.0; + } else if (rateHz > 30.0) { + rateHz = 30.0; + } + + if (rateHz != repeaterSource->getFrameRate()) { + ALOGI("setting frame rate to %.2f Hz", rateHz); + + repeaterSource->setFrameRate(rateHz); + } + } + } + } +} + +status_t WifiDisplaySource::PlaybackSession::setupMediaPacketizer( + bool enableAudio, bool enableVideo) { + mExtractor = new NuMediaExtractor(NuMediaExtractor::EntryPoint::OTHER); + + status_t err = mExtractor->setDataSource( + NULL /* httpService */, mMediaPath.c_str()); + + if (err != OK) { + return err; + } + + size_t n = mExtractor->countTracks(); + bool haveAudio = false; + bool haveVideo = false; + for (size_t i = 0; i < n; ++i) { + sp<AMessage> format; + err = mExtractor->getTrackFormat(i, &format); + + if (err != OK) { + continue; + } + + AString mime; + CHECK(format->findString("mime", &mime)); + + bool isAudio = !strncasecmp(mime.c_str(), "audio/", 6); + bool isVideo = !strncasecmp(mime.c_str(), "video/", 6); + + if (isAudio && enableAudio && !haveAudio) { + haveAudio = true; + } else if (isVideo && enableVideo && !haveVideo) { + haveVideo = true; + } else { + continue; + } + + err = mExtractor->selectTrack(i); + + size_t trackIndex = mTracks.size(); + + sp<AMessage> notify = new AMessage(kWhatTrackNotify, this); + notify->setSize("trackIndex", trackIndex); + + sp<Track> track = new Track(notify, format); + looper()->registerHandler(track); + + mTracks.add(trackIndex, track); + + mExtractorTrackToInternalTrack.add(i, trackIndex); + + if (isVideo) { + mVideoTrackIndex = trackIndex; + } + + uint32_t flags = MediaSender::FLAG_MANUALLY_PREPEND_SPS_PPS; + + ssize_t mediaSenderTrackIndex = + mMediaSender->addTrack(format, flags); + CHECK_GE(mediaSenderTrackIndex, 0); + + track->setMediaSenderTrackIndex(mediaSenderTrackIndex); + + if ((haveAudio || !enableAudio) && (haveVideo || !enableVideo)) { + break; + } + } + + return OK; +} + +void WifiDisplaySource::PlaybackSession::schedulePullExtractor() { + if (mPullExtractorPending) { + return; + } + + int64_t delayUs = 1000000; // default delay is 1 sec + int64_t sampleTimeUs; + status_t err = mExtractor->getSampleTime(&sampleTimeUs); + + if (err == OK) { + int64_t nowUs = ALooper::GetNowUs(); + + if (mFirstSampleTimeRealUs < 0ll) { + mFirstSampleTimeRealUs = nowUs; + mFirstSampleTimeUs = sampleTimeUs; + } + + int64_t whenUs = sampleTimeUs - mFirstSampleTimeUs + mFirstSampleTimeRealUs; + delayUs = whenUs - nowUs; + } else { + ALOGW("could not get sample time (%d)", err); + } + + sp<AMessage> msg = new AMessage(kWhatPullExtractorSample, this); + msg->setInt32("generation", mPullExtractorGeneration); + msg->post(delayUs); + + mPullExtractorPending = true; +} + +void WifiDisplaySource::PlaybackSession::onPullExtractor() { + sp<ABuffer> accessUnit = new ABuffer(1024 * 1024); + status_t err = mExtractor->readSampleData(accessUnit); + if (err != OK) { + // EOS. + return; + } + + int64_t timeUs; + CHECK_EQ((status_t)OK, mExtractor->getSampleTime(&timeUs)); + + accessUnit->meta()->setInt64( + "timeUs", mFirstSampleTimeRealUs + timeUs - mFirstSampleTimeUs); + + size_t trackIndex; + CHECK_EQ((status_t)OK, mExtractor->getSampleTrackIndex(&trackIndex)); + + sp<AMessage> msg = new AMessage(kWhatConverterNotify, this); + + msg->setSize( + "trackIndex", mExtractorTrackToInternalTrack.valueFor(trackIndex)); + + msg->setInt32("what", Converter::kWhatAccessUnit); + msg->setBuffer("accessUnit", accessUnit); + msg->post(); + + mExtractor->advance(); + + schedulePullExtractor(); +} + +status_t WifiDisplaySource::PlaybackSession::setupPacketizer( + bool enableAudio, + bool usePCMAudio, + bool enableVideo, + VideoFormats::ResolutionType videoResolutionType, + size_t videoResolutionIndex, + VideoFormats::ProfileType videoProfileType, + VideoFormats::LevelType videoLevelType) { + CHECK(enableAudio || enableVideo); + + if (!mMediaPath.empty()) { + return setupMediaPacketizer(enableAudio, enableVideo); + } + + if (enableVideo) { + status_t err = addVideoSource( + videoResolutionType, videoResolutionIndex, videoProfileType, + videoLevelType); + + if (err != OK) { + return err; + } + } + + if (!enableAudio) { + return OK; + } + + return addAudioSource(usePCMAudio); +} + +status_t WifiDisplaySource::PlaybackSession::addSource( + bool isVideo, const sp<MediaSource> &source, bool isRepeaterSource, + bool usePCMAudio, unsigned profileIdc, unsigned levelIdc, + unsigned constraintSet, size_t *numInputBuffers) { + CHECK(!usePCMAudio || !isVideo); + CHECK(!isRepeaterSource || isVideo); + CHECK(!profileIdc || isVideo); + CHECK(!levelIdc || isVideo); + CHECK(!constraintSet || isVideo); + + sp<ALooper> pullLooper = new ALooper; + pullLooper->setName("pull_looper"); + + pullLooper->start( + false /* runOnCallingThread */, + false /* canCallJava */, + PRIORITY_AUDIO); + + sp<ALooper> codecLooper = new ALooper; + codecLooper->setName("codec_looper"); + + codecLooper->start( + false /* runOnCallingThread */, + false /* canCallJava */, + PRIORITY_AUDIO); + + size_t trackIndex; + + sp<AMessage> notify; + + trackIndex = mTracks.size(); + + sp<AMessage> format; + status_t err = convertMetaDataToMessage(source->getFormat(), &format); + CHECK_EQ(err, (status_t)OK); + + if (isVideo) { + format->setString("mime", MEDIA_MIMETYPE_VIDEO_AVC); + format->setInt32( + "android._input-metadata-buffer-type", kMetadataBufferTypeANWBuffer); + format->setInt32("android._store-metadata-in-buffers-output", (mHDCP != NULL) + && (mHDCP->getCaps() & HDCPModule::HDCP_CAPS_ENCRYPT_NATIVE)); + format->setInt32( + "color-format", OMX_COLOR_FormatAndroidOpaque); + format->setInt32("profile-idc", profileIdc); + format->setInt32("level-idc", levelIdc); + format->setInt32("constraint-set", constraintSet); + } else { + if (usePCMAudio) { + format->setInt32("pcm-encoding", kAudioEncodingPcm16bit); + format->setString("mime", MEDIA_MIMETYPE_AUDIO_RAW); + } else { + format->setString("mime", MEDIA_MIMETYPE_AUDIO_AAC); + } + } + + notify = new AMessage(kWhatConverterNotify, this); + notify->setSize("trackIndex", trackIndex); + + sp<Converter> converter = new Converter(notify, codecLooper, format); + + looper()->registerHandler(converter); + + err = converter->init(); + if (err != OK) { + ALOGE("%s converter returned err %d", isVideo ? "video" : "audio", err); + + looper()->unregisterHandler(converter->id()); + return err; + } + + notify = new AMessage(Converter::kWhatMediaPullerNotify, converter); + notify->setSize("trackIndex", trackIndex); + + sp<MediaPuller> puller = new MediaPuller(source, notify); + pullLooper->registerHandler(puller); + + if (numInputBuffers != NULL) { + *numInputBuffers = converter->getInputBufferCount(); + } + + notify = new AMessage(kWhatTrackNotify, this); + notify->setSize("trackIndex", trackIndex); + + sp<Track> track = new Track( + notify, pullLooper, codecLooper, puller, converter); + + if (isRepeaterSource) { + track->setRepeaterSource(static_cast<RepeaterSource *>(source.get())); + } + + looper()->registerHandler(track); + + mTracks.add(trackIndex, track); + + if (isVideo) { + mVideoTrackIndex = trackIndex; + } + + uint32_t flags = 0; + if (converter->needToManuallyPrependSPSPPS()) { + flags |= MediaSender::FLAG_MANUALLY_PREPEND_SPS_PPS; + } + + ssize_t mediaSenderTrackIndex = + mMediaSender->addTrack(converter->getOutputFormat(), flags); + CHECK_GE(mediaSenderTrackIndex, 0); + + track->setMediaSenderTrackIndex(mediaSenderTrackIndex); + + return OK; +} + +status_t WifiDisplaySource::PlaybackSession::addVideoSource( + VideoFormats::ResolutionType videoResolutionType, + size_t videoResolutionIndex, + VideoFormats::ProfileType videoProfileType, + VideoFormats::LevelType videoLevelType) { + size_t width, height, framesPerSecond; + bool interlaced; + CHECK(VideoFormats::GetConfiguration( + videoResolutionType, + videoResolutionIndex, + &width, + &height, + &framesPerSecond, + &interlaced)); + + unsigned profileIdc, levelIdc, constraintSet; + CHECK(VideoFormats::GetProfileLevel( + videoProfileType, + videoLevelType, + &profileIdc, + &levelIdc, + &constraintSet)); + + sp<SurfaceMediaSource> source = new SurfaceMediaSource(width, height); + + source->setUseAbsoluteTimestamps(); + + sp<RepeaterSource> videoSource = + new RepeaterSource(source, framesPerSecond); + + size_t numInputBuffers; + status_t err = addSource( + true /* isVideo */, videoSource, true /* isRepeaterSource */, + false /* usePCMAudio */, profileIdc, levelIdc, constraintSet, + &numInputBuffers); + + if (err != OK) { + return err; + } + + err = source->setMaxAcquiredBufferCount(numInputBuffers); + CHECK_EQ(err, (status_t)OK); + + mProducer = source->getProducer(); + + return OK; +} + +status_t WifiDisplaySource::PlaybackSession::addAudioSource(bool usePCMAudio) { + audio_attributes_t attr = AUDIO_ATTRIBUTES_INITIALIZER; + attr.source = AUDIO_SOURCE_REMOTE_SUBMIX; + + sp<AudioSource> audioSource = new AudioSource( + &attr, + mOpPackageName, + 48000 /* sampleRate */, + 2 /* channelCount */); + + if (audioSource->initCheck() == OK) { + return addSource( + false /* isVideo */, audioSource, false /* isRepeaterSource */, + usePCMAudio, 0 /* profileIdc */, 0 /* levelIdc */, + 0 /* constraintSet */, NULL /* numInputBuffers */); + } + + ALOGW("Unable to instantiate audio source"); + + return OK; +} + +sp<IGraphicBufferProducer> WifiDisplaySource::PlaybackSession::getSurfaceTexture() { + return mProducer; +} + +void WifiDisplaySource::PlaybackSession::requestIDRFrame() { + for (size_t i = 0; i < mTracks.size(); ++i) { + const sp<Track> &track = mTracks.valueAt(i); + + track->requestIDRFrame(); + } +} + +void WifiDisplaySource::PlaybackSession::notifySessionDead() { + // Inform WifiDisplaySource of our premature death (wish). + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatSessionDead); + notify->post(); + + mWeAreDead = true; +} + +} // namespace android + diff --git a/media/libstagefright/wifi-display/source/PlaybackSession.h b/media/libstagefright/wifi-display/source/PlaybackSession.h new file mode 100644 index 0000000000..f6673df541 --- /dev/null +++ b/media/libstagefright/wifi-display/source/PlaybackSession.h @@ -0,0 +1,176 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PLAYBACK_SESSION_H_ + +#define PLAYBACK_SESSION_H_ + +#include "MediaSender.h" +#include "VideoFormats.h" +#include "WifiDisplaySource.h" + +#include <utils/String16.h> + +namespace android { + +struct ABuffer; +struct IHDCP; +class IGraphicBufferProducer; +struct MediaPuller; +struct MediaSource; +struct MediaSender; +struct NuMediaExtractor; + +// Encapsulates the state of an RTP/RTCP session in the context of wifi +// display. +struct WifiDisplaySource::PlaybackSession : public AHandler { + PlaybackSession( + const String16 &opPackageName, + const sp<ANetworkSession> &netSession, + const sp<AMessage> ¬ify, + const struct in_addr &interfaceAddr, + const sp<IHDCP> &hdcp, + const char *path = NULL); + + status_t init( + const char *clientIP, + int32_t clientRtp, + RTPSender::TransportMode rtpMode, + int32_t clientRtcp, + RTPSender::TransportMode rtcpMode, + bool enableAudio, + bool usePCMAudio, + bool enableVideo, + VideoFormats::ResolutionType videoResolutionType, + size_t videoResolutionIndex, + VideoFormats::ProfileType videoProfileType, + VideoFormats::LevelType videoLevelType); + + void destroyAsync(); + + int32_t getRTPPort() const; + + int64_t getLastLifesignUs() const; + void updateLiveness(); + + status_t play(); + status_t finishPlay(); + status_t pause(); + + sp<IGraphicBufferProducer> getSurfaceTexture(); + + void requestIDRFrame(); + + enum { + kWhatSessionDead, + kWhatBinaryData, + kWhatSessionEstablished, + kWhatSessionDestroyed, + }; + +protected: + virtual void onMessageReceived(const sp<AMessage> &msg); + virtual ~PlaybackSession(); + +private: + struct Track; + + enum { + kWhatMediaPullerNotify, + kWhatConverterNotify, + kWhatTrackNotify, + kWhatUpdateSurface, + kWhatPause, + kWhatResume, + kWhatMediaSenderNotify, + kWhatPullExtractorSample, + }; + + String16 mOpPackageName; + + sp<ANetworkSession> mNetSession; + sp<AMessage> mNotify; + in_addr mInterfaceAddr; + sp<IHDCP> mHDCP; + AString mMediaPath; + + sp<MediaSender> mMediaSender; + int32_t mLocalRTPPort; + + bool mWeAreDead; + bool mPaused; + + int64_t mLastLifesignUs; + + sp<IGraphicBufferProducer> mProducer; + + KeyedVector<size_t, sp<Track> > mTracks; + ssize_t mVideoTrackIndex; + + int64_t mPrevTimeUs; + + sp<NuMediaExtractor> mExtractor; + KeyedVector<size_t, size_t> mExtractorTrackToInternalTrack; + bool mPullExtractorPending; + int32_t mPullExtractorGeneration; + int64_t mFirstSampleTimeRealUs; + int64_t mFirstSampleTimeUs; + + status_t setupMediaPacketizer(bool enableAudio, bool enableVideo); + + status_t setupPacketizer( + bool enableAudio, + bool usePCMAudio, + bool enableVideo, + VideoFormats::ResolutionType videoResolutionType, + size_t videoResolutionIndex, + VideoFormats::ProfileType videoProfileType, + VideoFormats::LevelType videoLevelType); + + status_t addSource( + bool isVideo, + const sp<MediaSource> &source, + bool isRepeaterSource, + bool usePCMAudio, + unsigned profileIdc, + unsigned levelIdc, + unsigned contraintSet, + size_t *numInputBuffers); + + status_t addVideoSource( + VideoFormats::ResolutionType videoResolutionType, + size_t videoResolutionIndex, + VideoFormats::ProfileType videoProfileType, + VideoFormats::LevelType videoLevelType); + + status_t addAudioSource(bool usePCMAudio); + + status_t onMediaSenderInitialized(); + + void notifySessionDead(); + + void schedulePullExtractor(); + void onPullExtractor(); + + void onSinkFeedback(const sp<AMessage> &msg); + + DISALLOW_EVIL_CONSTRUCTORS(PlaybackSession); +}; + +} // namespace android + +#endif // PLAYBACK_SESSION_H_ + diff --git a/media/libstagefright/wifi-display/source/RepeaterSource.cpp b/media/libstagefright/wifi-display/source/RepeaterSource.cpp new file mode 100644 index 0000000000..e225a02773 --- /dev/null +++ b/media/libstagefright/wifi-display/source/RepeaterSource.cpp @@ -0,0 +1,219 @@ +//#define LOG_NDEBUG 0 +#define LOG_TAG "RepeaterSource" +#include <utils/Log.h> + +#include "RepeaterSource.h" + +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/ALooper.h> +#include <media/stagefright/foundation/AMessage.h> +#include <media/stagefright/MediaBufferBase.h> +#include <media/stagefright/MetaData.h> + +namespace android { + +RepeaterSource::RepeaterSource(const sp<MediaSource> &source, double rateHz) + : mStarted(false), + mSource(source), + mRateHz(rateHz), + mBuffer(NULL), + mResult(OK), + mLastBufferUpdateUs(-1ll), + mStartTimeUs(-1ll), + mFrameCount(0) { +} + +RepeaterSource::~RepeaterSource() { + CHECK(!mStarted); +} + +double RepeaterSource::getFrameRate() const { + return mRateHz; +} + +void RepeaterSource::setFrameRate(double rateHz) { + Mutex::Autolock autoLock(mLock); + + if (rateHz == mRateHz) { + return; + } + + if (mStartTimeUs >= 0ll) { + int64_t nextTimeUs = mStartTimeUs + (mFrameCount * 1000000ll) / mRateHz; + mStartTimeUs = nextTimeUs; + mFrameCount = 0; + } + mRateHz = rateHz; +} + +status_t RepeaterSource::start(MetaData *params) { + CHECK(!mStarted); + + status_t err = mSource->start(params); + + if (err != OK) { + return err; + } + + mBuffer = NULL; + mResult = OK; + mStartTimeUs = -1ll; + mFrameCount = 0; + + mLooper = new ALooper; + mLooper->setName("repeater_looper"); + mLooper->start(); + + mReflector = new AHandlerReflector<RepeaterSource>(this); + mLooper->registerHandler(mReflector); + + postRead(); + + mStarted = true; + + return OK; +} + +status_t RepeaterSource::stop() { + CHECK(mStarted); + + ALOGV("stopping"); + + status_t err = mSource->stop(); + + if (mLooper != NULL) { + mLooper->stop(); + mLooper.clear(); + + mReflector.clear(); + } + + if (mBuffer != NULL) { + ALOGV("releasing mbuf %p", mBuffer); + mBuffer->release(); + mBuffer = NULL; + } + + + ALOGV("stopped"); + + mStarted = false; + + return err; +} + +sp<MetaData> RepeaterSource::getFormat() { + return mSource->getFormat(); +} + +status_t RepeaterSource::read( + MediaBufferBase **buffer, const ReadOptions *options) { + int64_t seekTimeUs; + ReadOptions::SeekMode seekMode; + CHECK(options == NULL || !options->getSeekTo(&seekTimeUs, &seekMode)); + + for (;;) { + int64_t bufferTimeUs = -1ll; + + if (mStartTimeUs < 0ll) { + Mutex::Autolock autoLock(mLock); + while ((mLastBufferUpdateUs < 0ll || mBuffer == NULL) + && mResult == OK) { + mCondition.wait(mLock); + } + + ALOGV("now resuming."); + mStartTimeUs = ALooper::GetNowUs(); + bufferTimeUs = mStartTimeUs; + } else { + bufferTimeUs = mStartTimeUs + (mFrameCount * 1000000ll) / mRateHz; + + int64_t nowUs = ALooper::GetNowUs(); + int64_t delayUs = bufferTimeUs - nowUs; + + if (delayUs > 0ll) { + usleep(delayUs); + } + } + + bool stale = false; + + { + Mutex::Autolock autoLock(mLock); + if (mResult != OK) { + CHECK(mBuffer == NULL); + return mResult; + } + +#if SUSPEND_VIDEO_IF_IDLE + int64_t nowUs = ALooper::GetNowUs(); + if (nowUs - mLastBufferUpdateUs > 1000000ll) { + mLastBufferUpdateUs = -1ll; + stale = true; + } else +#endif + { + mBuffer->add_ref(); + *buffer = mBuffer; + (*buffer)->meta_data().setInt64(kKeyTime, bufferTimeUs); + ++mFrameCount; + } + } + + if (!stale) { + break; + } + + mStartTimeUs = -1ll; + mFrameCount = 0; + ALOGV("now dormant"); + } + + return OK; +} + +void RepeaterSource::postRead() { + (new AMessage(kWhatRead, mReflector))->post(); +} + +void RepeaterSource::onMessageReceived(const sp<AMessage> &msg) { + switch (msg->what()) { + case kWhatRead: + { + MediaBufferBase *buffer; + status_t err = mSource->read(&buffer); + + ALOGV("read mbuf %p", buffer); + + Mutex::Autolock autoLock(mLock); + if (mBuffer != NULL) { + mBuffer->release(); + mBuffer = NULL; + } + mBuffer = buffer; + mResult = err; + mLastBufferUpdateUs = ALooper::GetNowUs(); + + mCondition.broadcast(); + + if (err == OK) { + postRead(); + } + break; + } + + default: + TRESPASS(); + } +} + +void RepeaterSource::wakeUp() { + ALOGV("wakeUp"); + Mutex::Autolock autoLock(mLock); + if (mLastBufferUpdateUs < 0ll && mBuffer != NULL) { + mLastBufferUpdateUs = ALooper::GetNowUs(); + mCondition.broadcast(); + } +} + +} // namespace android diff --git a/media/libstagefright/wifi-display/source/RepeaterSource.h b/media/libstagefright/wifi-display/source/RepeaterSource.h new file mode 100644 index 0000000000..1647fb1c1f --- /dev/null +++ b/media/libstagefright/wifi-display/source/RepeaterSource.h @@ -0,0 +1,67 @@ +#ifndef REPEATER_SOURCE_H_ + +#define REPEATER_SOURCE_H_ + +#include <media/stagefright/foundation/ABase.h> +#include <media/stagefright/foundation/AHandlerReflector.h> +#include <media/stagefright/MediaSource.h> + +#define SUSPEND_VIDEO_IF_IDLE 0 + +namespace android { + +// This MediaSource delivers frames at a constant rate by repeating buffers +// if necessary. +struct RepeaterSource : public MediaSource { + RepeaterSource(const sp<MediaSource> &source, double rateHz); + + virtual status_t start(MetaData *params); + virtual status_t stop(); + virtual sp<MetaData> getFormat(); + + virtual status_t read( + MediaBufferBase **buffer, const ReadOptions *options); + + void onMessageReceived(const sp<AMessage> &msg); + + // If RepeaterSource is currently dormant, because SurfaceFlinger didn't + // send updates in a while, this is its wakeup call. + void wakeUp(); + + double getFrameRate() const; + void setFrameRate(double rateHz); + +protected: + virtual ~RepeaterSource(); + +private: + enum { + kWhatRead, + }; + + Mutex mLock; + Condition mCondition; + + bool mStarted; + + sp<MediaSource> mSource; + double mRateHz; + + sp<ALooper> mLooper; + sp<AHandlerReflector<RepeaterSource> > mReflector; + + MediaBufferBase *mBuffer; + status_t mResult; + int64_t mLastBufferUpdateUs; + + int64_t mStartTimeUs; + int32_t mFrameCount; + + void postRead(); + + DISALLOW_EVIL_CONSTRUCTORS(RepeaterSource); +}; + +} // namespace android + +#endif // REPEATER_SOURCE_H_ diff --git a/media/libstagefright/wifi-display/source/TSPacketizer.cpp b/media/libstagefright/wifi-display/source/TSPacketizer.cpp new file mode 100644 index 0000000000..7d30ae208e --- /dev/null +++ b/media/libstagefright/wifi-display/source/TSPacketizer.cpp @@ -0,0 +1,1055 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "TSPacketizer" +#include <utils/Log.h> + +#include "TSPacketizer.h" + +#include <media/stagefright/foundation/avc_utils.h> +#include <media/stagefright/foundation/ABuffer.h> +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> +#include <media/stagefright/foundation/hexdump.h> +#include <media/stagefright/MediaDefs.h> +#include <media/stagefright/MediaErrors.h> + +#include <arpa/inet.h> + +namespace android { + +struct TSPacketizer::Track : public RefBase { + Track(const sp<AMessage> &format, + unsigned PID, unsigned streamType, unsigned streamID); + + unsigned PID() const; + unsigned streamType() const; + unsigned streamID() const; + + // Returns the previous value. + unsigned incrementContinuityCounter(); + + bool isAudio() const; + bool isVideo() const; + + bool isH264() const; + bool isAAC() const; + bool lacksADTSHeader() const; + bool isPCMAudio() const; + + sp<ABuffer> prependCSD(const sp<ABuffer> &accessUnit) const; + sp<ABuffer> prependADTSHeader(const sp<ABuffer> &accessUnit) const; + + size_t countDescriptors() const; + sp<ABuffer> descriptorAt(size_t index) const; + + void finalize(); + void extractCSDIfNecessary(); + +protected: + virtual ~Track(); + +private: + sp<AMessage> mFormat; + + unsigned mPID; + unsigned mStreamType; + unsigned mStreamID; + unsigned mContinuityCounter; + + AString mMIME; + Vector<sp<ABuffer> > mCSD; + + Vector<sp<ABuffer> > mDescriptors; + + bool mAudioLacksATDSHeaders; + bool mFinalized; + bool mExtractedCSD; + + DISALLOW_EVIL_CONSTRUCTORS(Track); +}; + +TSPacketizer::Track::Track( + const sp<AMessage> &format, + unsigned PID, unsigned streamType, unsigned streamID) + : mFormat(format), + mPID(PID), + mStreamType(streamType), + mStreamID(streamID), + mContinuityCounter(0), + mAudioLacksATDSHeaders(false), + mFinalized(false), + mExtractedCSD(false) { + CHECK(format->findString("mime", &mMIME)); +} + +void TSPacketizer::Track::extractCSDIfNecessary() { + if (mExtractedCSD) { + return; + } + + if (!strcasecmp(mMIME.c_str(), MEDIA_MIMETYPE_VIDEO_AVC) + || !strcasecmp(mMIME.c_str(), MEDIA_MIMETYPE_AUDIO_AAC)) { + for (size_t i = 0;; ++i) { + sp<ABuffer> csd; + if (!mFormat->findBuffer(AStringPrintf("csd-%d", i).c_str(), &csd)) { + break; + } + + mCSD.push(csd); + } + + if (!strcasecmp(mMIME.c_str(), MEDIA_MIMETYPE_AUDIO_AAC)) { + int32_t isADTS; + if (!mFormat->findInt32("is-adts", &isADTS) || isADTS == 0) { + mAudioLacksATDSHeaders = true; + } + } + } + + mExtractedCSD = true; +} + +TSPacketizer::Track::~Track() { +} + +unsigned TSPacketizer::Track::PID() const { + return mPID; +} + +unsigned TSPacketizer::Track::streamType() const { + return mStreamType; +} + +unsigned TSPacketizer::Track::streamID() const { + return mStreamID; +} + +unsigned TSPacketizer::Track::incrementContinuityCounter() { + unsigned prevCounter = mContinuityCounter; + + if (++mContinuityCounter == 16) { + mContinuityCounter = 0; + } + + return prevCounter; +} + +bool TSPacketizer::Track::isAudio() const { + return !strncasecmp("audio/", mMIME.c_str(), 6); +} + +bool TSPacketizer::Track::isVideo() const { + return !strncasecmp("video/", mMIME.c_str(), 6); +} + +bool TSPacketizer::Track::isH264() const { + return !strcasecmp(mMIME.c_str(), MEDIA_MIMETYPE_VIDEO_AVC); +} + +bool TSPacketizer::Track::isAAC() const { + return !strcasecmp(mMIME.c_str(), MEDIA_MIMETYPE_AUDIO_AAC); +} + +bool TSPacketizer::Track::isPCMAudio() const { + return !strcasecmp(mMIME.c_str(), MEDIA_MIMETYPE_AUDIO_RAW); +} + +bool TSPacketizer::Track::lacksADTSHeader() const { + return mAudioLacksATDSHeaders; +} + +sp<ABuffer> TSPacketizer::Track::prependCSD( + const sp<ABuffer> &accessUnit) const { + size_t size = 0; + for (size_t i = 0; i < mCSD.size(); ++i) { + size += mCSD.itemAt(i)->size(); + } + + sp<ABuffer> dup = new ABuffer(accessUnit->size() + size); + size_t offset = 0; + for (size_t i = 0; i < mCSD.size(); ++i) { + const sp<ABuffer> &csd = mCSD.itemAt(i); + + memcpy(dup->data() + offset, csd->data(), csd->size()); + offset += csd->size(); + } + + memcpy(dup->data() + offset, accessUnit->data(), accessUnit->size()); + + return dup; +} + +sp<ABuffer> TSPacketizer::Track::prependADTSHeader( + const sp<ABuffer> &accessUnit) const { + CHECK_EQ(mCSD.size(), 1u); + + const uint8_t *codec_specific_data = mCSD.itemAt(0)->data(); + + const uint32_t aac_frame_length = accessUnit->size() + 7; + + sp<ABuffer> dup = new ABuffer(aac_frame_length); + + unsigned profile = (codec_specific_data[0] >> 3) - 1; + + unsigned sampling_freq_index = + ((codec_specific_data[0] & 7) << 1) + | (codec_specific_data[1] >> 7); + + unsigned channel_configuration = + (codec_specific_data[1] >> 3) & 0x0f; + + uint8_t *ptr = dup->data(); + + *ptr++ = 0xff; + *ptr++ = 0xf9; // b11111001, ID=1(MPEG-2), layer=0, protection_absent=1 + + *ptr++ = + profile << 6 + | sampling_freq_index << 2 + | ((channel_configuration >> 2) & 1); // private_bit=0 + + // original_copy=0, home=0, copyright_id_bit=0, copyright_id_start=0 + *ptr++ = + (channel_configuration & 3) << 6 + | aac_frame_length >> 11; + *ptr++ = (aac_frame_length >> 3) & 0xff; + *ptr++ = (aac_frame_length & 7) << 5; + + // adts_buffer_fullness=0, number_of_raw_data_blocks_in_frame=0 + *ptr++ = 0; + + memcpy(ptr, accessUnit->data(), accessUnit->size()); + + return dup; +} + +size_t TSPacketizer::Track::countDescriptors() const { + return mDescriptors.size(); +} + +sp<ABuffer> TSPacketizer::Track::descriptorAt(size_t index) const { + CHECK_LT(index, mDescriptors.size()); + return mDescriptors.itemAt(index); +} + +void TSPacketizer::Track::finalize() { + if (mFinalized) { + return; + } + + if (isH264()) { + { + // AVC video descriptor (40) + + sp<ABuffer> descriptor = new ABuffer(6); + uint8_t *data = descriptor->data(); + data[0] = 40; // descriptor_tag + data[1] = 4; // descriptor_length + + if (mCSD.size() > 0) { + CHECK_GE(mCSD.size(), 1u); + const sp<ABuffer> &sps = mCSD.itemAt(0); + CHECK(!memcmp("\x00\x00\x00\x01", sps->data(), 4)); + CHECK_GE(sps->size(), 7u); + // profile_idc, constraint_set*, level_idc + memcpy(&data[2], sps->data() + 4, 3); + } else { + int32_t profileIdc, levelIdc, constraintSet; + CHECK(mFormat->findInt32("profile-idc", &profileIdc)); + CHECK(mFormat->findInt32("level-idc", &levelIdc)); + CHECK(mFormat->findInt32("constraint-set", &constraintSet)); + CHECK_GE(profileIdc, 0); + CHECK_GE(levelIdc, 0); + data[2] = profileIdc; // profile_idc + data[3] = constraintSet; // constraint_set* + data[4] = levelIdc; // level_idc + } + + // AVC_still_present=0, AVC_24_hour_picture_flag=0, reserved + data[5] = 0x3f; + + mDescriptors.push_back(descriptor); + } + + { + // AVC timing and HRD descriptor (42) + + sp<ABuffer> descriptor = new ABuffer(4); + uint8_t *data = descriptor->data(); + data[0] = 42; // descriptor_tag + data[1] = 2; // descriptor_length + + // hrd_management_valid_flag = 0 + // reserved = 111111b + // picture_and_timing_info_present = 0 + + data[2] = 0x7e; + + // fixed_frame_rate_flag = 0 + // temporal_poc_flag = 0 + // picture_to_display_conversion_flag = 0 + // reserved = 11111b + data[3] = 0x1f; + + mDescriptors.push_back(descriptor); + } + } else if (isPCMAudio()) { + // LPCM audio stream descriptor (0x83) + + int32_t channelCount; + CHECK(mFormat->findInt32("channel-count", &channelCount)); + CHECK_EQ(channelCount, 2); + + int32_t sampleRate; + CHECK(mFormat->findInt32("sample-rate", &sampleRate)); + CHECK(sampleRate == 44100 || sampleRate == 48000); + + sp<ABuffer> descriptor = new ABuffer(4); + uint8_t *data = descriptor->data(); + data[0] = 0x83; // descriptor_tag + data[1] = 2; // descriptor_length + + unsigned sampling_frequency = (sampleRate == 44100) ? 1 : 2; + + data[2] = (sampling_frequency << 5) + | (3 /* reserved */ << 1) + | 0 /* emphasis_flag */; + + data[3] = + (1 /* number_of_channels = stereo */ << 5) + | 0xf /* reserved */; + + mDescriptors.push_back(descriptor); + } + + mFinalized = true; +} + +//////////////////////////////////////////////////////////////////////////////// + +TSPacketizer::TSPacketizer(uint32_t flags) + : mFlags(flags), + mPATContinuityCounter(0), + mPMTContinuityCounter(0) { + initCrcTable(); + + if (flags & (EMIT_HDCP20_DESCRIPTOR | EMIT_HDCP21_DESCRIPTOR)) { + int32_t hdcpVersion; + if (flags & EMIT_HDCP20_DESCRIPTOR) { + CHECK(!(flags & EMIT_HDCP21_DESCRIPTOR)); + hdcpVersion = 0x20; + } else { + CHECK(!(flags & EMIT_HDCP20_DESCRIPTOR)); + + // HDCP2.0 _and_ HDCP 2.1 specs say to set the version + // inside the HDCP descriptor to 0x20!!! + hdcpVersion = 0x20; + } + + // HDCP descriptor + sp<ABuffer> descriptor = new ABuffer(7); + uint8_t *data = descriptor->data(); + data[0] = 0x05; // descriptor_tag + data[1] = 5; // descriptor_length + data[2] = 'H'; + data[3] = 'D'; + data[4] = 'C'; + data[5] = 'P'; + data[6] = hdcpVersion; + + mProgramInfoDescriptors.push_back(descriptor); + } +} + +TSPacketizer::~TSPacketizer() { +} + +ssize_t TSPacketizer::addTrack(const sp<AMessage> &format) { + AString mime; + CHECK(format->findString("mime", &mime)); + + unsigned PIDStart; + bool isVideo = !strncasecmp("video/", mime.c_str(), 6); + bool isAudio = !strncasecmp("audio/", mime.c_str(), 6); + + if (isVideo) { + PIDStart = 0x1011; + } else if (isAudio) { + PIDStart = 0x1100; + } else { + return ERROR_UNSUPPORTED; + } + + unsigned streamType; + unsigned streamIDStart; + unsigned streamIDStop; + + if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_VIDEO_AVC)) { + streamType = 0x1b; + streamIDStart = 0xe0; + streamIDStop = 0xef; + } else if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_AUDIO_AAC)) { + streamType = 0x0f; + streamIDStart = 0xc0; + streamIDStop = 0xdf; + } else if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_AUDIO_RAW)) { + streamType = 0x83; + streamIDStart = 0xbd; + streamIDStop = 0xbd; + } else { + return ERROR_UNSUPPORTED; + } + + size_t numTracksOfThisType = 0; + unsigned PID = PIDStart; + + for (size_t i = 0; i < mTracks.size(); ++i) { + const sp<Track> &track = mTracks.itemAt(i); + + if (track->streamType() == streamType) { + ++numTracksOfThisType; + } + + if ((isAudio && track->isAudio()) || (isVideo && track->isVideo())) { + ++PID; + } + } + + unsigned streamID = streamIDStart + numTracksOfThisType; + if (streamID > streamIDStop) { + return -ERANGE; + } + + sp<Track> track = new Track(format, PID, streamType, streamID); + return mTracks.add(track); +} + +status_t TSPacketizer::extractCSDIfNecessary(size_t trackIndex) { + if (trackIndex >= mTracks.size()) { + return -ERANGE; + } + + const sp<Track> &track = mTracks.itemAt(trackIndex); + track->extractCSDIfNecessary(); + + return OK; +} + +status_t TSPacketizer::packetize( + size_t trackIndex, + const sp<ABuffer> &_accessUnit, + sp<ABuffer> *packets, + uint32_t flags, + const uint8_t *PES_private_data, size_t PES_private_data_len, + size_t numStuffingBytes) { + sp<ABuffer> accessUnit = _accessUnit; + + int64_t timeUs; + CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs)); + + packets->clear(); + + if (trackIndex >= mTracks.size()) { + return -ERANGE; + } + + const sp<Track> &track = mTracks.itemAt(trackIndex); + + if (track->isH264() && (flags & PREPEND_SPS_PPS_TO_IDR_FRAMES) + && IsIDR(accessUnit->data(), accessUnit->size())) { + // prepend codec specific data, i.e. SPS and PPS. + accessUnit = track->prependCSD(accessUnit); + } else if (track->isAAC() && track->lacksADTSHeader()) { + CHECK(!(flags & IS_ENCRYPTED)); + accessUnit = track->prependADTSHeader(accessUnit); + } + + // 0x47 + // transport_error_indicator = b0 + // payload_unit_start_indicator = b1 + // transport_priority = b0 + // PID + // transport_scrambling_control = b00 + // adaptation_field_control = b?? + // continuity_counter = b???? + // -- payload follows + // packet_startcode_prefix = 0x000001 + // stream_id + // PES_packet_length = 0x???? + // reserved = b10 + // PES_scrambling_control = b00 + // PES_priority = b0 + // data_alignment_indicator = b1 + // copyright = b0 + // original_or_copy = b0 + // PTS_DTS_flags = b10 (PTS only) + // ESCR_flag = b0 + // ES_rate_flag = b0 + // DSM_trick_mode_flag = b0 + // additional_copy_info_flag = b0 + // PES_CRC_flag = b0 + // PES_extension_flag = b0 + // PES_header_data_length = 0x05 + // reserved = b0010 (PTS) + // PTS[32..30] = b??? + // reserved = b1 + // PTS[29..15] = b??? ???? ???? ???? (15 bits) + // reserved = b1 + // PTS[14..0] = b??? ???? ???? ???? (15 bits) + // reserved = b1 + // the first fragment of "buffer" follows + + // Each transport packet (except for the last one contributing to the PES + // payload) must contain a multiple of 16 bytes of payload per HDCP spec. + bool alignPayload = + (mFlags & (EMIT_HDCP20_DESCRIPTOR | EMIT_HDCP21_DESCRIPTOR)); + + /* + a) The very first PES transport stream packet contains + + 4 bytes of TS header + ... padding + 14 bytes of static PES header + PES_private_data_len + 1 bytes (only if PES_private_data_len > 0) + numStuffingBytes bytes + + followed by the payload + + b) Subsequent PES transport stream packets contain + + 4 bytes of TS header + ... padding + + followed by the payload + */ + + size_t PES_packet_length = accessUnit->size() + 8 + numStuffingBytes; + if (PES_private_data_len > 0) { + PES_packet_length += PES_private_data_len + 1; + } + + size_t numTSPackets = 1; + + { + // Make sure the PES header fits into a single TS packet: + size_t PES_header_size = 14 + numStuffingBytes; + if (PES_private_data_len > 0) { + PES_header_size += PES_private_data_len + 1; + } + + CHECK_LE(PES_header_size, 188u - 4u); + + size_t sizeAvailableForPayload = 188 - 4 - PES_header_size; + size_t numBytesOfPayload = accessUnit->size(); + + if (numBytesOfPayload > sizeAvailableForPayload) { + numBytesOfPayload = sizeAvailableForPayload; + + if (alignPayload && numBytesOfPayload > 16) { + numBytesOfPayload -= (numBytesOfPayload % 16); + } + } + + size_t numPaddingBytes = sizeAvailableForPayload - numBytesOfPayload; + ALOGV("packet 1 contains %zd padding bytes and %zd bytes of payload", + numPaddingBytes, numBytesOfPayload); + + size_t numBytesOfPayloadRemaining = accessUnit->size() - numBytesOfPayload; + +#if 0 + // The following hopefully illustrates the logic that led to the + // more efficient computation in the #else block... + + while (numBytesOfPayloadRemaining > 0) { + size_t sizeAvailableForPayload = 188 - 4; + + size_t numBytesOfPayload = numBytesOfPayloadRemaining; + + if (numBytesOfPayload > sizeAvailableForPayload) { + numBytesOfPayload = sizeAvailableForPayload; + + if (alignPayload && numBytesOfPayload > 16) { + numBytesOfPayload -= (numBytesOfPayload % 16); + } + } + + size_t numPaddingBytes = sizeAvailableForPayload - numBytesOfPayload; + ALOGI("packet %zd contains %zd padding bytes and %zd bytes of payload", + numTSPackets + 1, numPaddingBytes, numBytesOfPayload); + + numBytesOfPayloadRemaining -= numBytesOfPayload; + ++numTSPackets; + } +#else + // This is how many bytes of payload each subsequent TS packet + // can contain at most. + sizeAvailableForPayload = 188 - 4; + size_t sizeAvailableForAlignedPayload = sizeAvailableForPayload; + if (alignPayload) { + // We're only going to use a subset of the available space + // since we need to make each fragment a multiple of 16 in size. + sizeAvailableForAlignedPayload -= + (sizeAvailableForAlignedPayload % 16); + } + + size_t numFullTSPackets = + numBytesOfPayloadRemaining / sizeAvailableForAlignedPayload; + + numTSPackets += numFullTSPackets; + + numBytesOfPayloadRemaining -= + numFullTSPackets * sizeAvailableForAlignedPayload; + + // numBytesOfPayloadRemaining < sizeAvailableForAlignedPayload + if (numFullTSPackets == 0 && numBytesOfPayloadRemaining > 0) { + // There wasn't enough payload left to form a full aligned payload, + // the last packet doesn't have to be aligned. + ++numTSPackets; + } else if (numFullTSPackets > 0 + && numBytesOfPayloadRemaining + + sizeAvailableForAlignedPayload > sizeAvailableForPayload) { + // The last packet emitted had a full aligned payload and together + // with the bytes remaining does exceed the unaligned payload + // size, so we need another packet. + ++numTSPackets; + } +#endif + } + + if (flags & EMIT_PAT_AND_PMT) { + numTSPackets += 2; + } + + if (flags & EMIT_PCR) { + ++numTSPackets; + } + + sp<ABuffer> buffer = new ABuffer(numTSPackets * 188); + uint8_t *packetDataStart = buffer->data(); + + if (flags & EMIT_PAT_AND_PMT) { + // Program Association Table (PAT): + // 0x47 + // transport_error_indicator = b0 + // payload_unit_start_indicator = b1 + // transport_priority = b0 + // PID = b0000000000000 (13 bits) + // transport_scrambling_control = b00 + // adaptation_field_control = b01 (no adaptation field, payload only) + // continuity_counter = b???? + // skip = 0x00 + // --- payload follows + // table_id = 0x00 + // section_syntax_indicator = b1 + // must_be_zero = b0 + // reserved = b11 + // section_length = 0x00d + // transport_stream_id = 0x0000 + // reserved = b11 + // version_number = b00001 + // current_next_indicator = b1 + // section_number = 0x00 + // last_section_number = 0x00 + // one program follows: + // program_number = 0x0001 + // reserved = b111 + // program_map_PID = kPID_PMT (13 bits!) + // CRC = 0x???????? + + if (++mPATContinuityCounter == 16) { + mPATContinuityCounter = 0; + } + + uint8_t *ptr = packetDataStart; + *ptr++ = 0x47; + *ptr++ = 0x40; + *ptr++ = 0x00; + *ptr++ = 0x10 | mPATContinuityCounter; + *ptr++ = 0x00; + + uint8_t *crcDataStart = ptr; + *ptr++ = 0x00; + *ptr++ = 0xb0; + *ptr++ = 0x0d; + *ptr++ = 0x00; + *ptr++ = 0x00; + *ptr++ = 0xc3; + *ptr++ = 0x00; + *ptr++ = 0x00; + *ptr++ = 0x00; + *ptr++ = 0x01; + *ptr++ = 0xe0 | (kPID_PMT >> 8); + *ptr++ = kPID_PMT & 0xff; + + CHECK_EQ(ptr - crcDataStart, 12); + uint32_t crc = htonl(crc32(crcDataStart, ptr - crcDataStart)); + memcpy(ptr, &crc, 4); + ptr += 4; + + size_t sizeLeft = packetDataStart + 188 - ptr; + memset(ptr, 0xff, sizeLeft); + + packetDataStart += 188; + + // Program Map (PMT): + // 0x47 + // transport_error_indicator = b0 + // payload_unit_start_indicator = b1 + // transport_priority = b0 + // PID = kPID_PMT (13 bits) + // transport_scrambling_control = b00 + // adaptation_field_control = b01 (no adaptation field, payload only) + // continuity_counter = b???? + // skip = 0x00 + // -- payload follows + // table_id = 0x02 + // section_syntax_indicator = b1 + // must_be_zero = b0 + // reserved = b11 + // section_length = 0x??? + // program_number = 0x0001 + // reserved = b11 + // version_number = b00001 + // current_next_indicator = b1 + // section_number = 0x00 + // last_section_number = 0x00 + // reserved = b111 + // PCR_PID = kPCR_PID (13 bits) + // reserved = b1111 + // program_info_length = 0x??? + // program_info_descriptors follow + // one or more elementary stream descriptions follow: + // stream_type = 0x?? + // reserved = b111 + // elementary_PID = b? ???? ???? ???? (13 bits) + // reserved = b1111 + // ES_info_length = 0x000 + // CRC = 0x???????? + + if (++mPMTContinuityCounter == 16) { + mPMTContinuityCounter = 0; + } + + ptr = packetDataStart; + *ptr++ = 0x47; + *ptr++ = 0x40 | (kPID_PMT >> 8); + *ptr++ = kPID_PMT & 0xff; + *ptr++ = 0x10 | mPMTContinuityCounter; + *ptr++ = 0x00; + + crcDataStart = ptr; + *ptr++ = 0x02; + + *ptr++ = 0x00; // section_length to be filled in below. + *ptr++ = 0x00; + + *ptr++ = 0x00; + *ptr++ = 0x01; + *ptr++ = 0xc3; + *ptr++ = 0x00; + *ptr++ = 0x00; + *ptr++ = 0xe0 | (kPID_PCR >> 8); + *ptr++ = kPID_PCR & 0xff; + + size_t program_info_length = 0; + for (size_t i = 0; i < mProgramInfoDescriptors.size(); ++i) { + program_info_length += mProgramInfoDescriptors.itemAt(i)->size(); + } + + CHECK_LT(program_info_length, 0x400u); + *ptr++ = 0xf0 | (program_info_length >> 8); + *ptr++ = (program_info_length & 0xff); + + for (size_t i = 0; i < mProgramInfoDescriptors.size(); ++i) { + const sp<ABuffer> &desc = mProgramInfoDescriptors.itemAt(i); + memcpy(ptr, desc->data(), desc->size()); + ptr += desc->size(); + } + + for (size_t i = 0; i < mTracks.size(); ++i) { + const sp<Track> &track = mTracks.itemAt(i); + + // Make sure all the decriptors have been added. + track->finalize(); + + *ptr++ = track->streamType(); + *ptr++ = 0xe0 | (track->PID() >> 8); + *ptr++ = track->PID() & 0xff; + + size_t ES_info_length = 0; + for (size_t i = 0; i < track->countDescriptors(); ++i) { + ES_info_length += track->descriptorAt(i)->size(); + } + CHECK_LE(ES_info_length, 0xfffu); + + *ptr++ = 0xf0 | (ES_info_length >> 8); + *ptr++ = (ES_info_length & 0xff); + + for (size_t i = 0; i < track->countDescriptors(); ++i) { + const sp<ABuffer> &descriptor = track->descriptorAt(i); + memcpy(ptr, descriptor->data(), descriptor->size()); + ptr += descriptor->size(); + } + } + + size_t section_length = ptr - (crcDataStart + 3) + 4 /* CRC */; + + crcDataStart[1] = 0xb0 | (section_length >> 8); + crcDataStart[2] = section_length & 0xff; + + crc = htonl(crc32(crcDataStart, ptr - crcDataStart)); + memcpy(ptr, &crc, 4); + ptr += 4; + + sizeLeft = packetDataStart + 188 - ptr; + memset(ptr, 0xff, sizeLeft); + + packetDataStart += 188; + } + + if (flags & EMIT_PCR) { + // PCR stream + // 0x47 + // transport_error_indicator = b0 + // payload_unit_start_indicator = b1 + // transport_priority = b0 + // PID = kPCR_PID (13 bits) + // transport_scrambling_control = b00 + // adaptation_field_control = b10 (adaptation field only, no payload) + // continuity_counter = b0000 (does not increment) + // adaptation_field_length = 183 + // discontinuity_indicator = b0 + // random_access_indicator = b0 + // elementary_stream_priority_indicator = b0 + // PCR_flag = b1 + // OPCR_flag = b0 + // splicing_point_flag = b0 + // transport_private_data_flag = b0 + // adaptation_field_extension_flag = b0 + // program_clock_reference_base = b????????????????????????????????? + // reserved = b111111 + // program_clock_reference_extension = b????????? + + int64_t nowUs = ALooper::GetNowUs(); + + uint64_t PCR = nowUs * 27; // PCR based on a 27MHz clock + uint64_t PCR_base = PCR / 300; + uint32_t PCR_ext = PCR % 300; + + uint8_t *ptr = packetDataStart; + *ptr++ = 0x47; + *ptr++ = 0x40 | (kPID_PCR >> 8); + *ptr++ = kPID_PCR & 0xff; + *ptr++ = 0x20; + *ptr++ = 0xb7; // adaptation_field_length + *ptr++ = 0x10; + *ptr++ = (PCR_base >> 25) & 0xff; + *ptr++ = (PCR_base >> 17) & 0xff; + *ptr++ = (PCR_base >> 9) & 0xff; + *ptr++ = ((PCR_base & 1) << 7) | 0x7e | ((PCR_ext >> 8) & 1); + *ptr++ = (PCR_ext & 0xff); + + size_t sizeLeft = packetDataStart + 188 - ptr; + memset(ptr, 0xff, sizeLeft); + + packetDataStart += 188; + } + + uint64_t PTS = (timeUs * 9ll) / 100ll; + + if (PES_packet_length >= 65536) { + // This really should only happen for video. + CHECK(track->isVideo()); + + // It's valid to set this to 0 for video according to the specs. + PES_packet_length = 0; + } + + size_t sizeAvailableForPayload = 188 - 4 - 14 - numStuffingBytes; + if (PES_private_data_len > 0) { + sizeAvailableForPayload -= PES_private_data_len + 1; + } + + size_t copy = accessUnit->size(); + + if (copy > sizeAvailableForPayload) { + copy = sizeAvailableForPayload; + + if (alignPayload && copy > 16) { + copy -= (copy % 16); + } + } + + size_t numPaddingBytes = sizeAvailableForPayload - copy; + + uint8_t *ptr = packetDataStart; + *ptr++ = 0x47; + *ptr++ = 0x40 | (track->PID() >> 8); + *ptr++ = track->PID() & 0xff; + + *ptr++ = (numPaddingBytes > 0 ? 0x30 : 0x10) + | track->incrementContinuityCounter(); + + if (numPaddingBytes > 0) { + *ptr++ = numPaddingBytes - 1; + if (numPaddingBytes >= 2) { + *ptr++ = 0x00; + memset(ptr, 0xff, numPaddingBytes - 2); + ptr += numPaddingBytes - 2; + } + } + + *ptr++ = 0x00; + *ptr++ = 0x00; + *ptr++ = 0x01; + *ptr++ = track->streamID(); + *ptr++ = PES_packet_length >> 8; + *ptr++ = PES_packet_length & 0xff; + *ptr++ = 0x84; + *ptr++ = (PES_private_data_len > 0) ? 0x81 : 0x80; + + size_t headerLength = 0x05 + numStuffingBytes; + if (PES_private_data_len > 0) { + headerLength += 1 + PES_private_data_len; + } + + *ptr++ = headerLength; + + *ptr++ = 0x20 | (((PTS >> 30) & 7) << 1) | 1; + *ptr++ = (PTS >> 22) & 0xff; + *ptr++ = (((PTS >> 15) & 0x7f) << 1) | 1; + *ptr++ = (PTS >> 7) & 0xff; + *ptr++ = ((PTS & 0x7f) << 1) | 1; + + if (PES_private_data_len > 0) { + *ptr++ = 0x8e; // PES_private_data_flag, reserved. + memcpy(ptr, PES_private_data, PES_private_data_len); + ptr += PES_private_data_len; + } + + for (size_t i = 0; i < numStuffingBytes; ++i) { + *ptr++ = 0xff; + } + + memcpy(ptr, accessUnit->data(), copy); + ptr += copy; + + CHECK_EQ(ptr, packetDataStart + 188); + packetDataStart += 188; + + size_t offset = copy; + while (offset < accessUnit->size()) { + // for subsequent fragments of "buffer": + // 0x47 + // transport_error_indicator = b0 + // payload_unit_start_indicator = b0 + // transport_priority = b0 + // PID = b0 0001 1110 ???? (13 bits) [0x1e0 + 1 + sourceIndex] + // transport_scrambling_control = b00 + // adaptation_field_control = b?? + // continuity_counter = b???? + // the fragment of "buffer" follows. + + size_t sizeAvailableForPayload = 188 - 4; + + size_t copy = accessUnit->size() - offset; + + if (copy > sizeAvailableForPayload) { + copy = sizeAvailableForPayload; + + if (alignPayload && copy > 16) { + copy -= (copy % 16); + } + } + + size_t numPaddingBytes = sizeAvailableForPayload - copy; + + uint8_t *ptr = packetDataStart; + *ptr++ = 0x47; + *ptr++ = 0x00 | (track->PID() >> 8); + *ptr++ = track->PID() & 0xff; + + *ptr++ = (numPaddingBytes > 0 ? 0x30 : 0x10) + | track->incrementContinuityCounter(); + + if (numPaddingBytes > 0) { + *ptr++ = numPaddingBytes - 1; + if (numPaddingBytes >= 2) { + *ptr++ = 0x00; + memset(ptr, 0xff, numPaddingBytes - 2); + ptr += numPaddingBytes - 2; + } + } + + memcpy(ptr, accessUnit->data() + offset, copy); + ptr += copy; + CHECK_EQ(ptr, packetDataStart + 188); + + offset += copy; + packetDataStart += 188; + } + + CHECK(packetDataStart == buffer->data() + buffer->capacity()); + + *packets = buffer; + + return OK; +} + +void TSPacketizer::initCrcTable() { + uint32_t poly = 0x04C11DB7; + + for (int i = 0; i < 256; i++) { + uint32_t crc = i << 24; + for (int j = 0; j < 8; j++) { + crc = (crc << 1) ^ ((crc & 0x80000000) ? (poly) : 0); + } + mCrcTable[i] = crc; + } +} + +uint32_t TSPacketizer::crc32(const uint8_t *start, size_t size) const { + uint32_t crc = 0xFFFFFFFF; + const uint8_t *p; + + for (p = start; p < start + size; ++p) { + crc = (crc << 8) ^ mCrcTable[((crc >> 24) ^ *p) & 0xFF]; + } + + return crc; +} + +sp<ABuffer> TSPacketizer::prependCSD( + size_t trackIndex, const sp<ABuffer> &accessUnit) const { + CHECK_LT(trackIndex, mTracks.size()); + + const sp<Track> &track = mTracks.itemAt(trackIndex); + CHECK(track->isH264() && IsIDR(accessUnit->data(), accessUnit->size())); + + int64_t timeUs; + CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs)); + + sp<ABuffer> accessUnit2 = track->prependCSD(accessUnit); + + accessUnit2->meta()->setInt64("timeUs", timeUs); + + return accessUnit2; +} + +} // namespace android + diff --git a/media/libstagefright/wifi-display/source/TSPacketizer.h b/media/libstagefright/wifi-display/source/TSPacketizer.h new file mode 100644 index 0000000000..0dcb179551 --- /dev/null +++ b/media/libstagefright/wifi-display/source/TSPacketizer.h @@ -0,0 +1,94 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TS_PACKETIZER_H_ + +#define TS_PACKETIZER_H_ + +#include <media/stagefright/foundation/ABase.h> +#include <utils/Errors.h> +#include <utils/RefBase.h> +#include <utils/Vector.h> + +namespace android { + +struct ABuffer; +struct AMessage; + +// Forms the packets of a transport stream given access units. +// Emits metadata tables (PAT and PMT) and timestamp stream (PCR) based +// on flags. +struct TSPacketizer : public RefBase { + enum { + EMIT_HDCP20_DESCRIPTOR = 1, + EMIT_HDCP21_DESCRIPTOR = 2, + }; + explicit TSPacketizer(uint32_t flags); + + // Returns trackIndex or error. + ssize_t addTrack(const sp<AMessage> &format); + + enum { + EMIT_PAT_AND_PMT = 1, + EMIT_PCR = 2, + IS_ENCRYPTED = 4, + PREPEND_SPS_PPS_TO_IDR_FRAMES = 8, + }; + status_t packetize( + size_t trackIndex, const sp<ABuffer> &accessUnit, + sp<ABuffer> *packets, + uint32_t flags, + const uint8_t *PES_private_data, size_t PES_private_data_len, + size_t numStuffingBytes = 0); + + status_t extractCSDIfNecessary(size_t trackIndex); + + // XXX to be removed once encoder config option takes care of this for + // encrypted mode. + sp<ABuffer> prependCSD( + size_t trackIndex, const sp<ABuffer> &accessUnit) const; + +protected: + virtual ~TSPacketizer(); + +private: + enum { + kPID_PMT = 0x100, + kPID_PCR = 0x1000, + }; + + struct Track; + + uint32_t mFlags; + Vector<sp<Track> > mTracks; + + Vector<sp<ABuffer> > mProgramInfoDescriptors; + + unsigned mPATContinuityCounter; + unsigned mPMTContinuityCounter; + + uint32_t mCrcTable[256]; + + void initCrcTable(); + uint32_t crc32(const uint8_t *start, size_t size) const; + + DISALLOW_EVIL_CONSTRUCTORS(TSPacketizer); +}; + +} // namespace android + +#endif // TS_PACKETIZER_H_ + diff --git a/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp b/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp new file mode 100644 index 0000000000..96aceb1df4 --- /dev/null +++ b/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp @@ -0,0 +1,1738 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "WifiDisplaySource" +#include <utils/Log.h> + +#include "WifiDisplaySource.h" +#include "PlaybackSession.h" +#include "Parameters.h" +#include "rtp/RTPSender.h" +#include <media/stagefright/FoundationUtils.h> + +#include <binder/IServiceManager.h> +#include <gui/IGraphicBufferProducer.h> +#include <media/IHDCP.h> +#include <media/IMediaPlayerService.h> +#include <media/IRemoteDisplayClient.h> +#include <media/stagefright/foundation/ABuffer.h> +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> +#include <media/stagefright/ParsedMessage.h> +#include <media/stagefright/MediaErrors.h> +#include <media/stagefright/Utils.h> + +#include <arpa/inet.h> +#include <cutils/properties.h> + +#include <ctype.h> + +namespace android { + +// static +const int64_t WifiDisplaySource::kReaperIntervalUs; +const int64_t WifiDisplaySource::kTeardownTriggerTimeouSecs; +const int64_t WifiDisplaySource::kPlaybackSessionTimeoutSecs; +const int64_t WifiDisplaySource::kPlaybackSessionTimeoutUs; +const AString WifiDisplaySource::sUserAgent = MakeUserAgent(); + +WifiDisplaySource::WifiDisplaySource( + const String16 &opPackageName, + const sp<ANetworkSession> &netSession, + const sp<IRemoteDisplayClient> &client, + const char *path) + : mOpPackageName(opPackageName), + mState(INITIALIZED), + mNetSession(netSession), + mClient(client), + mSessionID(0), + mStopReplyID(NULL), + mChosenRTPPort(-1), + mUsingPCMAudio(false), + mClientSessionID(0), + mReaperPending(false), + mNextCSeq(1), + mUsingHDCP(false), + mIsHDCP2_0(false), + mHDCPPort(0), + mHDCPInitializationComplete(false), + mSetupTriggerDeferred(false), + mPlaybackSessionEstablished(false) { + if (path != NULL) { + mMediaPath.setTo(path); + } + + mSupportedSourceVideoFormats.disableAll(); + + mSupportedSourceVideoFormats.setNativeResolution( + VideoFormats::RESOLUTION_CEA, 5); // 1280x720 p30 + + // Enable all resolutions up to 1280x720p30 + mSupportedSourceVideoFormats.enableResolutionUpto( + VideoFormats::RESOLUTION_CEA, 5, + VideoFormats::PROFILE_CHP, // Constrained High Profile + VideoFormats::LEVEL_32); // Level 3.2 +} + +WifiDisplaySource::~WifiDisplaySource() { +} + +static status_t PostAndAwaitResponse( + const sp<AMessage> &msg, sp<AMessage> *response) { + status_t err = msg->postAndAwaitResponse(response); + + if (err != OK) { + return err; + } + + if (response == NULL || !(*response)->findInt32("err", &err)) { + err = OK; + } + + return err; +} + +status_t WifiDisplaySource::start(const char *iface) { + CHECK_EQ(mState, INITIALIZED); + + sp<AMessage> msg = new AMessage(kWhatStart, this); + msg->setString("iface", iface); + + sp<AMessage> response; + return PostAndAwaitResponse(msg, &response); +} + +status_t WifiDisplaySource::stop() { + sp<AMessage> msg = new AMessage(kWhatStop, this); + + sp<AMessage> response; + return PostAndAwaitResponse(msg, &response); +} + +status_t WifiDisplaySource::pause() { + sp<AMessage> msg = new AMessage(kWhatPause, this); + + sp<AMessage> response; + return PostAndAwaitResponse(msg, &response); +} + +status_t WifiDisplaySource::resume() { + sp<AMessage> msg = new AMessage(kWhatResume, this); + + sp<AMessage> response; + return PostAndAwaitResponse(msg, &response); +} + +void WifiDisplaySource::onMessageReceived(const sp<AMessage> &msg) { + switch (msg->what()) { + case kWhatStart: + { + sp<AReplyToken> replyID; + CHECK(msg->senderAwaitsResponse(&replyID)); + + AString iface; + CHECK(msg->findString("iface", &iface)); + + status_t err = OK; + + ssize_t colonPos = iface.find(":"); + + unsigned long port; + + if (colonPos >= 0) { + const char *s = iface.c_str() + colonPos + 1; + + char *end; + port = strtoul(s, &end, 10); + + if (end == s || *end != '\0' || port > 65535) { + err = -EINVAL; + } else { + iface.erase(colonPos, iface.size() - colonPos); + } + } else { + port = kWifiDisplayDefaultPort; + } + + if (err == OK) { + if (inet_aton(iface.c_str(), &mInterfaceAddr) != 0) { + sp<AMessage> notify = new AMessage(kWhatRTSPNotify, this); + + err = mNetSession->createRTSPServer( + mInterfaceAddr, port, notify, &mSessionID); + } else { + err = -EINVAL; + } + } + + mState = AWAITING_CLIENT_CONNECTION; + + sp<AMessage> response = new AMessage; + response->setInt32("err", err); + response->postReply(replyID); + break; + } + + case kWhatRTSPNotify: + { + int32_t reason; + CHECK(msg->findInt32("reason", &reason)); + + switch (reason) { + case ANetworkSession::kWhatError: + { + int32_t sessionID; + CHECK(msg->findInt32("sessionID", &sessionID)); + + int32_t err; + CHECK(msg->findInt32("err", &err)); + + AString detail; + CHECK(msg->findString("detail", &detail)); + + ALOGE("An error occurred in session %d (%d, '%s/%s').", + sessionID, + err, + detail.c_str(), + strerror(-err)); + + mNetSession->destroySession(sessionID); + + if (sessionID == mClientSessionID) { + mClientSessionID = 0; + + mClient->onDisplayError( + IRemoteDisplayClient::kDisplayErrorUnknown); + } + break; + } + + case ANetworkSession::kWhatClientConnected: + { + int32_t sessionID; + CHECK(msg->findInt32("sessionID", &sessionID)); + + if (mClientSessionID > 0) { + ALOGW("A client tried to connect, but we already " + "have one."); + + mNetSession->destroySession(sessionID); + break; + } + + CHECK_EQ(mState, AWAITING_CLIENT_CONNECTION); + + CHECK(msg->findString("client-ip", &mClientInfo.mRemoteIP)); + CHECK(msg->findString("server-ip", &mClientInfo.mLocalIP)); + + if (mClientInfo.mRemoteIP == mClientInfo.mLocalIP) { + // Disallow connections from the local interface + // for security reasons. + mNetSession->destroySession(sessionID); + break; + } + + CHECK(msg->findInt32( + "server-port", &mClientInfo.mLocalPort)); + mClientInfo.mPlaybackSessionID = -1; + + mClientSessionID = sessionID; + + ALOGI("We now have a client (%d) connected.", sessionID); + + mState = AWAITING_CLIENT_SETUP; + + status_t err = sendM1(sessionID); + CHECK_EQ(err, (status_t)OK); + break; + } + + case ANetworkSession::kWhatData: + { + status_t err = onReceiveClientData(msg); + + if (err != OK) { + mClient->onDisplayError( + IRemoteDisplayClient::kDisplayErrorUnknown); + } + +#if 0 + // testing only. + char val[PROPERTY_VALUE_MAX]; + if (property_get("media.wfd.trigger", val, NULL)) { + if (!strcasecmp(val, "pause") && mState == PLAYING) { + mState = PLAYING_TO_PAUSED; + sendTrigger(mClientSessionID, TRIGGER_PAUSE); + } else if (!strcasecmp(val, "play") + && mState == PAUSED) { + mState = PAUSED_TO_PLAYING; + sendTrigger(mClientSessionID, TRIGGER_PLAY); + } + } +#endif + break; + } + + case ANetworkSession::kWhatNetworkStall: + { + break; + } + + default: + TRESPASS(); + } + break; + } + + case kWhatStop: + { + CHECK(msg->senderAwaitsResponse(&mStopReplyID)); + + CHECK_LT(mState, AWAITING_CLIENT_TEARDOWN); + + if (mState >= AWAITING_CLIENT_PLAY) { + // We have a session, i.e. a previous SETUP succeeded. + + status_t err = sendTrigger( + mClientSessionID, TRIGGER_TEARDOWN); + + if (err == OK) { + mState = AWAITING_CLIENT_TEARDOWN; + + (new AMessage(kWhatTeardownTriggerTimedOut, this))->post( + kTeardownTriggerTimeouSecs * 1000000ll); + + break; + } + + // fall through. + } + + finishStop(); + break; + } + + case kWhatPause: + { + sp<AReplyToken> replyID; + CHECK(msg->senderAwaitsResponse(&replyID)); + + status_t err = OK; + + if (mState != PLAYING) { + err = INVALID_OPERATION; + } else { + mState = PLAYING_TO_PAUSED; + sendTrigger(mClientSessionID, TRIGGER_PAUSE); + } + + sp<AMessage> response = new AMessage; + response->setInt32("err", err); + response->postReply(replyID); + break; + } + + case kWhatResume: + { + sp<AReplyToken> replyID; + CHECK(msg->senderAwaitsResponse(&replyID)); + + status_t err = OK; + + if (mState != PAUSED) { + err = INVALID_OPERATION; + } else { + mState = PAUSED_TO_PLAYING; + sendTrigger(mClientSessionID, TRIGGER_PLAY); + } + + sp<AMessage> response = new AMessage; + response->setInt32("err", err); + response->postReply(replyID); + break; + } + + case kWhatReapDeadClients: + { + mReaperPending = false; + + if (mClientSessionID == 0 + || mClientInfo.mPlaybackSession == NULL) { + break; + } + + if (mClientInfo.mPlaybackSession->getLastLifesignUs() + + kPlaybackSessionTimeoutUs < ALooper::GetNowUs()) { + ALOGI("playback session timed out, reaping."); + + mNetSession->destroySession(mClientSessionID); + mClientSessionID = 0; + + mClient->onDisplayError( + IRemoteDisplayClient::kDisplayErrorUnknown); + } else { + scheduleReaper(); + } + break; + } + + case kWhatPlaybackSessionNotify: + { + int32_t playbackSessionID; + CHECK(msg->findInt32("playbackSessionID", &playbackSessionID)); + + int32_t what; + CHECK(msg->findInt32("what", &what)); + + if (what == PlaybackSession::kWhatSessionDead) { + ALOGI("playback session wants to quit."); + + mClient->onDisplayError( + IRemoteDisplayClient::kDisplayErrorUnknown); + } else if (what == PlaybackSession::kWhatSessionEstablished) { + mPlaybackSessionEstablished = true; + + if (mClient != NULL) { + if (!mSinkSupportsVideo) { + mClient->onDisplayConnected( + NULL, // SurfaceTexture + 0, // width, + 0, // height, + mUsingHDCP + ? IRemoteDisplayClient::kDisplayFlagSecure + : 0, + 0); + } else { + size_t width, height; + + CHECK(VideoFormats::GetConfiguration( + mChosenVideoResolutionType, + mChosenVideoResolutionIndex, + &width, + &height, + NULL /* framesPerSecond */, + NULL /* interlaced */)); + + mClient->onDisplayConnected( + mClientInfo.mPlaybackSession + ->getSurfaceTexture(), + width, + height, + mUsingHDCP + ? IRemoteDisplayClient::kDisplayFlagSecure + : 0, + playbackSessionID); + } + } + + finishPlay(); + + if (mState == ABOUT_TO_PLAY) { + mState = PLAYING; + } + } else if (what == PlaybackSession::kWhatSessionDestroyed) { + disconnectClient2(); + } else { + CHECK_EQ(what, PlaybackSession::kWhatBinaryData); + + int32_t channel; + CHECK(msg->findInt32("channel", &channel)); + + sp<ABuffer> data; + CHECK(msg->findBuffer("data", &data)); + + CHECK_LE(channel, 0xff); + CHECK_LE(data->size(), 0xffffu); + + int32_t sessionID; + CHECK(msg->findInt32("sessionID", &sessionID)); + + char header[4]; + header[0] = '$'; + header[1] = channel; + header[2] = data->size() >> 8; + header[3] = data->size() & 0xff; + + mNetSession->sendRequest( + sessionID, header, sizeof(header)); + + mNetSession->sendRequest( + sessionID, data->data(), data->size()); + } + break; + } + + case kWhatKeepAlive: + { + int32_t sessionID; + CHECK(msg->findInt32("sessionID", &sessionID)); + + if (mClientSessionID != sessionID) { + // Obsolete event, client is already gone. + break; + } + + sendM16(sessionID); + break; + } + + case kWhatTeardownTriggerTimedOut: + { + if (mState == AWAITING_CLIENT_TEARDOWN) { + ALOGI("TEARDOWN trigger timed out, forcing disconnection."); + + CHECK(mStopReplyID != NULL); + finishStop(); + break; + } + break; + } + + case kWhatHDCPNotify: + { + int32_t msgCode, ext1, ext2; + CHECK(msg->findInt32("msg", &msgCode)); + CHECK(msg->findInt32("ext1", &ext1)); + CHECK(msg->findInt32("ext2", &ext2)); + + ALOGI("Saw HDCP notification code %d, ext1 %d, ext2 %d", + msgCode, ext1, ext2); + + switch (msgCode) { + case HDCPModule::HDCP_INITIALIZATION_COMPLETE: + { + mHDCPInitializationComplete = true; + + if (mSetupTriggerDeferred) { + mSetupTriggerDeferred = false; + + sendTrigger(mClientSessionID, TRIGGER_SETUP); + } + break; + } + + case HDCPModule::HDCP_SHUTDOWN_COMPLETE: + case HDCPModule::HDCP_SHUTDOWN_FAILED: + { + // Ugly hack to make sure that the call to + // HDCPObserver::notify is completely handled before + // we clear the HDCP instance and unload the shared + // library :( + (new AMessage(kWhatFinishStop2, this))->post(300000ll); + break; + } + + default: + { + ALOGE("HDCP failure, shutting down."); + + mClient->onDisplayError( + IRemoteDisplayClient::kDisplayErrorUnknown); + break; + } + } + break; + } + + case kWhatFinishStop2: + { + finishStop2(); + break; + } + + default: + TRESPASS(); + } +} + +void WifiDisplaySource::registerResponseHandler( + int32_t sessionID, int32_t cseq, HandleRTSPResponseFunc func) { + ResponseID id; + id.mSessionID = sessionID; + id.mCSeq = cseq; + mResponseHandlers.add(id, func); +} + +status_t WifiDisplaySource::sendM1(int32_t sessionID) { + AString request = "OPTIONS * RTSP/1.0\r\n"; + AppendCommonResponse(&request, mNextCSeq); + + request.append( + "Require: org.wfa.wfd1.0\r\n" + "\r\n"); + + status_t err = + mNetSession->sendRequest(sessionID, request.c_str(), request.size()); + + if (err != OK) { + return err; + } + + registerResponseHandler( + sessionID, mNextCSeq, &WifiDisplaySource::onReceiveM1Response); + + ++mNextCSeq; + + return OK; +} + +status_t WifiDisplaySource::sendM3(int32_t sessionID) { + AString body = + "wfd_content_protection\r\n" + "wfd_video_formats\r\n" + "wfd_audio_codecs\r\n" + "wfd_client_rtp_ports\r\n"; + + AString request = "GET_PARAMETER rtsp://localhost/wfd1.0 RTSP/1.0\r\n"; + AppendCommonResponse(&request, mNextCSeq); + + request.append("Content-Type: text/parameters\r\n"); + request.append(AStringPrintf("Content-Length: %d\r\n", body.size())); + request.append("\r\n"); + request.append(body); + + status_t err = + mNetSession->sendRequest(sessionID, request.c_str(), request.size()); + + if (err != OK) { + return err; + } + + registerResponseHandler( + sessionID, mNextCSeq, &WifiDisplaySource::onReceiveM3Response); + + ++mNextCSeq; + + return OK; +} + +status_t WifiDisplaySource::sendM4(int32_t sessionID) { + CHECK_EQ(sessionID, mClientSessionID); + + AString body; + + if (mSinkSupportsVideo) { + body.append("wfd_video_formats: "); + + VideoFormats chosenVideoFormat; + chosenVideoFormat.disableAll(); + chosenVideoFormat.setNativeResolution( + mChosenVideoResolutionType, mChosenVideoResolutionIndex); + chosenVideoFormat.setProfileLevel( + mChosenVideoResolutionType, mChosenVideoResolutionIndex, + mChosenVideoProfile, mChosenVideoLevel); + + body.append(chosenVideoFormat.getFormatSpec(true /* forM4Message */)); + body.append("\r\n"); + } + + if (mSinkSupportsAudio) { + body.append( + AStringPrintf("wfd_audio_codecs: %s\r\n", + (mUsingPCMAudio + ? "LPCM 00000002 00" // 2 ch PCM 48kHz + : "AAC 00000001 00"))); // 2 ch AAC 48kHz + } + + body.append( + AStringPrintf( + "wfd_presentation_URL: rtsp://%s/wfd1.0/streamid=0 none\r\n", + mClientInfo.mLocalIP.c_str())); + + body.append( + AStringPrintf( + "wfd_client_rtp_ports: %s\r\n", mWfdClientRtpPorts.c_str())); + + AString request = "SET_PARAMETER rtsp://localhost/wfd1.0 RTSP/1.0\r\n"; + AppendCommonResponse(&request, mNextCSeq); + + request.append("Content-Type: text/parameters\r\n"); + request.append(AStringPrintf("Content-Length: %d\r\n", body.size())); + request.append("\r\n"); + request.append(body); + + status_t err = + mNetSession->sendRequest(sessionID, request.c_str(), request.size()); + + if (err != OK) { + return err; + } + + registerResponseHandler( + sessionID, mNextCSeq, &WifiDisplaySource::onReceiveM4Response); + + ++mNextCSeq; + + return OK; +} + +status_t WifiDisplaySource::sendTrigger( + int32_t sessionID, TriggerType triggerType) { + AString body = "wfd_trigger_method: "; + switch (triggerType) { + case TRIGGER_SETUP: + body.append("SETUP"); + break; + case TRIGGER_TEARDOWN: + ALOGI("Sending TEARDOWN trigger."); + body.append("TEARDOWN"); + break; + case TRIGGER_PAUSE: + body.append("PAUSE"); + break; + case TRIGGER_PLAY: + body.append("PLAY"); + break; + default: + TRESPASS(); + } + + body.append("\r\n"); + + AString request = "SET_PARAMETER rtsp://localhost/wfd1.0 RTSP/1.0\r\n"; + AppendCommonResponse(&request, mNextCSeq); + + request.append("Content-Type: text/parameters\r\n"); + request.append(AStringPrintf("Content-Length: %d\r\n", body.size())); + request.append("\r\n"); + request.append(body); + + status_t err = + mNetSession->sendRequest(sessionID, request.c_str(), request.size()); + + if (err != OK) { + return err; + } + + registerResponseHandler( + sessionID, mNextCSeq, &WifiDisplaySource::onReceiveM5Response); + + ++mNextCSeq; + + return OK; +} + +status_t WifiDisplaySource::sendM16(int32_t sessionID) { + AString request = "GET_PARAMETER rtsp://localhost/wfd1.0 RTSP/1.0\r\n"; + AppendCommonResponse(&request, mNextCSeq); + + CHECK_EQ(sessionID, mClientSessionID); + request.append( + AStringPrintf("Session: %d\r\n", mClientInfo.mPlaybackSessionID)); + request.append("\r\n"); // Empty body + + status_t err = + mNetSession->sendRequest(sessionID, request.c_str(), request.size()); + + if (err != OK) { + return err; + } + + registerResponseHandler( + sessionID, mNextCSeq, &WifiDisplaySource::onReceiveM16Response); + + ++mNextCSeq; + + scheduleKeepAlive(sessionID); + + return OK; +} + +status_t WifiDisplaySource::onReceiveM1Response( + int32_t /* sessionID */, const sp<ParsedMessage> &msg) { + int32_t statusCode; + if (!msg->getStatusCode(&statusCode)) { + return ERROR_MALFORMED; + } + + if (statusCode != 200) { + return ERROR_UNSUPPORTED; + } + + return OK; +} + +// sink_audio_list := ("LPCM"|"AAC"|"AC3" HEXDIGIT*8 HEXDIGIT*2) +// (", " sink_audio_list)* +static void GetAudioModes(const char *s, const char *prefix, uint32_t *modes) { + *modes = 0; + + size_t prefixLen = strlen(prefix); + + while (*s != '0') { + if (!strncmp(s, prefix, prefixLen) && s[prefixLen] == ' ') { + unsigned latency; + if (sscanf(&s[prefixLen + 1], "%08x %02x", modes, &latency) != 2) { + *modes = 0; + } + + return; + } + + const char *commaPos = strchr(s, ','); + if (commaPos != NULL) { + s = commaPos + 1; + + while (isspace(*s)) { + ++s; + } + } else { + break; + } + } +} + +status_t WifiDisplaySource::onReceiveM3Response( + int32_t sessionID, const sp<ParsedMessage> &msg) { + int32_t statusCode; + if (!msg->getStatusCode(&statusCode)) { + return ERROR_MALFORMED; + } + + if (statusCode != 200) { + return ERROR_UNSUPPORTED; + } + + sp<Parameters> params = + Parameters::Parse(msg->getContent(), strlen(msg->getContent())); + + if (params == NULL) { + return ERROR_MALFORMED; + } + + AString value; + if (!params->findParameter("wfd_client_rtp_ports", &value)) { + ALOGE("Sink doesn't report its choice of wfd_client_rtp_ports."); + return ERROR_MALFORMED; + } + + unsigned port0 = 0, port1 = 0; + if (sscanf(value.c_str(), + "RTP/AVP/UDP;unicast %u %u mode=play", + &port0, + &port1) == 2 + || sscanf(value.c_str(), + "RTP/AVP/TCP;unicast %u %u mode=play", + &port0, + &port1) == 2) { + if (port0 == 0 || port0 > 65535 || port1 != 0) { + ALOGE("Sink chose its wfd_client_rtp_ports poorly (%s)", + value.c_str()); + + return ERROR_MALFORMED; + } + } else if (strcmp(value.c_str(), "RTP/AVP/TCP;interleaved mode=play")) { + ALOGE("Unsupported value for wfd_client_rtp_ports (%s)", + value.c_str()); + + return ERROR_UNSUPPORTED; + } + + mWfdClientRtpPorts = value; + mChosenRTPPort = port0; + + if (!params->findParameter("wfd_video_formats", &value)) { + ALOGE("Sink doesn't report its choice of wfd_video_formats."); + return ERROR_MALFORMED; + } + + mSinkSupportsVideo = false; + + if (!(value == "none")) { + mSinkSupportsVideo = true; + if (!mSupportedSinkVideoFormats.parseFormatSpec(value.c_str())) { + ALOGE("Failed to parse sink provided wfd_video_formats (%s)", + value.c_str()); + + return ERROR_MALFORMED; + } + + if (!VideoFormats::PickBestFormat( + mSupportedSinkVideoFormats, + mSupportedSourceVideoFormats, + &mChosenVideoResolutionType, + &mChosenVideoResolutionIndex, + &mChosenVideoProfile, + &mChosenVideoLevel)) { + ALOGE("Sink and source share no commonly supported video " + "formats."); + + return ERROR_UNSUPPORTED; + } + + size_t width, height, framesPerSecond; + bool interlaced; + CHECK(VideoFormats::GetConfiguration( + mChosenVideoResolutionType, + mChosenVideoResolutionIndex, + &width, + &height, + &framesPerSecond, + &interlaced)); + + ALOGI("Picked video resolution %zu x %zu %c%zu", + width, height, interlaced ? 'i' : 'p', framesPerSecond); + + ALOGI("Picked AVC profile %d, level %d", + mChosenVideoProfile, mChosenVideoLevel); + } else { + ALOGI("Sink doesn't support video at all."); + } + + if (!params->findParameter("wfd_audio_codecs", &value)) { + ALOGE("Sink doesn't report its choice of wfd_audio_codecs."); + return ERROR_MALFORMED; + } + + mSinkSupportsAudio = false; + + if (!(value == "none")) { + mSinkSupportsAudio = true; + + uint32_t modes; + GetAudioModes(value.c_str(), "AAC", &modes); + + bool supportsAAC = (modes & 1) != 0; // AAC 2ch 48kHz + + GetAudioModes(value.c_str(), "LPCM", &modes); + + bool supportsPCM = (modes & 2) != 0; // LPCM 2ch 48kHz + + if (supportsPCM + && property_get_bool("media.wfd.use-pcm-audio", false)) { + ALOGI("Using PCM audio."); + mUsingPCMAudio = true; + } else if (supportsAAC) { + ALOGI("Using AAC audio."); + mUsingPCMAudio = false; + } else if (supportsPCM) { + ALOGI("Using PCM audio."); + mUsingPCMAudio = true; + } else { + ALOGI("Sink doesn't support an audio format we do."); + return ERROR_UNSUPPORTED; + } + } else { + ALOGI("Sink doesn't support audio at all."); + } + + if (!mSinkSupportsVideo && !mSinkSupportsAudio) { + ALOGE("Sink supports neither video nor audio..."); + return ERROR_UNSUPPORTED; + } + + mUsingHDCP = false; + if (!params->findParameter("wfd_content_protection", &value)) { + ALOGI("Sink doesn't appear to support content protection."); + } else if (value == "none") { + ALOGI("Sink does not support content protection."); + } else { + mUsingHDCP = true; + + bool isHDCP2_0 = false; + if (value.startsWith("HDCP2.0 ")) { + isHDCP2_0 = true; + } else if (!value.startsWith("HDCP2.1 ")) { + ALOGE("malformed wfd_content_protection: '%s'", value.c_str()); + + return ERROR_MALFORMED; + } + + int32_t hdcpPort; + if (!ParsedMessage::GetInt32Attribute( + value.c_str() + 8, "port", &hdcpPort) + || hdcpPort < 1 || hdcpPort > 65535) { + return ERROR_MALFORMED; + } + + mIsHDCP2_0 = isHDCP2_0; + mHDCPPort = hdcpPort; + + status_t err = makeHDCP(); + if (err != OK) { + ALOGE("Unable to instantiate HDCP component. " + "Not using HDCP after all."); + + mUsingHDCP = false; + } + } + + return sendM4(sessionID); +} + +status_t WifiDisplaySource::onReceiveM4Response( + int32_t sessionID, const sp<ParsedMessage> &msg) { + int32_t statusCode; + if (!msg->getStatusCode(&statusCode)) { + return ERROR_MALFORMED; + } + + if (statusCode != 200) { + return ERROR_UNSUPPORTED; + } + + if (mUsingHDCP && !mHDCPInitializationComplete) { + ALOGI("Deferring SETUP trigger until HDCP initialization completes."); + + mSetupTriggerDeferred = true; + return OK; + } + + return sendTrigger(sessionID, TRIGGER_SETUP); +} + +status_t WifiDisplaySource::onReceiveM5Response( + int32_t /* sessionID */, const sp<ParsedMessage> &msg) { + int32_t statusCode; + if (!msg->getStatusCode(&statusCode)) { + return ERROR_MALFORMED; + } + + if (statusCode != 200) { + return ERROR_UNSUPPORTED; + } + + return OK; +} + +status_t WifiDisplaySource::onReceiveM16Response( + int32_t sessionID, const sp<ParsedMessage> & /* msg */) { + // If only the response was required to include a "Session:" header... + + CHECK_EQ(sessionID, mClientSessionID); + + if (mClientInfo.mPlaybackSession != NULL) { + mClientInfo.mPlaybackSession->updateLiveness(); + } + + return OK; +} + +void WifiDisplaySource::scheduleReaper() { + if (mReaperPending) { + return; + } + + mReaperPending = true; + (new AMessage(kWhatReapDeadClients, this))->post(kReaperIntervalUs); +} + +void WifiDisplaySource::scheduleKeepAlive(int32_t sessionID) { + // We need to send updates at least 5 secs before the timeout is set to + // expire, make sure the timeout is greater than 5 secs to begin with. + CHECK_GT(kPlaybackSessionTimeoutUs, 5000000ll); + + sp<AMessage> msg = new AMessage(kWhatKeepAlive, this); + msg->setInt32("sessionID", sessionID); + msg->post(kPlaybackSessionTimeoutUs - 5000000ll); +} + +status_t WifiDisplaySource::onReceiveClientData(const sp<AMessage> &msg) { + int32_t sessionID; + CHECK(msg->findInt32("sessionID", &sessionID)); + + sp<RefBase> obj; + CHECK(msg->findObject("data", &obj)); + + sp<ParsedMessage> data = + static_cast<ParsedMessage *>(obj.get()); + + ALOGV("session %d received '%s'", + sessionID, data->debugString().c_str()); + + AString method; + AString uri; + data->getRequestField(0, &method); + + int32_t cseq; + if (!data->findInt32("cseq", &cseq)) { + sendErrorResponse(sessionID, "400 Bad Request", -1 /* cseq */); + return ERROR_MALFORMED; + } + + if (method.startsWith("RTSP/")) { + // This is a response. + + ResponseID id; + id.mSessionID = sessionID; + id.mCSeq = cseq; + + ssize_t index = mResponseHandlers.indexOfKey(id); + + if (index < 0) { + ALOGW("Received unsolicited server response, cseq %d", cseq); + return ERROR_MALFORMED; + } + + HandleRTSPResponseFunc func = mResponseHandlers.valueAt(index); + mResponseHandlers.removeItemsAt(index); + + status_t err = (this->*func)(sessionID, data); + + if (err != OK) { + ALOGW("Response handler for session %d, cseq %d returned " + "err %d (%s)", + sessionID, cseq, err, strerror(-err)); + + return err; + } + + return OK; + } + + AString version; + data->getRequestField(2, &version); + if (!(version == AString("RTSP/1.0"))) { + sendErrorResponse(sessionID, "505 RTSP Version not supported", cseq); + return ERROR_UNSUPPORTED; + } + + status_t err; + if (method == "OPTIONS") { + err = onOptionsRequest(sessionID, cseq, data); + } else if (method == "SETUP") { + err = onSetupRequest(sessionID, cseq, data); + } else if (method == "PLAY") { + err = onPlayRequest(sessionID, cseq, data); + } else if (method == "PAUSE") { + err = onPauseRequest(sessionID, cseq, data); + } else if (method == "TEARDOWN") { + err = onTeardownRequest(sessionID, cseq, data); + } else if (method == "GET_PARAMETER") { + err = onGetParameterRequest(sessionID, cseq, data); + } else if (method == "SET_PARAMETER") { + err = onSetParameterRequest(sessionID, cseq, data); + } else { + sendErrorResponse(sessionID, "405 Method Not Allowed", cseq); + + err = ERROR_UNSUPPORTED; + } + + return err; +} + +status_t WifiDisplaySource::onOptionsRequest( + int32_t sessionID, + int32_t cseq, + const sp<ParsedMessage> &data) { + int32_t playbackSessionID; + sp<PlaybackSession> playbackSession = + findPlaybackSession(data, &playbackSessionID); + + if (playbackSession != NULL) { + playbackSession->updateLiveness(); + } + + AString response = "RTSP/1.0 200 OK\r\n"; + AppendCommonResponse(&response, cseq); + + response.append( + "Public: org.wfa.wfd1.0, SETUP, TEARDOWN, PLAY, PAUSE, " + "GET_PARAMETER, SET_PARAMETER\r\n"); + + response.append("\r\n"); + + status_t err = mNetSession->sendRequest(sessionID, response.c_str()); + + if (err == OK) { + err = sendM3(sessionID); + } + + return err; +} + +status_t WifiDisplaySource::onSetupRequest( + int32_t sessionID, + int32_t cseq, + const sp<ParsedMessage> &data) { + CHECK_EQ(sessionID, mClientSessionID); + if (mClientInfo.mPlaybackSessionID != -1) { + // We only support a single playback session per client. + // This is due to the reversed keep-alive design in the wfd specs... + sendErrorResponse(sessionID, "400 Bad Request", cseq); + return ERROR_MALFORMED; + } + + AString transport; + if (!data->findString("transport", &transport)) { + sendErrorResponse(sessionID, "400 Bad Request", cseq); + return ERROR_MALFORMED; + } + + RTPSender::TransportMode rtpMode = RTPSender::TRANSPORT_UDP; + + int clientRtp, clientRtcp; + if (transport.startsWith("RTP/AVP/TCP;")) { + AString interleaved; + if (ParsedMessage::GetAttribute( + transport.c_str(), "interleaved", &interleaved) + && sscanf(interleaved.c_str(), "%d-%d", + &clientRtp, &clientRtcp) == 2) { + rtpMode = RTPSender::TRANSPORT_TCP_INTERLEAVED; + } else { + bool badRequest = false; + + AString clientPort; + if (!ParsedMessage::GetAttribute( + transport.c_str(), "client_port", &clientPort)) { + badRequest = true; + } else if (sscanf(clientPort.c_str(), "%d-%d", + &clientRtp, &clientRtcp) == 2) { + } else if (sscanf(clientPort.c_str(), "%d", &clientRtp) == 1) { + // No RTCP. + clientRtcp = -1; + } else { + badRequest = true; + } + + if (badRequest) { + sendErrorResponse(sessionID, "400 Bad Request", cseq); + return ERROR_MALFORMED; + } + + rtpMode = RTPSender::TRANSPORT_TCP; + } + } else if (transport.startsWith("RTP/AVP;unicast;") + || transport.startsWith("RTP/AVP/UDP;unicast;")) { + bool badRequest = false; + + AString clientPort; + if (!ParsedMessage::GetAttribute( + transport.c_str(), "client_port", &clientPort)) { + badRequest = true; + } else if (sscanf(clientPort.c_str(), "%d-%d", + &clientRtp, &clientRtcp) == 2) { + } else if (sscanf(clientPort.c_str(), "%d", &clientRtp) == 1) { + // No RTCP. + clientRtcp = -1; + } else { + badRequest = true; + } + + if (badRequest) { + sendErrorResponse(sessionID, "400 Bad Request", cseq); + return ERROR_MALFORMED; + } +#if 1 + // The older LG dongles doesn't specify client_port=xxx apparently. + } else if (transport == "RTP/AVP/UDP;unicast") { + clientRtp = 19000; + clientRtcp = -1; +#endif + } else { + sendErrorResponse(sessionID, "461 Unsupported Transport", cseq); + return ERROR_UNSUPPORTED; + } + + int32_t playbackSessionID = makeUniquePlaybackSessionID(); + + sp<AMessage> notify = new AMessage(kWhatPlaybackSessionNotify, this); + notify->setInt32("playbackSessionID", playbackSessionID); + notify->setInt32("sessionID", sessionID); + + sp<PlaybackSession> playbackSession = + new PlaybackSession( + mOpPackageName, mNetSession, notify, mInterfaceAddr, mHDCP, mMediaPath.c_str()); + + looper()->registerHandler(playbackSession); + + AString uri; + data->getRequestField(1, &uri); + + if (strncasecmp("rtsp://", uri.c_str(), 7)) { + sendErrorResponse(sessionID, "400 Bad Request", cseq); + return ERROR_MALFORMED; + } + + if (!(uri.startsWith("rtsp://") && uri.endsWith("/wfd1.0/streamid=0"))) { + sendErrorResponse(sessionID, "404 Not found", cseq); + return ERROR_MALFORMED; + } + + RTPSender::TransportMode rtcpMode = RTPSender::TRANSPORT_UDP; + if (clientRtcp < 0) { + rtcpMode = RTPSender::TRANSPORT_NONE; + } + + status_t err = playbackSession->init( + mClientInfo.mRemoteIP.c_str(), + clientRtp, + rtpMode, + clientRtcp, + rtcpMode, + mSinkSupportsAudio, + mUsingPCMAudio, + mSinkSupportsVideo, + mChosenVideoResolutionType, + mChosenVideoResolutionIndex, + mChosenVideoProfile, + mChosenVideoLevel); + + if (err != OK) { + looper()->unregisterHandler(playbackSession->id()); + playbackSession.clear(); + } + + switch (err) { + case OK: + break; + case -ENOENT: + sendErrorResponse(sessionID, "404 Not Found", cseq); + return err; + default: + sendErrorResponse(sessionID, "403 Forbidden", cseq); + return err; + } + + mClientInfo.mPlaybackSessionID = playbackSessionID; + mClientInfo.mPlaybackSession = playbackSession; + + AString response = "RTSP/1.0 200 OK\r\n"; + AppendCommonResponse(&response, cseq, playbackSessionID); + + if (rtpMode == RTPSender::TRANSPORT_TCP_INTERLEAVED) { + response.append( + AStringPrintf( + "Transport: RTP/AVP/TCP;interleaved=%d-%d;", + clientRtp, clientRtcp)); + } else { + int32_t serverRtp = playbackSession->getRTPPort(); + + AString transportString = "UDP"; + if (rtpMode == RTPSender::TRANSPORT_TCP) { + transportString = "TCP"; + } + + if (clientRtcp >= 0) { + response.append( + AStringPrintf( + "Transport: RTP/AVP/%s;unicast;client_port=%d-%d;" + "server_port=%d-%d\r\n", + transportString.c_str(), + clientRtp, clientRtcp, serverRtp, serverRtp + 1)); + } else { + response.append( + AStringPrintf( + "Transport: RTP/AVP/%s;unicast;client_port=%d;" + "server_port=%d\r\n", + transportString.c_str(), + clientRtp, serverRtp)); + } + } + + response.append("\r\n"); + + err = mNetSession->sendRequest(sessionID, response.c_str()); + + if (err != OK) { + return err; + } + + mState = AWAITING_CLIENT_PLAY; + + scheduleReaper(); + scheduleKeepAlive(sessionID); + + return OK; +} + +status_t WifiDisplaySource::onPlayRequest( + int32_t sessionID, + int32_t cseq, + const sp<ParsedMessage> &data) { + int32_t playbackSessionID; + sp<PlaybackSession> playbackSession = + findPlaybackSession(data, &playbackSessionID); + + if (playbackSession == NULL) { + sendErrorResponse(sessionID, "454 Session Not Found", cseq); + return ERROR_MALFORMED; + } + + if (mState != AWAITING_CLIENT_PLAY + && mState != PAUSED_TO_PLAYING + && mState != PAUSED) { + ALOGW("Received PLAY request but we're in state %d", mState); + + sendErrorResponse( + sessionID, "455 Method Not Valid in This State", cseq); + + return INVALID_OPERATION; + } + + ALOGI("Received PLAY request."); + if (mPlaybackSessionEstablished) { + finishPlay(); + } else { + ALOGI("deferring PLAY request until session established."); + } + + AString response = "RTSP/1.0 200 OK\r\n"; + AppendCommonResponse(&response, cseq, playbackSessionID); + response.append("Range: npt=now-\r\n"); + response.append("\r\n"); + + status_t err = mNetSession->sendRequest(sessionID, response.c_str()); + + if (err != OK) { + return err; + } + + if (mState == PAUSED_TO_PLAYING || mPlaybackSessionEstablished) { + mState = PLAYING; + return OK; + } + + CHECK_EQ(mState, AWAITING_CLIENT_PLAY); + mState = ABOUT_TO_PLAY; + + return OK; +} + +void WifiDisplaySource::finishPlay() { + const sp<PlaybackSession> &playbackSession = + mClientInfo.mPlaybackSession; + + status_t err = playbackSession->play(); + CHECK_EQ(err, (status_t)OK); +} + +status_t WifiDisplaySource::onPauseRequest( + int32_t sessionID, + int32_t cseq, + const sp<ParsedMessage> &data) { + int32_t playbackSessionID; + sp<PlaybackSession> playbackSession = + findPlaybackSession(data, &playbackSessionID); + + if (playbackSession == NULL) { + sendErrorResponse(sessionID, "454 Session Not Found", cseq); + return ERROR_MALFORMED; + } + + ALOGI("Received PAUSE request."); + + if (mState != PLAYING_TO_PAUSED && mState != PLAYING) { + return INVALID_OPERATION; + } + + status_t err = playbackSession->pause(); + CHECK_EQ(err, (status_t)OK); + + AString response = "RTSP/1.0 200 OK\r\n"; + AppendCommonResponse(&response, cseq, playbackSessionID); + response.append("\r\n"); + + err = mNetSession->sendRequest(sessionID, response.c_str()); + + if (err != OK) { + return err; + } + + mState = PAUSED; + + return err; +} + +status_t WifiDisplaySource::onTeardownRequest( + int32_t sessionID, + int32_t cseq, + const sp<ParsedMessage> &data) { + ALOGI("Received TEARDOWN request."); + + int32_t playbackSessionID; + sp<PlaybackSession> playbackSession = + findPlaybackSession(data, &playbackSessionID); + + if (playbackSession == NULL) { + sendErrorResponse(sessionID, "454 Session Not Found", cseq); + return ERROR_MALFORMED; + } + + AString response = "RTSP/1.0 200 OK\r\n"; + AppendCommonResponse(&response, cseq, playbackSessionID); + response.append("Connection: close\r\n"); + response.append("\r\n"); + + mNetSession->sendRequest(sessionID, response.c_str()); + + if (mState == AWAITING_CLIENT_TEARDOWN) { + CHECK(mStopReplyID != NULL); + finishStop(); + } else { + mClient->onDisplayError(IRemoteDisplayClient::kDisplayErrorUnknown); + } + + return OK; +} + +void WifiDisplaySource::finishStop() { + ALOGV("finishStop"); + + mState = STOPPING; + + disconnectClientAsync(); +} + +void WifiDisplaySource::finishStopAfterDisconnectingClient() { + ALOGV("finishStopAfterDisconnectingClient"); + + if (mHDCP != NULL) { + ALOGI("Initiating HDCP shutdown."); + mHDCP->shutdownAsync(); + return; + } + + finishStop2(); +} + +void WifiDisplaySource::finishStop2() { + ALOGV("finishStop2"); + + if (mHDCP != NULL) { + mHDCP->setObserver(NULL); + mHDCPObserver.clear(); + mHDCP.clear(); + } + + if (mSessionID != 0) { + mNetSession->destroySession(mSessionID); + mSessionID = 0; + } + + ALOGI("We're stopped."); + mState = STOPPED; + + status_t err = OK; + + sp<AMessage> response = new AMessage; + response->setInt32("err", err); + response->postReply(mStopReplyID); +} + +status_t WifiDisplaySource::onGetParameterRequest( + int32_t sessionID, + int32_t cseq, + const sp<ParsedMessage> &data) { + int32_t playbackSessionID; + sp<PlaybackSession> playbackSession = + findPlaybackSession(data, &playbackSessionID); + + if (playbackSession == NULL) { + sendErrorResponse(sessionID, "454 Session Not Found", cseq); + return ERROR_MALFORMED; + } + + playbackSession->updateLiveness(); + + AString response = "RTSP/1.0 200 OK\r\n"; + AppendCommonResponse(&response, cseq, playbackSessionID); + response.append("\r\n"); + + status_t err = mNetSession->sendRequest(sessionID, response.c_str()); + return err; +} + +status_t WifiDisplaySource::onSetParameterRequest( + int32_t sessionID, + int32_t cseq, + const sp<ParsedMessage> &data) { + int32_t playbackSessionID; + sp<PlaybackSession> playbackSession = + findPlaybackSession(data, &playbackSessionID); + + if (playbackSession == NULL) { + sendErrorResponse(sessionID, "454 Session Not Found", cseq); + return ERROR_MALFORMED; + } + + if (strstr(data->getContent(), "wfd_idr_request\r\n")) { + playbackSession->requestIDRFrame(); + } + + playbackSession->updateLiveness(); + + AString response = "RTSP/1.0 200 OK\r\n"; + AppendCommonResponse(&response, cseq, playbackSessionID); + response.append("\r\n"); + + status_t err = mNetSession->sendRequest(sessionID, response.c_str()); + return err; +} + +// static +void WifiDisplaySource::AppendCommonResponse( + AString *response, int32_t cseq, int32_t playbackSessionID) { + time_t now = time(NULL); + struct tm *now2 = gmtime(&now); + char buf[128]; + strftime(buf, sizeof(buf), "%a, %d %b %Y %H:%M:%S %z", now2); + + response->append("Date: "); + response->append(buf); + response->append("\r\n"); + + response->append(AStringPrintf("Server: %s\r\n", sUserAgent.c_str())); + + if (cseq >= 0) { + response->append(AStringPrintf("CSeq: %d\r\n", cseq)); + } + + if (playbackSessionID >= 0ll) { + response->append( + AStringPrintf( + "Session: %d;timeout=%lld\r\n", + playbackSessionID, kPlaybackSessionTimeoutSecs)); + } +} + +void WifiDisplaySource::sendErrorResponse( + int32_t sessionID, + const char *errorDetail, + int32_t cseq) { + AString response; + response.append("RTSP/1.0 "); + response.append(errorDetail); + response.append("\r\n"); + + AppendCommonResponse(&response, cseq); + + response.append("\r\n"); + + mNetSession->sendRequest(sessionID, response.c_str()); +} + +int32_t WifiDisplaySource::makeUniquePlaybackSessionID() const { + return rand(); +} + +sp<WifiDisplaySource::PlaybackSession> WifiDisplaySource::findPlaybackSession( + const sp<ParsedMessage> &data, int32_t *playbackSessionID) const { + if (!data->findInt32("session", playbackSessionID)) { + // XXX the older dongles do not always include a "Session:" header. + *playbackSessionID = mClientInfo.mPlaybackSessionID; + return mClientInfo.mPlaybackSession; + } + + if (*playbackSessionID != mClientInfo.mPlaybackSessionID) { + return NULL; + } + + return mClientInfo.mPlaybackSession; +} + +void WifiDisplaySource::disconnectClientAsync() { + ALOGV("disconnectClient"); + + if (mClientInfo.mPlaybackSession == NULL) { + disconnectClient2(); + return; + } + + if (mClientInfo.mPlaybackSession != NULL) { + ALOGV("Destroying PlaybackSession"); + mClientInfo.mPlaybackSession->destroyAsync(); + } +} + +void WifiDisplaySource::disconnectClient2() { + ALOGV("disconnectClient2"); + + if (mClientInfo.mPlaybackSession != NULL) { + looper()->unregisterHandler(mClientInfo.mPlaybackSession->id()); + mClientInfo.mPlaybackSession.clear(); + } + + if (mClientSessionID != 0) { + mNetSession->destroySession(mClientSessionID); + mClientSessionID = 0; + } + + mClient->onDisplayDisconnected(); + + finishStopAfterDisconnectingClient(); +} + +struct WifiDisplaySource::HDCPObserver : public BnHDCPObserver { + explicit HDCPObserver(const sp<AMessage> ¬ify); + + virtual void notify( + int msg, int ext1, int ext2, const Parcel *obj); + +private: + sp<AMessage> mNotify; + + DISALLOW_EVIL_CONSTRUCTORS(HDCPObserver); +}; + +WifiDisplaySource::HDCPObserver::HDCPObserver( + const sp<AMessage> ¬ify) + : mNotify(notify) { +} + +void WifiDisplaySource::HDCPObserver::notify( + int msg, int ext1, int ext2, const Parcel * /* obj */) { + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("msg", msg); + notify->setInt32("ext1", ext1); + notify->setInt32("ext2", ext2); + notify->post(); +} + +status_t WifiDisplaySource::makeHDCP() { + sp<IServiceManager> sm = defaultServiceManager(); + sp<IBinder> binder = sm->getService(String16("media.player")); + + sp<IMediaPlayerService> service = + interface_cast<IMediaPlayerService>(binder); + + CHECK(service != NULL); + + mHDCP = service->makeHDCP(true /* createEncryptionModule */); + + if (mHDCP == NULL) { + return ERROR_UNSUPPORTED; + } + + sp<AMessage> notify = new AMessage(kWhatHDCPNotify, this); + mHDCPObserver = new HDCPObserver(notify); + + status_t err = mHDCP->setObserver(mHDCPObserver); + + if (err != OK) { + ALOGE("Failed to set HDCP observer."); + + mHDCPObserver.clear(); + mHDCP.clear(); + + return err; + } + + ALOGI("Initiating HDCP negotiation w/ host %s:%d", + mClientInfo.mRemoteIP.c_str(), mHDCPPort); + + err = mHDCP->initAsync(mClientInfo.mRemoteIP.c_str(), mHDCPPort); + + if (err != OK) { + return err; + } + + return OK; +} + +} // namespace android + diff --git a/media/libstagefright/wifi-display/source/WifiDisplaySource.h b/media/libstagefright/wifi-display/source/WifiDisplaySource.h new file mode 100644 index 0000000000..47047f8f7b --- /dev/null +++ b/media/libstagefright/wifi-display/source/WifiDisplaySource.h @@ -0,0 +1,278 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef WIFI_DISPLAY_SOURCE_H_ + +#define WIFI_DISPLAY_SOURCE_H_ + +#include "VideoFormats.h" + +#include <media/stagefright/foundation/AHandler.h> +#include <media/stagefright/ANetworkSession.h> + +#include <netinet/in.h> + +#include <utils/String16.h> + +namespace android { + +struct AReplyToken; +struct IHDCP; +class IRemoteDisplayClient; +struct ParsedMessage; + +// Represents the RTSP server acting as a wifi display source. +// Manages incoming connections, sets up Playback sessions as necessary. +struct WifiDisplaySource : public AHandler { + static const unsigned kWifiDisplayDefaultPort = 7236; + + WifiDisplaySource( + const String16 &opPackageName, + const sp<ANetworkSession> &netSession, + const sp<IRemoteDisplayClient> &client, + const char *path = NULL); + + status_t start(const char *iface); + status_t stop(); + + status_t pause(); + status_t resume(); + +protected: + virtual ~WifiDisplaySource(); + virtual void onMessageReceived(const sp<AMessage> &msg); + +private: + struct PlaybackSession; + struct HDCPObserver; + + enum State { + INITIALIZED, + AWAITING_CLIENT_CONNECTION, + AWAITING_CLIENT_SETUP, + AWAITING_CLIENT_PLAY, + ABOUT_TO_PLAY, + PLAYING, + PLAYING_TO_PAUSED, + PAUSED, + PAUSED_TO_PLAYING, + AWAITING_CLIENT_TEARDOWN, + STOPPING, + STOPPED, + }; + + enum { + kWhatStart, + kWhatRTSPNotify, + kWhatStop, + kWhatPause, + kWhatResume, + kWhatReapDeadClients, + kWhatPlaybackSessionNotify, + kWhatKeepAlive, + kWhatHDCPNotify, + kWhatFinishStop2, + kWhatTeardownTriggerTimedOut, + }; + + struct ResponseID { + int32_t mSessionID; + int32_t mCSeq; + + bool operator<(const ResponseID &other) const { + return mSessionID < other.mSessionID + || (mSessionID == other.mSessionID + && mCSeq < other.mCSeq); + } + }; + + typedef status_t (WifiDisplaySource::*HandleRTSPResponseFunc)( + int32_t sessionID, const sp<ParsedMessage> &msg); + + static const int64_t kReaperIntervalUs = 1000000ll; + + // We request that the dongle send us a "TEARDOWN" in order to + // perform an orderly shutdown. We're willing to wait up to 2 secs + // for this message to arrive, after that we'll force a disconnect + // instead. + static const int64_t kTeardownTriggerTimeouSecs = 2; + + static const int64_t kPlaybackSessionTimeoutSecs = 30; + + static const int64_t kPlaybackSessionTimeoutUs = + kPlaybackSessionTimeoutSecs * 1000000ll; + + static const AString sUserAgent; + + String16 mOpPackageName; + + State mState; + VideoFormats mSupportedSourceVideoFormats; + sp<ANetworkSession> mNetSession; + sp<IRemoteDisplayClient> mClient; + AString mMediaPath; + struct in_addr mInterfaceAddr; + int32_t mSessionID; + + sp<AReplyToken> mStopReplyID; + + AString mWfdClientRtpPorts; + int32_t mChosenRTPPort; // extracted from "wfd_client_rtp_ports" + + bool mSinkSupportsVideo; + VideoFormats mSupportedSinkVideoFormats; + + VideoFormats::ResolutionType mChosenVideoResolutionType; + size_t mChosenVideoResolutionIndex; + VideoFormats::ProfileType mChosenVideoProfile; + VideoFormats::LevelType mChosenVideoLevel; + + bool mSinkSupportsAudio; + + bool mUsingPCMAudio; + int32_t mClientSessionID; + + struct ClientInfo { + AString mRemoteIP; + AString mLocalIP; + int32_t mLocalPort; + int32_t mPlaybackSessionID; + sp<PlaybackSession> mPlaybackSession; + }; + ClientInfo mClientInfo; + + bool mReaperPending; + + int32_t mNextCSeq; + + KeyedVector<ResponseID, HandleRTSPResponseFunc> mResponseHandlers; + + // HDCP specific section >>>> + bool mUsingHDCP; + bool mIsHDCP2_0; + int32_t mHDCPPort; + sp<IHDCP> mHDCP; + sp<HDCPObserver> mHDCPObserver; + + bool mHDCPInitializationComplete; + bool mSetupTriggerDeferred; + + bool mPlaybackSessionEstablished; + + status_t makeHDCP(); + // <<<< HDCP specific section + + status_t sendM1(int32_t sessionID); + status_t sendM3(int32_t sessionID); + status_t sendM4(int32_t sessionID); + + enum TriggerType { + TRIGGER_SETUP, + TRIGGER_TEARDOWN, + TRIGGER_PAUSE, + TRIGGER_PLAY, + }; + + // M5 + status_t sendTrigger(int32_t sessionID, TriggerType triggerType); + + status_t sendM16(int32_t sessionID); + + status_t onReceiveM1Response( + int32_t sessionID, const sp<ParsedMessage> &msg); + + status_t onReceiveM3Response( + int32_t sessionID, const sp<ParsedMessage> &msg); + + status_t onReceiveM4Response( + int32_t sessionID, const sp<ParsedMessage> &msg); + + status_t onReceiveM5Response( + int32_t sessionID, const sp<ParsedMessage> &msg); + + status_t onReceiveM16Response( + int32_t sessionID, const sp<ParsedMessage> &msg); + + void registerResponseHandler( + int32_t sessionID, int32_t cseq, HandleRTSPResponseFunc func); + + status_t onReceiveClientData(const sp<AMessage> &msg); + + status_t onOptionsRequest( + int32_t sessionID, + int32_t cseq, + const sp<ParsedMessage> &data); + + status_t onSetupRequest( + int32_t sessionID, + int32_t cseq, + const sp<ParsedMessage> &data); + + status_t onPlayRequest( + int32_t sessionID, + int32_t cseq, + const sp<ParsedMessage> &data); + + status_t onPauseRequest( + int32_t sessionID, + int32_t cseq, + const sp<ParsedMessage> &data); + + status_t onTeardownRequest( + int32_t sessionID, + int32_t cseq, + const sp<ParsedMessage> &data); + + status_t onGetParameterRequest( + int32_t sessionID, + int32_t cseq, + const sp<ParsedMessage> &data); + + status_t onSetParameterRequest( + int32_t sessionID, + int32_t cseq, + const sp<ParsedMessage> &data); + + void sendErrorResponse( + int32_t sessionID, + const char *errorDetail, + int32_t cseq); + + static void AppendCommonResponse( + AString *response, int32_t cseq, int32_t playbackSessionID = -1ll); + + void scheduleReaper(); + void scheduleKeepAlive(int32_t sessionID); + + int32_t makeUniquePlaybackSessionID() const; + + sp<PlaybackSession> findPlaybackSession( + const sp<ParsedMessage> &data, int32_t *playbackSessionID) const; + + void finishStop(); + void disconnectClientAsync(); + void disconnectClient2(); + void finishStopAfterDisconnectingClient(); + void finishStop2(); + + void finishPlay(); + + DISALLOW_EVIL_CONSTRUCTORS(WifiDisplaySource); +}; + +} // namespace android + +#endif // WIFI_DISPLAY_SOURCE_H_ diff --git a/media/mediaserver/Android.bp b/media/mediaserver/Android.bp index afca7c4f71..1759ff66f6 100644 --- a/media/mediaserver/Android.bp +++ b/media/mediaserver/Android.bp @@ -11,6 +11,9 @@ cc_library_static { cc_binary { name: "mediaserver", + defaults: [ + "camera_in_mediaserver_defaults", + ], srcs: ["main_mediaserver.cpp"], diff --git a/media/mediaserver/main_mediaserver.cpp b/media/mediaserver/main_mediaserver.cpp index 316732bdc3..162153394d 100644 --- a/media/mediaserver/main_mediaserver.cpp +++ b/media/mediaserver/main_mediaserver.cpp @@ -27,6 +27,10 @@ #include "RegisterExtensions.h" // from LOCAL_C_INCLUDES +#ifdef NO_CAMERA_SERVER +#include "CameraService.h" +#include <hidl/HidlTransportSupport.h> +#endif #include "MediaPlayerService.h" #include "ResourceManagerService.h" @@ -36,12 +40,20 @@ int main(int argc __unused, char **argv __unused) { signal(SIGPIPE, SIG_IGN); +#ifdef NO_CAMERA_SERVER + // Set 3 threads for HIDL calls + hardware::configureRpcThreadpool(3, /*willjoin*/ false); +#endif + sp<ProcessState> proc(ProcessState::self()); sp<IServiceManager> sm(defaultServiceManager()); ALOGI("ServiceManager: %p", sm.get()); AIcu_initializeIcuOrDie(); MediaPlayerService::instantiate(); ResourceManagerService::instantiate(); +#ifdef NO_CAMERA_SERVER + CameraService::instantiate(); +#endif registerExtensions(); ::android::hardware::configureRpcThreadpool(16, false); ProcessState::self()->startThreadPool(); diff --git a/services/audioflinger/Effects.cpp b/services/audioflinger/Effects.cpp index 3dfeb83a4c..ff1f9ac136 100644 --- a/services/audioflinger/Effects.cpp +++ b/services/audioflinger/Effects.cpp @@ -2017,6 +2017,10 @@ void AudioFlinger::EffectChain::clearInputBuffer() { Mutex::Autolock _l(mLock); clearInputBuffer_l(); + + for (size_t i = 0; i < mEffects.size(); i++) { + mEffects[i]->reset_l(); + } } // Must be called with EffectChain::mLock locked diff --git a/services/audiopolicy/AudioPolicyInterface.h b/services/audiopolicy/AudioPolicyInterface.h index 8d0e5dbed7..a350f6cc22 100644 --- a/services/audiopolicy/AudioPolicyInterface.h +++ b/services/audiopolicy/AudioPolicyInterface.h @@ -421,6 +421,8 @@ public: // sessions to be preempted on modules that do not support sound trigger // recognition concurrently with audio capture. virtual void setSoundTriggerCaptureState(bool active) = 0; + + virtual void onOutputSessionEffectsUpdate(sp<AudioSessionInfo>& streamInfo, bool added) = 0; }; extern "C" AudioPolicyInterface* createAudioPolicyManager(AudioPolicyClientInterface *clientInterface); diff --git a/services/audiopolicy/common/include/policy.h b/services/audiopolicy/common/include/policy.h index 0537365ae2..4047d76c0d 100644 --- a/services/audiopolicy/common/include/policy.h +++ b/services/audiopolicy/common/include/policy.h @@ -35,7 +35,7 @@ static const uint32_t SONIFICATION_RESPECTFUL_AFTER_MUSIC_DELAY = 5000; // For mixed output and inputs, the policy will use max mixer sampling rates. // Do not limit sampling rate otherwise -#define SAMPLE_RATE_HZ_MAX 192000 +#define SAMPLE_RATE_HZ_MAX 384000 // Used when a client opens a capture stream, without specifying a desired sample rate. #define SAMPLE_RATE_HZ_DEFAULT 48000 diff --git a/services/audiopolicy/common/managerdefinitions/include/AudioOutputDescriptor.h b/services/audiopolicy/common/managerdefinitions/include/AudioOutputDescriptor.h index 39d11401be..daced38b93 100644 --- a/services/audiopolicy/common/managerdefinitions/include/AudioOutputDescriptor.h +++ b/services/audiopolicy/common/managerdefinitions/include/AudioOutputDescriptor.h @@ -305,6 +305,7 @@ public: DeviceVector mDevices; /**< current devices this output is routed to */ wp<AudioPolicyMix> mPolicyMix; // non NULL when used by a dynamic policy + audio_io_handle_t mIoHandle; // output handle protected: const sp<PolicyAudioPort> mPolicyAudioPort; @@ -404,7 +405,6 @@ public: DeviceVector filterSupportedDevices(const DeviceVector &devices) const; const sp<IOProfile> mProfile; // I/O profile this output derives from - audio_io_handle_t mIoHandle; // output handle uint32_t mLatency; // audio_output_flags_t mFlags; // sp<SwAudioOutputDescriptor> mOutput1; // used by duplicated outputs: first output diff --git a/services/audiopolicy/common/managerdefinitions/src/AudioOutputDescriptor.cpp b/services/audiopolicy/common/managerdefinitions/src/AudioOutputDescriptor.cpp index d6d472b82e..e61af6fc60 100644 --- a/services/audiopolicy/common/managerdefinitions/src/AudioOutputDescriptor.cpp +++ b/services/audiopolicy/common/managerdefinitions/src/AudioOutputDescriptor.cpp @@ -40,7 +40,7 @@ static const DeviceTypeSet& getAllOutRemoteDevices() { AudioOutputDescriptor::AudioOutputDescriptor(const sp<PolicyAudioPort>& policyAudioPort, AudioPolicyClientInterface *clientInterface) - : mPolicyAudioPort(policyAudioPort), mClientInterface(clientInterface) + : mIoHandle(AUDIO_IO_HANDLE_NONE), mPolicyAudioPort(policyAudioPort), mClientInterface(clientInterface) { if (mPolicyAudioPort.get() != nullptr) { mPolicyAudioPort->pickAudioProfile(mSamplingRate, mChannelMask, mFormat); @@ -278,7 +278,7 @@ void AudioOutputDescriptor::log(const char* indent) SwAudioOutputDescriptor::SwAudioOutputDescriptor(const sp<IOProfile>& profile, AudioPolicyClientInterface *clientInterface) : AudioOutputDescriptor(profile, clientInterface), - mProfile(profile), mIoHandle(AUDIO_IO_HANDLE_NONE), mLatency(0), + mProfile(profile), mLatency(0), mFlags((audio_output_flags_t)0), mOutput1(0), mOutput2(0), mDirectOpenCount(0), mDirectClientSession(AUDIO_SESSION_NONE) diff --git a/services/audiopolicy/engine/common/include/EngineBase.h b/services/audiopolicy/engine/common/include/EngineBase.h index 7f339dcfe4..136013a6d7 100755 --- a/services/audiopolicy/engine/common/include/EngineBase.h +++ b/services/audiopolicy/engine/common/include/EngineBase.h @@ -39,6 +39,11 @@ public: audio_mode_t getPhoneState() const override { return mPhoneState; } + void setDpConnAndAllowedForVoice(bool connAndAllowed) override + { + mDpConnAndAllowedForVoice = connAndAllowed; + } + status_t setForceUse(audio_policy_force_use_t usage, audio_policy_forced_cfg_t config) override { mForceUse[usage] = config; @@ -118,6 +123,11 @@ public: return is_state_in_call(getPhoneState()); } + inline bool getDpConnAndAllowedForVoice() const + { + return mDpConnAndAllowedForVoice; + } + VolumeSource toVolumeSource(audio_stream_type_t stream) const { return static_cast<VolumeSource>(getVolumeGroupForStreamType(stream)); @@ -135,6 +145,8 @@ private: VolumeGroupMap mVolumeGroups; LastRemovableMediaDevices mLastRemovableMediaDevices; audio_mode_t mPhoneState = AUDIO_MODE_NORMAL; /**< current phone state. */ + /* if display-port is connected and can be used for voip/voice */ + bool mDpConnAndAllowedForVoice; /** current forced use configuration. */ audio_policy_forced_cfg_t mForceUse[AUDIO_POLICY_FORCE_USE_CNT] = {}; diff --git a/services/audiopolicy/engine/interface/EngineInterface.h b/services/audiopolicy/engine/interface/EngineInterface.h index dfb20b5274..650c15f618 100644 --- a/services/audiopolicy/engine/interface/EngineInterface.h +++ b/services/audiopolicy/engine/interface/EngineInterface.h @@ -73,6 +73,14 @@ public: virtual audio_mode_t getPhoneState() const = 0; /** + * Set whether display-port is connected and is allowed to be used + * for voice usecases + * + * @param[in] connAndAllowed: if display-port is connected and can be used + */ + virtual void setDpConnAndAllowedForVoice(bool connAndAllowed) = 0; + + /** * Set Force Use config for a given usage. * * @param[in] usage for which a configuration shall be forced. diff --git a/services/audiopolicy/enginedefault/src/Engine.cpp b/services/audiopolicy/enginedefault/src/Engine.cpp index b14d2bbb0b..37f1a9815f 100755 --- a/services/audiopolicy/enginedefault/src/Engine.cpp +++ b/services/audiopolicy/enginedefault/src/Engine.cpp @@ -250,6 +250,10 @@ DeviceVector Engine::getDevicesForStrategyInt(legacy_strategy strategy, AUDIO_DEVICE_OUT_BLUETOOTH_A2DP_HEADPHONES}); if (!devices.isEmpty()) break; } + if (getDpConnAndAllowedForVoice() && isInCall()) { + devices = availableOutputDevices.getDevicesFromType(AUDIO_DEVICE_OUT_AUX_DIGITAL); + if (!devices.isEmpty()) break; + } devices = availableOutputDevices.getFirstDevicesFromTypes({ AUDIO_DEVICE_OUT_WIRED_HEADPHONE, AUDIO_DEVICE_OUT_WIRED_HEADSET, AUDIO_DEVICE_OUT_LINE, AUDIO_DEVICE_OUT_USB_HEADSET, @@ -340,6 +344,16 @@ DeviceVector Engine::getDevicesForStrategyInt(legacy_strategy strategy, } } } + // if display-port is connected and being used in voice usecase, + // play ringtone over speaker and display-port + if ((strategy == STRATEGY_SONIFICATION) && getDpConnAndAllowedForVoice()) { + DeviceVector devices2 = availableOutputDevices.getDevicesFromType( + AUDIO_DEVICE_OUT_AUX_DIGITAL); + if (!devices2.isEmpty()) { + devices.add(devices2); + break; + } + } // The second device used for sonification is the same as the device used by media strategy FALLTHROUGH_INTENDED; @@ -371,6 +385,13 @@ DeviceVector Engine::getDevicesForStrategyInt(legacy_strategy strategy, // FIXME: STRATEGY_REROUTING follow STRATEGY_MEDIA for now case STRATEGY_REROUTING: case STRATEGY_MEDIA: { + if (isInCall() && devices.isEmpty()) { + // when in call, get the device for Phone strategy + devices = getDevicesForStrategyInt( + STRATEGY_PHONE, availableOutputDevices, availableInputDevices, outputs); + break; + } + DeviceVector devices2; if (strategy != STRATEGY_SONIFICATION) { // no sonification on remote submix (e.g. WFD) @@ -407,7 +428,8 @@ DeviceVector Engine::getDevicesForStrategyInt(legacy_strategy strategy, getLastRemovableMediaDevices(GROUP_WIRED)); } } - if ((devices2.isEmpty()) && (strategy != STRATEGY_SONIFICATION)) { + if ((devices2.isEmpty()) && (strategy != STRATEGY_SONIFICATION) && + (devices.isEmpty())) { // no sonification on aux digital (e.g. HDMI) devices2 = availableOutputDevices.getDevicesFromType(AUDIO_DEVICE_OUT_AUX_DIGITAL); } @@ -416,6 +438,12 @@ DeviceVector Engine::getDevicesForStrategyInt(legacy_strategy strategy, devices2 = availableOutputDevices.getDevicesFromType( AUDIO_DEVICE_OUT_ANLG_DOCK_HEADSET); } + if ((devices2.isEmpty()) && (strategy != STRATEGY_SONIFICATION) && + (devices.isEmpty())) { + // no sonification on WFD sink + devices2 = availableOutputDevices.getDevicesFromType( + AUDIO_DEVICE_OUT_PROXY); + } if (devices2.isEmpty()) { devices2 = availableOutputDevices.getDevicesFromType(AUDIO_DEVICE_OUT_SPEAKER); } diff --git a/services/audiopolicy/managerdefault/AudioPolicyManager.h b/services/audiopolicy/managerdefault/AudioPolicyManager.h index b588f898d4..b2a670e6a4 100644 --- a/services/audiopolicy/managerdefault/AudioPolicyManager.h +++ b/services/audiopolicy/managerdefault/AudioPolicyManager.h @@ -506,7 +506,7 @@ protected: // when a device is disconnected, checks if an output is not used any more and // returns its handle if any. // transfers the audio tracks and effects from one output thread to another accordingly. - status_t checkOutputsForDevice(const sp<DeviceDescriptor>& device, + virtual status_t checkOutputsForDevice(const sp<DeviceDescriptor>& device, audio_policy_dev_state_t state, SortedVector<audio_io_handle_t>& outputs); @@ -539,7 +539,7 @@ protected: * Must be called before updateDevicesAndOutputs() * @param attr to be considered */ - void checkOutputForAttributes(const audio_attributes_t &attr); + virtual void checkOutputForAttributes(const audio_attributes_t &attr); bool followsSameRouting(const audio_attributes_t &lAttr, const audio_attributes_t &rAttr) const; @@ -685,10 +685,10 @@ protected: uint32_t delayMs); bool isDeviceOfModule(const sp<DeviceDescriptor>& devDesc, const char *moduleId) const; - status_t startSource(const sp<SwAudioOutputDescriptor>& outputDesc, + virtual status_t startSource(const sp<SwAudioOutputDescriptor>& outputDesc, const sp<TrackClientDescriptor>& client, uint32_t *delayMs); - status_t stopSource(const sp<SwAudioOutputDescriptor>& outputDesc, + virtual status_t stopSource(const sp<SwAudioOutputDescriptor>& outputDesc, const sp<TrackClientDescriptor>& client); void clearAudioPatches(uid_t uid); @@ -787,7 +787,7 @@ protected: std::unordered_set<audio_format_t> mManualSurroundFormats; std::unordered_map<uid_t, audio_flags_mask_t> mAllowedCapturePolicies; -private: +protected: void onNewAudioModulesAvailableInt(DeviceVector *newDevices); // Add or remove AC3 DTS encodings based on user preferences. @@ -840,7 +840,7 @@ private: std::vector<sp<AudioPolicyMix>> *secondaryMixes, output_type_t *outputType); // internal method to return the output handle for the given device and format - audio_io_handle_t getOutputForDevices( + virtual audio_io_handle_t getOutputForDevices( const DeviceVector &devices, audio_session_t session, audio_stream_type_t stream, @@ -885,7 +885,7 @@ private: bool isValidAttributes(const audio_attributes_t *paa); // Called by setDeviceConnectionState(). - status_t setDeviceConnectionStateInt(audio_devices_t deviceType, + virtual status_t setDeviceConnectionStateInt(audio_devices_t deviceType, audio_policy_dev_state_t state, const char *device_address, const char *device_name, diff --git a/services/audiopolicy/service/AudioPolicyClientImpl.cpp b/services/audiopolicy/service/AudioPolicyClientImpl.cpp index 9fa7a5348b..177c3cbe78 100644 --- a/services/audiopolicy/service/AudioPolicyClientImpl.cpp +++ b/services/audiopolicy/service/AudioPolicyClientImpl.cpp @@ -236,6 +236,13 @@ void AudioPolicyService::AudioPolicyClient::onAudioVolumeGroupChanged(volume_gro mAudioPolicyService->onAudioVolumeGroupChanged(group, flags); } +void AudioPolicyService::AudioPolicyClient::onOutputSessionEffectsUpdate( + sp<AudioSessionInfo>& info, bool added) +{ + mAudioPolicyService->onOutputSessionEffectsUpdate(info, added); +} + + audio_unique_id_t AudioPolicyService::AudioPolicyClient::newAudioUniqueId(audio_unique_id_use_t use) { return AudioSystem::newAudioUniqueId(use); diff --git a/services/audiopolicy/service/AudioPolicyEffects.cpp b/services/audiopolicy/service/AudioPolicyEffects.cpp index b738633924..a5e5bb84e3 100644 --- a/services/audiopolicy/service/AudioPolicyEffects.cpp +++ b/services/audiopolicy/service/AudioPolicyEffects.cpp @@ -31,6 +31,7 @@ #include <utils/SortedVector.h> #include <cutils/config_utils.h> #include <binder/IPCThreadState.h> +#include "AudioPolicyService.h" #include "AudioPolicyEffects.h" namespace android { @@ -39,7 +40,8 @@ namespace android { // AudioPolicyEffects Implementation // ---------------------------------------------------------------------------- -AudioPolicyEffects::AudioPolicyEffects() +AudioPolicyEffects::AudioPolicyEffects(AudioPolicyService *audioPolicyService) : + mAudioPolicyService(audioPolicyService) { status_t loadResult = loadAudioEffectXmlConfig(); if (loadResult == NO_ERROR) { @@ -238,6 +240,8 @@ status_t AudioPolicyEffects::addOutputSessionEffects(audio_io_handle_t output, { status_t status = NO_ERROR; + ALOGV("addOutputSessionEffects %d", audioSession); + Mutex::Autolock _l(mLock); // create audio processors according to stream // FIXME: should we have specific post processing settings for internal streams? @@ -245,6 +249,22 @@ status_t AudioPolicyEffects::addOutputSessionEffects(audio_io_handle_t output, if (stream >= AUDIO_STREAM_PUBLIC_CNT) { stream = AUDIO_STREAM_MUSIC; } + + // send the streaminfo notification only once + ssize_t sidx = mOutputAudioSessionInfo.indexOfKey(audioSession); + if (sidx >= 0) { + // AudioSessionInfo is existing and we just need to increase ref count + sp<AudioSessionInfo> info = mOutputAudioSessionInfo.valueAt(sidx); + info->mRefCount++; + + if (info->mRefCount == 1) { + mAudioPolicyService->onOutputSessionEffectsUpdate(info, true); + } + ALOGV("addOutputSessionEffects(): session info %d refCount=%d", audioSession, info->mRefCount); + } else { + ALOGV("addOutputSessionEffects(): no output stream info found for stream"); + } + ssize_t index = mOutputStreams.indexOfKey(stream); if (index < 0) { ALOGV("addOutputSessionEffects(): no output processing needed for this stream"); @@ -290,6 +310,86 @@ status_t AudioPolicyEffects::addOutputSessionEffects(audio_io_handle_t output, return status; } +status_t AudioPolicyEffects::releaseOutputAudioSessionInfo(audio_io_handle_t /* output */, + audio_stream_type_t stream, + audio_session_t session) +{ + if (uint32_t(stream) >= AUDIO_STREAM_CNT) { + return BAD_VALUE; + } + + Mutex::Autolock _l(mLock); + + ssize_t idx = mOutputAudioSessionInfo.indexOfKey(session); + if (idx >= 0) { + sp<AudioSessionInfo> info = mOutputAudioSessionInfo.valueAt(idx); + if (info->mRefCount == 0) { + mOutputAudioSessionInfo.removeItemsAt(idx); + } + ALOGV("releaseOutputAudioSessionInfo() sessionId=%d refcount=%d", + session, info->mRefCount); + } else { + ALOGV("releaseOutputAudioSessionInfo() no session info found"); + } + return NO_ERROR; +} + +status_t AudioPolicyEffects::updateOutputAudioSessionInfo(audio_io_handle_t /* output */, + audio_stream_type_t stream, + audio_session_t session, + audio_output_flags_t flags, + const audio_config_t *config, uid_t uid) +{ + if (uint32_t(stream) >= AUDIO_STREAM_CNT) { + return BAD_VALUE; + } + + Mutex::Autolock _l(mLock); + + // TODO: Handle other stream types based on client registration + if (stream != AUDIO_STREAM_MUSIC) { + return NO_ERROR; + } + + // update AudioSessionInfo. This is used in the stream open/close path + // to notify userspace applications about session creation and + // teardown, allowing the app to make decisions about effects for + // a particular stream. This is independent of the current + // output_session_processing feature which forcibly attaches a + // static list of effects to a stream. + ssize_t idx = mOutputAudioSessionInfo.indexOfKey(session); + sp<AudioSessionInfo> info; + if (idx < 0) { + info = new AudioSessionInfo(session, stream, flags, config->channel_mask, uid); + mOutputAudioSessionInfo.add(session, info); + } else { + // the streaminfo may actually change + info = mOutputAudioSessionInfo.valueAt(idx); + info->mFlags = flags; + info->mChannelMask = config->channel_mask; + } + + ALOGV("updateOutputAudioSessionInfo() sessionId=%d, flags=0x%x, channel_mask=0x%x uid=%d refCount=%d", + info->mSessionId, info->mFlags, info->mChannelMask, info->mUid, info->mRefCount); + + return NO_ERROR; +} + +status_t AudioPolicyEffects::listAudioSessions(audio_stream_type_t streams, + Vector< sp<AudioSessionInfo>> &sessions) +{ + ALOGV("listAudioSessions() streams %d", streams); + + for (unsigned int i = 0; i < mOutputAudioSessionInfo.size(); i++) { + sp<AudioSessionInfo> info = mOutputAudioSessionInfo.valueAt(i); + if (streams == -1 || info->mStream == streams) { + sessions.push_back(info); + } + } + + return NO_ERROR; +} + status_t AudioPolicyEffects::releaseOutputSessionEffects(audio_io_handle_t output, audio_stream_type_t stream, audio_session_t audioSession) @@ -299,7 +399,19 @@ status_t AudioPolicyEffects::releaseOutputSessionEffects(audio_io_handle_t outpu (void) stream; // argument not used for now Mutex::Autolock _l(mLock); - ssize_t index = mOutputSessions.indexOfKey(audioSession); + ssize_t index = mOutputAudioSessionInfo.indexOfKey(audioSession); + if (index >= 0) { + sp<AudioSessionInfo> info = mOutputAudioSessionInfo.valueAt(index); + info->mRefCount--; + if (info->mRefCount == 0) { + mAudioPolicyService->onOutputSessionEffectsUpdate(info, false); + } + ALOGV("releaseOutputSessionEffects(): session=%d refCount=%d", info->mSessionId, info->mRefCount); + } else { + ALOGV("releaseOutputSessionEffects: no stream info was attached to this stream"); + } + + index = mOutputSessions.indexOfKey(audioSession); if (index < 0) { ALOGV("releaseOutputSessionEffects: no output processing was attached to this stream"); return NO_ERROR; diff --git a/services/audiopolicy/service/AudioPolicyEffects.h b/services/audiopolicy/service/AudioPolicyEffects.h index 81c728df8d..137e37f768 100644 --- a/services/audiopolicy/service/AudioPolicyEffects.h +++ b/services/audiopolicy/service/AudioPolicyEffects.h @@ -31,6 +31,8 @@ namespace android { +class AudioPolicyService; + // ---------------------------------------------------------------------------- // AudioPolicyEffects class @@ -44,7 +46,7 @@ public: // The constructor will parse audio_effects.conf // First it will look whether vendor specific file exists, // otherwise it will parse the system default file. - AudioPolicyEffects(); + AudioPolicyEffects(AudioPolicyService *audioPolicyService); virtual ~AudioPolicyEffects(); // NOTE: methods on AudioPolicyEffects should never be called with the AudioPolicyService @@ -106,6 +108,19 @@ public: // Remove the default stream effect from wherever it's attached. status_t removeStreamDefaultEffect(audio_unique_id_t id); + status_t updateOutputAudioSessionInfo(audio_io_handle_t output, + audio_stream_type_t stream, + audio_session_t audioSession, + audio_output_flags_t flags, + const audio_config_t *config, uid_t uid); + + status_t releaseOutputAudioSessionInfo(audio_io_handle_t output, + audio_stream_type_t stream, + audio_session_t audioSession); + + status_t listAudioSessions(audio_stream_type_t streams, + Vector< sp<AudioSessionInfo>> &sessions); + private: void initDefaultDeviceEffects(); @@ -276,6 +291,11 @@ private: * We must store the reference of the furture garantee real asynchronous operation. */ std::future<void> mDefaultDeviceEffectFuture; + + // Stream info for session events + KeyedVector< audio_session_t, sp<AudioSessionInfo> > mOutputAudioSessionInfo; + + AudioPolicyService *mAudioPolicyService; }; } // namespace android diff --git a/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp b/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp index df27f6e519..751759af50 100644 --- a/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp +++ b/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp @@ -235,61 +235,72 @@ status_t AudioPolicyService::getOutputForAttr(audio_attributes_t *attr, } ALOGV("%s()", __func__); - Mutex::Autolock _l(mLock); + sp<AudioPolicyEffects> audioPolicyEffects; + { + Mutex::Autolock _l(mLock); - const uid_t callingUid = IPCThreadState::self()->getCallingUid(); - if (!isAudioServerOrMediaServerUid(callingUid) || uid == (uid_t)-1) { - ALOGW_IF(uid != (uid_t)-1 && uid != callingUid, - "%s uid %d tried to pass itself off as %d", __func__, callingUid, uid); - uid = callingUid; - } - if (!mPackageManager.allowPlaybackCapture(uid)) { - attr->flags |= AUDIO_FLAG_NO_MEDIA_PROJECTION; - } - if (((attr->flags & (AUDIO_FLAG_BYPASS_INTERRUPTION_POLICY|AUDIO_FLAG_BYPASS_MUTE)) != 0) - && !bypassInterruptionPolicyAllowed(pid, uid)) { - attr->flags &= ~(AUDIO_FLAG_BYPASS_INTERRUPTION_POLICY|AUDIO_FLAG_BYPASS_MUTE); - } - AutoCallerClear acc; - AudioPolicyInterface::output_type_t outputType; - result = mAudioPolicyManager->getOutputForAttr(attr, output, session, stream, uid, - config, - &flags, selectedDeviceId, portId, - secondaryOutputs, - &outputType); - - // FIXME: Introduce a way to check for the the telephony device before opening the output - if (result == NO_ERROR) { - // enforce permission (if any) required for each type of input - switch (outputType) { - case AudioPolicyInterface::API_OUTPUT_LEGACY: - break; - case AudioPolicyInterface::API_OUTPUT_TELEPHONY_TX: - if (!modifyPhoneStateAllowed(pid, uid)) { - ALOGE("%s() permission denied: modify phone state not allowed for uid %d", - __func__, uid); - result = PERMISSION_DENIED; - } - break; - case AudioPolicyInterface::API_OUT_MIX_PLAYBACK: - if (!modifyAudioRoutingAllowed(pid, uid)) { - ALOGE("%s() permission denied: modify audio routing not allowed for uid %d", - __func__, uid); - result = PERMISSION_DENIED; + const uid_t callingUid = IPCThreadState::self()->getCallingUid(); + if (!isAudioServerOrMediaServerUid(callingUid) || uid == (uid_t)-1) { + ALOGW_IF(uid != (uid_t)-1 && uid != callingUid, + "%s uid %d tried to pass itself off as %d", __func__, callingUid, uid); + uid = callingUid; + } + if (!mPackageManager.allowPlaybackCapture(uid)) { + attr->flags |= AUDIO_FLAG_NO_MEDIA_PROJECTION; + } + if (((attr->flags & (AUDIO_FLAG_BYPASS_INTERRUPTION_POLICY|AUDIO_FLAG_BYPASS_MUTE)) != 0) + && !bypassInterruptionPolicyAllowed(pid, uid)) { + attr->flags &= ~(AUDIO_FLAG_BYPASS_INTERRUPTION_POLICY|AUDIO_FLAG_BYPASS_MUTE); + } + AutoCallerClear acc; + AudioPolicyInterface::output_type_t outputType; + result = mAudioPolicyManager->getOutputForAttr(attr, output, session, stream, uid, + config, + &flags, selectedDeviceId, portId, + secondaryOutputs, + &outputType); + + // FIXME: Introduce a way to check for the the telephony device before opening the output + if (result == NO_ERROR) { + // enforce permission (if any) required for each type of input + switch (outputType) { + case AudioPolicyInterface::API_OUTPUT_LEGACY: + break; + case AudioPolicyInterface::API_OUTPUT_TELEPHONY_TX: + if (!modifyPhoneStateAllowed(pid, uid)) { + ALOGE("%s() permission denied: modify phone state not allowed for uid %d", + __func__, uid); + result = PERMISSION_DENIED; + } + break; + case AudioPolicyInterface::API_OUT_MIX_PLAYBACK: + if (!modifyAudioRoutingAllowed(pid, uid)) { + ALOGE("%s() permission denied: modify audio routing not allowed for uid %d", + __func__, uid); + result = PERMISSION_DENIED; + } + break; + case AudioPolicyInterface::API_OUTPUT_INVALID: + default: + LOG_ALWAYS_FATAL("%s() encountered an invalid output type %d", + __func__, (int)outputType); } - break; - case AudioPolicyInterface::API_OUTPUT_INVALID: - default: - LOG_ALWAYS_FATAL("%s() encountered an invalid output type %d", - __func__, (int)outputType); } + + if (result == NO_ERROR) { + sp <AudioPlaybackClient> client = + new AudioPlaybackClient(*attr, *output, uid, pid, session, *portId, *selectedDeviceId, *stream); + mAudioPlaybackClients.add(*portId, client); + } + + audioPolicyEffects = mAudioPolicyEffects; } - if (result == NO_ERROR) { - sp <AudioPlaybackClient> client = - new AudioPlaybackClient(*attr, *output, uid, pid, session, *portId, *selectedDeviceId, *stream); - mAudioPlaybackClients.add(*portId, client); + if (result == NO_ERROR && audioPolicyEffects != 0) { + audioPolicyEffects->updateOutputAudioSessionInfo(*output, *stream, + session, flags, config, uid); } + return result; } @@ -393,11 +404,20 @@ void AudioPolicyService::doReleaseOutput(audio_port_handle_t portId) audioPolicyEffects->releaseOutputSessionEffects( client->io, client->stream, client->session); } - Mutex::Autolock _l(mLock); - mAudioPlaybackClients.removeItem(portId); + { + Mutex::Autolock _l(mLock); + mAudioPlaybackClients.removeItem(portId); - // called from internal thread: no need to clear caller identity - mAudioPolicyManager->releaseOutput(portId); + audioPolicyEffects = mAudioPolicyEffects; + + // called from internal thread: no need to clear caller identity + mAudioPolicyManager->releaseOutput(portId); + } + + if (audioPolicyEffects != 0) { + audioPolicyEffects->releaseOutputAudioSessionInfo(client->io, + client->stream, client->session); + } } status_t AudioPolicyService::getInputForAttr(const audio_attributes_t *attr, @@ -455,7 +475,7 @@ status_t AudioPolicyService::getInputForAttr(const audio_attributes_t *attr, // check calling permissions. // Capturing from FM_TUNER source is controlled by captureAudioOutputAllowed() only as this // does not affect users privacy as does capturing from an actual microphone. - if (!(recordingAllowed(opPackageName, pid, uid) || attr->source == AUDIO_SOURCE_FM_TUNER)) { + if (!isAudioServerOrMediaServerUid(callingUid) && !(recordingAllowed(opPackageName, pid, uid) || attr->source == AUDIO_SOURCE_FM_TUNER)) { ALOGE("%s permission denied: recording not allowed for uid %d pid %d", __func__, uid, pid); return PERMISSION_DENIED; @@ -503,7 +523,7 @@ status_t AudioPolicyService::getInputForAttr(const audio_attributes_t *attr, case AudioPolicyInterface::API_INPUT_TELEPHONY_RX: // FIXME: use the same permission as for remote submix for now. case AudioPolicyInterface::API_INPUT_MIX_CAPTURE: - if (!canCaptureOutput) { + if (!isAudioServerOrMediaServerUid(callingUid) && !canCaptureOutput) { ALOGE("getInputForAttr() permission denied: capture not allowed"); status = PERMISSION_DENIED; } @@ -572,7 +592,7 @@ status_t AudioPolicyService::startInput(audio_port_handle_t portId) } // check calling permissions - if (!(startRecording(client->opPackageName, client->pid, client->uid, + if (!isAudioServerOrMediaServerUid(IPCThreadState::self()->getCallingUid()) && !(startRecording(client->opPackageName, client->pid, client->uid, client->attributes.source == AUDIO_SOURCE_HOTWORD) || client->attributes.source == AUDIO_SOURCE_FM_TUNER)) { ALOGE("%s permission denied: recording not allowed for uid %d pid %d", @@ -1534,4 +1554,24 @@ status_t AudioPolicyService::registerSoundTriggerCaptureStateListener( return NO_ERROR; } +status_t AudioPolicyService::listAudioSessions(audio_stream_type_t streams, + Vector< sp<AudioSessionInfo>> &sessions) +{ + sp<AudioPolicyEffects> audioPolicyEffects; + { + Mutex::Autolock _l(mLock); + if (mAudioPolicyManager == NULL) { + return NO_INIT; + } + audioPolicyEffects = mAudioPolicyEffects; + } + + if (audioPolicyEffects != 0) { + return audioPolicyEffects->listAudioSessions(streams, sessions); + } + + // no errors here if effects are not available + return NO_ERROR; +} + } // namespace android diff --git a/services/audiopolicy/service/AudioPolicyService.cpp b/services/audiopolicy/service/AudioPolicyService.cpp index a6e8989722..fbd7614c08 100644 --- a/services/audiopolicy/service/AudioPolicyService.cpp +++ b/services/audiopolicy/service/AudioPolicyService.cpp @@ -78,7 +78,7 @@ void AudioPolicyService::onFirstRef() mAudioPolicyManager = createAudioPolicyManager(mAudioPolicyClient); } // load audio processing modules - sp<AudioPolicyEffects> audioPolicyEffects = new AudioPolicyEffects(); + sp<AudioPolicyEffects> audioPolicyEffects = new AudioPolicyEffects(this); sp<UidPolicy> uidPolicy = new UidPolicy(this); sp<SensorPrivacyPolicy> sensorPrivacyPolicy = new SensorPrivacyPolicy(this); { @@ -293,6 +293,21 @@ status_t AudioPolicyService::clientSetAudioPortConfig(const struct audio_port_co return mAudioCommandThread->setAudioPortConfigCommand(config, delayMs); } +void AudioPolicyService::onOutputSessionEffectsUpdate(sp<AudioSessionInfo>& info, bool added) +{ + ALOGV("AudioPolicyService::onOutputSessionEffectsUpdate(%d, %d, %d)", + info->mStream, info->mSessionId, added); + mOutputCommandThread->effectSessionUpdateCommand(info, added); +} + +void AudioPolicyService::doOnOutputSessionEffectsUpdate(sp<AudioSessionInfo>& info, bool added) +{ + Mutex::Autolock _l(mNotificationClientsLock); + for (size_t i = 0; i < mNotificationClients.size(); i++) { + mNotificationClients.valueAt(i)->onOutputSessionEffectsUpdate(info, added); + } +} + AudioPolicyService::NotificationClient::NotificationClient(const sp<AudioPolicyService>& service, const sp<IAudioPolicyServiceClient>& client, uid_t uid, @@ -337,6 +352,13 @@ void AudioPolicyService::NotificationClient::onAudioVolumeGroupChanged(volume_gr } } +void AudioPolicyService::NotificationClient::onOutputSessionEffectsUpdate( + sp<AudioSessionInfo>& info, bool added) +{ + if (mAudioPolicyServiceClient != 0) { + mAudioPolicyServiceClient->onOutputSessionEffectsUpdate(info, added); + } +} void AudioPolicyService::NotificationClient::onDynamicPolicyMixStateUpdate( const String8& regId, int32_t state) @@ -1366,6 +1388,20 @@ bool AudioPolicyService::AudioCommandThread::threadLoop() svc->doOnNewAudioModulesAvailable(); mLock.lock(); } break; + case EFFECT_SESSION_UPDATE: { + EffectSessionUpdateData *data = + (EffectSessionUpdateData *)command->mParam.get(); + ALOGV("AudioCommandThread() processing effect session update %d %d %d", + data->mAudioSessionInfo->mStream, data->mAudioSessionInfo->mSessionId, + data->mAdded); + svc = mService.promote(); + if (svc == 0) { + break; + } + mLock.unlock(); + svc->doOnOutputSessionEffectsUpdate(data->mAudioSessionInfo, data->mAdded); + mLock.lock(); + } break; default: ALOGW("AudioCommandThread() unknown command %d", command->mCommand); @@ -1662,6 +1698,20 @@ void AudioPolicyService::AudioCommandThread::audioModulesUpdateCommand() sendCommand(command); } +void AudioPolicyService::AudioCommandThread::effectSessionUpdateCommand( + sp<AudioSessionInfo>& streamInfo, bool added) +{ + sp<AudioCommand> command = new AudioCommand(); + command->mCommand = EFFECT_SESSION_UPDATE; + EffectSessionUpdateData *data = new EffectSessionUpdateData(); + data->mAudioSessionInfo = streamInfo; + data->mAdded = added; + command->mParam = data; + ALOGV("AudioCommandThread() sending effect session update (id=%d) for stream %d (added=%d)", + streamInfo->mStream, streamInfo->mSessionId, added); + sendCommand(command); +} + status_t AudioPolicyService::AudioCommandThread::sendCommand(sp<AudioCommand>& command, int delayMs) { { diff --git a/services/audiopolicy/service/AudioPolicyService.h b/services/audiopolicy/service/AudioPolicyService.h index 869a963d05..fc61cce52c 100644 --- a/services/audiopolicy/service/AudioPolicyService.h +++ b/services/audiopolicy/service/AudioPolicyService.h @@ -288,6 +288,9 @@ public: status_t doStopOutput(audio_port_handle_t portId); void doReleaseOutput(audio_port_handle_t portId); + virtual status_t listAudioSessions(audio_stream_type_t stream, + Vector< sp<AudioSessionInfo>>& sessions); + status_t clientCreateAudioPatch(const struct audio_patch *patch, audio_patch_handle_t *handle, int delayMs); @@ -327,6 +330,9 @@ public: audio_session_t sessionId, bool suspended); + void onOutputSessionEffectsUpdate(sp<AudioSessionInfo>& info, bool added); + void doOnOutputSessionEffectsUpdate(sp<AudioSessionInfo>& info, bool added); + private: AudioPolicyService() ANDROID_API; virtual ~AudioPolicyService(); @@ -475,6 +481,7 @@ private: RECORDING_CONFIGURATION_UPDATE, SET_EFFECT_SUSPENDED, AUDIO_MODULES_UPDATE, + EFFECT_SESSION_UPDATE, }; AudioCommandThread (String8 name, const wp<AudioPolicyService>& service); @@ -522,6 +529,8 @@ private: bool suspended); void audioModulesUpdateCommand(); void insertCommand_l(AudioCommand *command, int delayMs = 0); + void effectSessionUpdateCommand(sp<AudioSessionInfo>& info, bool added); + private: class AudioCommandData; @@ -625,6 +634,12 @@ private: bool mSuspended; }; + class EffectSessionUpdateData : public AudioCommandData { + public: + sp<AudioSessionInfo> mAudioSessionInfo; + bool mAdded; + }; + Mutex mLock; Condition mWaitWorkCV; Vector < sp<AudioCommand> > mAudioCommands; // list of pending commands @@ -743,6 +758,8 @@ private: void setSoundTriggerCaptureState(bool active) override; + virtual void onOutputSessionEffectsUpdate(sp<AudioSessionInfo>& info, bool added); + private: AudioPolicyService *mAudioPolicyService; }; @@ -771,6 +788,8 @@ private: audio_source_t source); void setAudioPortCallbacksEnabled(bool enabled); void setAudioVolumeGroupCallbacksEnabled(bool enabled); + void onOutputSessionEffectsUpdate(sp<AudioSessionInfo>& info, + bool added); uid_t uid() { return mUid; diff --git a/services/camera/libcameraservice/Android.bp b/services/camera/libcameraservice/Android.bp index 501d92226f..92e9817e16 100644 --- a/services/camera/libcameraservice/Android.bp +++ b/services/camera/libcameraservice/Android.bp @@ -18,6 +18,11 @@ cc_library_shared { name: "libcameraservice", + defaults: [ + "no_cameraserver_defaults", + "qti_camera_device_defaults", + "needs_camera_boottime_defaults", + ], // Camera service source diff --git a/services/camera/libcameraservice/CameraService.cpp b/services/camera/libcameraservice/CameraService.cpp index e2fcb5a97c..fcccd262fe 100644 --- a/services/camera/libcameraservice/CameraService.cpp +++ b/services/camera/libcameraservice/CameraService.cpp @@ -252,8 +252,14 @@ void CameraService::pingCameraServiceProxy() { proxyBinder->pingForUserUpdate(); } -void CameraService::broadcastTorchModeStatus(const String8& cameraId, TorchModeStatus status, - SystemCameraKind systemCameraKind) { +void CameraService::broadcastTorchModeStatus(const String8& cameraId, TorchModeStatus status) { + SystemCameraKind systemCameraKind = SystemCameraKind::PUBLIC; + status_t res = getSystemCameraKind(cameraId, &systemCameraKind); + if (res != OK) { + ALOGE("%s: Could not get system camera kind for camera id %s", __FUNCTION__, + cameraId.string()); + return; + } Mutex::Autolock lock(mStatusListenerLock); for (auto& i : mListenerList) { if (shouldSkipStatusUpdates(systemCameraKind, i->isVendorListener(), i->getListenerPid(), @@ -347,7 +353,7 @@ void CameraService::addStates(const String8 id) { Mutex::Autolock al(mTorchStatusMutex); mTorchStatusMap.add(id, TorchModeStatus::AVAILABLE_OFF); - broadcastTorchModeStatus(id, TorchModeStatus::AVAILABLE_OFF, deviceKind); + broadcastTorchModeStatus(id, TorchModeStatus::AVAILABLE_OFF); } updateCameraNumAndIds(); @@ -508,19 +514,12 @@ void CameraService::disconnectClient(const String8& id, sp<BasicClient> clientTo void CameraService::onTorchStatusChanged(const String8& cameraId, TorchModeStatus newStatus) { - SystemCameraKind systemCameraKind = SystemCameraKind::PUBLIC; - status_t res = getSystemCameraKind(cameraId, &systemCameraKind); - if (res != OK) { - ALOGE("%s: Could not get system camera kind for camera id %s", __FUNCTION__, - cameraId.string()); - return; - } Mutex::Autolock al(mTorchStatusMutex); - onTorchStatusChangedLocked(cameraId, newStatus, systemCameraKind); + onTorchStatusChangedLocked(cameraId, newStatus); } void CameraService::onTorchStatusChangedLocked(const String8& cameraId, - TorchModeStatus newStatus, SystemCameraKind systemCameraKind) { + TorchModeStatus newStatus) { ALOGI("%s: Torch status changed for cameraId=%s, newStatus=%d", __FUNCTION__, cameraId.string(), newStatus); @@ -569,7 +568,7 @@ void CameraService::onTorchStatusChangedLocked(const String8& cameraId, } } } - broadcastTorchModeStatus(cameraId, newStatus, systemCameraKind); + broadcastTorchModeStatus(cameraId, newStatus); } static bool hasPermissionsForSystemCamera(int callingPid, int callingUid) { @@ -963,7 +962,11 @@ int32_t CameraService::mapToInterface(StatusInternal status) { Status CameraService::initializeShimMetadata(int cameraId) { int uid = CameraThreadState::getCallingUid(); +#ifdef NO_CAMERA_SERVER + String16 internalPackageName("media"); +#else String16 internalPackageName("cameraserver"); +#endif String8 id = String8::format("%d", cameraId); Status ret = Status::ok(); sp<Client> tmp = nullptr; @@ -1044,7 +1047,9 @@ Status CameraService::getLegacyParametersLazy(int cameraId, static bool isTrustedCallingUid(uid_t uid) { switch (uid) { case AID_MEDIA: // mediaserver +#ifndef NO_CAMERA_SERVER case AID_CAMERASERVER: // cameraserver +#endif case AID_RADIO: // telephony return true; default: @@ -1177,6 +1182,7 @@ Status CameraService::validateClientPermissionsLocked(const String8& cameraId, clientName8.string(), clientUid, clientPid, cameraId.string()); } +#ifndef NO_CAMERA_SERVER // Make sure the UID is in an active state to use the camera if (!mUidPolicy->isUidActive(callingUid, String16(clientName8))) { int32_t procState = mUidPolicy->getProcState(callingUid); @@ -1188,6 +1194,7 @@ Status CameraService::validateClientPermissionsLocked(const String8& cameraId, clientName8.string(), clientUid, clientPid, cameraId.string(), callingUid, procState); } +#endif // If sensor privacy is enabled then prevent access to the camera if (mSensorPrivacyPolicy->isSensorPrivacyEnabled()) { @@ -2437,7 +2444,8 @@ bool CameraService::evictClientIdByRemote(const wp<IBinder>& remote) { ret = true; } } - + //clear the evicted client list before acquring service lock again. + evicted.clear(); // Reacquire mServiceLock mServiceLock.lock(); @@ -3797,7 +3805,7 @@ void CameraService::updateStatus(StatusInternal status, const String8& cameraId, TorchModeStatus::AVAILABLE_OFF : TorchModeStatus::NOT_AVAILABLE; if (torchStatus != newTorchStatus) { - onTorchStatusChangedLocked(cameraId, newTorchStatus, deviceKind); + onTorchStatusChangedLocked(cameraId, newTorchStatus); } } } diff --git a/services/camera/libcameraservice/CameraService.h b/services/camera/libcameraservice/CameraService.h index 5c4c96bae7..685ed5ee1d 100644 --- a/services/camera/libcameraservice/CameraService.h +++ b/services/camera/libcameraservice/CameraService.h @@ -995,8 +995,7 @@ private: // handle torch mode status change and invoke callbacks. mTorchStatusMutex // should be locked. void onTorchStatusChangedLocked(const String8& cameraId, - hardware::camera::common::V1_0::TorchModeStatus newStatus, - SystemCameraKind systemCameraKind); + hardware::camera::common::V1_0::TorchModeStatus newStatus); // get a camera's torch status. mTorchStatusMutex should be locked. status_t getTorchStatusLocked(const String8 &cameraId, @@ -1085,8 +1084,7 @@ private: static void pingCameraServiceProxy(); void broadcastTorchModeStatus(const String8& cameraId, - hardware::camera::common::V1_0::TorchModeStatus status, - SystemCameraKind systemCameraKind); + hardware::camera::common::V1_0::TorchModeStatus status); void disconnectClient(const String8& id, sp<BasicClient> clientToDisconnect); diff --git a/services/camera/libcameraservice/api1/CameraClient.cpp b/services/camera/libcameraservice/api1/CameraClient.cpp index 892996c3af..43da23eabc 100644 --- a/services/camera/libcameraservice/api1/CameraClient.cpp +++ b/services/camera/libcameraservice/api1/CameraClient.cpp @@ -55,6 +55,9 @@ CameraClient::CameraClient(const sp<CameraService>& cameraService, mPreviewCallbackFlag = CAMERA_FRAME_CALLBACK_FLAG_NOOP; mOrientation = getOrientation(0, mCameraFacing == CAMERA_FACING_FRONT); mPlayShutterSound = true; + + mLongshotEnabled = false; + mBurstCnt = 0; LOG1("CameraClient::CameraClient X (pid %d, id %d)", callingPid, cameraId); } @@ -672,6 +675,10 @@ status_t CameraClient::takePicture(int msgType) { CAMERA_MSG_COMPRESSED_IMAGE); enableMsgType(picMsgType); + mBurstCnt = mHardware->getParameters().getInt("num-snaps-per-shutter"); + if(mBurstCnt <= 0) + mBurstCnt = 1; + LOG1("mBurstCnt = %d", mBurstCnt); return mHardware->takePicture(); } @@ -755,6 +762,20 @@ status_t CameraClient::sendCommand(int32_t cmd, int32_t arg1, int32_t arg2) { } else if (cmd == CAMERA_CMD_PING) { // If mHardware is 0, checkPidAndHardware will return error. return OK; + } else if (cmd == CAMERA_CMD_HISTOGRAM_ON) { + enableMsgType(CAMERA_MSG_STATS_DATA); + } else if (cmd == CAMERA_CMD_HISTOGRAM_OFF) { + disableMsgType(CAMERA_MSG_STATS_DATA); + } else if (cmd == CAMERA_CMD_METADATA_ON) { + enableMsgType(CAMERA_MSG_META_DATA); + } else if (cmd == CAMERA_CMD_METADATA_OFF) { + disableMsgType(CAMERA_MSG_META_DATA); + } else if ( cmd == CAMERA_CMD_LONGSHOT_ON ) { + mLongshotEnabled = true; + } else if ( cmd == CAMERA_CMD_LONGSHOT_OFF ) { + mLongshotEnabled = false; + disableMsgType(CAMERA_MSG_SHUTTER); + disableMsgType(CAMERA_MSG_COMPRESSED_IMAGE); } return mHardware->sendCommand(cmd, arg1, arg2); @@ -954,7 +975,9 @@ void CameraClient::handleShutter(void) { c->notifyCallback(CAMERA_MSG_SHUTTER, 0, 0); if (!lockIfMessageWanted(CAMERA_MSG_SHUTTER)) return; } - disableMsgType(CAMERA_MSG_SHUTTER); + if ( !mLongshotEnabled ) { + disableMsgType(CAMERA_MSG_SHUTTER); + } // Shutters only happen in response to takePicture, so mark device as // idle now, until preview is restarted @@ -1040,7 +1063,13 @@ void CameraClient::handleRawPicture(const sp<IMemory>& mem) { // picture callback - compressed picture ready void CameraClient::handleCompressedPicture(const sp<IMemory>& mem) { - disableMsgType(CAMERA_MSG_COMPRESSED_IMAGE); + if (mBurstCnt) + mBurstCnt--; + + if (!mBurstCnt && !mLongshotEnabled) { + LOG1("handleCompressedPicture mBurstCnt = %d", mBurstCnt); + disableMsgType(CAMERA_MSG_COMPRESSED_IMAGE); + } sp<hardware::ICameraClient> c = mRemoteCallback; mLock.unlock(); diff --git a/services/camera/libcameraservice/api1/CameraClient.h b/services/camera/libcameraservice/api1/CameraClient.h index a7eb960ca7..12d6ad5a01 100644 --- a/services/camera/libcameraservice/api1/CameraClient.h +++ b/services/camera/libcameraservice/api1/CameraClient.h @@ -183,6 +183,9 @@ private: // This function keeps trying to grab mLock, or give up if the message // is found to be disabled. It returns true if mLock is grabbed. bool lockIfMessageWanted(int32_t msgType); + + bool mLongshotEnabled; + int mBurstCnt; }; } diff --git a/services/camera/libcameraservice/device1/CameraHardwareInterface.cpp b/services/camera/libcameraservice/device1/CameraHardwareInterface.cpp index 62ef681668..61cada61a9 100644 --- a/services/camera/libcameraservice/device1/CameraHardwareInterface.cpp +++ b/services/camera/libcameraservice/device1/CameraHardwareInterface.cpp @@ -136,6 +136,27 @@ hardware::Return<void> CameraHardwareInterface::dataCallback( return hardware::Void(); } +#ifdef QTI_CAMERA_DEVICE +hardware::Return<void> CameraHardwareInterface::QDataCallback( + DataCallbackMsg msgType, uint32_t data, uint32_t bufferIndex, + const vendor::qti::hardware::camera::device::V1_0::QCameraFrameMetadata& metadata) { + camera_memory_t* mem = nullptr; + { + std::lock_guard<std::mutex> lock(mHidlMemPoolMapLock); + if (mHidlMemPoolMap.count(data) == 0) { + ALOGE("%s: memory pool ID %d not found", __FUNCTION__, data); + return hardware::Void(); + } + mem = mHidlMemPoolMap.at(data); + } + camera_frame_metadata_t md; + md.number_of_faces = metadata.faces.size(); + md.faces = (camera_face_t*) metadata.faces.data(); + sDataCb((int32_t) msgType, mem, bufferIndex, &md, this); + return hardware::Void(); +} +#endif + hardware::Return<void> CameraHardwareInterface::dataCallbackTimestamp( DataCallbackMsg msgType, uint32_t data, uint32_t bufferIndex, int64_t timestamp) { @@ -591,12 +612,16 @@ void CameraHardwareInterface::releaseRecordingFrame(const sp<IMemory>& mem) // Either document why it is safe in this case or address the // issue (e.g. by copying). VideoNativeHandleMetadata* md = (VideoNativeHandleMetadata*) mem->unsecurePointer(); - // Caching the handle here because md->pHandle will be subject to HAL's edit - native_handle_t* nh = md->pHandle; - hidl_handle frame = nh; - mHidlDevice->releaseRecordingFrameHandle(heapId, bufferIndex, frame); - native_handle_close(nh); - native_handle_delete(nh); + if (md->eType == kMetadataBufferTypeNativeHandleSource) { + // Caching the handle here because md->pHandle will be subject to HAL's edit + native_handle_t* nh = md->pHandle; + hidl_handle frame = nh; + mHidlDevice->releaseRecordingFrameHandle(heapId, bufferIndex, frame); + native_handle_close(nh); + native_handle_delete(nh); + } else { + mHidlDevice->releaseRecordingFrame(heapId, bufferIndex); + } } else { mHidlDevice->releaseRecordingFrame(heapId, bufferIndex); } diff --git a/services/camera/libcameraservice/device1/CameraHardwareInterface.h b/services/camera/libcameraservice/device1/CameraHardwareInterface.h index e519b04cdd..4a20b6a597 100644 --- a/services/camera/libcameraservice/device1/CameraHardwareInterface.h +++ b/services/camera/libcameraservice/device1/CameraHardwareInterface.h @@ -29,6 +29,9 @@ #include <hardware/camera.h> #include <common/CameraProviderManager.h> +#ifdef QTI_CAMERA_DEVICE +#include <vendor/qti/hardware/camera/device/1.0/IQCameraDeviceCallback.h> +#endif namespace android { @@ -85,7 +88,11 @@ typedef void (*data_callback_timestamp_batch)( class CameraHardwareInterface : public virtual RefBase, +#ifdef QTI_CAMERA_DEVICE + public virtual vendor::qti::hardware::camera::device::V1_0::IQCameraDeviceCallback, +#else public virtual hardware::camera::device::V1_0::ICameraDeviceCallback, +#endif public virtual hardware::camera::device::V1_0::ICameraDevicePreviewCallback { public: @@ -395,6 +402,12 @@ private: hardware::camera::device::V1_0::DataCallbackMsg msgType, const hardware::hidl_vec< hardware::camera::device::V1_0::HandleTimestampMessage>&) override; +#ifdef QTI_CAMERA_DEVICE + hardware::Return<void> QDataCallback( + hardware::camera::device::V1_0::DataCallbackMsg msgType, + uint32_t data, uint32_t bufferIndex, + const vendor::qti::hardware::camera::device::V1_0::QCameraFrameMetadata& metadata) override; +#endif /** * Implementation of android::hardware::camera::device::V1_0::ICameraDevicePreviewCallback diff --git a/services/camera/libcameraservice/device3/Camera3Device.cpp b/services/camera/libcameraservice/device3/Camera3Device.cpp index d5f136b84d..fc3a5b4901 100644 --- a/services/camera/libcameraservice/device3/Camera3Device.cpp +++ b/services/camera/libcameraservice/device3/Camera3Device.cpp @@ -321,6 +321,11 @@ status_t Camera3Device::initializeCommonLocked() { mTimestampOffset = getMonoToBoottimeOffset(); } +#ifdef TARGET_CAMERA_BOOTTIME_TIMESTAMP + // Always calculate the offset if requested + mTimestampOffset = getMonoToBoottimeOffset(); +#endif + // Will the HAL be sending in early partial result metadata? camera_metadata_entry partialResultsCount = mDeviceInfo.find(ANDROID_REQUEST_PARTIAL_RESULT_COUNT); |