commit 552a391396d56a8208b876644b364bb55c71f7c7 Author: dvab-sarma Date: Fri Jun 27 00:01:59 2025 -0500 android-16.0 diff --git a/Android.bp b/Android.bp new file mode 100644 index 0000000..8d38a08 --- /dev/null +++ b/Android.bp @@ -0,0 +1,92 @@ +// Copyright (C) 2022 Michael Goffioul +// Copyright (C) 2025 KonstaKANG +// +// SPDX-License-Identifier: Apache-2.0 + +cc_defaults { + name: "ffmpeg_codec2_defaults", + relative_install_path: "hw", + vendor: true, + srcs: [ + "C2FFMPEGAudioDecodeComponent.cpp", + "C2FFMPEGAudioDecodeInterface.cpp", + "C2FFMPEGComponentInterface.cpp", + "C2FFMPEGComponentStore.cpp", + "C2FFMPEGVideoDecodeComponent.cpp", + "C2FFMPEGVideoDecodeInterface.cpp", + ], + shared_libs: [ + "libavcodec", + "libavservices_minijail", + "libavutil", + "libbase", + "libcodec2_soft_common", + "libcodec2_vndk", + "libffmpeg_utils", + "liblog", + "libstagefright_foundation", + "libswresample", + "libswscale", + "libutils", + ], +} + +cc_binary { + name: "android.hardware.media.c2@1.2-service-ffmpeg", + init_rc: ["android.hardware.media.c2@1.2-service-ffmpeg.rc"], + vintf_fragments: ["android.hardware.media.c2@1.2-service-ffmpeg.xml"], + srcs: [ + "main-hidl.cpp", + ], + shared_libs: [ + "android.hardware.media.c2@1.2", + "libbinder", + "libcodec2_hidl@1.2", + "libhidlbase", + ], + defaults: ["ffmpeg_codec2_defaults"], +} + +cc_binary { + name: "android.hardware.media.c2-service-ffmpeg", + srcs: [ + "main.cpp", + ], + shared_libs: [ + "android.hardware.media.c2-V1-ndk", + "libbinder_ndk", + "libcodec2_aidl", + "libcodec2_hal_common", + ], + defaults: ["ffmpeg_codec2_defaults"], + installable: false, +} + +prebuilt_etc { + name: "android.hardware.media.c2-service-ffmpeg.rc", + src: "android.hardware.media.c2-service-ffmpeg.rc", + installable: false, +} + +prebuilt_etc { + name: "android.hardware.media.c2-service-ffmpeg.xml", + src: "android.hardware.media.c2-service-ffmpeg.xml", + sub_dir: "vintf", + installable: false, +} + +apex { + name: "com.android.hardware.media.c2.ffmpeg", + manifest: "apex_manifest.json", + file_contexts: "apex_file_contexts", + key: "com.android.hardware.key", + certificate: ":com.android.hardware.certificate", + updatable: false, + vendor: true, + binaries: ["android.hardware.media.c2-service-ffmpeg"], + + prebuilts: [ + "android.hardware.media.c2-service-ffmpeg.rc", + "android.hardware.media.c2-service-ffmpeg.xml", + ], +} diff --git a/C2FFMPEGAudioDecodeComponent.cpp b/C2FFMPEGAudioDecodeComponent.cpp new file mode 100644 index 0000000..92327c0 --- /dev/null +++ b/C2FFMPEGAudioDecodeComponent.cpp @@ -0,0 +1,730 @@ +/* + * Copyright 2022 Michael Goffioul + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "C2FFMPEGAudioDecodeComponent" +#include +#include + +#include +#include "C2FFMPEGAudioDecodeComponent.h" +#include + +#define DEBUG_FRAMES 0 +#define DEBUG_EXTRADATA 0 + +namespace android { + +static enum AVSampleFormat convertFormatToFFMPEG(C2Config::pcm_encoding_t encoding) { + switch (encoding) { + case C2Config::PCM_8: return AV_SAMPLE_FMT_U8; + case C2Config::PCM_16: return AV_SAMPLE_FMT_S16; + case C2Config::PCM_32: return AV_SAMPLE_FMT_S32; + case C2Config::PCM_FLOAT: return AV_SAMPLE_FMT_FLT; + default: return AV_SAMPLE_FMT_NONE; + } +} + +__unused +static C2Config::pcm_encoding_t convertFormatToC2(enum AVSampleFormat format) { + switch (format) { + case AV_SAMPLE_FMT_U8: return C2Config::PCM_8; + case AV_SAMPLE_FMT_S16: return C2Config::PCM_16; + case AV_SAMPLE_FMT_S32: return C2Config::PCM_32; + case AV_SAMPLE_FMT_FLT: return C2Config::PCM_FLOAT; + default: return C2Config::PCM_16; + } +} + +// Helper structures to encapsulate the specific codec behaviors. +// Currently only used to process extradata. + +struct CodecHelper { + virtual ~CodecHelper() {} + virtual c2_status_t onCodecConfig(AVCodecContext* mCtx, C2ReadView* inBuffer); + virtual c2_status_t onOpen(AVCodecContext* mCtx); + virtual c2_status_t onOpened(AVCodecContext* mCtx); +}; + +c2_status_t CodecHelper::onCodecConfig(AVCodecContext* mCtx, C2ReadView* inBuffer) { + int orig_extradata_size = mCtx->extradata_size; + int add_extradata_size = inBuffer->capacity(); + +#if DEBUG_EXTRADATA + ALOGD("CodecHelper::onCodecConfig: add = %u, current = %d", add_extradata_size, orig_extradata_size); +#endif + mCtx->extradata_size += add_extradata_size; + mCtx->extradata = (uint8_t *) realloc(mCtx->extradata, mCtx->extradata_size + AV_INPUT_BUFFER_PADDING_SIZE); + if (! mCtx->extradata) { + ALOGE("CodecHelper::onCodecConfig: ffmpeg audio decoder failed to alloc extradata memory."); + return C2_NO_MEMORY; + } + memcpy(mCtx->extradata + orig_extradata_size, inBuffer->data(), add_extradata_size); + memset(mCtx->extradata + mCtx->extradata_size, 0, AV_INPUT_BUFFER_PADDING_SIZE); + + return C2_OK; +} + +c2_status_t CodecHelper::onOpen(AVCodecContext* mCtx) { +#if DEBUG_EXTRADATA + ALOGD("CodecHelper::onOpen: extradata = %d", mCtx->extradata_size); +#else + // Silence compilation warning. + (void)mCtx; +#endif + return C2_OK; +} + +c2_status_t CodecHelper::onOpened(AVCodecContext* mCtx) { + (void)mCtx; + return C2_OK; +} + +struct VorbisCodecHelper : public CodecHelper { + VorbisCodecHelper(); + ~VorbisCodecHelper(); + c2_status_t onCodecConfig(AVCodecContext* mCtx, C2ReadView* rView); + c2_status_t onOpen(AVCodecContext* mCtx); + + uint8_t* mHeader[3]; + int mHeaderLen[3]; +}; + +VorbisCodecHelper::VorbisCodecHelper() + : CodecHelper(), + mHeader{ NULL, NULL, NULL }, + mHeaderLen{ 0, 0, 0 } { +} + +VorbisCodecHelper::~VorbisCodecHelper() { + for (int i = 0; i < 3; i++) { + if (mHeader[i]) { + av_free(mHeader[i]); + mHeader[i] = NULL; + } + mHeaderLen[i] = 0; + } +} + +c2_status_t VorbisCodecHelper::onCodecConfig(AVCodecContext* mCtx __unused, C2ReadView* inBuffer) { + const uint8_t* data = inBuffer->data(); + int len = inBuffer->capacity(); + int index = 0; + + switch (data[0]) { + case 1: index = 0; break; + case 3: index = 1; break; + case 5: index = 2; break; + default: + ALOGE("VorbisCodecHelper::onCodecConfig: invalid vorbis codec config (%02x)", data[0]); + return C2_BAD_VALUE; + } + + if (mHeader[index]) { + ALOGW("VorbisCodecHelper::onCodecConfig: overwriting header[%d]", index); + av_free(mHeader[index]); + } + mHeader[index] = (uint8_t*)av_mallocz(len); + if (! mHeader[index]) { + ALOGE("VorbisCodecHelper::onCodecConfig: oom for vorbis extradata"); + return C2_NO_MEMORY; + } + memcpy(mHeader[index], data, len); + mHeaderLen[index] = len; + +#if DEBUG_EXTRADATA + ALOGD("VorbisCodecHelper::onCodecConfig: found header[%d] = %d", index, len); +#endif + + return C2_OK; +} + +c2_status_t VorbisCodecHelper::onOpen(AVCodecContext* mCtx) { + // Don't generate extradata twice + if (! mCtx->extradata) { + if (! setup_vorbis_extradata(&mCtx->extradata, + &mCtx->extradata_size, + (const uint8_t**)mHeader, + mHeaderLen)) { + return C2_NO_MEMORY; + } +#if DEBUG_EXTRADATA + ALOGD("VorbisCodecHelper::onOpen: extradata = %d", mCtx->extradata_size); +#endif + } + return C2_OK; +} + +struct Ac3CodecHelper : public CodecHelper { + c2_status_t onOpened(AVCodecContext* mCtx); +}; + +c2_status_t Ac3CodecHelper::onOpened(AVCodecContext* mCtx) { + int err = av_opt_set_chlayout(mCtx->priv_data, "downmix", &mCtx->ch_layout, 0); + if (err < 0) { + ALOGE("Ac3CodecHelper::onOpened: failed to set downmix = %d: %s (%08x)", + mCtx->ch_layout.nb_channels, av_err2str(err), err); + } else { + ALOGD("Ac3CodecHelper::onOpened: set downmix = %d", mCtx->ch_layout.nb_channels); + } + return C2_OK; +} + +CodecHelper* createCodecHelper(enum AVCodecID codec_id) { + switch (codec_id) { + case AV_CODEC_ID_AC3: + case AV_CODEC_ID_EAC3: + return new Ac3CodecHelper(); + case AV_CODEC_ID_VORBIS: + return new VorbisCodecHelper(); + default: + return new CodecHelper(); + } +} + +C2FFMPEGAudioDecodeComponent::C2FFMPEGAudioDecodeComponent( + const C2FFMPEGComponentInfo* componentInfo, + const std::shared_ptr& intf) + : SimpleC2Component(std::make_shared>(componentInfo->name, 0, intf)), + mInfo(componentInfo), + mIntf(intf), + mCodecID(componentInfo->codecID), + mCtx(NULL), + mFrame(NULL), + mPacket(NULL), + mFFMPEGInitialized(false), + mCodecAlreadyOpened(false), + mEOSSignalled(false), + mSwrCtx(NULL), + mTargetSampleFormat(AV_SAMPLE_FMT_NONE), + mTargetSampleRate(44100), + mTargetChannels(1) { + ALOGD("C2FFMPEGAudioDecodeComponent: mediaType = %s", componentInfo->mediaType); +} + +C2FFMPEGAudioDecodeComponent::~C2FFMPEGAudioDecodeComponent() { + ALOGD("~C2FFMPEGAudioDecodeComponent: mCtx = %p", mCtx); + onRelease(); +} + +c2_status_t C2FFMPEGAudioDecodeComponent::initDecoder() { + if (! mFFMPEGInitialized) { + if (initFFmpeg() != C2_OK) { + ALOGE("initDecoder: FFMPEG initialization failed."); + return C2_NO_INIT; + } + mFFMPEGInitialized = true; + } + + mCtx = avcodec_alloc_context3(NULL); + if (! mCtx) { + ALOGE("initDecoder: avcodec_alloc_context failed."); + return C2_NO_MEMORY; + } + + mCtx->codec_type = AVMEDIA_TYPE_AUDIO; + mCtx->codec_id = mCodecID; + + updateAudioParameters(); + + av_channel_layout_default(&mCtx->ch_layout, mTargetChannels); + mCtx->sample_rate = mTargetSampleRate; + mCtx->bit_rate = 0; + mCtx->sample_fmt = mTargetSampleFormat; + + // Avoid resampling if possible, ask the codec for the target format. + mCtx->request_sample_fmt = mCtx->sample_fmt; + + mCodecHelper = createCodecHelper(mCtx->codec_id); + + ALOGD("initDecoder: %p [%s], %s - sr=%d, ch=%d, fmt=%s", + mCtx, avcodec_get_name(mCtx->codec_id), mInfo->mediaType, + mCtx->sample_rate, mCtx->ch_layout.nb_channels, av_get_sample_fmt_name(mCtx->sample_fmt)); + + return C2_OK; +} + +c2_status_t C2FFMPEGAudioDecodeComponent::openDecoder() { + if (mCodecAlreadyOpened) { + return C2_OK; + } + + mCodecHelper->onOpen(mCtx); + + // Find decoder + mCtx->codec = avcodec_find_decoder(mCtx->codec_id); + if (! mCtx->codec) { + ALOGE("openDecoder: ffmpeg audio decoder failed to find codec %d", mCtx->codec_id); + return C2_NOT_FOUND; + } + + // Configure decoder + mCtx->workaround_bugs = 1; + mCtx->idct_algo = 0; + mCtx->skip_frame = AVDISCARD_DEFAULT; + mCtx->skip_idct = AVDISCARD_DEFAULT; + mCtx->skip_loop_filter = AVDISCARD_DEFAULT; + mCtx->error_concealment = 3; + + mCtx->flags |= AV_CODEC_FLAG_BITEXACT; + + ALOGD("openDecoder: begin to open ffmpeg audio decoder(%s), mCtx sample_rate: %d, channels: %d", + avcodec_get_name(mCtx->codec_id), mCtx->sample_rate, mCtx->ch_layout.nb_channels); + + int err = avcodec_open2(mCtx, mCtx->codec, NULL); + if (err < 0) { + ALOGE("openDecoder: ffmpeg audio decoder failed to initialize.(%s)", av_err2str(err)); + return C2_NO_INIT; + } + mCodecAlreadyOpened = true; + + mCodecHelper->onOpened(mCtx); + + ALOGD("openDecoder: open ffmpeg audio decoder(%s) success, mCtx sample_rate: %d, " + "channels: %d, sample_fmt: %s, bits_per_coded_sample: %d, bits_per_raw_sample: %d", + avcodec_get_name(mCtx->codec_id), + mCtx->sample_rate, mCtx->ch_layout.nb_channels, + av_get_sample_fmt_name(mCtx->sample_fmt), + mCtx->bits_per_coded_sample, mCtx->bits_per_raw_sample); + + mFrame = av_frame_alloc(); + if (! mFrame) { + ALOGE("openDecoder: oom for audio frame"); + return C2_NO_MEMORY; + } + + return C2_OK; +} + +void C2FFMPEGAudioDecodeComponent::deInitDecoder() { + ALOGD("deInitDecoder: %p", mCtx); + if (mCtx) { + if (avcodec_is_open(mCtx)) { + avcodec_flush_buffers(mCtx); + } + avcodec_free_context(&mCtx); + mCodecAlreadyOpened = false; + } + if (mFrame) { + av_frame_free(&mFrame); + mFrame = NULL; + } + if (mPacket) { + av_packet_free(&mPacket); + mPacket = NULL; + } + if (mSwrCtx) { + swr_free(&mSwrCtx); + } + if (mCodecHelper) { + delete mCodecHelper; + mCodecHelper = NULL; + } + mEOSSignalled = false; +} + +c2_status_t C2FFMPEGAudioDecodeComponent::processCodecConfig(C2ReadView* inBuffer) { +#if DEBUG_EXTRADATA + ALOGD("processCodecConfig: inBuffer = %d", inBuffer->capacity()); +#endif + if (! mCodecAlreadyOpened) { + return mCodecHelper->onCodecConfig(mCtx, inBuffer); + } else { + ALOGW("processCodecConfig: decoder is already opened, ignoring %d bytes", inBuffer->capacity()); + } + + return C2_OK; +} + +c2_status_t C2FFMPEGAudioDecodeComponent::sendInputBuffer( + C2ReadView *inBuffer, int64_t timestamp) { + if (!mPacket) { + mPacket = av_packet_alloc(); + if (!mPacket) { + ALOGE("sendInputBuffer: oom for audio packet"); + return C2_NO_MEMORY; + } + } + + mPacket->data = inBuffer ? const_cast(inBuffer->data()) : NULL; + mPacket->size = inBuffer ? inBuffer->capacity() : 0; + mPacket->pts = timestamp; + mPacket->dts = timestamp; + + int err = avcodec_send_packet(mCtx, mPacket); + av_packet_unref(mPacket); + + if (err < 0) { + ALOGE("sendInputBuffer: failed to send data to decoder err = %d", err); + // Don't report error to client. + } + + return C2_OK; +} + +c2_status_t C2FFMPEGAudioDecodeComponent::receiveFrame(bool* hasFrame) { + int err = avcodec_receive_frame(mCtx, mFrame); + + if (err == 0) { + *hasFrame = true; + } else if (err == AVERROR(EAGAIN) || err == AVERROR_EOF) { + *hasFrame = false; + } else { + ALOGE("receiveFrame: failed to receive frame from decoder err = %d", err); + // Don't report error to client. + } + + return C2_OK; +} + +c2_status_t C2FFMPEGAudioDecodeComponent::getOutputBuffer(C2WriteView* outBuffer) { + if (! mSwrCtx || + mSwrCtx->in_sample_fmt != mFrame->format || + mSwrCtx->in_sample_rate != mFrame->sample_rate || + av_channel_layout_compare(&mSwrCtx->in_ch_layout, &mFrame->ch_layout) != 0 || + mSwrCtx->out_sample_fmt != mTargetSampleFormat || + mSwrCtx->out_sample_rate != mTargetSampleRate || + mSwrCtx->out_ch_layout.nb_channels != mTargetChannels) { + if (mSwrCtx) { + swr_free(&mSwrCtx); + } + + AVChannelLayout newLayout; + + av_channel_layout_default(&newLayout, mTargetChannels); + swr_alloc_set_opts2(&mSwrCtx, + &newLayout, mTargetSampleFormat, mTargetSampleRate, + &mFrame->ch_layout, (enum AVSampleFormat)mFrame->format, mFrame->sample_rate, + 0, NULL); + av_channel_layout_uninit(&newLayout); + if (! mSwrCtx || swr_init(mSwrCtx) < 0) { + ALOGE("getOutputBuffer: cannot create audio converter - sr=%d, ch=%d, fmt=%s => sr=%d, ch=%d, fmt=%s", + mFrame->sample_rate, mFrame->ch_layout.nb_channels, av_get_sample_fmt_name((enum AVSampleFormat)mFrame->format), + mTargetSampleRate, mTargetChannels, av_get_sample_fmt_name(mTargetSampleFormat)); + if (mSwrCtx) { + swr_free(&mSwrCtx); + } + return C2_NO_MEMORY; + } + + ALOGD("getOutputBuffer: created audio converter - sr=%d, ch=%d, fmt=%s => sr=%d, ch=%d, fmt=%s", + mFrame->sample_rate, mFrame->ch_layout.nb_channels, av_get_sample_fmt_name((enum AVSampleFormat)mFrame->format), + mTargetSampleRate, mTargetChannels, av_get_sample_fmt_name(mTargetSampleFormat)); + } + + uint8_t* out[1] = { outBuffer->data() }; + int ret = swr_convert(mSwrCtx, out, mFrame->nb_samples, (const uint8_t**)mFrame->extended_data, mFrame->nb_samples); + + if (ret < 0) { + ALOGE("getOutputBuffer: audio conversion failed"); + return C2_CORRUPTED; + } else if (ret != mFrame->nb_samples) { + ALOGW("getOutputBuffer: audio conversion truncated!"); + } + +#if DEBUG_FRAMES + ALOGD("getOutputBuffer: audio converted - sr=%d, ch=%d, fmt=%s, #=%d => sr=%d, ch=%d, fmt=%s, #=%d(%d)", + mFrame->sample_rate, mFrame->ch_layout.nb_channels, av_get_sample_fmt_name((enum AVSampleFormat)mFrame->format), mFrame->nb_samples, + mTargetSampleRate, mTargetChannels, av_get_sample_fmt_name(mTargetSampleFormat), mFrame->nb_samples, outBuffer->capacity()); +#endif + + return C2_OK; +} + +void C2FFMPEGAudioDecodeComponent::updateAudioParameters() { + mTargetSampleFormat = convertFormatToFFMPEG(mIntf->getPcmEncodingInfo()); + mTargetSampleRate = mIntf->getSampleRate(); + mTargetChannels = mIntf->getChannelCount(); +} + +c2_status_t C2FFMPEGAudioDecodeComponent::onInit() { + ALOGD("onInit"); + return initDecoder(); +} + +c2_status_t C2FFMPEGAudioDecodeComponent::onStop() { + ALOGD("onStop"); + return C2_OK; +} + +void C2FFMPEGAudioDecodeComponent::onReset() { + ALOGD("onReset"); + deInitDecoder(); + initDecoder(); +} + +void C2FFMPEGAudioDecodeComponent::onRelease() { + ALOGD("onRelease"); + deInitDecoder(); + if (mFFMPEGInitialized) { + deInitFFmpeg(); + mFFMPEGInitialized = false; + } +} + +c2_status_t C2FFMPEGAudioDecodeComponent::onFlush_sm() { + ALOGD("onFlush_sm"); + if (mCtx && avcodec_is_open(mCtx)) { + // Make sure that the next buffer output does not still + // depend on fragments from the last one decoded. + avcodec_flush_buffers(mCtx); + mEOSSignalled = false; + } + return C2_OK; +} + +void C2FFMPEGAudioDecodeComponent::process( + const std::unique_ptr &work, + const std::shared_ptr& pool +) { + size_t inSize = 0u; + bool eos = (work->input.flags & C2FrameData::FLAG_END_OF_STREAM); + C2ReadView rView = mDummyReadView; + bool hasInputBuffer = false; + bool hasFrame = false; + + if (! work->input.buffers.empty()) { + rView = work->input.buffers[0]->data().linearBlocks().front().map().get(); + inSize = rView.capacity(); + hasInputBuffer = true; + } + +#if DEBUG_FRAMES + ALOGD("process: input flags=%08x ts=%lu idx=%lu #buf=%lu[%lu] #conf=%lu #info=%lu", + work->input.flags, work->input.ordinal.timestamp.peeku(), work->input.ordinal.frameIndex.peeku(), + work->input.buffers.size(), inSize, work->input.configUpdate.size(), work->input.infoBuffers.size()); +#endif + + if (mEOSSignalled) { + ALOGE("process: ignoring work while EOS reached"); + work->workletsProcessed = 0u; + work->result = C2_BAD_VALUE; + return; + } + + if (hasInputBuffer && rView.error()) { + ALOGE("process: read view map failed err = %d", rView.error()); + work->workletsProcessed = 0u; + work->result = rView.error(); + return; + } + + // In all cases the work is marked as completed. + // NOTE: This has an impact on the drain operation. + + work->result = C2_OK; + work->worklets.front()->output.flags = (C2FrameData::flags_t)0; + work->worklets.front()->output.buffers.clear(); + work->worklets.front()->output.ordinal = work->input.ordinal; + work->workletsProcessed = 1u; + + if (inSize || (eos && mCodecAlreadyOpened)) { + c2_status_t err = C2_OK; + + if (work->input.flags & C2FrameData::FLAG_CODEC_CONFIG) { + work->result = processCodecConfig(&rView); + return; + } + + if (! mCodecAlreadyOpened) { + err = openDecoder(); + if (err != C2_OK) { + work->result = err; + return; + } + } + + err = sendInputBuffer(&rView, work->input.ordinal.timestamp.peekll()); + if (err != C2_OK) { + work->result = err; + return; + } + + while (true) { + hasFrame = false; + err = receiveFrame(&hasFrame); + if (err != C2_OK) { + work->result = err; + return; + } + + if (! hasFrame) { + break; + } + +#if DEBUG_FRAMES + ALOGD("process: got frame pts=%" PRId64 " dts=%" PRId64 " ts=%" PRId64 " - sr=%d, ch=%d, fmt=%s, #=%d", + mFrame->pts, mFrame->pkt_dts, mFrame->best_effort_timestamp, + mFrame->sample_rate, mFrame->ch_layout.nb_channels, av_get_sample_fmt_name((enum AVSampleFormat)mFrame->format), + mFrame->nb_samples); +#endif + // Always target the sample format on output port. Even if we can trigger a config update + // for the sample format, Android does not support planar formats, so if the codec uses + // such format (e.g. AC3), conversion is needed. Technically we can limit the conversion to + // planer->packed, but that means Android would also do its own conversion to the wanted + // format on output port. To avoid double conversion, target directly the wanted format. + + bool needConfigUpdate = (mFrame->sample_rate != mTargetSampleRate || + mFrame->ch_layout.nb_channels != mTargetChannels); + bool needResampling = (needConfigUpdate || + mFrame->format != mTargetSampleFormat || + // We only support sending audio data to Android in native order. + mFrame->ch_layout.order != AV_CHANNEL_ORDER_NATIVE); + + if (needConfigUpdate) { + ALOGD("process: audio params changed - sr=%d, ch=%d, fmt=%s => sr=%d, ch=%d, fmt=%s", + mTargetSampleRate, mTargetChannels, av_get_sample_fmt_name(mTargetSampleFormat), + mFrame->sample_rate, mFrame->ch_layout.nb_channels, av_get_sample_fmt_name(mTargetSampleFormat)); + + if (work->worklets.front()->output.buffers.size() > 0) { + // Not sure if this would ever happen, nor how to handle it... + ALOGW("process: audio params changed with non empty output buffers pending"); + } + + C2StreamSampleRateInfo::output sampleRate(0u, mFrame->sample_rate); + C2StreamChannelCountInfo::output channelCount(0u, mFrame->ch_layout.nb_channels); + std::vector> failures; + + err = mIntf->config({ &sampleRate, &channelCount }, C2_MAY_BLOCK, &failures); + if (err == C2_OK) { + work->worklets.front()->output.configUpdate.push_back(C2Param::Copy(sampleRate)); + work->worklets.front()->output.configUpdate.push_back(C2Param::Copy(channelCount)); + updateAudioParameters(); + } else { + ALOGE("process: config update failed err = %d", err); + work->result = C2_CORRUPTED; + return; + } + } + + std::shared_ptr block; + int len = av_samples_get_buffer_size(NULL, mTargetChannels, mFrame->nb_samples, mTargetSampleFormat, 0); + + err = pool->fetchLinearBlock(len, { C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE }, &block); + if (err != C2_OK) { + ALOGE("process: failed to fetch linear block for #=%d err = %d", + mFrame->nb_samples, err); + work->result = C2_CORRUPTED; + return; + } + + C2WriteView wView = block->map().get(); + + err = wView.error(); + if (err != C2_OK) { + ALOGE("process: write view map failed err = %d", err); + work->result = C2_CORRUPTED; + return; + } + + if (needResampling) { + err = getOutputBuffer(&wView); + if (err != C2_OK) { + work->result = err; + return; + } + } + else { +#if DEBUG_FRAMES + ALOGD("process: no audio conversion needed"); +#endif + memcpy(wView.data(), mFrame->data[0], mFrame->linesize[0]); + } + + std::shared_ptr buffer = createLinearBuffer(std::move(block), 0, len); + + if (mCtx->codec->capabilities & AV_CODEC_CAP_SUBFRAMES) { + auto fillWork = [buffer, &work, this](const std::unique_ptr& clone) { + clone->worklets.front()->output.configUpdate = std::move(work->worklets.front()->output.configUpdate); + clone->worklets.front()->output.buffers.clear(); + clone->worklets.front()->output.buffers.push_back(buffer); + clone->worklets.front()->output.ordinal = clone->input.ordinal; + if (mFrame->best_effort_timestamp != AV_NOPTS_VALUE) { + work->worklets.front()->output.ordinal.timestamp = mFrame->best_effort_timestamp; + } + clone->worklets.front()->output.flags = C2FrameData::FLAG_INCOMPLETE; + clone->workletsProcessed = 1u; + clone->result = C2_OK; + }; + +#if DEBUG_FRAMES + ALOGD("process: send subframe buffer ts=%" PRIu64 " idx=%" PRIu64, + work->input.ordinal.timestamp.peeku(), work->input.ordinal.frameIndex.peeku()); +#endif + cloneAndSend(work->input.ordinal.frameIndex.peeku(), work, fillWork); + } + else { + work->worklets.front()->output.buffers.push_back(buffer); + if (mFrame->best_effort_timestamp != AV_NOPTS_VALUE) { + work->worklets.front()->output.ordinal.timestamp = mFrame->best_effort_timestamp; + } + break; + } + } + } +#if DEBUG_FRAMES + else { + ALOGW("process: ignoring empty work"); + } +#endif + + if (eos) { + mEOSSignalled = true; + work->worklets.front()->output.flags = C2FrameData::FLAG_END_OF_STREAM; + } +} + +c2_status_t C2FFMPEGAudioDecodeComponent::drain( + uint32_t drainMode, + const std::shared_ptr& /* pool */ +) { + ALOGD("drain: mode = %u", drainMode); + + if (drainMode == NO_DRAIN) { + ALOGW("drain: NO_DRAIN is no-op"); + return C2_OK; + } + if (drainMode == DRAIN_CHAIN) { + ALOGW("drain: DRAIN_CHAIN not supported"); + return C2_OMITTED; + } + if (! mCodecAlreadyOpened) { + ALOGW("drain: codec not opened yet"); + return C2_OK; + } + + bool hasFrame = false; + c2_status_t err = C2_OK; + + while (err == C2_OK) { + hasFrame = false; + err = sendInputBuffer(NULL, 0); + if (err == C2_OK) { + err = receiveFrame(&hasFrame); + if (hasFrame) { + ALOGW("drain: skip frame pts=%" PRId64 " dts=%" PRId64 " ts=%" PRId64 " - sr=%d, ch=%d, fmt=%s, #=%d", + mFrame->pts, mFrame->pkt_dts, mFrame->best_effort_timestamp, + mFrame->sample_rate, mFrame->ch_layout.nb_channels, av_get_sample_fmt_name((enum AVSampleFormat)mFrame->format), + mFrame->nb_samples); + } else { + err = C2_NOT_FOUND; + } + } + } + + return C2_OK; +} + +} // namespace android diff --git a/C2FFMPEGAudioDecodeComponent.h b/C2FFMPEGAudioDecodeComponent.h new file mode 100644 index 0000000..a737209 --- /dev/null +++ b/C2FFMPEGAudioDecodeComponent.h @@ -0,0 +1,79 @@ +/* + * Copyright 2022 Michael Goffioul + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef C2_FFMPEG_AUDIO_DECODE_COMPONENT_H +#define C2_FFMPEG_AUDIO_DECODE_COMPONENT_H + +#include +#include "C2FFMPEGCommon.h" +#include "C2FFMPEGAudioDecodeInterface.h" + +namespace android { + +struct CodecHelper; + +class C2FFMPEGAudioDecodeComponent : public SimpleC2Component { +public: + explicit C2FFMPEGAudioDecodeComponent( + const C2FFMPEGComponentInfo* componentInfo, + const std::shared_ptr& intf); + virtual ~C2FFMPEGAudioDecodeComponent(); + +protected: + c2_status_t onInit() override; + c2_status_t onStop() override; + void onReset() override; + void onRelease() override; + c2_status_t onFlush_sm() override; + void process( + const std::unique_ptr &work, + const std::shared_ptr &pool) override; + c2_status_t drain( + uint32_t drainMode, + const std::shared_ptr &pool) override; + +private: + c2_status_t initDecoder(); + c2_status_t openDecoder(); + void deInitDecoder(); + c2_status_t processCodecConfig(C2ReadView* inBuffer); + c2_status_t sendInputBuffer(C2ReadView* inBuffer, int64_t timestamp); + c2_status_t receiveFrame(bool* hasFrame); + c2_status_t getOutputBuffer(C2WriteView* outBuffer); + void updateAudioParameters(); + +private: + const C2FFMPEGComponentInfo* mInfo; + std::shared_ptr mIntf; + enum AVCodecID mCodecID; + AVCodecContext* mCtx; + AVFrame* mFrame; + AVPacket* mPacket; + bool mFFMPEGInitialized; + bool mCodecAlreadyOpened; + bool mEOSSignalled; + // Audio resampling + struct SwrContext* mSwrCtx; + enum AVSampleFormat mTargetSampleFormat; + int mTargetSampleRate; + int mTargetChannels; + // Misc + CodecHelper* mCodecHelper; +}; + +} // namespace android + +#endif // C2_FFMPEG_AUDIO_DECODE_COMPONENT_H diff --git a/C2FFMPEGAudioDecodeInterface.cpp b/C2FFMPEGAudioDecodeInterface.cpp new file mode 100644 index 0000000..1dc3bd2 --- /dev/null +++ b/C2FFMPEGAudioDecodeInterface.cpp @@ -0,0 +1,97 @@ +/* + * Copyright 2022 Michael Goffioul + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "C2FFMPEGAudioDecodeInterface" +#include + +#include +#include "C2FFMPEGAudioDecodeInterface.h" + +#define MAX_CHANNEL_COUNT 8 + +namespace android { + +constexpr size_t kDefaultOutputPortDelay = 2; +constexpr size_t kMaxOutputPortDelay = 16; + +C2FFMPEGAudioDecodeInterface::C2FFMPEGAudioDecodeInterface( + const C2FFMPEGComponentInfo* componentInfo, + const std::shared_ptr& helper) + : SimpleInterface::BaseParams( + helper, + componentInfo->name, + C2Component::KIND_DECODER, + C2Component::DOMAIN_AUDIO, + componentInfo->mediaType) { + noPrivateBuffers(); + noInputReferences(); + noOutputReferences(); + noInputLatency(); + noTimeStretch(); + setDerivedInstance(this); + + addParameter( + DefineParam(mActualOutputDelay, C2_PARAMKEY_OUTPUT_DELAY) + .withDefault(new C2PortActualDelayTuning::output(kDefaultOutputPortDelay)) + .withFields({C2F(mActualOutputDelay, value).inRange(0, kMaxOutputPortDelay)}) + .withSetter(Setter::StrictValueWithNoDeps) + .build()); + + addParameter( + DefineParam(mSampleRate, C2_PARAMKEY_SAMPLE_RATE) + .withDefault(new C2StreamSampleRateInfo::output(0u, 44100)) + .withFields({C2F(mSampleRate, value).oneOf({ + 7350, 8000, 11025, 12000, 16000, 22050, 24000, 32000, + 44100, 48000, 64000, 88200, 96000, 192000 + })}) + .withSetter(Setter::NonStrictValueWithNoDeps) + .build()); + + addParameter( + DefineParam(mBitrate, C2_PARAMKEY_BITRATE) + .withDefault(new C2StreamBitrateInfo::input(0u, 64000)) + .withFields({C2F(mBitrate, value).inRange(8000, 320000)}) + .withSetter(Setter::NonStrictValueWithNoDeps) + .build()); + + addParameter( + DefineParam(mChannelCount, C2_PARAMKEY_CHANNEL_COUNT) + .withDefault(new C2StreamChannelCountInfo::output(0u, 2)) + .withFields({C2F(mChannelCount, value).inRange(1, MAX_CHANNEL_COUNT)}) + .withSetter(Setter::StrictValueWithNoDeps) + .build()); + + addParameter( + DefineParam(mPcmEncodingInfo, C2_PARAMKEY_PCM_ENCODING) + .withDefault(new C2StreamPcmEncodingInfo::output(0u, C2Config::PCM_16)) + .withFields({C2F(mPcmEncodingInfo, value).oneOf({ + C2Config::PCM_16, + C2Config::PCM_8, + C2Config::PCM_FLOAT, + C2Config::PCM_32}) + }) + .withSetter((Setter::StrictValueWithNoDeps)) + .build()); + + if (strcasecmp(componentInfo->mediaType, MEDIA_MIMETYPE_AUDIO_FLAC) == 0) { + addParameter( + DefineParam(mInputMaxBufSize, C2_PARAMKEY_INPUT_MAX_BUFFER_SIZE) + .withConstValue(new C2StreamMaxBufferSizeInfo::input(0u, 32768)) + .build()); + } +} + +} // namespace android diff --git a/C2FFMPEGAudioDecodeInterface.h b/C2FFMPEGAudioDecodeInterface.h new file mode 100644 index 0000000..988e92c --- /dev/null +++ b/C2FFMPEGAudioDecodeInterface.h @@ -0,0 +1,46 @@ +/* + * Copyright 2022 Michael Goffioul + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef C2_FFMPEG_AUDIO_DECODE_INTERFACE_H +#define C2_FFMPEG_AUDIO_DECODE_INTERFACE_H + +#include +#include "C2FFMPEGCommon.h" + +namespace android { + +class C2FFMPEGAudioDecodeInterface : public SimpleInterface::BaseParams { +public: + explicit C2FFMPEGAudioDecodeInterface( + const C2FFMPEGComponentInfo* componentInfo, + const std::shared_ptr& helper); + + uint32_t getSampleRate() const { return mSampleRate->value; } + uint32_t getChannelCount() const { return mChannelCount->value; } + uint32_t getBitrate() const { return mBitrate->value; } + C2Config::pcm_encoding_t getPcmEncodingInfo() const { return mPcmEncodingInfo->value; } + +private: + std::shared_ptr mSampleRate; + std::shared_ptr mChannelCount; + std::shared_ptr mBitrate; + std::shared_ptr mPcmEncodingInfo; + std::shared_ptr mInputMaxBufSize; +}; + +} // namespace android + +#endif // C2_FFMPEG_AUDIO_DECODE_INTERFACE_H diff --git a/C2FFMPEGCommon.h b/C2FFMPEGCommon.h new file mode 100644 index 0000000..f1a85b8 --- /dev/null +++ b/C2FFMPEGCommon.h @@ -0,0 +1,33 @@ +/* + * Copyright 2022 Michael Goffioul + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef C2_FFMPEG_COMPONENT_COMMON_H +#define C2_FFMPEG_COMPONENT_COMMON_H + +#include +#include "ffmpeg_utils.h" + +namespace android { + +typedef struct { + const char* name; + const char* mediaType; + enum AVCodecID codecID; +} C2FFMPEGComponentInfo; + +} // namespace android + +#endif // C2_FFMPEG_COMPONENT_COMMON_H diff --git a/C2FFMPEGComponentInterface.cpp b/C2FFMPEGComponentInterface.cpp new file mode 100644 index 0000000..d6ae8d6 --- /dev/null +++ b/C2FFMPEGComponentInterface.cpp @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2022 Michael Goffioul + * Copyright (C) 2025 KonstaKANG + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "C2FFMPEGComponentInterface.h" + +namespace android { + +C2FFMPEGComponentInterface::C2FFMPEGComponentInterface(const std::shared_ptr &helper) + : C2InterfaceHelper(helper) { + setDerivedInstance(this); + + addParameter( + DefineParam(mIonUsageInfo, "ion-usage") + .withDefault(new C2StoreIonUsageInfo()) + .withFields({ + C2F(mIonUsageInfo, usage).flags( + {C2MemoryUsage::CPU_READ | C2MemoryUsage::CPU_WRITE}), + C2F(mIonUsageInfo, capacity).inRange(0, UINT32_MAX, 1024), + C2F(mIonUsageInfo, heapMask).any(), + C2F(mIonUsageInfo, allocFlags).flags({}), + C2F(mIonUsageInfo, minAlignment).equalTo(0) + }) + .withSetter(SetIonUsage) + .build()); + + addParameter( + DefineParam(mDmaBufUsageInfo, "dmabuf-usage") + .withDefault(C2StoreDmaBufUsageInfo::AllocUnique(0)) + .withFields({ + C2F(mDmaBufUsageInfo, m.usage).flags({C2MemoryUsage::CPU_READ | C2MemoryUsage::CPU_WRITE}), + C2F(mDmaBufUsageInfo, m.capacity).inRange(0, UINT32_MAX, 1024), + C2F(mDmaBufUsageInfo, m.allocFlags).flags({}), + C2F(mDmaBufUsageInfo, m.heapName).any(), + }) + .withSetter(SetDmaBufUsage) + .build()); +} + +C2FFMPEGComponentInterface::~C2FFMPEGComponentInterface() = default; + +C2R C2FFMPEGComponentInterface::SetIonUsage(bool /* mayBlock */, C2P &me) { + // Vendor's TODO: put appropriate mapping logic + me.set().heapMask = ~0; + me.set().allocFlags = 0; + me.set().minAlignment = 0; + return C2R::Ok(); +} + +C2R C2FFMPEGComponentInterface::SetDmaBufUsage(bool /* mayBlock */, C2P &me) { + // Vendor's TODO: put appropriate mapping logic + strncpy(me.set().m.heapName, "system", me.v.flexCount()); + me.set().m.allocFlags = 0; + return C2R::Ok(); +} + +} // namespace android diff --git a/C2FFMPEGComponentInterface.h b/C2FFMPEGComponentInterface.h new file mode 100644 index 0000000..6f689ee --- /dev/null +++ b/C2FFMPEGComponentInterface.h @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2022 Michael Goffioul + * Copyright (C) 2025 KonstaKANG + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef C2_FFMPEG_COMPONENT_INTERFACE_H +#define C2_FFMPEG_COMPONENT_INTERFACE_H + +#include +#include + +namespace android { + +class C2FFMPEGComponentInterface : public C2InterfaceHelper { +public: + C2FFMPEGComponentInterface(const std::shared_ptr &helper); + virtual ~C2FFMPEGComponentInterface(); + +private: + static C2R SetIonUsage(bool /* mayBlock */, C2P &me); + static C2R SetDmaBufUsage(bool /* mayBlock */, C2P &me); + std::shared_ptr mIonUsageInfo; + std::shared_ptr mDmaBufUsageInfo; +}; + +} // namespace android + +#endif // C2_FFMPEG_COMPONENT_INTERFACE_H diff --git a/C2FFMPEGComponentStore.cpp b/C2FFMPEGComponentStore.cpp new file mode 100644 index 0000000..55ba1ea --- /dev/null +++ b/C2FFMPEGComponentStore.cpp @@ -0,0 +1,195 @@ +/* + * Copyright (C) 2022 Michael Goffioul + * Copyright (C) 2025 KonstaKANG + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "C2FFMPEGComponentStore" +#include + +#include "C2FFMPEGCommon.h" +#include "C2FFMPEGAudioDecodeComponent.h" +#include "C2FFMPEGAudioDecodeInterface.h" +#include "C2FFMPEGVideoDecodeComponent.h" +#include "C2FFMPEGVideoDecodeInterface.h" + +#include "C2FFMPEGComponentStore.h" + +#define RANK_DISABLED 0xFFFFFFFF + +namespace android { + +static const C2FFMPEGComponentInfo kFFMPEGVideoComponents[] = { + { "c2.ffmpeg.av1.decoder" , MEDIA_MIMETYPE_VIDEO_AV1 , AV_CODEC_ID_AV1 }, + { "c2.ffmpeg.h263.decoder" , MEDIA_MIMETYPE_VIDEO_H263 , AV_CODEC_ID_H263 }, + { "c2.ffmpeg.h264.decoder" , MEDIA_MIMETYPE_VIDEO_AVC , AV_CODEC_ID_H264 }, + { "c2.ffmpeg.hevc.decoder" , MEDIA_MIMETYPE_VIDEO_HEVC , AV_CODEC_ID_HEVC }, + { "c2.ffmpeg.mpeg2.decoder" , MEDIA_MIMETYPE_VIDEO_MPEG2 , AV_CODEC_ID_MPEG2VIDEO }, + { "c2.ffmpeg.mpeg4.decoder" , MEDIA_MIMETYPE_VIDEO_MPEG4 , AV_CODEC_ID_MPEG4 }, + { "c2.ffmpeg.vp8.decoder" , MEDIA_MIMETYPE_VIDEO_VP8 , AV_CODEC_ID_VP8 }, + { "c2.ffmpeg.vp9.decoder" , MEDIA_MIMETYPE_VIDEO_VP9 , AV_CODEC_ID_VP9 }, +}; + +static const size_t kNumVideoComponents = + (sizeof(kFFMPEGVideoComponents) / sizeof(kFFMPEGVideoComponents[0])); + +static const C2FFMPEGComponentInfo kFFMPEGAudioComponents[] = { + { "c2.ffmpeg.aac.decoder" , MEDIA_MIMETYPE_AUDIO_AAC , AV_CODEC_ID_AAC }, + { "c2.ffmpeg.ac3.decoder" , MEDIA_MIMETYPE_AUDIO_AC3 , AV_CODEC_ID_AC3 }, + { "c2.ffmpeg.alac.decoder" , MEDIA_MIMETYPE_AUDIO_ALAC , AV_CODEC_ID_ALAC }, + { "c2.ffmpeg.flac.decoder" , MEDIA_MIMETYPE_AUDIO_FLAC , AV_CODEC_ID_FLAC }, + { "c2.ffmpeg.mp2.decoder" , MEDIA_MIMETYPE_AUDIO_MPEG_LAYER_II, AV_CODEC_ID_MP2 }, + { "c2.ffmpeg.mp3.decoder" , MEDIA_MIMETYPE_AUDIO_MPEG , AV_CODEC_ID_MP3 }, + { "c2.ffmpeg.vorbis.decoder", MEDIA_MIMETYPE_AUDIO_VORBIS , AV_CODEC_ID_VORBIS }, +}; + +static const size_t kNumAudioComponents = + (sizeof(kFFMPEGAudioComponents) / sizeof(kFFMPEGAudioComponents[0])); + +C2FFMPEGComponentStore::C2FFMPEGComponentStore() + : mReflectorHelper(std::make_shared()), + mInterface(mReflectorHelper) { +} + +C2FFMPEGComponentStore::~C2FFMPEGComponentStore() = default; + +C2String C2FFMPEGComponentStore::getName() const { + return "ffmpeg"; +} + +c2_status_t C2FFMPEGComponentStore::createComponent( + C2String name, + std::shared_ptr* const component) { + ALOGD("createComponent: %s", name.c_str()); + for (int i = 0; i < kNumAudioComponents; i++) { + auto info = &kFFMPEGAudioComponents[i]; + if (name == info->name) { + component->reset(); + *component = std::shared_ptr( + new C2FFMPEGAudioDecodeComponent( + info, std::make_shared(info, mReflectorHelper))); + return C2_OK; + } + } + for (int i = 0; i < kNumVideoComponents; i++) { + auto info = &kFFMPEGVideoComponents[i]; + if (name == info->name) { + component->reset(); + *component = std::shared_ptr( + new C2FFMPEGVideoDecodeComponent( + info, std::make_shared(info, mReflectorHelper))); + return C2_OK; + } + } + return C2_NOT_FOUND; +} + +c2_status_t C2FFMPEGComponentStore::createInterface( + C2String name, + std::shared_ptr* const interface) { + ALOGD("createInterface: %s", name.c_str()); + for (int i = 0; i < kNumAudioComponents; i++) { + auto info = &kFFMPEGAudioComponents[i]; + if (name == info->name) { + interface->reset(); + *interface = std::shared_ptr( + new SimpleInterface( + info->name, 0, std::make_shared(info, mReflectorHelper))); + return C2_OK; + } + } + for (int i = 0; i < kNumVideoComponents; i++) { + auto info = &kFFMPEGVideoComponents[i]; + if (name == info->name) { + interface->reset(); + *interface = std::shared_ptr( + new SimpleInterface( + info->name, 0, std::make_shared(info, mReflectorHelper))); + return C2_OK; + } + } + ALOGE("createInterface: unknown component = %s", name.c_str()); + return C2_NOT_FOUND; +} + +std::vector> + C2FFMPEGComponentStore::listComponents() { + std::vector> ret; + // FIXME: Prefer OMX codecs for the time being... + uint32_t defaultRank = ::android::base::GetUintProperty("persist.vendor.ffmpeg_codec2.rank", 0x110u); + uint32_t defaultRankAudio = ::android::base::GetUintProperty("persist.vendor.ffmpeg_codec2.rank.audio", defaultRank); + uint32_t defaultRankVideo = ::android::base::GetUintProperty("persist.vendor.ffmpeg_codec2.rank.video", defaultRank); + ALOGD("listComponents: defaultRank=%x, defaultRankAudio=%x, defaultRankVideo=%x", + defaultRank, defaultRankAudio, defaultRankVideo); + if (defaultRank != RANK_DISABLED) { + if (defaultRankAudio != RANK_DISABLED) { + for (int i = 0; i < kNumAudioComponents; i++) { + auto traits = std::make_shared(); + traits->name = kFFMPEGAudioComponents[i].name; + traits->domain = C2Component::DOMAIN_AUDIO; + traits->kind = C2Component::KIND_DECODER; + traits->mediaType = kFFMPEGAudioComponents[i].mediaType; + traits->rank = defaultRankAudio; + ret.push_back(traits); + } + } + if (defaultRankVideo != RANK_DISABLED) { + for (int i = 0; i < kNumVideoComponents; i++) { + auto traits = std::make_shared(); + traits->name = kFFMPEGVideoComponents[i].name; + traits->domain = C2Component::DOMAIN_VIDEO; + traits->kind = C2Component::KIND_DECODER; + traits->mediaType = kFFMPEGVideoComponents[i].mediaType; + traits->rank = defaultRankVideo; + ret.push_back(traits); + } + } + } + return ret; +} + +c2_status_t C2FFMPEGComponentStore::copyBuffer( + std::shared_ptr /* src */, + std::shared_ptr /* dst */) { + return C2_OMITTED; +} + +c2_status_t C2FFMPEGComponentStore::query_sm( + const std::vector& stackParams, + const std::vector& heapParamIndices, + std::vector>* const heapParams) const { + return mInterface.query(stackParams, heapParamIndices, C2_MAY_BLOCK, heapParams); +} + +c2_status_t C2FFMPEGComponentStore::config_sm( + const std::vector& params, + std::vector>* const failures) { + return mInterface.config(params, C2_MAY_BLOCK, failures); +} + +std::shared_ptr C2FFMPEGComponentStore::getParamReflector() const { + return mReflectorHelper; +} + +c2_status_t C2FFMPEGComponentStore::querySupportedParams_nb( + std::vector>* const params) const { + return mInterface.querySupportedParams(params); +} + +c2_status_t C2FFMPEGComponentStore::querySupportedValues_sm( + std::vector& fields) const { + return mInterface.querySupportedValues(fields, C2_MAY_BLOCK); +} + +} // namespace android diff --git a/C2FFMPEGComponentStore.h b/C2FFMPEGComponentStore.h new file mode 100644 index 0000000..09f1076 --- /dev/null +++ b/C2FFMPEGComponentStore.h @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2022 Michael Goffioul + * Copyright (C) 2025 KonstaKANG + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef C2_FFMPEG_COMPONENT_STORE_H +#define C2_FFMPEG_COMPONENT_STORE_H + +#include "C2FFMPEGComponentInterface.h" + +namespace android { + +class C2FFMPEGComponentStore : public C2ComponentStore { +public: + C2FFMPEGComponentStore(); + virtual ~C2FFMPEGComponentStore() override; + + virtual C2String getName() const override; + virtual c2_status_t createComponent( + C2String name, + std::shared_ptr* const component) override; + virtual c2_status_t createInterface( + C2String name, + std::shared_ptr* const interface) override; + virtual std::vector> + listComponents() override; + virtual c2_status_t copyBuffer( + std::shared_ptr /* src */, + std::shared_ptr /* dst */) override; + virtual c2_status_t query_sm( + const std::vector& stackParams, + const std::vector& heapParamIndices, + std::vector>* const heapParams) const override; + virtual c2_status_t config_sm( + const std::vector& params, + std::vector>* const failures) override; + virtual std::shared_ptr getParamReflector() const override; + virtual c2_status_t querySupportedParams_nb( + std::vector>* const params) const override; + virtual c2_status_t querySupportedValues_sm( + std::vector& fields) const override; + +private: + std::shared_ptr mReflectorHelper; + C2FFMPEGComponentInterface mInterface; +}; + +} // namespace android + +#endif // C2_FFMPEG_COMPONENT_STORE_H diff --git a/C2FFMPEGVideoDecodeComponent.cpp b/C2FFMPEGVideoDecodeComponent.cpp new file mode 100644 index 0000000..1d33485 --- /dev/null +++ b/C2FFMPEGVideoDecodeComponent.cpp @@ -0,0 +1,696 @@ +/* + * Copyright 2022 Michael Goffioul + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "C2FFMPEGVideoDecodeComponent" +#include +#include +#include + +#include +#include "C2FFMPEGVideoDecodeComponent.h" +#include "ffmpeg_hwaccel.h" + +#define DEBUG_FRAMES 0 +#define DEBUG_WORKQUEUE 0 +#define DEBUG_EXTRADATA 0 + +namespace android { + +C2FFMPEGVideoDecodeComponent::C2FFMPEGVideoDecodeComponent( + const C2FFMPEGComponentInfo* componentInfo, + const std::shared_ptr& intf) + : SimpleC2Component(std::make_shared>(componentInfo->name, 0, intf)), + mInfo(componentInfo), + mIntf(intf), + mCodecID(componentInfo->codecID), + mCtx(NULL), + mImgConvertCtx(NULL), + mFrame(NULL), + mPacket(NULL), + mFFMPEGInitialized(false), + mCodecAlreadyOpened(false), + mExtradataReady(false), + mEOSSignalled(false) { + ALOGD("C2FFMPEGVideoDecodeComponent: mediaType = %s", componentInfo->mediaType); +} + +C2FFMPEGVideoDecodeComponent::~C2FFMPEGVideoDecodeComponent() { + ALOGD("~C2FFMPEGVideoDecodeComponent: mCtx = %p", mCtx); + onRelease(); +} + +c2_status_t C2FFMPEGVideoDecodeComponent::initDecoder() { + if (! mFFMPEGInitialized) { + if (initFFmpeg() != C2_OK) { + ALOGE("initDecoder: FFMPEG initialization failed."); + return C2_NO_INIT; + } + mFFMPEGInitialized = true; + } + + mCtx = avcodec_alloc_context3(NULL); + if (! mCtx) { + ALOGE("initDecoder: avcodec_alloc_context failed."); + return C2_NO_MEMORY; + } + + C2StreamPictureSizeInfo::output size(0u, 320, 240); + c2_status_t err = mIntf->query({ &size }, {}, C2_DONT_BLOCK, nullptr); + if (err != C2_OK) { + ALOGE("initDecoder: cannot query picture size, err = %d", err); + } + + mCtx->codec_type = AVMEDIA_TYPE_VIDEO; + mCtx->codec_id = mCodecID; + mCtx->extradata_size = 0; + mCtx->extradata = NULL; + mCtx->width = size.width; + mCtx->height = size.height; + + ALOGD("initDecoder: %p [%s], %d x %d, %s", + mCtx, avcodec_get_name(mCtx->codec_id), size.width, size.height, mInfo->mediaType); + + return C2_OK; +} + +c2_status_t C2FFMPEGVideoDecodeComponent::openDecoder() { + if (mCodecAlreadyOpened) { + return C2_OK; + } + + // Can't change extradata after opening the decoder. +#if DEBUG_EXTRADATA + ALOGD("openDecoder: extradata_size = %d", mCtx->extradata_size); +#endif + mExtradataReady = true; + + // Find decoder again as codec_id may have changed. + if (mCtx->codec_id == AV_CODEC_ID_H264 && + base::GetBoolProperty("persist.vendor.ffmpeg_codec2.v4l2.h264", false)) { + mCtx->codec = avcodec_find_decoder_by_name("h264_v4l2m2m"); + } else { + mCtx->codec = avcodec_find_decoder(mCtx->codec_id); + } + + if (! mCtx->codec) { + ALOGE("openDecoder: ffmpeg video decoder failed to find codec %d", mCtx->codec_id); + return C2_NOT_FOUND; + } + + // Configure decoder. + mCtx->workaround_bugs = 1; + mCtx->idct_algo = 0; + mCtx->skip_frame = AVDISCARD_DEFAULT; + mCtx->skip_idct = AVDISCARD_DEFAULT; + mCtx->skip_loop_filter = AVDISCARD_DEFAULT; + mCtx->error_concealment = 3; + mCtx->thread_count = base::GetIntProperty("debug.ffmpeg_codec2.threads", 0); + + if (base::GetBoolProperty("debug.ffmpeg_codec2.fast", false)) { + mCtx->flags2 |= AV_CODEC_FLAG2_FAST; + } + + ffmpeg_hwaccel_init(mCtx); + + ALOGD("openDecoder: opening ffmpeg decoder(%s): threads = %d, hw = %s", + avcodec_get_name(mCtx->codec_id), mCtx->thread_count, mCtx->hw_device_ctx ? "yes" : "no"); + + int err = avcodec_open2(mCtx, mCtx->codec, NULL); + if (err < 0) { + ALOGE("openDecoder: ffmpeg video decoder failed to initialize. (%s)", av_err2str(err)); + return C2_NO_INIT; + } + mCodecAlreadyOpened = true; + + ALOGD("openDecoder: open ffmpeg video decoder(%s) success, caps = %08x", + avcodec_get_name(mCtx->codec_id), mCtx->codec->capabilities); + + mFrame = av_frame_alloc(); + if (! mFrame) { + ALOGE("openDecoder: oom for video frame"); + return C2_NO_MEMORY; + } + + return C2_OK; +} + +void C2FFMPEGVideoDecodeComponent::deInitDecoder() { + ALOGD("%p deInitDecoder: %p", this, mCtx); + if (mCtx) { + if (avcodec_is_open(mCtx)) { + avcodec_flush_buffers(mCtx); + } + ffmpeg_hwaccel_deinit(mCtx); + avcodec_free_context(&mCtx); + mCodecAlreadyOpened = false; + } + if (mFrame) { + av_frame_free(&mFrame); + mFrame = NULL; + } + if (mPacket) { + av_packet_free(&mPacket); + mPacket = NULL; + } + if (mImgConvertCtx) { + sws_freeContext(mImgConvertCtx); + mImgConvertCtx = NULL; + } + mEOSSignalled = false; + mExtradataReady = false; + mPendingWorkQueue.clear(); +} + +c2_status_t C2FFMPEGVideoDecodeComponent::processCodecConfig(C2ReadView* inBuffer) { + int orig_extradata_size = mCtx->extradata_size; + int add_extradata_size = inBuffer->capacity(); + +#if DEBUG_EXTRADATA + ALOGD("processCodecConfig: add = %u, current = %d", add_extradata_size, orig_extradata_size); +#endif + if (! mExtradataReady) { + mCtx->extradata_size += add_extradata_size; + mCtx->extradata = (uint8_t *) realloc(mCtx->extradata, mCtx->extradata_size + AV_INPUT_BUFFER_PADDING_SIZE); + if (! mCtx->extradata) { + ALOGE("processCodecConfig: ffmpeg video decoder failed to alloc extradata memory."); + return C2_NO_MEMORY; + } + memcpy(mCtx->extradata + orig_extradata_size, inBuffer->data(), add_extradata_size); + memset(mCtx->extradata + mCtx->extradata_size, 0, AV_INPUT_BUFFER_PADDING_SIZE); + } + else { + ALOGW("processCodecConfig: decoder is already opened, ignoring..."); + } + + return C2_OK; +} + +c2_status_t C2FFMPEGVideoDecodeComponent::sendInputBuffer( + C2ReadView *inBuffer, int64_t timestamp) { + if (!mPacket) { + mPacket = av_packet_alloc(); + if (!mPacket) { + ALOGE("sendInputBuffer: oom for video packet"); + return C2_NO_MEMORY; + } + } + + mPacket->data = inBuffer ? const_cast(inBuffer->data()) : NULL; + mPacket->size = inBuffer ? inBuffer->capacity() : 0; + mPacket->pts = timestamp; + mPacket->dts = AV_NOPTS_VALUE; + + int err = avcodec_send_packet(mCtx, mPacket); + av_packet_unref(mPacket); + + if (err < 0) { + ALOGE("sendInputBuffer: failed to send data (%d) to decoder: %s (%08x)", + inBuffer->capacity(), av_err2str(err), err); + if (err == AVERROR(EAGAIN)) { + // Frames must be read first, notify main decoding loop. + ALOGD("sendInputBuffer: returning C2_BAD_STATE"); + return C2_BAD_STATE; + } + // Otherwise don't send error to client. + } + + return C2_OK; +} + +c2_status_t C2FFMPEGVideoDecodeComponent::receiveFrame(bool* hasPicture) { + int err = avcodec_receive_frame(mCtx, mFrame); + + *hasPicture = false; + if (err == 0) { + err = ffmpeg_hwaccel_get_frame(mCtx, mFrame); + if (err == 0) { + *hasPicture = true; + } else { + ALOGE("receiveFrame: failed to receive frame from HW decoder err = %d", err); + // Don't send error to client, skip frame! + } + } else if (err != AVERROR(EAGAIN) && err != AVERROR_EOF) { + ALOGE("receiveFrame: failed to receive frame from decoder err = %d", err); + // Don't report error to client. + } + + return C2_OK; +} + +c2_status_t C2FFMPEGVideoDecodeComponent::getOutputBuffer(C2GraphicView* outBuffer) { + uint8_t* data[4]; + int linesize[4]; + C2PlanarLayout layout = outBuffer->layout(); + struct SwsContext* currentImgConvertCtx = mImgConvertCtx; + + data[0] = outBuffer->data()[C2PlanarLayout::PLANE_Y]; + data[1] = outBuffer->data()[C2PlanarLayout::PLANE_U]; + data[2] = outBuffer->data()[C2PlanarLayout::PLANE_V]; + linesize[0] = layout.planes[C2PlanarLayout::PLANE_Y].rowInc; + linesize[1] = layout.planes[C2PlanarLayout::PLANE_U].rowInc; + linesize[2] = layout.planes[C2PlanarLayout::PLANE_V].rowInc; + + mImgConvertCtx = sws_getCachedContext(currentImgConvertCtx, + mFrame->width, mFrame->height, (AVPixelFormat)mFrame->format, + mFrame->width, mFrame->height, AV_PIX_FMT_YUV420P, + SWS_BICUBIC, NULL, NULL, NULL); + if (mImgConvertCtx && mImgConvertCtx != currentImgConvertCtx) { + ALOGD("getOutputBuffer: created video converter - %s => %s", + av_get_pix_fmt_name((AVPixelFormat)mFrame->format), av_get_pix_fmt_name(AV_PIX_FMT_YUV420P)); + + } else if (! mImgConvertCtx) { + ALOGE("getOutputBuffer: cannot initialize the conversion context"); + return C2_NO_MEMORY; + } + + sws_scale(mImgConvertCtx, mFrame->data, mFrame->linesize, + 0, mFrame->height, data, linesize); + + return C2_OK; +} + +static void fillEmptyWork(const std::unique_ptr& work) { + work->worklets.front()->output.flags = + (C2FrameData::flags_t)(work->input.flags & C2FrameData::FLAG_END_OF_STREAM); + work->worklets.front()->output.buffers.clear(); + work->worklets.front()->output.ordinal = work->input.ordinal; + work->workletsProcessed = 1u; + work->result = C2_OK; +#if DEBUG_WORKQUEUE + ALOGD("WorkQueue: drop idx=%" PRIu64 ", ts=%" PRIu64, + work->input.ordinal.frameIndex.peeku(), work->input.ordinal.timestamp.peeku()); +#endif +} + +static bool comparePendingWork(const PendingWork& w1, const PendingWork& w2) { + return w1.second < w2.second; +} + +void C2FFMPEGVideoDecodeComponent::pushPendingWork(const std::unique_ptr& work) { + uint32_t outputDelay = mIntf->getOutputDelay(); + + if (mPendingWorkQueue.size() >= outputDelay) { + uint32_t newOutputDelay = outputDelay; + std::vector> configUpdate; + + switch (mCtx->codec_id) { + case AV_CODEC_ID_HEVC: + case AV_CODEC_ID_H264: + // Increase output delay step-wise. + if (outputDelay >= 18u) { + newOutputDelay = 34u; + } else if (outputDelay >= 8u) { + newOutputDelay = 18u; + } else { + newOutputDelay = 8u; + } + break; + default: + // Other codecs use constant output delay. + break; + } + + if (newOutputDelay != outputDelay) { + C2PortActualDelayTuning::output delay(newOutputDelay); + std::vector> failures; + int err; + + err = mIntf->config({ &delay }, C2_MAY_BLOCK, &failures); + if (err == C2_OK) { + ALOGD("WorkQueue: queue full, output delay set to %u", newOutputDelay); + configUpdate.push_back(C2Param::Copy(delay)); + } else { + ALOGE("WorkQueue: output delay update to %u failed err = %d", + newOutputDelay, err); + } + } + + auto fillEmptyWorkWithConfigUpdate = [&configUpdate](const std::unique_ptr& work) { + fillEmptyWork(work); + work->worklets.front()->output.configUpdate = std::move(configUpdate); + }; + + finish(mPendingWorkQueue.front().first, fillEmptyWorkWithConfigUpdate); + mPendingWorkQueue.pop_front(); + } +#if DEBUG_WORKQUEUE + ALOGD("WorkQueue: push idx=%" PRIu64 ", ts=%" PRIu64, + work->input.ordinal.frameIndex.peeku(), work->input.ordinal.timestamp.peeku()); +#endif + mPendingWorkQueue.push_back(PendingWork(work->input.ordinal.frameIndex.peeku(), + work->input.ordinal.timestamp.peeku())); + std::sort(mPendingWorkQueue.begin(), mPendingWorkQueue.end(), comparePendingWork); +} + +void C2FFMPEGVideoDecodeComponent::popPendingWork(const std::unique_ptr& work) { + uint64_t index = work->input.ordinal.frameIndex.peeku(); + auto it = std::find_if(mPendingWorkQueue.begin(), mPendingWorkQueue.end(), + [index](const PendingWork& pWork) { return index == pWork.first; }); + +#if DEBUG_WORKQUEUE + ALOGD("WorkQueue: pop idx=%" PRIu64 ", ts=%" PRIu64, + work->input.ordinal.frameIndex.peeku(), work->input.ordinal.timestamp.peeku()); +#endif + + if (it != mPendingWorkQueue.end()) { + mPendingWorkQueue.erase(it); + } +#if DEBUG_WORKQUEUE + else { + ALOGD("WorkQueue: pop work not found idx=%" PRIu64 ", ts=%" PRIu64, + work->input.ordinal.frameIndex.peeku(), work->input.ordinal.timestamp.peeku()); + } +#endif + prunePendingWorksUntil(work); +} + +void C2FFMPEGVideoDecodeComponent::prunePendingWorksUntil(const std::unique_ptr& work) { +#if DEBUG_WORKQUEUE + ALOGD("WorkQueue: prune until idx=%" PRIu64 ", ts=%" PRIu64, + work->input.ordinal.frameIndex.peeku(), work->input.ordinal.timestamp.peeku()); +#endif + // Drop all works with a PTS earlier than provided argument. + while (mPendingWorkQueue.size() > 0 && + mPendingWorkQueue.front().second < work->input.ordinal.timestamp.peeku()) { + finish(mPendingWorkQueue.front().first, fillEmptyWork); + mPendingWorkQueue.pop_front(); + } +} + +c2_status_t C2FFMPEGVideoDecodeComponent::onInit() { + ALOGD("onInit"); + return initDecoder(); +} + +c2_status_t C2FFMPEGVideoDecodeComponent::onStop() { + ALOGD("onStop"); + return C2_OK; +} + +void C2FFMPEGVideoDecodeComponent::onReset() { + ALOGD("onReset"); + deInitDecoder(); + initDecoder(); +} + +void C2FFMPEGVideoDecodeComponent::onRelease() { + ALOGD("onRelease"); + deInitDecoder(); + if (mFFMPEGInitialized) { + deInitFFmpeg(); + mFFMPEGInitialized = false; + } +} + +c2_status_t C2FFMPEGVideoDecodeComponent::onFlush_sm() { + ALOGD("onFlush_sm"); + if (mCtx && avcodec_is_open(mCtx)) { + // Make sure that the next buffer output does not still + // depend on fragments from the last one decoded. + avcodec_flush_buffers(mCtx); + mEOSSignalled = false; + } + return C2_OK; +} + +c2_status_t C2FFMPEGVideoDecodeComponent::outputFrame( + const std::unique_ptr& work, + const std::shared_ptr &pool +) { + c2_status_t err; + std::vector> configUpdate; + +#if DEBUG_FRAMES + ALOGD("outputFrame: pts=%" PRId64 " dts=%" PRId64 " ts=%" PRId64 " - %d x %d (%x)", + mFrame->pts, mFrame->pkt_dts, mFrame->best_effort_timestamp, mFrame->width, mFrame->height, mFrame->format); +#endif + + if (mFrame->width != mIntf->getWidth() || mFrame->height != mIntf->getHeight()) { + ALOGD("outputFrame: video params changed - %d x %d (%x)", mFrame->width, mFrame->height, mFrame->format); + + C2StreamPictureSizeInfo::output size(0u, mFrame->width, mFrame->height); + std::vector> failures; + + err = mIntf->config({ &size }, C2_MAY_BLOCK, &failures); + if (err == OK) { + configUpdate.push_back(C2Param::Copy(size)); + mCtx->width = mFrame->width; + mCtx->height = mFrame->height; + } else { + ALOGE("outputFrame: config update failed err = %d", err); + return C2_CORRUPTED; + } + } + + std::shared_ptr block; + + err = pool->fetchGraphicBlock(mFrame->width, mFrame->height, HAL_PIXEL_FORMAT_YV12, + { C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE }, &block); + + if (err != C2_OK) { + ALOGE("outputFrame: failed to fetch graphic block %d x %d (%x) err = %d", + mFrame->width, mFrame->height, HAL_PIXEL_FORMAT_YV12, err); + return C2_CORRUPTED; + } + + C2GraphicView wView = block->map().get(); + + err = wView.error(); + if (err != C2_OK) { + ALOGE("outputFrame: graphic view map failed err = %d", err); + return C2_CORRUPTED; + } + + err = getOutputBuffer(&wView); + if (err == C2_OK) { + std::shared_ptr buffer = createGraphicBuffer(std::move(block), C2Rect(mFrame->width, mFrame->height)); + + buffer->setInfo(mIntf->getPixelFormatInfo()); + + if (work && c2_cntr64_t(mFrame->best_effort_timestamp) == work->input.ordinal.frameIndex) { + prunePendingWorksUntil(work); + work->worklets.front()->output.configUpdate = std::move(configUpdate); + work->worklets.front()->output.buffers.clear(); + work->worklets.front()->output.buffers.push_back(buffer); + work->worklets.front()->output.ordinal = work->input.ordinal; + work->workletsProcessed = 1u; + work->result = C2_OK; + } else { + auto fillWork = [buffer, &configUpdate, this](const std::unique_ptr& work) { + popPendingWork(work); + work->worklets.front()->output.configUpdate = std::move(configUpdate); + work->worklets.front()->output.flags = (C2FrameData::flags_t)0; + work->worklets.front()->output.buffers.clear(); + work->worklets.front()->output.buffers.push_back(buffer); + work->worklets.front()->output.ordinal = work->input.ordinal; + work->workletsProcessed = 1u; + work->result = C2_OK; +#if DEBUG_FRAMES + ALOGD("outputFrame: work(finish) idx=%" PRIu64 ", processed=%u, result=%d", + work->input.ordinal.frameIndex.peeku(), work->workletsProcessed, work->result); +#endif + }; + + finish(mFrame->best_effort_timestamp, fillWork); + } + } else { + return err; + } + + return C2_OK; +} + +void C2FFMPEGVideoDecodeComponent::process( + const std::unique_ptr &work, + const std::shared_ptr &pool +) { + size_t inSize = 0u; + bool eos = (work->input.flags & C2FrameData::FLAG_END_OF_STREAM); + C2ReadView rView = mDummyReadView; + bool hasInputBuffer = false; + + if (! work->input.buffers.empty()) { + rView = work->input.buffers[0]->data().linearBlocks().front().map().get(); + inSize = rView.capacity(); + hasInputBuffer = true; + } + +#if DEBUG_FRAMES + ALOGD("process: input flags=%08x ts=%lu idx=%lu #buf=%lu[%lu] #conf=%lu #info=%lu", + work->input.flags, work->input.ordinal.timestamp.peeku(), work->input.ordinal.frameIndex.peeku(), + work->input.buffers.size(), inSize, work->input.configUpdate.size(), work->input.infoBuffers.size()); +#endif + + if (mEOSSignalled) { + ALOGE("process: ignoring work while EOS reached"); + work->workletsProcessed = 0u; + work->result = C2_BAD_VALUE; + return; + } + + if (hasInputBuffer && rView.error()) { + ALOGE("process: read view map failed err = %d", rView.error()); + work->workletsProcessed = 0u; + work->result = rView.error(); + return; + } + + // In all cases the work is marked as completed. + // + // There is not always a 1:1 mapping between input and output frames, in particular for + // interlaced content. Keeping the corresponding worklets in the queue quickly fills it + // in and stalls the decoder. But there's no obvious mechanism to determine, from + // FFMPEG API, whether a given packet will produce an output frame and the worklet should + // be kept around so it can be completed when the frame is produced. + // + // NOTE: This has an impact on the drain operation. + + work->result = C2_OK; + work->worklets.front()->output.flags = (C2FrameData::flags_t)0; + work->workletsProcessed = 0u; + + if (inSize || (eos && mCodecAlreadyOpened)) { + c2_status_t err = C2_OK; + + if (work->input.flags & C2FrameData::FLAG_CODEC_CONFIG) { + work->workletsProcessed = 1u; + work->result = processCodecConfig(&rView); + return; + } + + if (! mCodecAlreadyOpened) { + err = openDecoder(); + if (err != C2_OK) { + work->workletsProcessed = 1u; + work->result = err; + return; + } + } + + bool inputConsumed = false; + bool outputAvailable = true; + bool hasPicture = false; +#if DEBUG_FRAMES + int outputFrameCount = 0; +#endif + + while (!inputConsumed || outputAvailable) { + if (!inputConsumed) { + err = sendInputBuffer(&rView, work->input.ordinal.frameIndex.peekll()); + if (err == C2_OK) { + inputConsumed = true; + outputAvailable = true; + work->input.buffers.clear(); + } else if (err != C2_BAD_STATE) { + work->workletsProcessed = 1u; + work->result = err; + return; + } + } + + if (outputAvailable) { + hasPicture = false; + err = receiveFrame(&hasPicture); + if (err != C2_OK) { + work->workletsProcessed = 1u; + work->result = err; + return; + } + + if (hasPicture) { + err = outputFrame(work, pool); + if (err != C2_OK) { + work->workletsProcessed = 1u; + work->result = err; + return; + } +#if DEBUG_FRAMES + else { + outputFrameCount++; + } +#endif + } + else { +#if DEBUG_FRAMES + if (!outputFrameCount) { + ALOGD("process: no frame"); + } +#endif + outputAvailable = false; + } + } + } + } +#if DEBUG_FRAMES + else { + ALOGD("process: empty work"); + } +#endif + + if (eos) { + mEOSSignalled = true; + work->worklets.front()->output.flags = C2FrameData::FLAG_END_OF_STREAM; + work->workletsProcessed = 1u; + } + + if (work->workletsProcessed == 0u) { + pushPendingWork(work); + } + +#if DEBUG_FRAMES + ALOGD("process: work(end) idx=%" PRIu64 ", processed=%u, result=%d", + work->input.ordinal.frameIndex.peeku(), work->workletsProcessed, work->result); +#endif +} + +c2_status_t C2FFMPEGVideoDecodeComponent::drain( + uint32_t drainMode, + const std::shared_ptr& pool +) { + ALOGD("drain: mode = %u", drainMode); + + if (drainMode == NO_DRAIN) { + ALOGW("drain: NO_DRAIN is no-op"); + return C2_OK; + } + if (drainMode == DRAIN_CHAIN) { + ALOGW("drain: DRAIN_CHAIN not supported"); + return C2_OMITTED; + } + if (! mCodecAlreadyOpened) { + ALOGW("drain: codec not opened yet"); + return C2_OK; + } + + bool hasPicture = false; + c2_status_t err = C2_OK; + + err = sendInputBuffer(NULL, 0); + while (err == C2_OK) { + hasPicture = false; + err = receiveFrame(&hasPicture); + if (hasPicture) { + // Ignore errors at this point, just drain the decoder. + outputFrame(nullptr, pool); + } else { + err = C2_NOT_FOUND; + } + } + + return C2_OK; +} + +} // namespace android diff --git a/C2FFMPEGVideoDecodeComponent.h b/C2FFMPEGVideoDecodeComponent.h new file mode 100644 index 0000000..241ddcf --- /dev/null +++ b/C2FFMPEGVideoDecodeComponent.h @@ -0,0 +1,83 @@ +/* + * Copyright 2022 Michael Goffioul + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef C2_FFMPEG_VIDEO_DECODE_COMPONENT_H +#define C2_FFMPEG_VIDEO_DECODE_COMPONENT_H + +#include +#include +#include +#include "C2FFMPEGCommon.h" +#include "C2FFMPEGVideoDecodeInterface.h" + +namespace android { + +typedef std::pair PendingWork; + +class C2FFMPEGVideoDecodeComponent : public SimpleC2Component { +public: + explicit C2FFMPEGVideoDecodeComponent( + const C2FFMPEGComponentInfo* componentInfo, + const std::shared_ptr& intf); + virtual ~C2FFMPEGVideoDecodeComponent(); + +protected: + c2_status_t onInit() override; + c2_status_t onStop() override; + void onReset() override; + void onRelease() override; + c2_status_t onFlush_sm() override; + void process( + const std::unique_ptr &work, + const std::shared_ptr &pool) override; + c2_status_t drain( + uint32_t drainMode, + const std::shared_ptr &pool) override; + +private: + c2_status_t initDecoder(); + c2_status_t openDecoder(); + void deInitDecoder(); + c2_status_t processCodecConfig(C2ReadView* inBuffer); + c2_status_t sendInputBuffer(C2ReadView* inBuffer, int64_t timestamp); + c2_status_t receiveFrame(bool* hasPicture); + c2_status_t getOutputBuffer(C2GraphicView* outBuffer); + c2_status_t outputFrame( + const std::unique_ptr &work, + const std::shared_ptr &pool); + + void pushPendingWork(const std::unique_ptr& work); + void popPendingWork(const std::unique_ptr& work); + void prunePendingWorksUntil(const std::unique_ptr& work); + +private: + const C2FFMPEGComponentInfo* mInfo; + std::shared_ptr mIntf; + enum AVCodecID mCodecID; + AVCodecContext* mCtx; + struct SwsContext *mImgConvertCtx; + AVFrame* mFrame; + AVPacket* mPacket; + bool mFFMPEGInitialized; + bool mCodecAlreadyOpened; + bool mExtradataReady; + bool mEOSSignalled; + std::deque mPendingWorkQueue; +}; + +} // namespace android + +#endif // C2_FFMPEG_VIDEO_DECODE_COMPONENT_H diff --git a/C2FFMPEGVideoDecodeInterface.cpp b/C2FFMPEGVideoDecodeInterface.cpp new file mode 100644 index 0000000..0809261 --- /dev/null +++ b/C2FFMPEGVideoDecodeInterface.cpp @@ -0,0 +1,275 @@ +/* + * Copyright 2022 Michael Goffioul + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "C2FFMPEGVideoDecodeInterface" +#include +#include +#include + +#include +#include "C2FFMPEGVideoDecodeInterface.h" + +namespace android { + +constexpr size_t kMaxDimension = 4080; + +C2FFMPEGVideoDecodeInterface::C2FFMPEGVideoDecodeInterface( + const C2FFMPEGComponentInfo* componentInfo, + const std::shared_ptr& helper) + : SimpleInterface::BaseParams( + helper, + componentInfo->name, + C2Component::KIND_DECODER, + C2Component::DOMAIN_VIDEO, + componentInfo->mediaType) { + noPrivateBuffers(); + noInputReferences(); + noOutputReferences(); + noInputLatency(); + noTimeStretch(); + setDerivedInstance(this); + + addParameter( + DefineParam(mAttrib, C2_PARAMKEY_COMPONENT_ATTRIBUTES) + .withConstValue(new C2ComponentAttributesSetting(C2Component::ATTRIB_IS_TEMPORAL)) + .build()); + + addParameter( + DefineParam(mSize, C2_PARAMKEY_PICTURE_SIZE) + .withDefault(new C2StreamPictureSizeInfo::output(0u, 320, 240)) + .withFields({ + C2F(mSize, width).inRange(16, kMaxDimension, 2), + C2F(mSize, height).inRange(16, kMaxDimension, 2), + }) + .withSetter(SizeSetter) + .build()); + + if (strcasecmp(componentInfo->mediaType, MEDIA_MIMETYPE_VIDEO_MPEG2) == 0) { + addParameter( + DefineParam(mActualOutputDelay, C2_PARAMKEY_OUTPUT_DELAY) + .withConstValue(new C2PortActualDelayTuning::output(3u)) + .build()); + + addParameter( + DefineParam(mProfileLevel, C2_PARAMKEY_PROFILE_LEVEL) + .withDefault(new C2StreamProfileLevelInfo::input(0u, + C2Config::PROFILE_MP2V_SIMPLE, C2Config::LEVEL_MP2V_HIGH)) + .withFields({ + C2F(mProfileLevel, profile).oneOf({ + C2Config::PROFILE_MP2V_SIMPLE, + C2Config::PROFILE_MP2V_MAIN}), + C2F(mProfileLevel, level).oneOf({ + C2Config::LEVEL_MP2V_LOW, + C2Config::LEVEL_MP2V_MAIN, + C2Config::LEVEL_MP2V_HIGH_1440, + C2Config::LEVEL_MP2V_HIGH}) + }) + .withSetter(ProfileLevelSetter, mSize) + .build()); + } + + else if (strcasecmp(componentInfo->mediaType, MEDIA_MIMETYPE_VIDEO_AVC) == 0) { + addParameter( + DefineParam(mActualOutputDelay, C2_PARAMKEY_OUTPUT_DELAY) + .withDefault(new C2PortActualDelayTuning::output(8u)) + .withFields({C2F(mActualOutputDelay, value).inRange(0, 34u)}) + .withSetter(Setter::StrictValueWithNoDeps) + .build()); + + addParameter( + DefineParam(mProfileLevel, C2_PARAMKEY_PROFILE_LEVEL) + .withDefault(new C2StreamProfileLevelInfo::input(0u, + C2Config::PROFILE_AVC_CONSTRAINED_BASELINE, C2Config::LEVEL_AVC_5_2)) + .withFields({ + C2F(mProfileLevel, profile).oneOf({ + C2Config::PROFILE_AVC_CONSTRAINED_BASELINE, + C2Config::PROFILE_AVC_BASELINE, + C2Config::PROFILE_AVC_MAIN, + C2Config::PROFILE_AVC_CONSTRAINED_HIGH, + C2Config::PROFILE_AVC_PROGRESSIVE_HIGH, + C2Config::PROFILE_AVC_HIGH}), + C2F(mProfileLevel, level).oneOf({ + C2Config::LEVEL_AVC_1, C2Config::LEVEL_AVC_1B, C2Config::LEVEL_AVC_1_1, + C2Config::LEVEL_AVC_1_2, C2Config::LEVEL_AVC_1_3, + C2Config::LEVEL_AVC_2, C2Config::LEVEL_AVC_2_1, C2Config::LEVEL_AVC_2_2, + C2Config::LEVEL_AVC_3, C2Config::LEVEL_AVC_3_1, C2Config::LEVEL_AVC_3_2, + C2Config::LEVEL_AVC_4, C2Config::LEVEL_AVC_4_1, C2Config::LEVEL_AVC_4_2, + C2Config::LEVEL_AVC_5, C2Config::LEVEL_AVC_5_1, C2Config::LEVEL_AVC_5_2 + }) + }) + .withSetter(ProfileLevelSetter, mSize) + .build()); + } + + else if (strcasecmp(componentInfo->mediaType, MEDIA_MIMETYPE_VIDEO_HEVC) == 0) { + addParameter( + DefineParam(mActualOutputDelay, C2_PARAMKEY_OUTPUT_DELAY) + .withDefault(new C2PortActualDelayTuning::output(8u)) + .withFields({C2F(mActualOutputDelay, value).inRange(0, 34u)}) + .withSetter(Setter::StrictValueWithNoDeps) + .build()); + + addParameter( + DefineParam(mProfileLevel, C2_PARAMKEY_PROFILE_LEVEL) + .withDefault(new C2StreamProfileLevelInfo::input(0u, + C2Config::PROFILE_HEVC_MAIN, C2Config::LEVEL_HEVC_MAIN_5_1)) + .withFields({ + C2F(mProfileLevel, profile).oneOf({ + C2Config::PROFILE_HEVC_MAIN, + C2Config::PROFILE_HEVC_MAIN_10, + C2Config::PROFILE_HEVC_MAIN_STILL}), + C2F(mProfileLevel, level).oneOf({ + C2Config::LEVEL_HEVC_MAIN_1, + C2Config::LEVEL_HEVC_MAIN_2, C2Config::LEVEL_HEVC_MAIN_2_1, + C2Config::LEVEL_HEVC_MAIN_3, C2Config::LEVEL_HEVC_MAIN_3_1, + C2Config::LEVEL_HEVC_MAIN_4, C2Config::LEVEL_HEVC_MAIN_4_1, + C2Config::LEVEL_HEVC_MAIN_5, C2Config::LEVEL_HEVC_MAIN_5_1, + C2Config::LEVEL_HEVC_MAIN_5_2, C2Config::LEVEL_HEVC_HIGH_4, + C2Config::LEVEL_HEVC_HIGH_4_1, C2Config::LEVEL_HEVC_HIGH_5, + C2Config::LEVEL_HEVC_HIGH_5_1, C2Config::LEVEL_HEVC_HIGH_5_2 + }) + }) + .withSetter(ProfileLevelSetter, mSize) + .build()); + } + + else if (strcasecmp(componentInfo->mediaType, MEDIA_MIMETYPE_VIDEO_AV1) == 0) { + addParameter( + DefineParam(mActualOutputDelay, C2_PARAMKEY_OUTPUT_DELAY) + .withDefault(new C2PortActualDelayTuning::output(8u)) + .withFields({C2F(mActualOutputDelay, value).inRange(0, 34u)}) + .withSetter(Setter::StrictValueWithNoDeps) + .build()); + + addParameter( + DefineParam(mProfileLevel, C2_PARAMKEY_PROFILE_LEVEL) + .withDefault(new C2StreamProfileLevelInfo::input(0u, + C2Config::PROFILE_AV1_0, C2Config::LEVEL_AV1_2_1)) + .withFields({ + C2F(mProfileLevel, profile).oneOf({ + C2Config::PROFILE_AV1_0, + C2Config::PROFILE_AV1_1}), + C2F(mProfileLevel, level).oneOf({ + C2Config::LEVEL_AV1_2, C2Config::LEVEL_AV1_2_1, + C2Config::LEVEL_AV1_2_2, C2Config::LEVEL_AV1_2_3, + C2Config::LEVEL_AV1_3, C2Config::LEVEL_AV1_3_1, + C2Config::LEVEL_AV1_3_2, C2Config::LEVEL_AV1_3_3, + C2Config::LEVEL_AV1_4, C2Config::LEVEL_AV1_4_1, + C2Config::LEVEL_AV1_4_2, C2Config::LEVEL_AV1_4_3, + C2Config::LEVEL_AV1_5, C2Config::LEVEL_AV1_5_1, + C2Config::LEVEL_AV1_5_2, C2Config::LEVEL_AV1_5_3 + }) + }) + .withSetter(ProfileLevelSetter, mSize) + .build()); + } + + else { + int nthreads = base::GetIntProperty("debug.ffmpeg_codec2.threads", 0); + + if (nthreads <= 0) { + nthreads = std::thread::hardware_concurrency(); + } + + addParameter( + DefineParam(mActualOutputDelay, C2_PARAMKEY_OUTPUT_DELAY) + .withConstValue(new C2PortActualDelayTuning::output(2 * nthreads)) + .build()); + + if (strcasecmp(componentInfo->mediaType, MEDIA_MIMETYPE_VIDEO_VP9) == 0) { + addParameter( + DefineParam(mProfileLevel, C2_PARAMKEY_PROFILE_LEVEL) + .withDefault(new C2StreamProfileLevelInfo::input(0u, + C2Config::PROFILE_VP9_0, C2Config::LEVEL_VP9_5)) + .withFields({ + C2F(mProfileLevel, profile).oneOf({ + C2Config::PROFILE_VP9_0, + C2Config::PROFILE_VP9_2}), + C2F(mProfileLevel, level).oneOf({ + C2Config::LEVEL_VP9_1, + C2Config::LEVEL_VP9_1_1, + C2Config::LEVEL_VP9_2, + C2Config::LEVEL_VP9_2_1, + C2Config::LEVEL_VP9_3, + C2Config::LEVEL_VP9_3_1, + C2Config::LEVEL_VP9_4, + C2Config::LEVEL_VP9_4_1, + C2Config::LEVEL_VP9_5, + }) + }) + .withSetter(ProfileLevelSetter, mSize) + .build()); + } + } + + C2ChromaOffsetStruct locations[1] = { C2ChromaOffsetStruct::ITU_YUV_420_0() }; + std::shared_ptr defaultColorInfo = + C2StreamColorInfo::output::AllocShared( + 1u, 0u, 8u /* bitDepth */, C2Color::YUV_420); + memcpy(defaultColorInfo->m.locations, locations, sizeof(locations)); + + defaultColorInfo = + C2StreamColorInfo::output::AllocShared( + { C2ChromaOffsetStruct::ITU_YUV_420_0() }, + 0u, 8u /* bitDepth */, C2Color::YUV_420); + helper->addStructDescriptors(); + + addParameter( + DefineParam(mColorInfo, C2_PARAMKEY_CODED_COLOR_INFO) + .withConstValue(defaultColorInfo) + .build()); + + addParameter( + DefineParam(mPixelFormat, C2_PARAMKEY_PIXEL_FORMAT) + .withConstValue(new C2StreamPixelFormatInfo::output( + 0u, HAL_PIXEL_FORMAT_YV12)) + .build()); + + addParameter( + DefineParam(mConsumerUsage, C2_PARAMKEY_OUTPUT_STREAM_USAGE) + .withDefault(new C2StreamUsageTuning::output( + 0u, GRALLOC_USAGE_HW_TEXTURE | GRALLOC_USAGE_HW_COMPOSER)) + .withFields({C2F(mConsumerUsage, value).any()}) + .withSetter(Setter::StrictValueWithNoDeps) + .build()); +} + +C2R C2FFMPEGVideoDecodeInterface::SizeSetter( + bool /* mayBlock */, + const C2P &oldMe, + C2P &me) { + C2R res = C2R::Ok(); + + if (!me.F(me.v.width).supportsAtAll(me.v.width)) { + res = res.plus(C2SettingResultBuilder::BadValue(me.F(me.v.width))); + me.set().width = oldMe.v.width; + } + if (!me.F(me.v.height).supportsAtAll(me.v.height)) { + res = res.plus(C2SettingResultBuilder::BadValue(me.F(me.v.height))); + me.set().height = oldMe.v.height; + } + + return res; +} + +C2R C2FFMPEGVideoDecodeInterface::ProfileLevelSetter( + bool /* mayBlock */, + C2P& /* me */, + const C2P& /* size */) { + return C2R::Ok(); +} + +} // namespace android diff --git a/C2FFMPEGVideoDecodeInterface.h b/C2FFMPEGVideoDecodeInterface.h new file mode 100644 index 0000000..e1dd2be --- /dev/null +++ b/C2FFMPEGVideoDecodeInterface.h @@ -0,0 +1,58 @@ +/* + * Copyright 2022 Michael Goffioul + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef C2_FFMPEG_VIDEO_DECODE_INTERFACE_H +#define C2_FFMPEG_VIDEO_DECODE_INTERFACE_H + +#include +#include "C2FFMPEGCommon.h" + +namespace android { + +class C2FFMPEGVideoDecodeInterface : public SimpleInterface::BaseParams { +public: + explicit C2FFMPEGVideoDecodeInterface( + const C2FFMPEGComponentInfo* componentInfo, + const std::shared_ptr& helper); + + uint32_t getWidth() const { return mSize->width; } + uint32_t getHeight() const { return mSize->height; } + uint64_t getConsumerUsage() const { return mConsumerUsage->value; } + const std::shared_ptr& + getPixelFormatInfo() const { return mPixelFormat; } + uint32_t getOutputDelay() const { return mActualOutputDelay->value; } + +private: + static C2R SizeSetter( + bool mayBlock, + const C2P &oldMe, + C2P &me); + static C2R ProfileLevelSetter( + bool mayBlock, + C2P &me, + const C2P &size); + +private: + std::shared_ptr mSize; + std::shared_ptr mProfileLevel; + std::shared_ptr mColorInfo; + std::shared_ptr mPixelFormat; + std::shared_ptr mConsumerUsage; +}; + +} // namespace android + +#endif // C2_FFMPEG_VIDEO_DECODE_INTERFACE_H diff --git a/android.hardware.media.c2-service-ffmpeg.rc b/android.hardware.media.c2-service-ffmpeg.rc new file mode 100644 index 0000000..21c5308 --- /dev/null +++ b/android.hardware.media.c2-service-ffmpeg.rc @@ -0,0 +1,22 @@ +# +# Copyright 2022 Michael Goffioul +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +service android-hardware-media-c2-ffmpeg-hal /apex/com.android.hardware.media.c2.ffmpeg/bin/hw/android.hardware.media.c2-service-ffmpeg + class hal + user media + group mediadrm drmrpc + ioprio rt 4 + task_profiles ProcessCapacityHigh diff --git a/android.hardware.media.c2-service-ffmpeg.xml b/android.hardware.media.c2-service-ffmpeg.xml new file mode 100644 index 0000000..13a836b --- /dev/null +++ b/android.hardware.media.c2-service-ffmpeg.xml @@ -0,0 +1,22 @@ + + + + + android.hardware.media.c2 + 1 + IComponentStore/ffmpeg + + diff --git a/android.hardware.media.c2@1.2-service-ffmpeg.rc b/android.hardware.media.c2@1.2-service-ffmpeg.rc new file mode 100644 index 0000000..ddca68c --- /dev/null +++ b/android.hardware.media.c2@1.2-service-ffmpeg.rc @@ -0,0 +1,22 @@ +# +# Copyright 2022 Michael Goffioul +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +service android-hardware-media-c2-ffmpeg-hal-1-2 /vendor/bin/hw/android.hardware.media.c2@1.2-service-ffmpeg + class hal + user media + group mediadrm drmrpc + ioprio rt 4 + task_profiles ProcessCapacityHigh diff --git a/android.hardware.media.c2@1.2-service-ffmpeg.xml b/android.hardware.media.c2@1.2-service-ffmpeg.xml new file mode 100644 index 0000000..a59f7d9 --- /dev/null +++ b/android.hardware.media.c2@1.2-service-ffmpeg.xml @@ -0,0 +1,22 @@ + + + + + android.hardware.media.c2 + hwbinder + @1.2::IComponentStore/ffmpeg + + diff --git a/apex_file_contexts b/apex_file_contexts new file mode 100644 index 0000000..70d017c --- /dev/null +++ b/apex_file_contexts @@ -0,0 +1,3 @@ +(/.*)? u:object_r:vendor_file:s0 +/etc(/.*)? u:object_r:vendor_configs_file:s0 +/bin/hw/android\.hardware\.media\.c2-service-ffmpeg u:object_r:mediacodec_exec:s0 diff --git a/apex_manifest.json b/apex_manifest.json new file mode 100644 index 0000000..7c52795 --- /dev/null +++ b/apex_manifest.json @@ -0,0 +1,4 @@ +{ + "name": "com.android.hardware.media.c2.ffmpeg", + "version": 1 +} diff --git a/ffmpeg_utils/Android.bp b/ffmpeg_utils/Android.bp new file mode 100644 index 0000000..71787e2 --- /dev/null +++ b/ffmpeg_utils/Android.bp @@ -0,0 +1,26 @@ +// Copyright (C) 2017 The Android-x86 Open Source Project +// Copyright (C) 2025 KonstaKANG +// +// SPDX-License-Identifier: Apache-2.0 + +cc_library_shared { + name: "libffmpeg_utils", + vendor: true, + export_include_dirs: [ + ".", + ], + srcs: [ + "ffmpeg_hwaccel.c", + "ffmpeg_utils.cpp", + ], + shared_libs: [ + "libavcodec", + "libavformat", + "libavutil", + "libcutils", + "liblog", + "libswresample", + "libswscale", + "libutils", + ], +} diff --git a/ffmpeg_utils/ffmpeg_hwaccel.c b/ffmpeg_utils/ffmpeg_hwaccel.c new file mode 100644 index 0000000..f6cdcf1 --- /dev/null +++ b/ffmpeg_utils/ffmpeg_hwaccel.c @@ -0,0 +1,97 @@ +#define DEBUG_HWACCEL 0 +#define LOG_TAG "HWACCEL" +#include +#include + +#include "ffmpeg_hwaccel.h" +#include "libavutil/opt.h" + +int ffmpeg_hwaccel_init(AVCodecContext *avctx) { + if (avctx->codec_id != AV_CODEC_ID_HEVC || !property_get_bool("persist.vendor.ffmpeg_codec2.v4l2.h265", 0)) + return 0; + + // Find codec information. At this point, AVCodecContext.codec may not be + // set yet, so retrieve our own version using AVCodecContext.codec_id. + const AVCodec* codec = avcodec_find_decoder(avctx->codec_id); + if (!codec) { + ALOGE("ffmpeg_hwaccel_init: codec not found = %d", avctx->codec_id); + return 0; + } + + // Find a working HW configuration for this codec. + for (int i = 0;; i++) { + const AVCodecHWConfig* config = avcodec_get_hw_config(codec, i); + if (!config) { + // No more HW configs available. + break; + } + + // Try to initialize HW device. + if (av_hwdevice_ctx_create(&avctx->hw_device_ctx, config->device_type, NULL, NULL, 0) < 0) { + // Initialization failed, skip this HW config. + ALOGD_IF(DEBUG_HWACCEL, "ffmpeg_hwaccel_init: failed to initialize HW device %s", + av_hwdevice_get_type_name(config->device_type)); + continue; + } + + // Use refcounted frames. + av_opt_set_int(avctx, "refcounted_frames", 1, 0); + // Don't use multithreading. + avctx->thread_count = 1; + + // HW device created, stop here. + ALOGD("ffmpeg_hwaccel_init: %p [%s], hw device = %s", avctx, codec->name, + av_hwdevice_get_type_name(config->device_type)); + break; + } + + if (!avctx->hw_device_ctx) { + ALOGD("ffmpeg_hwaccel_init: no HW accel found for codec = %s", codec->name); + } + + return 0; +} + +void ffmpeg_hwaccel_deinit(AVCodecContext *avctx __unused) { +} + +int ffmpeg_hwaccel_get_frame(AVCodecContext *avctx __unused, AVFrame *frame) { + if (!frame->hw_frames_ctx) { + // Frame is not hw-accel + return 0; + } + + AVFrame* output; + int err; + + output = av_frame_alloc(); + if (!output) { + return AVERROR(ENOMEM); + } + + output->format = AV_PIX_FMT_NV12; + + err = av_hwframe_transfer_data(output, frame, 0); + if (err < 0) { + ALOGE("ffmpeg_hwaccel_get_frame failed to transfer data: %s (%08x)", + av_err2str(err), err); + goto fail; + } + + err = av_frame_copy_props(output, frame); + if (err < 0) { + ALOGE("ffmpeg_hwaccel_get_frame failed to copy frame properties: %s (%08x)", + av_err2str(err), err); + goto fail; + } + + av_frame_unref(frame); + av_frame_move_ref(frame, output); + av_frame_free(&output); + + return 0; + +fail: + av_frame_free(&output); + return err; +} diff --git a/ffmpeg_utils/ffmpeg_hwaccel.h b/ffmpeg_utils/ffmpeg_hwaccel.h new file mode 100644 index 0000000..0d520d1 --- /dev/null +++ b/ffmpeg_utils/ffmpeg_hwaccel.h @@ -0,0 +1,18 @@ +#ifndef FFMPEG_HWACCEL_H +#define FFMPEG_HWACCEL_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "libavcodec/avcodec.h" + +extern int ffmpeg_hwaccel_init(AVCodecContext *avctx); +extern void ffmpeg_hwaccel_deinit(AVCodecContext *avctx); +extern int ffmpeg_hwaccel_get_frame(AVCodecContext *avctx, AVFrame *frame); + +#ifdef __cplusplus +}; +#endif + +#endif diff --git a/ffmpeg_utils/ffmpeg_utils.cpp b/ffmpeg_utils/ffmpeg_utils.cpp new file mode 100644 index 0000000..d8a61e6 --- /dev/null +++ b/ffmpeg_utils/ffmpeg_utils.cpp @@ -0,0 +1,200 @@ +/* + * Copyright 2012 Michael Chen + * Copyright 2015 The CyanogenMod Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "FFMPEG" + +#include +#include + +#include "ffmpeg_utils.h" + +#define LOG_BUF_SIZE 1024 + +static int flags = AV_LOG_SKIP_REPEATED; + +static pthread_mutex_t s_init_mutex = PTHREAD_MUTEX_INITIALIZER; +static int s_ref_count = 0; + +namespace android { + +static void sanitize(uint8_t *line){ + while (*line) { + if (*line < 0x08 || (*line > 0x0D && *line < 0x20)) + *line='?'; + line++; + } +} + +// TODO, remove static variables to support multi-instances +void nam_av_log_callback(void* ptr, int level, const char* fmt, va_list vl) +{ + static int print_prefix = 1; + static int count; + static char prev[LOG_BUF_SIZE]; + char line[LOG_BUF_SIZE]; + + if (level > av_log_get_level()) + return; + av_log_format_line(ptr, level, fmt, vl, line, sizeof(line), &print_prefix); + + if (print_prefix && (flags & AV_LOG_SKIP_REPEATED) && !strcmp(line, prev)){ + count++; + return; + } + if (count > 0) { + ALOGI("Last message repeated %d times\n", count); + count = 0; + } + strcpy(prev, line); + sanitize((uint8_t *)line); + + static char g_msg[LOG_BUF_SIZE]; + static int g_msg_len = 0; + + int saw_lf, check_len; + + do { + check_len = g_msg_len + strlen(line) + 1; + if (check_len <= LOG_BUF_SIZE) { + /* lf: Line feed ('\n') */ + saw_lf = (strchr(line, '\n') != NULL) ? 1 : 0; + strncpy(g_msg + g_msg_len, line, strlen(line)); + g_msg_len += strlen(line); + if (!saw_lf) { + /* skip */ + return; + } else { + /* attach the line feed */ + g_msg_len += 1; + g_msg[g_msg_len] = '\n'; + } + } else { + /* trace is fragmented */ + g_msg_len += 1; + g_msg[g_msg_len] = '\n'; + } + ALOGI("%s", g_msg); + /* reset g_msg and g_msg_len */ + memset(g_msg, 0, LOG_BUF_SIZE); + g_msg_len = 0; + } while (check_len > LOG_BUF_SIZE); +} + +static int parseLogLevel(const char* s) { + if (strcmp(s, "quiet") == 0) + return AV_LOG_QUIET; + else if (strcmp(s, "panic") == 0) + return AV_LOG_PANIC; + else if (strcmp(s, "fatal") == 0) + return AV_LOG_FATAL; + else if (strcmp(s, "error") == 0) + return AV_LOG_ERROR; + else if (strcmp(s, "warning") == 0) + return AV_LOG_WARNING; + else if (strcmp(s, "info") == 0) + return AV_LOG_INFO; + else if (strcmp(s, "verbose") == 0) + return AV_LOG_VERBOSE; + else if (strcmp(s, "debug") == 0) + return AV_LOG_DEBUG; + else if (strcmp(s, "trace") == 0) + return AV_LOG_TRACE; + else { + ALOGE("unsupported loglevel: %s", s); + return AV_LOG_INFO; + } +} + +/* + * To set ffmpeg log level, type this command on the console before starting playback: + * setprop debug.ffmpeg.loglevel [quiet|panic|fatal|error|warning|info|verbose|debug|trace] + */ +status_t initFFmpeg() +{ + status_t ret = OK; + char pval[PROPERTY_VALUE_MAX]; + + pthread_mutex_lock(&s_init_mutex); + + if (property_get("debug.ffmpeg.loglevel", pval, "info")) { + av_log_set_level(parseLogLevel(pval)); + } else { + av_log_set_level(AV_LOG_INFO); + } + + if (s_ref_count == 0) { + av_log_set_callback(nam_av_log_callback); + + /* global ffmpeg initialization */ + avformat_network_init(); + + ALOGI("FFMPEG initialized: %s", av_version_info()); + } + + // update counter + s_ref_count++; + + pthread_mutex_unlock(&s_init_mutex); + + return ret; +} + +void deInitFFmpeg() +{ + pthread_mutex_lock(&s_init_mutex); + + // update counter + s_ref_count--; + + if (s_ref_count == 0) { + avformat_network_deinit(); + ALOGD("FFMPEG deinitialized"); + } + + pthread_mutex_unlock(&s_init_mutex); +} + +bool setup_vorbis_extradata(uint8_t **extradata, int *extradata_size, + const uint8_t *header_start[3], const int header_len[3]) +{ + uint8_t *p = NULL; + int len = 0; + int i = 0; + + len = header_len[0] + header_len[1] + header_len[2]; + p = *extradata = (uint8_t *)av_mallocz(64 + len + len/255); + if (!p) { + ALOGE("oom for vorbis extradata"); + return false; + } + + *p++ = 2; + p += av_xiphlacing(p, header_len[0]); + p += av_xiphlacing(p, header_len[1]); + for (i = 0; i < 3; i++) { + if (header_len[i] > 0) { + memcpy(p, header_start[i], header_len[i]); + p += header_len[i]; + } + } + *extradata_size = p - *extradata; + + return true; +} + +} // namespace android + diff --git a/ffmpeg_utils/ffmpeg_utils.h b/ffmpeg_utils/ffmpeg_utils.h new file mode 100644 index 0000000..be67aac --- /dev/null +++ b/ffmpeg_utils/ffmpeg_utils.h @@ -0,0 +1,46 @@ +/* + * Copyright 2012 Michael Chen + * Copyright 2015 The CyanogenMod Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FFMPEG_UTILS_H_ +#define FFMPEG_UTILS_H_ + +#include + +extern "C" { + +#include "libavformat/avformat.h" +#include "libavcodec/avcodec.h" +#include "libswscale/swscale.h" +#include "libswresample/swresample.h" +#include "libavutil/opt.h" +#include "libavutil/pixdesc.h" + +} + +namespace android { + +void nam_av_log_callback(void* ptr, int level, const char* fmt, va_list vl); + +status_t initFFmpeg(); +void deInitFFmpeg(); + +bool setup_vorbis_extradata(uint8_t **extradata, int *extradata_size, + const uint8_t *header_start[3], const int header_len[3]); + +} // namespace android + +#endif // FFMPEG_UTILS_H_ diff --git a/main-hidl.cpp b/main-hidl.cpp new file mode 100644 index 0000000..613b66f --- /dev/null +++ b/main-hidl.cpp @@ -0,0 +1,81 @@ +/* + * Copyright (C) 2022 Michael Goffioul + * Copyright (C) 2025 KonstaKANG + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "android.hardware.media.c2@1.2-service-ffmpeg" + +#include "C2FFMPEGComponentStore.h" + +#include +#include +#include +#include +#include + +using namespace ::android; +using namespace ::android::hardware::media::c2::V1_2; + +// This is the absolute on-device path of the prebuild_etc module +// "android.hardware.media.c2-ffmpeg.policy" in Android.bp. +static constexpr char kBaseSeccompPolicyPath[] = + "/vendor/etc/seccomp_policy/" + "android.hardware.media.c2-ffmpeg.policy"; + +// Additional seccomp permissions can be added in this file. +// This file does not exist by default. +static constexpr char kExtSeccompPolicyPath[] = + "/vendor/etc/seccomp_policy/" + "android.hardware.media.c2-ffmpeg-extended.policy"; + +int main() { + LOG(DEBUG) << "android.hardware.media.c2@1.2-service-ffmpeg starting..."; + + // Set up minijail to limit system calls. + signal(SIGPIPE, SIG_IGN); + SetUpMinijail(kBaseSeccompPolicyPath, kExtSeccompPolicyPath); + + ProcessState::self()->startThreadPool(); + // Extra threads may be needed to handle a stacked IPC sequence that + // contains alternating binder and hwbinder calls. (See b/35283480.) + hardware::configureRpcThreadpool(8, true /* callerWillJoin */); + + // Create IComponentStore service. + sp store; + + // TODO: Replace this with + // store = new utils::ComponentStore( + // /* implementation of C2ComponentStore */); + LOG(DEBUG) << "Instantiating Codec2's IComponentStore service..."; + store = new utils::ComponentStore( + std::make_shared()); + + if (store == nullptr) { + LOG(ERROR) << "Cannot create Codec2's IComponentStore service."; + } else { + constexpr char const* serviceName = "ffmpeg"; + if (store->registerAsService(serviceName) != OK) { + LOG(ERROR) << "Cannot register Codec2's IComponentStore service" + " with instance name << \"" + << serviceName << "\"."; + } else { + LOG(DEBUG) << "Codec2's IComponentStore service registered. " + "Instance name: \"" << serviceName << "\"."; + } + } + + hardware::joinRpcThreadpool(); + return 0; +} diff --git a/main.cpp b/main.cpp new file mode 100644 index 0000000..63f4178 --- /dev/null +++ b/main.cpp @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2022 Michael Goffioul + * Copyright (C) 2025 KonstaKANG + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "android.hardware.media.c2-service-ffmpeg" + +#include "C2FFMPEGComponentStore.h" + +#include +#include +#include +#include +#include + +using namespace ::aidl::android::hardware::media::c2; + +// This is the absolute on-device path of the prebuild_etc module +// "android.hardware.media.c2-ffmpeg.policy" in Android.bp. +static constexpr char kBaseSeccompPolicyPath[] = + "/vendor/etc/seccomp_policy/" + "android.hardware.media.c2-ffmpeg.policy"; + +// Additional seccomp permissions can be added in this file. +// This file does not exist by default. +static constexpr char kExtSeccompPolicyPath[] = + "/vendor/etc/seccomp_policy/" + "android.hardware.media.c2-ffmpeg-extended.policy"; + +int main() { + LOG(DEBUG) << "android.hardware.media.c2-service-ffmpeg starting..."; + + // Set up minijail to limit system calls. + signal(SIGPIPE, SIG_IGN); + android::SetUpMinijail(kBaseSeccompPolicyPath, kExtSeccompPolicyPath); + + // Extra threads may be needed to handle a stacked IPC sequence that + // contains alternating binder and hwbinder calls. (See b/35283480.) + ABinderProcess_setThreadPoolMaxThreadCount(8); + ABinderProcess_startThreadPool(); + + // Create IComponentStore service. + std::shared_ptr store = ndk::SharedRefBase::make( + std::make_shared()); + + const std::string instance = std::string() + IComponentStore::descriptor + "/ffmpeg"; + binder_status_t status = AServiceManager_addService(store->asBinder().get(), instance.c_str()); + CHECK(status == STATUS_OK); + + ABinderProcess_joinThreadPool(); + return EXIT_FAILURE; // should not reach +}