/* * This file is part of FFmpeg. * * FFmpeg is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * FFmpeg is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with FFmpeg; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ #include "config.h" #include "h264dec.h" #include "hwaccel_internal.h" #include "hwconfig.h" #include "internal.h" #include "v4l2_request.h" typedef struct V4L2RequestContextH264 { V4L2RequestContext base; enum v4l2_stateless_h264_decode_mode decode_mode; enum v4l2_stateless_h264_start_code start_code; } V4L2RequestContextH264; typedef struct V4L2RequestControlsH264 { V4L2RequestPictureContext pic; struct v4l2_ctrl_h264_sps sps; struct v4l2_ctrl_h264_pps pps; struct v4l2_ctrl_h264_scaling_matrix scaling_matrix; struct v4l2_ctrl_h264_decode_params decode_params; struct v4l2_ctrl_h264_slice_params slice_params; struct v4l2_ctrl_h264_pred_weights pred_weights; bool pred_weights_required; bool first_slice; int num_slices; } V4L2RequestControlsH264; static uint8_t nalu_slice_start_code[] = { 0x00, 0x00, 0x01 }; static void fill_weight_factors(struct v4l2_h264_weight_factors *weight_factors, int list, const H264SliceContext *sl) { for (int i = 0; i < sl->ref_count[list]; i++) { if (sl->pwt.luma_weight_flag[list]) { weight_factors->luma_weight[i] = sl->pwt.luma_weight[i][list][0]; weight_factors->luma_offset[i] = sl->pwt.luma_weight[i][list][1]; } else { weight_factors->luma_weight[i] = 1 << sl->pwt.luma_log2_weight_denom; weight_factors->luma_offset[i] = 0; } for (int j = 0; j < 2; j++) { if (sl->pwt.chroma_weight_flag[list]) { weight_factors->chroma_weight[i][j] = sl->pwt.chroma_weight[i][list][j][0]; weight_factors->chroma_offset[i][j] = sl->pwt.chroma_weight[i][list][j][1]; } else { weight_factors->chroma_weight[i][j] = 1 << sl->pwt.chroma_log2_weight_denom; weight_factors->chroma_offset[i][j] = 0; } } } } static void fill_dpb_entry(struct v4l2_h264_dpb_entry *entry, const H264Picture *pic, int long_idx) { entry->reference_ts = ff_v4l2_request_get_capture_timestamp(pic->f); entry->pic_num = pic->pic_id; entry->frame_num = pic->long_ref ? long_idx : pic->frame_num; entry->fields = pic->reference & V4L2_H264_FRAME_REF; entry->flags = V4L2_H264_DPB_ENTRY_FLAG_VALID; if (entry->fields) entry->flags |= V4L2_H264_DPB_ENTRY_FLAG_ACTIVE; if (pic->long_ref) entry->flags |= V4L2_H264_DPB_ENTRY_FLAG_LONG_TERM; if (pic->field_picture) entry->flags |= V4L2_H264_DPB_ENTRY_FLAG_FIELD; if (pic->field_poc[0] != INT_MAX) entry->top_field_order_cnt = pic->field_poc[0]; if (pic->field_poc[1] != INT_MAX) entry->bottom_field_order_cnt = pic->field_poc[1]; } static void fill_dpb(struct v4l2_ctrl_h264_decode_params *decode_params, const H264Context *h) { int entries = 0; for (int i = 0; i < h->short_ref_count; i++) { const H264Picture *pic = h->short_ref[i]; if (pic && (pic->field_poc[0] != INT_MAX || pic->field_poc[1] != INT_MAX)) fill_dpb_entry(&decode_params->dpb[entries++], pic, pic->pic_id); } if (!h->long_ref_count) return; for (int i = 0; i < FF_ARRAY_ELEMS(h->long_ref); i++) { const H264Picture *pic = h->long_ref[i]; if (pic && (pic->field_poc[0] != INT_MAX || pic->field_poc[1] != INT_MAX)) fill_dpb_entry(&decode_params->dpb[entries++], pic, i); } } static void fill_ref_list(struct v4l2_h264_reference *reference, struct v4l2_ctrl_h264_decode_params *decode_params, const H264Ref *ref) { uint64_t timestamp; if (!ref->parent) return; timestamp = ff_v4l2_request_get_capture_timestamp(ref->parent->f); for (uint8_t i = 0; i < FF_ARRAY_ELEMS(decode_params->dpb); i++) { struct v4l2_h264_dpb_entry *entry = &decode_params->dpb[i]; if ((entry->flags & V4L2_H264_DPB_ENTRY_FLAG_VALID) && entry->reference_ts == timestamp) { reference->fields = ref->reference & V4L2_H264_FRAME_REF; reference->index = i; return; } } } static void fill_sps(struct v4l2_ctrl_h264_sps *ctrl, const H264Context *h) { const SPS *sps = h->ps.sps; *ctrl = (struct v4l2_ctrl_h264_sps) { .profile_idc = sps->profile_idc, .constraint_set_flags = sps->constraint_set_flags, .level_idc = sps->level_idc, .seq_parameter_set_id = sps->sps_id, .chroma_format_idc = sps->chroma_format_idc, .bit_depth_luma_minus8 = sps->bit_depth_luma - 8, .bit_depth_chroma_minus8 = sps->bit_depth_chroma - 8, .log2_max_frame_num_minus4 = sps->log2_max_frame_num - 4, .pic_order_cnt_type = sps->poc_type, .log2_max_pic_order_cnt_lsb_minus4 = sps->log2_max_poc_lsb - 4, .max_num_ref_frames = sps->ref_frame_count, .num_ref_frames_in_pic_order_cnt_cycle = sps->poc_cycle_length, .offset_for_non_ref_pic = sps->offset_for_non_ref_pic, .offset_for_top_to_bottom_field = sps->offset_for_top_to_bottom_field, .pic_width_in_mbs_minus1 = h->mb_width - 1, .pic_height_in_map_units_minus1 = sps->frame_mbs_only_flag ? h->mb_height - 1 : h->mb_height / 2 - 1, }; if (sps->poc_cycle_length > 0 && sps->poc_cycle_length <= 255) memcpy(ctrl->offset_for_ref_frame, sps->offset_for_ref_frame, sps->poc_cycle_length * sizeof(ctrl->offset_for_ref_frame[0])); if (sps->residual_color_transform_flag) ctrl->flags |= V4L2_H264_SPS_FLAG_SEPARATE_COLOUR_PLANE; if (sps->transform_bypass) ctrl->flags |= V4L2_H264_SPS_FLAG_QPPRIME_Y_ZERO_TRANSFORM_BYPASS; if (sps->delta_pic_order_always_zero_flag) ctrl->flags |= V4L2_H264_SPS_FLAG_DELTA_PIC_ORDER_ALWAYS_ZERO; if (sps->gaps_in_frame_num_allowed_flag) ctrl->flags |= V4L2_H264_SPS_FLAG_GAPS_IN_FRAME_NUM_VALUE_ALLOWED; if (sps->frame_mbs_only_flag) ctrl->flags |= V4L2_H264_SPS_FLAG_FRAME_MBS_ONLY; if (sps->mb_aff) ctrl->flags |= V4L2_H264_SPS_FLAG_MB_ADAPTIVE_FRAME_FIELD; if (sps->direct_8x8_inference_flag) ctrl->flags |= V4L2_H264_SPS_FLAG_DIRECT_8X8_INFERENCE; } static void fill_pps(struct v4l2_ctrl_h264_pps *ctrl, const H264Context *h) { const SPS *sps = h->ps.sps; const PPS *pps = h->ps.pps; const H264SliceContext *sl = &h->slice_ctx[0]; int qp_bd_offset = 6 * (sps->bit_depth_luma - 8); *ctrl = (struct v4l2_ctrl_h264_pps) { .pic_parameter_set_id = sl->pps_id, .seq_parameter_set_id = pps->sps_id, .num_slice_groups_minus1 = pps->slice_group_count - 1, .num_ref_idx_l0_default_active_minus1 = pps->ref_count[0] - 1, .num_ref_idx_l1_default_active_minus1 = pps->ref_count[1] - 1, .weighted_bipred_idc = pps->weighted_bipred_idc, .pic_init_qp_minus26 = pps->init_qp - 26 - qp_bd_offset, .pic_init_qs_minus26 = pps->init_qs - 26 - qp_bd_offset, .chroma_qp_index_offset = pps->chroma_qp_index_offset[0], .second_chroma_qp_index_offset = pps->chroma_qp_index_offset[1], }; if (pps->cabac) ctrl->flags |= V4L2_H264_PPS_FLAG_ENTROPY_CODING_MODE; if (pps->pic_order_present) ctrl->flags |= V4L2_H264_PPS_FLAG_BOTTOM_FIELD_PIC_ORDER_IN_FRAME_PRESENT; if (pps->weighted_pred) ctrl->flags |= V4L2_H264_PPS_FLAG_WEIGHTED_PRED; if (pps->deblocking_filter_parameters_present) ctrl->flags |= V4L2_H264_PPS_FLAG_DEBLOCKING_FILTER_CONTROL_PRESENT; if (pps->constrained_intra_pred) ctrl->flags |= V4L2_H264_PPS_FLAG_CONSTRAINED_INTRA_PRED; if (pps->redundant_pic_cnt_present) ctrl->flags |= V4L2_H264_PPS_FLAG_REDUNDANT_PIC_CNT_PRESENT; if (pps->transform_8x8_mode) ctrl->flags |= V4L2_H264_PPS_FLAG_TRANSFORM_8X8_MODE; /* FFmpeg always provide a scaling matrix */ ctrl->flags |= V4L2_H264_PPS_FLAG_SCALING_MATRIX_PRESENT; } static int v4l2_request_h264_start_frame(AVCodecContext *avctx, av_unused const uint8_t *buffer, av_unused uint32_t size) { const H264Context *h = avctx->priv_data; const PPS *pps = h->ps.pps; const SPS *sps = h->ps.sps; const H264SliceContext *sl = &h->slice_ctx[0]; V4L2RequestControlsH264 *controls = h->cur_pic_ptr->hwaccel_picture_private; int ret; ret = ff_v4l2_request_start_frame(avctx, &controls->pic, h->cur_pic_ptr->f); if (ret) return ret; fill_sps(&controls->sps, h); fill_pps(&controls->pps, h); memcpy(controls->scaling_matrix.scaling_list_4x4, pps->scaling_matrix4, sizeof(controls->scaling_matrix.scaling_list_4x4)); memcpy(controls->scaling_matrix.scaling_list_8x8[0], pps->scaling_matrix8[0], sizeof(controls->scaling_matrix.scaling_list_8x8[0])); memcpy(controls->scaling_matrix.scaling_list_8x8[1], pps->scaling_matrix8[3], sizeof(controls->scaling_matrix.scaling_list_8x8[1])); if (sps->chroma_format_idc == 3) { memcpy(controls->scaling_matrix.scaling_list_8x8[2], pps->scaling_matrix8[1], sizeof(controls->scaling_matrix.scaling_list_8x8[2])); memcpy(controls->scaling_matrix.scaling_list_8x8[3], pps->scaling_matrix8[4], sizeof(controls->scaling_matrix.scaling_list_8x8[3])); memcpy(controls->scaling_matrix.scaling_list_8x8[4], pps->scaling_matrix8[2], sizeof(controls->scaling_matrix.scaling_list_8x8[4])); memcpy(controls->scaling_matrix.scaling_list_8x8[5], pps->scaling_matrix8[5], sizeof(controls->scaling_matrix.scaling_list_8x8[5])); } controls->decode_params = (struct v4l2_ctrl_h264_decode_params) { .nal_ref_idc = h->nal_ref_idc, .frame_num = h->poc.frame_num, .top_field_order_cnt = h->cur_pic_ptr->field_poc[0] != INT_MAX ? h->cur_pic_ptr->field_poc[0] : 0, .bottom_field_order_cnt = h->cur_pic_ptr->field_poc[1] != INT_MAX ? h->cur_pic_ptr->field_poc[1] : 0, .idr_pic_id = sl->idr_pic_id, .pic_order_cnt_lsb = sl->poc_lsb, .delta_pic_order_cnt_bottom = sl->delta_poc_bottom, .delta_pic_order_cnt0 = sl->delta_poc[0], .delta_pic_order_cnt1 = sl->delta_poc[1], /* Size in bits of dec_ref_pic_marking() syntax element. */ .dec_ref_pic_marking_bit_size = sl->ref_pic_marking_bit_size, /* Size in bits of pic order count syntax. */ .pic_order_cnt_bit_size = sl->pic_order_cnt_bit_size, .slice_group_change_cycle = 0, /* slice group not supported by FFmpeg */ }; if (h->picture_idr) controls->decode_params.flags |= V4L2_H264_DECODE_PARAM_FLAG_IDR_PIC; if (FIELD_PICTURE(h)) controls->decode_params.flags |= V4L2_H264_DECODE_PARAM_FLAG_FIELD_PIC; if (h->picture_structure == PICT_BOTTOM_FIELD) controls->decode_params.flags |= V4L2_H264_DECODE_PARAM_FLAG_BOTTOM_FIELD; #if defined(V4L2_H264_DECODE_PARAM_FLAG_PFRAME) if (sl->slice_type_nos == AV_PICTURE_TYPE_P) controls->decode_params.flags |= V4L2_H264_DECODE_PARAM_FLAG_PFRAME; #endif #if defined(V4L2_H264_DECODE_PARAM_FLAG_BFRAME) if (sl->slice_type_nos == AV_PICTURE_TYPE_B) controls->decode_params.flags |= V4L2_H264_DECODE_PARAM_FLAG_BFRAME; #endif fill_dpb(&controls->decode_params, h); controls->first_slice = true; controls->num_slices = 0; return 0; } static int v4l2_request_h264_queue_decode(AVCodecContext *avctx, bool last_slice) { const H264Context *h = avctx->priv_data; V4L2RequestContextH264 *ctx = avctx->internal->hwaccel_priv_data; V4L2RequestControlsH264 *controls = h->cur_pic_ptr->hwaccel_picture_private; struct v4l2_ext_control control[] = { { .id = V4L2_CID_STATELESS_H264_SPS, .ptr = &controls->sps, .size = sizeof(controls->sps), }, { .id = V4L2_CID_STATELESS_H264_PPS, .ptr = &controls->pps, .size = sizeof(controls->pps), }, { .id = V4L2_CID_STATELESS_H264_SCALING_MATRIX, .ptr = &controls->scaling_matrix, .size = sizeof(controls->scaling_matrix), }, { .id = V4L2_CID_STATELESS_H264_DECODE_PARAMS, .ptr = &controls->decode_params, .size = sizeof(controls->decode_params), }, { .id = V4L2_CID_STATELESS_H264_SLICE_PARAMS, .ptr = &controls->slice_params, .size = sizeof(controls->slice_params), }, { .id = V4L2_CID_STATELESS_H264_PRED_WEIGHTS, .ptr = &controls->pred_weights, .size = sizeof(controls->pred_weights), }, }; if (ctx->decode_mode == V4L2_STATELESS_H264_DECODE_MODE_SLICE_BASED) { int count = FF_ARRAY_ELEMS(control) - (controls->pred_weights_required ? 0 : 1); return ff_v4l2_request_decode_slice(avctx, &controls->pic, control, count, controls->first_slice, last_slice); } return ff_v4l2_request_decode_frame(avctx, &controls->pic, control, FF_ARRAY_ELEMS(control) - 2); } static int v4l2_request_h264_decode_slice(AVCodecContext *avctx, const uint8_t *buffer, uint32_t size) { const H264Context *h = avctx->priv_data; const PPS *pps = h->ps.pps; const H264SliceContext *sl = &h->slice_ctx[0]; V4L2RequestContextH264 *ctx = avctx->internal->hwaccel_priv_data; V4L2RequestControlsH264 *controls = h->cur_pic_ptr->hwaccel_picture_private; int i, ret, count; if (ctx->decode_mode == V4L2_STATELESS_H264_DECODE_MODE_SLICE_BASED && controls->num_slices) { ret = v4l2_request_h264_queue_decode(avctx, false); if (ret) return ret; ff_v4l2_request_reset_picture(avctx, &controls->pic); controls->first_slice = 0; } if (ctx->start_code == V4L2_STATELESS_H264_START_CODE_ANNEX_B) { ret = ff_v4l2_request_append_output(avctx, &controls->pic, nalu_slice_start_code, 3); if (ret) return ret; } ret = ff_v4l2_request_append_output(avctx, &controls->pic, buffer, size); if (ret) return ret; if (ctx->decode_mode != V4L2_STATELESS_H264_DECODE_MODE_SLICE_BASED) return 0; controls->slice_params = (struct v4l2_ctrl_h264_slice_params) { /* Offset in bits to slice_data() from the beginning of this slice. */ .header_bit_size = get_bits_count(&sl->gb), .first_mb_in_slice = sl->first_mb_addr, .slice_type = ff_h264_get_slice_type(sl), .colour_plane_id = 0, /* separate colour plane not supported by FFmpeg */ .redundant_pic_cnt = sl->redundant_pic_count, .cabac_init_idc = sl->cabac_init_idc, .slice_qp_delta = sl->qscale - pps->init_qp, .slice_qs_delta = 0, /* not implemented by FFmpeg */ .disable_deblocking_filter_idc = sl->deblocking_filter < 2 ? !sl->deblocking_filter : sl->deblocking_filter, .slice_alpha_c0_offset_div2 = sl->slice_alpha_c0_offset / 2, .slice_beta_offset_div2 = sl->slice_beta_offset / 2, .num_ref_idx_l0_active_minus1 = sl->list_count > 0 ? sl->ref_count[0] - 1 : 0, .num_ref_idx_l1_active_minus1 = sl->list_count > 1 ? sl->ref_count[1] - 1 : 0, }; if (sl->slice_type == AV_PICTURE_TYPE_B && sl->direct_spatial_mv_pred) controls->slice_params.flags |= V4L2_H264_SLICE_FLAG_DIRECT_SPATIAL_MV_PRED; /* V4L2_H264_SLICE_FLAG_SP_FOR_SWITCH: not implemented by FFmpeg */ controls->pred_weights_required = V4L2_H264_CTRL_PRED_WEIGHTS_REQUIRED(&controls->pps, &controls->slice_params); if (controls->pred_weights_required) { controls->pred_weights.chroma_log2_weight_denom = sl->pwt.chroma_log2_weight_denom; controls->pred_weights.luma_log2_weight_denom = sl->pwt.luma_log2_weight_denom; } count = sl->list_count > 0 ? sl->ref_count[0] : 0; for (i = 0; i < count; i++) fill_ref_list(&controls->slice_params.ref_pic_list0[i], &controls->decode_params, &sl->ref_list[0][i]); if (count && controls->pred_weights_required) fill_weight_factors(&controls->pred_weights.weight_factors[0], 0, sl); count = sl->list_count > 1 ? sl->ref_count[1] : 0; for (i = 0; i < count; i++) fill_ref_list(&controls->slice_params.ref_pic_list1[i], &controls->decode_params, &sl->ref_list[1][i]); if (count && controls->pred_weights_required) fill_weight_factors(&controls->pred_weights.weight_factors[1], 1, sl); controls->num_slices++; return 0; } static int v4l2_request_h264_end_frame(AVCodecContext *avctx) { return v4l2_request_h264_queue_decode(avctx, true); } static int v4l2_request_h264_post_probe(AVCodecContext *avctx) { V4L2RequestContextH264 *ctx = avctx->internal->hwaccel_priv_data; struct v4l2_ext_control control[] = { { .id = V4L2_CID_STATELESS_H264_DECODE_MODE, }, { .id = V4L2_CID_STATELESS_H264_START_CODE, }, }; ctx->decode_mode = ff_v4l2_request_query_control_default_value(avctx, V4L2_CID_STATELESS_H264_DECODE_MODE); if (ctx->decode_mode != V4L2_STATELESS_H264_DECODE_MODE_SLICE_BASED && ctx->decode_mode != V4L2_STATELESS_H264_DECODE_MODE_FRAME_BASED) { av_log(ctx, AV_LOG_VERBOSE, "Unsupported decode mode: %d\n", ctx->decode_mode); return AVERROR(EINVAL); } ctx->start_code = ff_v4l2_request_query_control_default_value(avctx, V4L2_CID_STATELESS_H264_START_CODE); if (ctx->start_code != V4L2_STATELESS_H264_START_CODE_NONE && ctx->start_code != V4L2_STATELESS_H264_START_CODE_ANNEX_B) { av_log(ctx, AV_LOG_VERBOSE, "Unsupported start code: %d\n", ctx->start_code); return AVERROR(EINVAL); } // TODO: check V4L2_CID_MPEG_VIDEO_H264_PROFILE control // TODO: check V4L2_CID_MPEG_VIDEO_H264_LEVEL control control[0].value = ctx->decode_mode; control[1].value = ctx->start_code; return ff_v4l2_request_set_controls(avctx, control, FF_ARRAY_ELEMS(control)); } static int v4l2_request_h264_init(AVCodecContext *avctx) { V4L2RequestContextH264 *ctx = avctx->internal->hwaccel_priv_data; const H264Context *h = avctx->priv_data; struct v4l2_ctrl_h264_sps sps; struct v4l2_ext_control control[] = { { .id = V4L2_CID_STATELESS_H264_SPS, .ptr = &sps, .size = sizeof(sps), }, }; fill_sps(&sps, h); ctx->base.post_probe = v4l2_request_h264_post_probe; return ff_v4l2_request_init(avctx, V4L2_PIX_FMT_H264_SLICE, 4 * 1024 * 1024, control, FF_ARRAY_ELEMS(control)); } const FFHWAccel ff_h264_v4l2request_hwaccel = { .p.name = "h264_v4l2request", .p.type = AVMEDIA_TYPE_VIDEO, .p.id = AV_CODEC_ID_H264, .p.pix_fmt = AV_PIX_FMT_DRM_PRIME, .start_frame = v4l2_request_h264_start_frame, .decode_slice = v4l2_request_h264_decode_slice, .end_frame = v4l2_request_h264_end_frame, .flush = ff_v4l2_request_flush, .frame_priv_data_size = sizeof(V4L2RequestControlsH264), .init = v4l2_request_h264_init, .uninit = ff_v4l2_request_uninit, .priv_data_size = sizeof(V4L2RequestContextH264), .frame_params = ff_v4l2_request_frame_params, };