From 06787d947d4e73f327fbb5dba56da5755d2be906 Mon Sep 17 00:00:00 2001 From: Sil Vilerino Date: Mon, 12 Feb 2024 21:08:07 -0500 Subject: [PATCH] d3d12: Support writing H264_SEI_SCALABILITY_INFO header on demand Reviewed-By: Pohsiang (John) Hsu Part-of: --- src/gallium/drivers/d3d12/d3d12_video_enc.h | 1 + .../drivers/d3d12/d3d12_video_enc_h264.cpp | 35 ++++- ...2_video_encoder_bitstream_builder_h264.cpp | 18 +++ ...d12_video_encoder_bitstream_builder_h264.h | 6 + .../d3d12_video_encoder_nalu_writer_h264.cpp | 126 ++++++++++++++++++ .../d3d12_video_encoder_nalu_writer_h264.h | 24 ++++ 6 files changed, 205 insertions(+), 5 deletions(-) diff --git a/src/gallium/drivers/d3d12/d3d12_video_enc.h b/src/gallium/drivers/d3d12/d3d12_video_enc.h index cf830bdad0f..d413dfaec0c 100644 --- a/src/gallium/drivers/d3d12/d3d12_video_enc.h +++ b/src/gallium/drivers/d3d12/d3d12_video_enc.h @@ -136,6 +136,7 @@ enum d3d12_video_encoder_config_dirty_flags d3d12_video_encoder_config_dirty_flag_video_header = 0x1000, d3d12_video_encoder_config_dirty_flag_picture_header = 0x2000, d3d12_video_encoder_config_dirty_flag_aud_header = 0x4000, + d3d12_video_encoder_config_dirty_flag_sei_header = 0x8000, }; DEFINE_ENUM_FLAG_OPERATORS(d3d12_video_encoder_config_dirty_flags); diff --git a/src/gallium/drivers/d3d12/d3d12_video_enc_h264.cpp b/src/gallium/drivers/d3d12/d3d12_video_enc_h264.cpp index 84deac6ac10..07bbe8a3f21 100644 --- a/src/gallium/drivers/d3d12/d3d12_video_enc_h264.cpp +++ b/src/gallium/drivers/d3d12/d3d12_video_enc_h264.cpp @@ -931,6 +931,8 @@ d3d12_video_encoder_update_current_encoder_config_state_h264(struct d3d12_video_ pD3D12Enc->m_currentEncodeConfig.m_ConfigDirtyFlags |= d3d12_video_encoder_config_dirty_flag_picture_header; else if (header->type == PIPE_H264_NAL_AUD) pD3D12Enc->m_currentEncodeConfig.m_ConfigDirtyFlags |= d3d12_video_encoder_config_dirty_flag_aud_header; + else if (header->type == NAL_TYPE_SEI) + pD3D12Enc->m_currentEncodeConfig.m_ConfigDirtyFlags |= d3d12_video_encoder_config_dirty_flag_sei_header; } // Set input format @@ -1188,6 +1190,29 @@ d3d12_video_encoder_build_codec_headers_h264(struct d3d12_video_encoder *pD3D12E uint32_t active_seq_parameter_set_id = pH264BitstreamBuilder->get_active_sps().seq_parameter_set_id; + size_t writtenSEIBytesCount = 0; + bool forceWriteSEI = (pD3D12Enc->m_currentEncodeConfig.m_ConfigDirtyFlags & d3d12_video_encoder_config_dirty_flag_sei_header); + // We only support H264_SEI_SCALABILITY_INFO, so check num_temporal_layers > 1 + if (forceWriteSEI && + (pD3D12Enc->m_currentEncodeConfig.m_encoderCodecSpecificSequenceStateDescH264.num_temporal_layers > 1)) + { + std::vector sei_messages; + H264_SEI_MESSAGE scalability_info_sei = { }; + memset(&scalability_info_sei, 0, sizeof(scalability_info_sei)); + scalability_info_sei.payload_type = H264_SEI_SCALABILITY_INFO; + scalability_info_sei.scalability_info.num_layers_minus1 = pD3D12Enc->m_currentEncodeConfig.m_encoderCodecSpecificSequenceStateDescH264.num_temporal_layers - 1; + // Only support Identity temporal_id for now + for (uint32_t i = 0; i <= scalability_info_sei.scalability_info.num_layers_minus1; i++) + scalability_info_sei.scalability_info.temporal_id[i] = i; + + sei_messages.push_back(scalability_info_sei); + pH264BitstreamBuilder->write_sei_messages(sei_messages, + pD3D12Enc->m_BitstreamHeadersBuffer, + pD3D12Enc->m_BitstreamHeadersBuffer.begin() + writtenAUDBytesCount, + writtenSEIBytesCount); + pWrittenCodecUnitsSizes.push_back(writtenSEIBytesCount); + } + size_t writtenSPSBytesCount = 0; if (writeNewSPS) { H264_SPS sps = pH264BitstreamBuilder->build_sps(pD3D12Enc->m_currentEncodeConfig.m_encoderCodecSpecificSequenceStateDescH264, @@ -1200,7 +1225,7 @@ d3d12_video_encoder_build_codec_headers_h264(struct d3d12_video_encoder *pD3D12E pD3D12Enc->m_currentEncodeConfig.m_currentResolution, pD3D12Enc->m_currentEncodeConfig.m_FrameCroppingCodecConfig, pD3D12Enc->m_BitstreamHeadersBuffer, - pD3D12Enc->m_BitstreamHeadersBuffer.begin() + writtenAUDBytesCount, + pD3D12Enc->m_BitstreamHeadersBuffer.begin() + writtenAUDBytesCount + writtenSEIBytesCount, writtenSPSBytesCount); pH264BitstreamBuilder->set_active_sps(sps); pWrittenCodecUnitsSizes.push_back(writtenSPSBytesCount); @@ -1220,8 +1245,8 @@ d3d12_video_encoder_build_codec_headers_h264(struct d3d12_video_encoder *pD3D12E bool forceWritePPS = (pD3D12Enc->m_currentEncodeConfig.m_ConfigDirtyFlags & d3d12_video_encoder_config_dirty_flag_picture_header); if (forceWritePPS || d3d12_video_encoder_needs_new_pps_h264(pD3D12Enc, writeNewSPS, tentative_pps, active_pps)) { pH264BitstreamBuilder->set_active_pps(tentative_pps); - pD3D12Enc->m_BitstreamHeadersBuffer.resize(writtenAUDBytesCount + writtenSPSBytesCount + writtenPPSBytesCount); - memcpy(&pD3D12Enc->m_BitstreamHeadersBuffer.data()[writtenAUDBytesCount + writtenSPSBytesCount], pD3D12Enc->m_StagingHeadersBuffer.data(), writtenPPSBytesCount); + pD3D12Enc->m_BitstreamHeadersBuffer.resize(writtenAUDBytesCount + writtenSEIBytesCount + writtenSPSBytesCount + writtenPPSBytesCount); + memcpy(&pD3D12Enc->m_BitstreamHeadersBuffer.data()[writtenAUDBytesCount + writtenSEIBytesCount + writtenSPSBytesCount], pD3D12Enc->m_StagingHeadersBuffer.data(), writtenPPSBytesCount); pWrittenCodecUnitsSizes.push_back(writtenPPSBytesCount); } else { writtenPPSBytesCount = 0; @@ -1229,8 +1254,8 @@ d3d12_video_encoder_build_codec_headers_h264(struct d3d12_video_encoder *pD3D12E } // Shrink buffer to fit the headers - if (pD3D12Enc->m_BitstreamHeadersBuffer.size() > (writtenAUDBytesCount + writtenSPSBytesCount + writtenPPSBytesCount)) { - pD3D12Enc->m_BitstreamHeadersBuffer.resize(writtenAUDBytesCount + writtenSPSBytesCount + writtenPPSBytesCount); + if (pD3D12Enc->m_BitstreamHeadersBuffer.size() > (writtenAUDBytesCount + writtenSEIBytesCount + writtenSPSBytesCount + writtenPPSBytesCount)) { + pD3D12Enc->m_BitstreamHeadersBuffer.resize(writtenAUDBytesCount + writtenSEIBytesCount + writtenSPSBytesCount + writtenPPSBytesCount); } assert(std::accumulate(pWrittenCodecUnitsSizes.begin(), pWrittenCodecUnitsSizes.end(), 0u) == diff --git a/src/gallium/drivers/d3d12/d3d12_video_encoder_bitstream_builder_h264.cpp b/src/gallium/drivers/d3d12/d3d12_video_encoder_bitstream_builder_h264.cpp index 66e7fcf4ef1..f83f4d8b70d 100644 --- a/src/gallium/drivers/d3d12/d3d12_video_encoder_bitstream_builder_h264.cpp +++ b/src/gallium/drivers/d3d12/d3d12_video_encoder_bitstream_builder_h264.cpp @@ -191,6 +191,24 @@ d3d12_video_bitstream_builder_h264::write_aud(std::vector & hea m_h264Encoder.write_access_unit_delimiter_nalu(headerBitstream, placingPositionStart, writtenBytes); } +void +d3d12_video_bitstream_builder_h264::write_sei_messages(const std::vector& sei_messages, + std::vector & headerBitstream, + std::vector::iterator placingPositionStart, + size_t & writtenBytes) +{ + uint64_t byte_offset_placing_start = std::distance(headerBitstream.begin(), placingPositionStart); + writtenBytes = 0; + + for (auto& message : sei_messages) + { + size_t WrittenBytesCurrentSei = 0; + m_h264Encoder.write_sei_nalu(message, headerBitstream, headerBitstream.begin() + byte_offset_placing_start, WrittenBytesCurrentSei); + byte_offset_placing_start += WrittenBytesCurrentSei; + writtenBytes += WrittenBytesCurrentSei; + } +} + H264_PPS d3d12_video_bitstream_builder_h264::build_pps(const enum pipe_video_profile & profile, const D3D12_VIDEO_ENCODER_CODEC_CONFIGURATION_H264 & codecConfig, diff --git a/src/gallium/drivers/d3d12/d3d12_video_encoder_bitstream_builder_h264.h b/src/gallium/drivers/d3d12/d3d12_video_encoder_bitstream_builder_h264.h index d78775b93c0..79613f7d390 100644 --- a/src/gallium/drivers/d3d12/d3d12_video_encoder_bitstream_builder_h264.h +++ b/src/gallium/drivers/d3d12/d3d12_video_encoder_bitstream_builder_h264.h @@ -67,6 +67,12 @@ class d3d12_video_bitstream_builder_h264 : public d3d12_video_bitstream_builder_ std::vector::iterator placingPositionStart, size_t & writtenBytes); + void write_sei_messages(const std::vector& sei_messages, + std::vector & headerBitstream, + std::vector::iterator placingPositionStart, + size_t & writtenBytes); + + void print_pps(const H264_PPS &pps); void print_sps(const H264_SPS &sps); diff --git a/src/gallium/drivers/d3d12/d3d12_video_encoder_nalu_writer_h264.cpp b/src/gallium/drivers/d3d12/d3d12_video_encoder_nalu_writer_h264.cpp index 6957550303e..4db05747842 100644 --- a/src/gallium/drivers/d3d12/d3d12_video_encoder_nalu_writer_h264.cpp +++ b/src/gallium/drivers/d3d12/d3d12_video_encoder_nalu_writer_h264.cpp @@ -522,3 +522,129 @@ d3d12_video_nalu_writer_h264::write_access_unit_delimiter_nalu(std::vector & headerBitstream, + std::vector::iterator placingPositionStart, + size_t & writtenBytes) +{ + // Fill byte buffer with sei_message() payload + d3d12_video_encoder_bitstream sei_payload_bitstream; + if (!sei_payload_bitstream.create_bitstream(2 * sizeof(H264_SEI_MESSAGE))) { + debug_printf("sei_payload_bitstream.create_bitstream(2 * sizeof(H264_SEI_MESSAGE) failed.\n"); + assert(false); + } + + switch (sei_message.payload_type) + { + case H264_SEI_SCALABILITY_INFO: + { + sei_payload_bitstream.put_bits(1, 0); // temporal_id_nesting_flag + sei_payload_bitstream.put_bits(1, 0); // priority_layer_info_present_flag + sei_payload_bitstream.put_bits(1, 0); // priority_id_setting_flag + sei_payload_bitstream.exp_Golomb_ue(sei_message.scalability_info.num_layers_minus1); + for (uint32_t i = 0; i <= sei_message.scalability_info.num_layers_minus1; i++) + { + sei_payload_bitstream.exp_Golomb_ue(i); // layer_id[i] + sei_payload_bitstream.put_bits(6, 0); // priority_id[i] + sei_payload_bitstream.put_bits(1, 0); // discardable_flag[i] + sei_payload_bitstream.put_bits(3, 0); // dependency_id[i] + sei_payload_bitstream.put_bits(4, 0); // quality_id[i] + sei_payload_bitstream.put_bits(3, sei_message.scalability_info.temporal_id[i]); // temporal_id[i] + sei_payload_bitstream.put_bits(1, 0); // sub_pic_layer_flag[i] + sei_payload_bitstream.put_bits(1, 0); // sub_region_layer_flag[i] + sei_payload_bitstream.put_bits(1, 0); // iroi_division_info_present_flag[i] + sei_payload_bitstream.put_bits(1, 0); // profile_level_info_present_flag[i] + sei_payload_bitstream.put_bits(1, 0); // bitrate_info_present_flag[i] + sei_payload_bitstream.put_bits(1, 0); // frm_rate_info_present_flag[i] + sei_payload_bitstream.put_bits(1, 0); // frm_size_info_present_flag[i] + sei_payload_bitstream.put_bits(1, 0); // layer_dependency_info_present_flag[i] + sei_payload_bitstream.put_bits(1, 0); // parameter_sets_info_present_flag[i] + sei_payload_bitstream.put_bits(1, 0); // bitstream_restriction_info_present_flag[i] + sei_payload_bitstream.put_bits(1, 0); // exact_inter_layer_pred_flag[i] + sei_payload_bitstream.put_bits(1, 0); // layer_conversion_flag[i] + sei_payload_bitstream.put_bits(1, 0); // layer_output_flag[i] + sei_payload_bitstream.exp_Golomb_ue(0); // layer_dependency_info_src_layer_id_delta [i] + sei_payload_bitstream.exp_Golomb_ue(0); // parameter_sets_info_src_layer_id_delta [i] + } + } break; + default: + debug_printf("[d3d12_video_nalu_writer_h264::write_sei_nalu] Unsupported sei_message.payload_type.\n"); + assert(false); + return; + break; + } + + // Add trailing bits after sei_message() bits in sei_payload_bitstream and flush + if(!sei_payload_bitstream.is_byte_aligned()) + rbsp_trailing(&sei_payload_bitstream); + sei_payload_bitstream.flush(); + + // Set payload_size from bitstream data written + uint32_t payload_size = sei_payload_bitstream.get_byte_count(); + + // + // Wrap sei_payload_bitstream in RBSP and NALU + // + + d3d12_video_encoder_bitstream rbsp, nalu; + if (!rbsp.create_bitstream(2 * sizeof(H264_SEI_MESSAGE))) { + debug_printf("rbsp.create_bitstream(2 * sizeof(H264_SEI_MESSAGE)) failed.\n"); + assert(false); + } + + if (!nalu.create_bitstream(2 * sizeof(H264_SEI_MESSAGE))) { + debug_printf("nalu.create_bitstream(2 * sizeof(H264_SEI_MESSAGE)) failed.\n"); + assert(false); + } + + rbsp.set_start_code_prevention(true); + + // + // Write payload_type to bitstream + // + uint32_t payload_type = static_cast(sei_message.payload_type); + while(payload_type >= 255) + { + rbsp.put_bits(8, 255 /* payload_type */); + payload_type -= 255; + } + rbsp.put_bits(8, payload_type); + + // + // Write payload_size to bitstream + // + while(payload_size >= 255) + { + rbsp.put_bits(8, 255 /* payload_size */); + payload_size -= 255; + } + rbsp.put_bits(8, payload_size); + + rbsp.flush(); + rbsp.append_byte_stream(&sei_payload_bitstream); + + rbsp_trailing(&rbsp); + rbsp.flush(); + if (wrap_rbsp_into_nalu(&nalu, &rbsp, NAL_REFIDC_NONREF, NAL_TYPE_SEI) <= 0u) { + + debug_printf( + "wrap_rbsp_into_nalu(&nalu, &rbsp, NAL_REFIDC_NONREF, NAL_TYPE_ACCESS_UNIT_DELIMITER) didn't write any bytes.\n"); + assert(false); + } + + // Deep copy nalu into headerBitstream, nalu gets out of scope here and its destructor frees the nalu object buffer + // memory. + uint8_t *naluBytes = nalu.get_bitstream_buffer(); + size_t naluByteSize = nalu.get_byte_count(); + + auto startDstIndex = std::distance(headerBitstream.begin(), placingPositionStart); + if (headerBitstream.size() < (startDstIndex + naluByteSize)) { + headerBitstream.resize(startDstIndex + naluByteSize); + } + + std::copy_n(&naluBytes[0], naluByteSize, &headerBitstream.data()[startDstIndex]); + + writtenBytes = naluByteSize; +} diff --git a/src/gallium/drivers/d3d12/d3d12_video_encoder_nalu_writer_h264.h b/src/gallium/drivers/d3d12/d3d12_video_encoder_nalu_writer_h264.h index b2ea73ef1ac..17e9cf98266 100644 --- a/src/gallium/drivers/d3d12/d3d12_video_encoder_nalu_writer_h264.h +++ b/src/gallium/drivers/d3d12/d3d12_video_encoder_nalu_writer_h264.h @@ -55,6 +55,26 @@ enum H264_NALU_TYPE /* 24...31 UNSPECIFIED */ }; +enum H264_SEI_TYPE +{ + H264_SEI_SCALABILITY_INFO = 24, +}; + +typedef struct +{ + uint32_t num_layers_minus1; // ue(v) + uint32_t temporal_id[/* as per codec spec num_layers_minus1 range */ 2048]; // u(3) +} H264_SEI_SCALABILITYINFO; + +typedef struct H264_SEI_MESSAGE +{ + H264_SEI_TYPE payload_type; + union + { + H264_SEI_SCALABILITYINFO scalability_info; + }; +} H264_SEI_MESSAGE; + typedef struct H264_HRD_PARAMS { uint32_t cpb_cnt_minus1; @@ -190,6 +210,10 @@ class d3d12_video_nalu_writer_h264 void write_access_unit_delimiter_nalu(std::vector & headerBitstream, std::vector::iterator placingPositionStart, size_t & writtenBytes); + void write_sei_nalu(H264_SEI_MESSAGE sei_message, + std::vector & headerBitstream, + std::vector::iterator placingPositionStart, + size_t & writtenBytes); private: // Writes from structure into bitstream with RBSP trailing but WITHOUT NAL unit wrap (eg. nal_idc_type, etc)