From eb67a45ca82bc01ac843c853fd3c17f2a90e0250 Mon Sep 17 00:00:00 2001 From: ameerj Date: Mon, 26 Oct 2020 23:07:36 -0400 Subject: video_core: NVDEC Implementation This commit aims to implement the NVDEC (Nvidia Decoder) functionality, with video frame decoding being handled by the FFmpeg library. The process begins with Ioctl commands being sent to the NVDEC and VIC (Video Image Composer) emulated devices. These allocate the necessary GPU buffers for the frame data, along with providing information on the incoming video data. A Submit command then signals the GPU to process and decode the frame data. To decode the frame, the respective codec's header must be manually composed from the information provided by NVDEC, then sent with the raw frame data to the ffmpeg library. Currently, H264 and VP9 are supported, with VP9 having some minor artifacting issues related mainly to the reference frame composition in its uncompressed header. Async GPU is not properly implemented at the moment. Co-Authored-By: David <25727384+ogniK5377@users.noreply.github.com> --- src/video_core/command_classes/codecs/h264.h | 130 +++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 src/video_core/command_classes/codecs/h264.h (limited to 'src/video_core/command_classes/codecs/h264.h') diff --git a/src/video_core/command_classes/codecs/h264.h b/src/video_core/command_classes/codecs/h264.h new file mode 100644 index 000000000..21752dd90 --- /dev/null +++ b/src/video_core/command_classes/codecs/h264.h @@ -0,0 +1,130 @@ +// MIT License +// +// Copyright (c) Ryujinx Team and Contributors +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +// associated documentation files (the "Software"), to deal in the Software without restriction, +// including without limitation the rights to use, copy, modify, merge, publish, distribute, +// sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all copies or +// substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +// NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// + +#pragma once + +#include +#include "common/common_funcs.h" +#include "common/common_types.h" +#include "video_core/command_classes/nvdec_common.h" + +namespace Tegra { +class GPU; +namespace Decoder { + +class H264BitWriter { +public: + H264BitWriter(); + ~H264BitWriter(); + + /// The following Write methods are based on clause 9.1 in the H.264 specification. + /// WriteSe and WriteUe write in the Exp-Golomb-coded syntax + void WriteU(s32 value, s32 value_sz); + void WriteSe(s32 value); + void WriteUe(s32 value); + + /// Finalize the bitstream + void End(); + + /// append a bit to the stream, equivalent value to the state parameter + void WriteBit(bool state); + + /// Based on section 7.3.2.1.1.1 and Table 7-4 in the H.264 specification + /// Writes the scaling matrices of the sream + void WriteScalingList(const std::vector& list, s32 start, s32 count); + + /// Return the bitstream as a vector. + std::vector& GetByteArray(); + const std::vector& GetByteArray() const; + +private: + // ZigZag LUTs from libavcodec. + static constexpr std::array zig_zag_direct{ + 0, 1, 8, 16, 9, 2, 3, 10, 17, 24, 32, 25, 18, 11, 4, 5, 12, 19, 26, 33, 40, 48, + 41, 34, 27, 20, 13, 6, 7, 14, 21, 28, 35, 42, 49, 56, 57, 50, 43, 36, 29, 22, 15, 23, + 30, 37, 44, 51, 58, 59, 52, 45, 38, 31, 39, 46, 53, 60, 61, 54, 47, 55, 62, 63, + }; + + static constexpr std::array zig_zag_scan{ + 0 + 0 * 4, 1 + 0 * 4, 0 + 1 * 4, 0 + 2 * 4, 1 + 1 * 4, 2 + 0 * 4, 3 + 0 * 4, 2 + 1 * 4, + 1 + 2 * 4, 0 + 3 * 4, 1 + 3 * 4, 2 + 2 * 4, 3 + 1 * 4, 3 + 2 * 4, 2 + 3 * 4, 3 + 3 * 4, + }; + + void WriteBits(s32 value, s32 bit_count); + void WriteExpGolombCodedInt(s32 value); + void WriteExpGolombCodedUInt(u32 value); + s32 GetFreeBufferBits(); + void Flush(); + + s32 buffer_size{8}; + + s32 buffer{}; + s32 buffer_pos{}; + std::vector byte_array; +}; + +class H264 { +public: + explicit H264(GPU& gpu); + ~H264(); + + /// Compose the H264 header of the frame for FFmpeg decoding + std::vector& ComposeFrameHeader(NvdecCommon::NvdecRegisters& state, + bool is_first_frame = false); + +private: + struct H264ParameterSet { + u32 log2_max_pic_order_cnt{}; + u32 delta_pic_order_always_zero_flag{}; + u32 frame_mbs_only_flag{}; + u32 pic_width_in_mbs{}; + u32 pic_height_in_map_units{}; + INSERT_PADDING_WORDS(1); + u32 entropy_coding_mode_flag{}; + u32 bottom_field_pic_order_flag{}; + u32 num_refidx_l0_default_active{}; + u32 num_refidx_l1_default_active{}; + u32 deblocking_filter_control_flag{}; + u32 redundant_pic_count_flag{}; + u32 transform_8x8_mode_flag{}; + INSERT_PADDING_WORDS(9); + u64 flags{}; + u32 frame_number{}; + u32 frame_number2{}; + }; + static_assert(sizeof(H264ParameterSet) == 0x68, "H264ParameterSet is an invalid size"); + + struct H264DecoderContext { + INSERT_PADDING_BYTES(0x48); + u32 frame_data_size{}; + INSERT_PADDING_BYTES(0xc); + H264ParameterSet h264_parameter_set{}; + INSERT_PADDING_BYTES(0x100); + std::array scaling_matrix_4; + std::array scaling_matrix_8; + }; + static_assert(sizeof(H264DecoderContext) == 0x2a0, "H264DecoderContext is an invalid size"); + + std::vector frame; + GPU& gpu; +}; + +} // namespace Decoder +} // namespace Tegra -- cgit v1.2.3 From c04203b786d87ecb66811341e7ea776452664e91 Mon Sep 17 00:00:00 2001 From: Lioncash Date: Tue, 27 Oct 2020 02:09:17 -0400 Subject: nvdec: Tidy up header includes Prevents a few unnecessary inclusions. --- src/video_core/command_classes/codecs/h264.h | 12 ------------ 1 file changed, 12 deletions(-) (limited to 'src/video_core/command_classes/codecs/h264.h') diff --git a/src/video_core/command_classes/codecs/h264.h b/src/video_core/command_classes/codecs/h264.h index 21752dd90..c36a54399 100644 --- a/src/video_core/command_classes/codecs/h264.h +++ b/src/video_core/command_classes/codecs/h264.h @@ -55,18 +55,6 @@ public: const std::vector& GetByteArray() const; private: - // ZigZag LUTs from libavcodec. - static constexpr std::array zig_zag_direct{ - 0, 1, 8, 16, 9, 2, 3, 10, 17, 24, 32, 25, 18, 11, 4, 5, 12, 19, 26, 33, 40, 48, - 41, 34, 27, 20, 13, 6, 7, 14, 21, 28, 35, 42, 49, 56, 57, 50, 43, 36, 29, 22, 15, 23, - 30, 37, 44, 51, 58, 59, 52, 45, 38, 31, 39, 46, 53, 60, 61, 54, 47, 55, 62, 63, - }; - - static constexpr std::array zig_zag_scan{ - 0 + 0 * 4, 1 + 0 * 4, 0 + 1 * 4, 0 + 2 * 4, 1 + 1 * 4, 2 + 0 * 4, 3 + 0 * 4, 2 + 1 * 4, - 1 + 2 * 4, 0 + 3 * 4, 1 + 3 * 4, 2 + 2 * 4, 3 + 1 * 4, 3 + 2 * 4, 2 + 3 * 4, 3 + 3 * 4, - }; - void WriteBits(s32 value, s32 bit_count); void WriteExpGolombCodedInt(s32 value); void WriteExpGolombCodedUInt(u32 value); -- cgit v1.2.3 From cce14b4cd75fd3bbab548375a0a547ec499283e2 Mon Sep 17 00:00:00 2001 From: Lioncash Date: Tue, 27 Oct 2020 02:50:03 -0400 Subject: h264: Make WriteUe take a u32 Enforces the type of the desired value in calling code. --- src/video_core/command_classes/codecs/h264.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/video_core/command_classes/codecs/h264.h') diff --git a/src/video_core/command_classes/codecs/h264.h b/src/video_core/command_classes/codecs/h264.h index c36a54399..f2292fd2f 100644 --- a/src/video_core/command_classes/codecs/h264.h +++ b/src/video_core/command_classes/codecs/h264.h @@ -38,7 +38,7 @@ public: /// WriteSe and WriteUe write in the Exp-Golomb-coded syntax void WriteU(s32 value, s32 value_sz); void WriteSe(s32 value); - void WriteUe(s32 value); + void WriteUe(u32 value); /// Finalize the bitstream void End(); -- cgit v1.2.3 From 4f0f481f63d0d5587a6d6d319f9815de87ed79ec Mon Sep 17 00:00:00 2001 From: Lioncash Date: Fri, 30 Oct 2020 16:13:29 -0400 Subject: nvdec: Make use of [[nodiscard]] where applicable Prevents bugs from occurring where the results of a function are accidentally discarded --- src/video_core/command_classes/codecs/h264.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'src/video_core/command_classes/codecs/h264.h') diff --git a/src/video_core/command_classes/codecs/h264.h b/src/video_core/command_classes/codecs/h264.h index f2292fd2f..273449495 100644 --- a/src/video_core/command_classes/codecs/h264.h +++ b/src/video_core/command_classes/codecs/h264.h @@ -51,14 +51,14 @@ public: void WriteScalingList(const std::vector& list, s32 start, s32 count); /// Return the bitstream as a vector. - std::vector& GetByteArray(); - const std::vector& GetByteArray() const; + [[nodiscard]] std::vector& GetByteArray(); + [[nodiscard]] const std::vector& GetByteArray() const; private: void WriteBits(s32 value, s32 bit_count); void WriteExpGolombCodedInt(s32 value); void WriteExpGolombCodedUInt(u32 value); - s32 GetFreeBufferBits(); + [[nodiscard]] s32 GetFreeBufferBits(); void Flush(); s32 buffer_size{8}; @@ -74,8 +74,8 @@ public: ~H264(); /// Compose the H264 header of the frame for FFmpeg decoding - std::vector& ComposeFrameHeader(NvdecCommon::NvdecRegisters& state, - bool is_first_frame = false); + [[nodiscard]] const std::vector& ComposeFrameHeader(NvdecCommon::NvdecRegisters& state, + bool is_first_frame = false); private: struct H264ParameterSet { -- cgit v1.2.3 From eab041866b7c766aa38258aecef8a00c03612459 Mon Sep 17 00:00:00 2001 From: ameerj Date: Wed, 25 Nov 2020 17:10:44 -0500 Subject: Queue decoded frames, cleanup decoders --- src/video_core/command_classes/codecs/h264.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/video_core/command_classes/codecs/h264.h') diff --git a/src/video_core/command_classes/codecs/h264.h b/src/video_core/command_classes/codecs/h264.h index 273449495..0f3a1d9f3 100644 --- a/src/video_core/command_classes/codecs/h264.h +++ b/src/video_core/command_classes/codecs/h264.h @@ -74,8 +74,8 @@ public: ~H264(); /// Compose the H264 header of the frame for FFmpeg decoding - [[nodiscard]] const std::vector& ComposeFrameHeader(NvdecCommon::NvdecRegisters& state, - bool is_first_frame = false); + [[nodiscard]] const std::vector& ComposeFrameHeader( + const NvdecCommon::NvdecRegisters& state, bool is_first_frame = false); private: struct H264ParameterSet { -- cgit v1.2.3