NVDEC (H264): Use separate contexts per channel and decode frames in DTS order (#2671)

* Use separate NVDEC contexts per channel (for FFMPEG)

* Remove NVDEC -> VIC frame override hack

* Add missing bottom_field_pic_order_in_frame_present_flag

* Make FFMPEG logging static

* nit: Remove empty lines

* New FFMPEG decoding approach -- call h264_decode_frame directly, trim surface cache to reduce memory usage

* Fix case

* Silence warnings

* PR feedback

* Per-decoder rather than per-codec ownership of surfaces on the cache

1 files changed, 37 insertions, 16 deletions

diff --git a/Ryujinx.Graphics.Nvdec.H264/FFmpegContext.cs b/Ryujinx.Graphics.Nvdec.H264/FFmpegContext.cs
index 17de1a03..7676093f 100644
--- a/Ryujinx.Graphics.Nvdec.H264/FFmpegContext.cs
+++ b/Ryujinx.Graphics.Nvdec.H264/FFmpegContext.cs
@@ -9,30 +9,34 @@ namespace Ryujinx.Graphics.Nvdec.H264
 {
     unsafe class FFmpegContext : IDisposable
     {
-        private readonly av_log_set_callback_callback _logFunc;
+        private readonly AVCodec_decode _h264Decode;
+        private static readonly av_log_set_callback_callback _logFunc;
         private readonly AVCodec* _codec;
         private AVPacket* _packet;
         private AVCodecContext* _context;
 
         public FFmpegContext()
         {
-            _logFunc = Log;
-
-            // Redirect log output
-            ffmpeg.av_log_set_level(ffmpeg.AV_LOG_MAX_OFFSET);
-            ffmpeg.av_log_set_callback(_logFunc);
-
             _codec = ffmpeg.avcodec_find_decoder(AVCodecID.AV_CODEC_ID_H264);
             _context = ffmpeg.avcodec_alloc_context3(_codec);
+            _context->debug |= ffmpeg.FF_DEBUG_MMCO;
 
             ffmpeg.avcodec_open2(_context, _codec, null);
 
             _packet = ffmpeg.av_packet_alloc();
+
+            _h264Decode = Marshal.GetDelegateForFunctionPointer<AVCodec_decode>(_codec->decode.Pointer);
         }
 
         static FFmpegContext()
         {
             SetRootPath();
+
+            _logFunc = Log;
+
+            // Redirect log output.
+            ffmpeg.av_log_set_level(ffmpeg.AV_LOG_MAX_OFFSET);
+            ffmpeg.av_log_set_callback(_logFunc);
         }
 
         private static void SetRootPath()
@@ -64,7 +68,7 @@ namespace Ryujinx.Graphics.Nvdec.H264
             }
         }
 
-        private void Log(void* p0, int level, string format, byte* vl)
+        private static void Log(void* p0, int level, string format, byte* vl)
         {
             if (level > ffmpeg.av_log_get_level())
             {
@@ -102,23 +106,40 @@ namespace Ryujinx.Graphics.Nvdec.H264
 
         public int DecodeFrame(Surface output, ReadOnlySpan<byte> bitstream)
         {
-            // Ensure the packet is clean before proceeding
-            ffmpeg.av_packet_unref(_packet);
+            int result;
+            int gotFrame;
 
             fixed (byte* ptr = bitstream)
             {
                 _packet->data = ptr;
                 _packet->size = bitstream.Length;
+                result = _h264Decode(_context, output.Frame, &gotFrame, _packet);
+            }
+
+            if (gotFrame == 0)
+            {
+                ffmpeg.av_frame_unref(output.Frame);
 
-                int rc = ffmpeg.avcodec_send_packet(_context, _packet);
+                // If the frame was not delivered, it was probably delayed.
+                // Get the next delayed frame by passing a 0 length packet.
+                _packet->data = null;
+                _packet->size = 0;
+                result = _h264Decode(_context, output.Frame, &gotFrame, _packet);
 
-                if (rc != 0)
-                {
-                    return rc;
-                }
+                // We need to set B frames to 0 as we already consumed all delayed frames.
+                // This prevents the decoder from trying to return a delayed frame next time.
+                _context->has_b_frames = 0;
+            }
+
+            ffmpeg.av_packet_unref(_packet);
+
+            if (gotFrame == 0)
+            {
+                ffmpeg.av_frame_unref(output.Frame);
+                return -1;
             }
 
-            return ffmpeg.avcodec_receive_frame(_context, output.Frame);
+            return result < 0 ? result : 0;
         }
 
         public void Dispose()