nvdec: Adding Vp8 codec support (#2707)

* first try

* second try

* working update

* Final impl

* Fixes nits

* Fix everything

* remove leftover

* Update FFmpegContext.cs

* Update Surface.cs

* Addresses gdkchan feedback

* bool not byte

* Addresses gdkchan feedback

7 files changed, 607 insertions, 0 deletions

diff --git a/Ryujinx.Graphics.Nvdec.FFmpeg/FFmpegContext.cs b/Ryujinx.Graphics.Nvdec.FFmpeg/FFmpegContext.cs
new file mode 100644
index 00000000..1e3b88bb
--- /dev/null
+++ b/Ryujinx.Graphics.Nvdec.FFmpeg/FFmpegContext.cs
@@ -0,0 +1,162 @@
+using FFmpeg.AutoGen;
+using Ryujinx.Common.Logging;
+using System;
+using System.Diagnostics;
+using System.IO;
+using System.Runtime.InteropServices;
+
+namespace Ryujinx.Graphics.Nvdec.FFmpeg
+{
+    unsafe class FFmpegContext : IDisposable
+    {
+        private readonly AVCodec_decode _decodeFrame;
+        private static readonly av_log_set_callback_callback _logFunc;
+        private readonly AVCodec* _codec;
+        private AVPacket* _packet;
+        private AVCodecContext* _context;
+
+        public FFmpegContext(AVCodecID codecId)
+        {
+            _codec = ffmpeg.avcodec_find_decoder(codecId);
+            _context = ffmpeg.avcodec_alloc_context3(_codec);
+
+            ffmpeg.avcodec_open2(_context, _codec, null);
+
+            _packet = ffmpeg.av_packet_alloc();
+
+            _decodeFrame = Marshal.GetDelegateForFunctionPointer<AVCodec_decode>(_codec->decode.Pointer);
+        }
+
+        static FFmpegContext()
+        {
+            SetRootPath();
+
+            _logFunc = Log;
+
+            // Redirect log output.
+            ffmpeg.av_log_set_level(ffmpeg.AV_LOG_MAX_OFFSET);
+            ffmpeg.av_log_set_callback(_logFunc);
+        }
+
+        private static void SetRootPath()
+        {
+            if (OperatingSystem.IsLinux())
+            {
+                // Configure FFmpeg search path
+                Process lddProcess = Process.Start(new ProcessStartInfo
+                {
+                    FileName               = "/bin/sh",
+                    Arguments              = "-c \"ldd $(which ffmpeg 2>/dev/null) | grep libavfilter\" 2>/dev/null",
+                    UseShellExecute        = false,
+                    RedirectStandardOutput = true
+                });
+
+                string lddOutput = lddProcess.StandardOutput.ReadToEnd();
+
+                lddProcess.WaitForExit();
+                lddProcess.Close();
+
+                if (lddOutput.Contains(" => "))
+                {
+                    ffmpeg.RootPath = Path.GetDirectoryName(lddOutput.Split(" => ")[1]);
+                }
+                else
+                {
+                    Logger.Error?.PrintMsg(LogClass.FFmpeg, "FFmpeg wasn't found. Make sure that you have it installed and up to date.");
+                }
+            }
+        }
+
+        private static void Log(void* p0, int level, string format, byte* vl)
+        {
+            if (level > ffmpeg.av_log_get_level())
+            {
+                return;
+            }
+
+            int lineSize = 1024;
+            byte* lineBuffer = stackalloc byte[lineSize];
+            int printPrefix = 1;
+
+            ffmpeg.av_log_format_line(p0, level, format, vl, lineBuffer, lineSize, &printPrefix);
+
+            string line = Marshal.PtrToStringAnsi((IntPtr)lineBuffer).Trim();
+
+            switch (level)
+            {
+                case ffmpeg.AV_LOG_PANIC:
+                case ffmpeg.AV_LOG_FATAL:
+                case ffmpeg.AV_LOG_ERROR:
+                    Logger.Error?.Print(LogClass.FFmpeg, line);
+                    break;
+                case ffmpeg.AV_LOG_WARNING:
+                    Logger.Warning?.Print(LogClass.FFmpeg, line);
+                    break;
+                case ffmpeg.AV_LOG_INFO:
+                    Logger.Info?.Print(LogClass.FFmpeg, line);
+                    break;
+                case ffmpeg.AV_LOG_VERBOSE:
+                case ffmpeg.AV_LOG_DEBUG:
+                case ffmpeg.AV_LOG_TRACE:
+                    Logger.Debug?.Print(LogClass.FFmpeg, line);
+                    break;
+            }
+        }
+
+        public int DecodeFrame(Surface output, ReadOnlySpan<byte> bitstream)
+        {
+            ffmpeg.av_frame_unref(output.Frame);
+
+            int result;
+            int gotFrame;
+
+            fixed (byte* ptr = bitstream)
+            {
+                _packet->data = ptr;
+                _packet->size = bitstream.Length;
+                result = _decodeFrame(_context, output.Frame, &gotFrame, _packet);
+            }
+
+            if (gotFrame == 0)
+            {
+                ffmpeg.av_frame_unref(output.Frame);
+
+                // If the frame was not delivered, it was probably delayed.
+                // Get the next delayed frame by passing a 0 length packet.
+                _packet->data = null;
+                _packet->size = 0;
+                result = _decodeFrame(_context, output.Frame, &gotFrame, _packet);
+
+                // We need to set B frames to 0 as we already consumed all delayed frames.
+                // This prevents the decoder from trying to return a delayed frame next time.
+                _context->has_b_frames = 0;
+            }
+
+            ffmpeg.av_packet_unref(_packet);
+
+            if (gotFrame == 0)
+            {
+                ffmpeg.av_frame_unref(output.Frame);
+
+                return -1;
+            }
+
+            return result < 0 ? result : 0;
+        }
+
+        public void Dispose()
+        {
+            fixed (AVPacket** ppPacket = &_packet)
+            {
+                ffmpeg.av_packet_free(ppPacket);
+            }
+
+            ffmpeg.avcodec_close(_context);
+
+            fixed (AVCodecContext** ppContext = &_context)
+            {
+                ffmpeg.avcodec_free_context(ppContext);
+            }
+        }
+    }
+}
diff --git a/Ryujinx.Graphics.Nvdec.FFmpeg/H264/Decoder.cs b/Ryujinx.Graphics.Nvdec.FFmpeg/H264/Decoder.cs
new file mode 100644
index 00000000..8deda42a
--- /dev/null
+++ b/Ryujinx.Graphics.Nvdec.FFmpeg/H264/Decoder.cs
@@ -0,0 +1,56 @@
+using FFmpeg.AutoGen;
+using Ryujinx.Graphics.Video;
+using System;
+
+namespace Ryujinx.Graphics.Nvdec.FFmpeg.H264
+{
+    public sealed class Decoder : IH264Decoder
+    {
+        public bool IsHardwareAccelerated => false;
+
+        private const int WorkBufferSize = 0x200;
+
+        private readonly byte[] _workBuffer = new byte[WorkBufferSize];
+
+        private FFmpegContext _context = new FFmpegContext(AVCodecID.AV_CODEC_ID_H264);
+
+        private int _oldOutputWidth;
+        private int _oldOutputHeight;
+
+        public ISurface CreateSurface(int width, int height)
+        {
+            return new Surface(width, height);
+        }
+
+        public bool Decode(ref H264PictureInfo pictureInfo, ISurface output, ReadOnlySpan<byte> bitstream)
+        {
+            Surface outSurf = (Surface)output;
+
+            if (outSurf.RequestedWidth != _oldOutputWidth ||
+                outSurf.RequestedHeight != _oldOutputHeight)
+            {
+                _context.Dispose();
+                _context = new FFmpegContext(AVCodecID.AV_CODEC_ID_H264);
+
+                _oldOutputWidth = outSurf.RequestedWidth;
+                _oldOutputHeight = outSurf.RequestedHeight;
+            }
+
+            Span<byte> bs = Prepend(bitstream, SpsAndPpsReconstruction.Reconstruct(ref pictureInfo, _workBuffer));
+
+            return _context.DecodeFrame(outSurf, bs) == 0;
+        }
+
+        private static byte[] Prepend(ReadOnlySpan<byte> data, ReadOnlySpan<byte> prep)
+        {
+            byte[] output = new byte[data.Length + prep.Length];
+
+            prep.CopyTo(output);
+            data.CopyTo(new Span<byte>(output).Slice(prep.Length));
+
+            return output;
+        }
+
+        public void Dispose() => _context.Dispose();
+    }
+}
diff --git a/Ryujinx.Graphics.Nvdec.FFmpeg/H264/H264BitStreamWriter.cs b/Ryujinx.Graphics.Nvdec.FFmpeg/H264/H264BitStreamWriter.cs
new file mode 100644
index 00000000..3d3b3293
--- /dev/null
+++ b/Ryujinx.Graphics.Nvdec.FFmpeg/H264/H264BitStreamWriter.cs
@@ -0,0 +1,121 @@
+using System;
+using System.Numerics;
+
+namespace Ryujinx.Graphics.Nvdec.FFmpeg.H264
+{
+    struct H264BitStreamWriter
+    {
+        private const int BufferSize = 8;
+
+        private readonly byte[] _workBuffer;
+
+        private int _offset;
+        private int _buffer;
+        private int _bufferPos;
+
+        public H264BitStreamWriter(byte[] workBuffer)
+        {
+            _workBuffer = workBuffer;
+            _offset = 0;
+            _buffer = 0;
+            _bufferPos = 0;
+        }
+
+        public void WriteBit(bool value)
+        {
+            WriteBits(value ? 1 : 0, 1);
+        }
+
+        public void WriteBits(int value, int valueSize)
+        {
+            int valuePos = 0;
+
+            int remaining = valueSize;
+
+            while (remaining > 0)
+            {
+                int copySize = remaining;
+
+                int free = GetFreeBufferBits();
+
+                if (copySize > free)
+                {
+                    copySize = free;
+                }
+
+                int mask = (1 << copySize) - 1;
+
+                int srcShift = (valueSize - valuePos) - copySize;
+                int dstShift = (BufferSize - _bufferPos) - copySize;
+
+                _buffer |= ((value >> srcShift) & mask) << dstShift;
+
+                valuePos += copySize;
+                _bufferPos += copySize;
+                remaining -= copySize;
+            }
+        }
+
+        private int GetFreeBufferBits()
+        {
+            if (_bufferPos == BufferSize)
+            {
+                Flush();
+            }
+
+            return BufferSize - _bufferPos;
+        }
+
+        public void Flush()
+        {
+            if (_bufferPos != 0)
+            {
+                _workBuffer[_offset++] = (byte)_buffer;
+
+                _buffer = 0;
+                _bufferPos = 0;
+            }
+        }
+
+        public void End()
+        {
+            WriteBit(true);
+
+            Flush();
+        }
+
+        public Span<byte> AsSpan()
+        {
+            return new Span<byte>(_workBuffer).Slice(0, _offset);
+        }
+
+        public void WriteU(uint value, int valueSize) => WriteBits((int)value, valueSize);
+        public void WriteSe(int value) => WriteExpGolombCodedInt(value);
+        public void WriteUe(uint value) => WriteExpGolombCodedUInt(value);
+
+        private void WriteExpGolombCodedInt(int value)
+        {
+            int sign = value <= 0 ? 0 : 1;
+
+            if (value < 0)
+            {
+                value = -value;
+            }
+
+            value = (value << 1) - sign;
+
+            WriteExpGolombCodedUInt((uint)value);
+        }
+
+        private void WriteExpGolombCodedUInt(uint value)
+        {
+            int size = 32 - BitOperations.LeadingZeroCount(value + 1);
+
+            WriteBits(1, size);
+
+            value -= (1u << (size - 1)) - 1;
+
+            WriteBits((int)value, size - 1);
+        }
+    }
+}
\ No newline at end of file
diff --git a/Ryujinx.Graphics.Nvdec.FFmpeg/H264/SpsAndPpsReconstruction.cs b/Ryujinx.Graphics.Nvdec.FFmpeg/H264/SpsAndPpsReconstruction.cs
new file mode 100644
index 00000000..5c16ef3d
--- /dev/null
+++ b/Ryujinx.Graphics.Nvdec.FFmpeg/H264/SpsAndPpsReconstruction.cs
@@ -0,0 +1,159 @@
+using Ryujinx.Common.Memory;
+using Ryujinx.Graphics.Video;
+using System;
+
+namespace Ryujinx.Graphics.Nvdec.FFmpeg.H264
+{
+    static class SpsAndPpsReconstruction
+    {
+        public static Span<byte> Reconstruct(ref H264PictureInfo pictureInfo, byte[] workBuffer)
+        {
+            H264BitStreamWriter writer = new H264BitStreamWriter(workBuffer);
+
+            // Sequence Parameter Set.
+            writer.WriteU(1, 24);
+            writer.WriteU(0, 1);
+            writer.WriteU(3, 2);
+            writer.WriteU(7, 5);
+            writer.WriteU(100, 8); // Profile idc
+            writer.WriteU(0, 8); // Reserved
+            writer.WriteU(31, 8); // Level idc
+            writer.WriteUe(0); // Seq parameter set id
+            writer.WriteUe(pictureInfo.ChromaFormatIdc);
+
+            if (pictureInfo.ChromaFormatIdc == 3)
+            {
+                writer.WriteBit(false); // Separate colour plane flag
+            }
+
+            writer.WriteUe(0); // Bit depth luma minus 8
+            writer.WriteUe(0); // Bit depth chroma minus 8
+            writer.WriteBit(pictureInfo.QpprimeYZeroTransformBypassFlag);
+            writer.WriteBit(false); // Scaling matrix present flag
+
+            writer.WriteUe(pictureInfo.Log2MaxFrameNumMinus4);
+            writer.WriteUe(pictureInfo.PicOrderCntType);
+
+            if (pictureInfo.PicOrderCntType == 0)
+            {
+                writer.WriteUe(pictureInfo.Log2MaxPicOrderCntLsbMinus4);
+            }
+            else if (pictureInfo.PicOrderCntType == 1)
+            {
+                writer.WriteBit(pictureInfo.DeltaPicOrderAlwaysZeroFlag);
+
+                writer.WriteSe(0); // Offset for non-ref pic
+                writer.WriteSe(0); // Offset for top to bottom field
+                writer.WriteUe(0); // Num ref frames in pic order cnt cycle
+            }
+
+            writer.WriteUe(16); // Max num ref frames
+            writer.WriteBit(false); // Gaps in frame num value allowed flag
+            writer.WriteUe(pictureInfo.PicWidthInMbsMinus1);
+            writer.WriteUe(pictureInfo.PicHeightInMapUnitsMinus1);
+            writer.WriteBit(pictureInfo.FrameMbsOnlyFlag);
+
+            if (!pictureInfo.FrameMbsOnlyFlag)
+            {
+                writer.WriteBit(pictureInfo.MbAdaptiveFrameFieldFlag);
+            }
+
+            writer.WriteBit(pictureInfo.Direct8x8InferenceFlag);
+            writer.WriteBit(false); // Frame cropping flag
+            writer.WriteBit(false); // VUI parameter present flag
+
+            writer.End();
+
+            // Picture Parameter Set.
+            writer.WriteU(1, 24);
+            writer.WriteU(0, 1);
+            writer.WriteU(3, 2);
+            writer.WriteU(8, 5);
+
+            writer.WriteUe(0); // Pic parameter set id
+            writer.WriteUe(0); // Seq parameter set id
+
+            writer.WriteBit(pictureInfo.EntropyCodingModeFlag);
+            writer.WriteBit(pictureInfo.PicOrderPresentFlag);
+            writer.WriteUe(0); // Num slice groups minus 1
+            writer.WriteUe(pictureInfo.NumRefIdxL0ActiveMinus1);
+            writer.WriteUe(pictureInfo.NumRefIdxL1ActiveMinus1);
+            writer.WriteBit(pictureInfo.WeightedPredFlag);
+            writer.WriteU(pictureInfo.WeightedBipredIdc, 2);
+            writer.WriteSe(pictureInfo.PicInitQpMinus26);
+            writer.WriteSe(0); // Pic init qs minus 26
+            writer.WriteSe(pictureInfo.ChromaQpIndexOffset);
+            writer.WriteBit(pictureInfo.DeblockingFilterControlPresentFlag);
+            writer.WriteBit(pictureInfo.ConstrainedIntraPredFlag);
+            writer.WriteBit(pictureInfo.RedundantPicCntPresentFlag);
+            writer.WriteBit(pictureInfo.Transform8x8ModeFlag);
+
+            writer.WriteBit(pictureInfo.ScalingMatrixPresent);
+
+            if (pictureInfo.ScalingMatrixPresent)
+            {
+                for (int index = 0; index < 6; index++)
+                {
+                    writer.WriteBit(true);
+
+                    WriteScalingList(ref writer, pictureInfo.ScalingLists4x4[index]);
+                }
+
+                if (pictureInfo.Transform8x8ModeFlag)
+                {
+                    for (int index = 0; index < 2; index++)
+                    {
+                        writer.WriteBit(true);
+
+                        WriteScalingList(ref writer, pictureInfo.ScalingLists8x8[index]);
+                    }
+                }
+            }
+
+            writer.WriteSe(pictureInfo.SecondChromaQpIndexOffset);
+
+            writer.End();
+
+            return writer.AsSpan();
+        }
+
+        // ZigZag LUTs from libavcodec.
+        private static readonly byte[] ZigZagDirect = new byte[]
+        {
+            0,   1,  8, 16,  9,  2,  3, 10,
+            17, 24, 32, 25, 18, 11,  4,  5,
+            12, 19, 26, 33, 40, 48, 41, 34,
+            27, 20, 13,  6,  7, 14, 21, 28,
+            35, 42, 49, 56, 57, 50, 43, 36,
+            29, 22, 15, 23, 30, 37, 44, 51,
+            58, 59, 52, 45, 38, 31, 39, 46,
+            53, 60, 61, 54, 47, 55, 62, 63
+        };
+
+        private static readonly byte[] ZigZagScan = new byte[]
+        {
+            0 + 0 * 4, 1 + 0 * 4, 0 + 1 * 4, 0 + 2 * 4,
+            1 + 1 * 4, 2 + 0 * 4, 3 + 0 * 4, 2 + 1 * 4,
+            1 + 2 * 4, 0 + 3 * 4, 1 + 3 * 4, 2 + 2 * 4,
+            3 + 1 * 4, 3 + 2 * 4, 2 + 3 * 4, 3 + 3 * 4
+        };
+
+        private static void WriteScalingList(ref H264BitStreamWriter writer, IArray<byte> list)
+        {
+            byte[] scan = list.Length == 16 ? ZigZagScan : ZigZagDirect;
+
+            int lastScale = 8;
+
+            for (int index = 0; index < list.Length; index++)
+            {
+                byte value = list[scan[index]];
+
+                int deltaScale = value - lastScale;
+
+                writer.WriteSe(deltaScale);
+
+                lastScale = value;
+            }
+        }
+    }
+}
diff --git a/Ryujinx.Graphics.Nvdec.FFmpeg/Ryujinx.Graphics.Nvdec.FFmpeg.csproj b/Ryujinx.Graphics.Nvdec.FFmpeg/Ryujinx.Graphics.Nvdec.FFmpeg.csproj
new file mode 100644
index 00000000..b437f36e
--- /dev/null
+++ b/Ryujinx.Graphics.Nvdec.FFmpeg/Ryujinx.Graphics.Nvdec.FFmpeg.csproj
@@ -0,0 +1,17 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net5.0</TargetFramework>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="FFmpeg.AutoGen" Version="4.4.1" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Ryujinx.Common\Ryujinx.Common.csproj" />
+    <ProjectReference Include="..\Ryujinx.Graphics.Video\Ryujinx.Graphics.Video.csproj" />
+  </ItemGroup>
+
+</Project>
diff --git a/Ryujinx.Graphics.Nvdec.FFmpeg/Surface.cs b/Ryujinx.Graphics.Nvdec.FFmpeg/Surface.cs
new file mode 100644
index 00000000..20cee4a1
--- /dev/null
+++ b/Ryujinx.Graphics.Nvdec.FFmpeg/Surface.cs
@@ -0,0 +1,39 @@
+using FFmpeg.AutoGen;
+using Ryujinx.Graphics.Video;
+using System;
+
+namespace Ryujinx.Graphics.Nvdec.FFmpeg
+{
+    unsafe class Surface : ISurface
+    {
+        public AVFrame* Frame { get; }
+
+        public int RequestedWidth { get; }
+        public int RequestedHeight { get; }
+
+        public Plane YPlane => new Plane((IntPtr)Frame->data[0], Stride * Height);
+        public Plane UPlane => new Plane((IntPtr)Frame->data[1], UvStride * UvHeight);
+        public Plane VPlane => new Plane((IntPtr)Frame->data[2], UvStride * UvHeight);
+
+        public int Width => Frame->width;
+        public int Height => Frame->height;
+        public int Stride => Frame->linesize[0];
+        public int UvWidth => (Frame->width + 1) >> 1;
+        public int UvHeight => (Frame->height + 1) >> 1;
+        public int UvStride => Frame->linesize[1];
+
+        public Surface(int width, int height)
+        {
+            RequestedWidth = width;
+            RequestedHeight = height;
+
+            Frame = ffmpeg.av_frame_alloc();
+        }
+
+        public void Dispose()
+        {
+            ffmpeg.av_frame_unref(Frame);
+            ffmpeg.av_free(Frame);
+        }
+    }
+}
diff --git a/Ryujinx.Graphics.Nvdec.FFmpeg/Vp8/Decoder.cs b/Ryujinx.Graphics.Nvdec.FFmpeg/Vp8/Decoder.cs
new file mode 100644
index 00000000..f12de287
--- /dev/null
+++ b/Ryujinx.Graphics.Nvdec.FFmpeg/Vp8/Decoder.cs
@@ -0,0 +1,53 @@
+using FFmpeg.AutoGen;
+using Ryujinx.Graphics.Video;
+using System;
+
+namespace Ryujinx.Graphics.Nvdec.FFmpeg.Vp8
+{
+    public sealed class Decoder : IDecoder
+    {
+        public bool IsHardwareAccelerated => false;
+
+        private readonly FFmpegContext _context = new FFmpegContext(AVCodecID.AV_CODEC_ID_VP8);
+
+        public ISurface CreateSurface(int width, int height)
+        {
+            return new Surface(width, height);
+        }
+
+        public bool Decode(ref Vp8PictureInfo pictureInfo, ISurface output, ReadOnlySpan<byte> bitstream)
+        {
+            Surface outSurf = (Surface)output;
+
+            int uncompHeaderSize = pictureInfo.KeyFrame ? 10 : 3;
+
+            byte[] frame = new byte[bitstream.Length + uncompHeaderSize];
+
+            uint firstPartSizeShifted = pictureInfo.FirstPartSize << 5;
+
+            frame[0] = (byte)(pictureInfo.KeyFrame ? 0 : 1);
+            frame[0] |= (byte)((pictureInfo.Version & 7) << 1);
+            frame[0] |= 1 << 4;
+            frame[0] |= (byte)firstPartSizeShifted;
+            frame[1] |= (byte)(firstPartSizeShifted >> 8);
+            frame[2] |= (byte)(firstPartSizeShifted >> 16);
+
+            if (pictureInfo.KeyFrame)
+            {
+                frame[3] = 0x9d;
+                frame[4] = 0x01;
+                frame[5] = 0x2a;
+                frame[6] = (byte)pictureInfo.FrameWidth;
+                frame[7] = (byte)((pictureInfo.FrameWidth >> 8) & 0x3F);
+                frame[8] = (byte)pictureInfo.FrameHeight;
+                frame[9] = (byte)((pictureInfo.FrameHeight >> 8) & 0x3F);
+            }
+
+            bitstream.CopyTo(new Span<byte>(frame).Slice(uncompHeaderSize));
+
+            return _context.DecodeFrame(outSurf, frame) == 0;
+        }
+
+        public void Dispose() => _context.Dispose();
+    }
+}
\ No newline at end of file