New NVDEC and VIC implementation (#1384)

* Initial NVDEC and VIC implementation

* Update FFmpeg.AutoGen to 4.3.0

* Add nvdec dependencies for Windows

* Unify some VP9 structures

* Rename VP9 structure fields

* Improvements to Video API

* XML docs for Common.Memory

* Remove now unused or redundant overloads from MemoryAccessor

* NVDEC UV surface read/write scalar paths

* Add FIXME comments about hacky things/stuff that will need to be fixed in the future

* Cleaned up VP9 memory allocation

* Remove some debug logs

* Rename some VP9 structs

* Remove unused struct

* No need to compile Ryujinx.Graphics.Host1x with unsafe anymore

* Name AsyncWorkQueue threads to make debugging easier

* Make Vp9PictureInfo a ref struct

* LayoutConverter no longer needs the depth argument (broken by rebase)

* Pooling of VP9 buffers, plus fix a memory leak on VP9

* Really wish VS could rename projects properly...

* Address feedback

* Remove using

* Catch OperationCanceledException

* Add licensing informations

* Add THIRDPARTY.md to release too

Co-authored-by: Thog <me@thog.eu>

12 files changed, 278 insertions, 0 deletions

diff --git a/Ryujinx.Graphics.Video/H264PictureInfo.cs b/Ryujinx.Graphics.Video/H264PictureInfo.cs
new file mode 100644
index 00000000..3b2c2fff
--- /dev/null
+++ b/Ryujinx.Graphics.Video/H264PictureInfo.cs
@@ -0,0 +1,47 @@
+using Ryujinx.Common.Memory;
+
+namespace Ryujinx.Graphics.Video
+{
+    public struct H264PictureInfo
+    {
+        public Array2<int> FieldOrderCnt;
+        public bool IsReference;
+        public ushort ChromaFormatIdc;
+        public ushort FrameNum;
+        public bool FieldPicFlag;
+        public bool BottomFieldFlag;
+        public uint NumRefFrames;
+        public bool MbAdaptiveFrameFieldFlag;
+        public bool ConstrainedIntraPredFlag;
+        public bool WeightedPredFlag;
+        public uint WeightedBipredIdc;
+        public bool FrameMbsOnlyFlag;
+        public bool Transform8x8ModeFlag;
+        public int ChromaQpIndexOffset;
+        public int SecondChromaQpIndexOffset;
+        public int PicInitQpMinus26;
+        public uint NumRefIdxL0ActiveMinus1;
+        public uint NumRefIdxL1ActiveMinus1;
+        public uint Log2MaxFrameNumMinus4;
+        public uint PicOrderCntType;
+        public uint Log2MaxPicOrderCntLsbMinus4;
+        public bool DeltaPicOrderAlwaysZeroFlag;
+        public bool Direct8x8InferenceFlag;
+        public bool EntropyCodingModeFlag;
+        public bool PicOrderPresentFlag;
+        public bool DeblockingFilterControlPresentFlag;
+        public bool RedundantPicCntPresentFlag;
+        public uint NumSliceGroupsMinus1;
+        public uint SliceGroupMapType;
+        public uint SliceGroupChangeRateMinus1;
+        // TODO: Slice group map
+        public bool FmoAsoEnable;
+        public bool ScalingMatrixPresent;
+        public Array6<Array16<byte>> ScalingLists4x4;
+        public Array2<Array64<byte>> ScalingLists8x8;
+        public uint FrameType;
+        public uint PicWidthInMbsMinus1;
+        public uint PicHeightInMapUnitsMinus1;
+        public bool QpprimeYZeroTransformBypassFlag;
+    }
+}
diff --git a/Ryujinx.Graphics.Video/IDecoder.cs b/Ryujinx.Graphics.Video/IDecoder.cs
new file mode 100644
index 00000000..5957af08
--- /dev/null
+++ b/Ryujinx.Graphics.Video/IDecoder.cs
@@ -0,0 +1,11 @@
+using System;
+
+namespace Ryujinx.Graphics.Video
+{
+    public interface IDecoder : IDisposable
+    {
+        bool IsHardwareAccelerated { get; }
+
+        ISurface CreateSurface(int width, int height);
+    }
+}
diff --git a/Ryujinx.Graphics.Video/IH264Decoder.cs b/Ryujinx.Graphics.Video/IH264Decoder.cs
new file mode 100644
index 00000000..127b9412
--- /dev/null
+++ b/Ryujinx.Graphics.Video/IH264Decoder.cs
@@ -0,0 +1,9 @@
+using System;
+
+namespace Ryujinx.Graphics.Video
+{
+    public interface IH264Decoder : IDecoder
+    {
+        bool Decode(ref H264PictureInfo pictureInfo, ISurface output, ReadOnlySpan<byte> bitstream);
+    }
+}
diff --git a/Ryujinx.Graphics.Video/ISurface.cs b/Ryujinx.Graphics.Video/ISurface.cs
new file mode 100644
index 00000000..fb66f31a
--- /dev/null
+++ b/Ryujinx.Graphics.Video/ISurface.cs
@@ -0,0 +1,18 @@
+using System;
+
+namespace Ryujinx.Graphics.Video
+{
+    public interface ISurface : IDisposable
+    {
+        Plane YPlane { get; }
+        Plane UPlane { get; }
+        Plane VPlane { get; }
+
+        int Width { get; }
+        int Height { get; }
+        int Stride { get; }
+        int UvWidth { get; }
+        int UvHeight { get; }
+        int UvStride { get; }
+    }
+}
diff --git a/Ryujinx.Graphics.Video/IVp9Decoder.cs b/Ryujinx.Graphics.Video/IVp9Decoder.cs
new file mode 100644
index 00000000..ac79bc42
--- /dev/null
+++ b/Ryujinx.Graphics.Video/IVp9Decoder.cs
@@ -0,0 +1,14 @@
+using System;
+
+namespace Ryujinx.Graphics.Video
+{
+    public interface IVp9Decoder : IDecoder
+    {
+        bool Decode(
+            ref Vp9PictureInfo pictureInfo,
+            ISurface output,
+            ReadOnlySpan<byte> bitstream,
+            ReadOnlySpan<Vp9MvRef> mvsIn,
+            Span<Vp9MvRef> mvsOut);
+    }
+}
diff --git a/Ryujinx.Graphics.Video/Plane.cs b/Ryujinx.Graphics.Video/Plane.cs
new file mode 100644
index 00000000..c0aca59c
--- /dev/null
+++ b/Ryujinx.Graphics.Video/Plane.cs
@@ -0,0 +1,42 @@
+using System;
+using System.Diagnostics.CodeAnalysis;
+
+namespace Ryujinx.Graphics.Video
+{
+    public struct Plane : IEquatable<Plane>
+    {
+        public IntPtr Pointer { get; }
+        public int Length { get; }
+
+        public Plane(IntPtr pointer, int length)
+        {
+            Pointer = pointer;
+            Length = length;
+        }
+
+        public override bool Equals(object obj)
+        {
+            return obj is Plane other && Equals(other);
+        }
+
+        public bool Equals([AllowNull] Plane other)
+        {
+            return Pointer == other.Pointer && Length == other.Length;
+        }
+
+        public override int GetHashCode()
+        {
+            return HashCode.Combine(Pointer, Length);
+        }
+
+        public static bool operator ==(Plane left, Plane right)
+        {
+            return left.Equals(right);
+        }
+
+        public static bool operator !=(Plane left, Plane right)
+        {
+            return !(left == right);
+        }
+    }
+}
diff --git a/Ryujinx.Graphics.Video/Ryujinx.Graphics.Video.csproj b/Ryujinx.Graphics.Video/Ryujinx.Graphics.Video.csproj
new file mode 100644
index 00000000..6710726c
--- /dev/null
+++ b/Ryujinx.Graphics.Video/Ryujinx.Graphics.Video.csproj
@@ -0,0 +1,11 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Ryujinx.Common\Ryujinx.Common.csproj" />
+  </ItemGroup>
+
+</Project>
diff --git a/Ryujinx.Graphics.Video/Vp9BackwardUpdates.cs b/Ryujinx.Graphics.Video/Vp9BackwardUpdates.cs
new file mode 100644
index 00000000..a3aa4de7
--- /dev/null
+++ b/Ryujinx.Graphics.Video/Vp9BackwardUpdates.cs
@@ -0,0 +1,32 @@
+using Ryujinx.Common.Memory;
+
+namespace Ryujinx.Graphics.Video
+{
+    public struct Vp9BackwardUpdates
+    {
+        public Array4<Array10<uint>> YMode;
+        public Array10<Array10<uint>> UvMode;
+        public Array16<Array4<uint>> Partition;
+        public Array4<Array2<Array2<Array6<Array6<Array4<uint>>>>>> Coef;
+        public Array4<Array2<Array2<Array6<Array6<uint>>>>> EobBranch;
+        public Array4<Array3<uint>> SwitchableInterp;
+        public Array7<Array4<uint>> InterMode;
+        public Array4<Array2<uint>> IntraInter;
+        public Array5<Array2<uint>> CompInter;
+        public Array5<Array2<Array2<uint>>> SingleRef;
+        public Array5<Array2<uint>> CompRef;
+        public Array2<Array4<uint>> Tx32x32;
+        public Array2<Array3<uint>> Tx16x16;
+        public Array2<Array2<uint>> Tx8x8;
+        public Array3<Array2<uint>> Skip;
+        public Array4<uint> Joints;
+        public Array2<Array2<uint>> Sign;
+        public Array2<Array11<uint>> Classes;
+        public Array2<Array2<uint>> Class0;
+        public Array2<Array10<Array2<uint>>> Bits;
+        public Array2<Array2<Array4<uint>>> Class0Fp;
+        public Array2<Array4<uint>> Fp;
+        public Array2<Array2<uint>> Class0Hp;
+        public Array2<Array2<uint>> Hp;
+    }
+}
diff --git a/Ryujinx.Graphics.Video/Vp9EntropyProbs.cs b/Ryujinx.Graphics.Video/Vp9EntropyProbs.cs
new file mode 100644
index 00000000..10b997a5
--- /dev/null
+++ b/Ryujinx.Graphics.Video/Vp9EntropyProbs.cs
@@ -0,0 +1,36 @@
+using Ryujinx.Common.Memory;
+
+namespace Ryujinx.Graphics.Video
+{
+    public struct Vp9EntropyProbs
+    {
+        public Array10<Array10<Array9<byte>>> KfYModeProb;
+        public Array7<byte> SegTreeProb;
+        public Array3<byte> SegPredProb;
+        public Array10<Array9<byte>> KfUvModeProb;
+        public Array4<Array9<byte>> YModeProb;
+        public Array10<Array9<byte>> UvModeProb;
+        public Array16<Array3<byte>> KfPartitionProb;
+        public Array16<Array3<byte>> PartitionProb;
+        public Array4<Array2<Array2<Array6<Array6<Array3<byte>>>>>> CoefProbs;
+        public Array4<Array2<byte>> SwitchableInterpProb;
+        public Array7<Array3<byte>> InterModeProb;
+        public Array4<byte> IntraInterProb;
+        public Array5<byte> CompInterProb;
+        public Array5<Array2<byte>> SingleRefProb;
+        public Array5<byte> CompRefProb;
+        public Array2<Array3<byte>> Tx32x32Prob;
+        public Array2<Array2<byte>> Tx16x16Prob;
+        public Array2<Array1<byte>> Tx8x8Prob;
+        public Array3<byte> SkipProb;
+        public Array3<byte> Joints;
+        public Array2<byte> Sign;
+        public Array2<Array10<byte>> Classes;
+        public Array2<Array1<byte>> Class0;
+        public Array2<Array10<byte>> Bits;
+        public Array2<Array2<Array3<byte>>> Class0Fp;
+        public Array2<Array3<byte>> Fp;
+        public Array2<byte> Class0Hp;
+        public Array2<byte> Hp;
+    }
+}
diff --git a/Ryujinx.Graphics.Video/Vp9Mv.cs b/Ryujinx.Graphics.Video/Vp9Mv.cs
new file mode 100644
index 00000000..9de41058
--- /dev/null
+++ b/Ryujinx.Graphics.Video/Vp9Mv.cs
@@ -0,0 +1,8 @@
+namespace Ryujinx.Graphics.Video
+{
+    public struct Vp9Mv
+    {
+        public short Row;
+        public short Col;
+    }
+}
diff --git a/Ryujinx.Graphics.Video/Vp9MvRef.cs b/Ryujinx.Graphics.Video/Vp9MvRef.cs
new file mode 100644
index 00000000..6f2d8e81
--- /dev/null
+++ b/Ryujinx.Graphics.Video/Vp9MvRef.cs
@@ -0,0 +1,11 @@
+using Ryujinx.Common.Memory;
+
+namespace Ryujinx.Graphics.Video
+{
+    // This must match the structure used by NVDEC, do not modify.
+    public struct Vp9MvRef
+    {
+        public Array2<Vp9Mv> Mvs;
+        public Array2<int> RefFrames;
+    }
+}
diff --git a/Ryujinx.Graphics.Video/Vp9PictureInfo.cs b/Ryujinx.Graphics.Video/Vp9PictureInfo.cs
new file mode 100644
index 00000000..a5cc2b45
--- /dev/null
+++ b/Ryujinx.Graphics.Video/Vp9PictureInfo.cs
@@ -0,0 +1,39 @@
+using Ryujinx.Common.Memory;
+
+namespace Ryujinx.Graphics.Video
+{
+    public ref struct Vp9PictureInfo
+    {
+        public ISurface LastReference;
+        public ISurface GoldenReference;
+        public ISurface AltReference;
+        public bool IsKeyFrame;
+        public bool IntraOnly;
+        public Array4<sbyte> RefFrameSignBias;
+        public int BaseQIndex;
+        public int YDcDeltaQ;
+        public int UvDcDeltaQ;
+        public int UvAcDeltaQ;
+        public bool Lossless;
+        public int TransformMode;
+        public bool AllowHighPrecisionMv;
+        public int InterpFilter;
+        public int ReferenceMode;
+        public sbyte CompFixedRef;
+        public Array2<sbyte> CompVarRef;
+        public int Log2TileCols;
+        public int Log2TileRows;
+        public bool SegmentEnabled;
+        public bool SegmentMapUpdate;
+        public bool SegmentMapTemporalUpdate;
+        public int SegmentAbsDelta;
+        public Array8<uint> SegmentFeatureEnable;
+        public Array8<Array4<short>> SegmentFeatureData;
+        public bool ModeRefDeltaEnabled;
+        public bool UsePrevInFindMvRefs;
+        public Array4<sbyte> RefDeltas;
+        public Array2<sbyte> ModeDeltas;
+        public Vp9EntropyProbs Entropy;
+        public Vp9BackwardUpdates BackwardUpdateCounts;
+    }
+}