Initial tessellation shader support (#2534)

* Initial tessellation shader support

* Nits

* Re-arrange built-in table

* This is not needed anymore

* PR feedback

5 files changed, 111 insertions, 62 deletions

diff --git a/Ryujinx.Graphics.Shader/Translation/AttributeConsts.cs b/Ryujinx.Graphics.Shader/Translation/AttributeConsts.cs
index 3d0d216e..128013d8 100644
--- a/Ryujinx.Graphics.Shader/Translation/AttributeConsts.cs
+++ b/Ryujinx.Graphics.Shader/Translation/AttributeConsts.cs
@@ -2,36 +2,45 @@ namespace Ryujinx.Graphics.Shader.Translation
 {
     static class AttributeConsts
     {
-        public const int Layer         = 0x064;
-        public const int PointSize     = 0x06c;
-        public const int PositionX     = 0x070;
-        public const int PositionY     = 0x074;
-        public const int PositionZ     = 0x078;
-        public const int PositionW     = 0x07c;
-        public const int ClipDistance0 = 0x2c0;
-        public const int ClipDistance1 = 0x2c4;
-        public const int ClipDistance2 = 0x2c8;
-        public const int ClipDistance3 = 0x2cc;
-        public const int ClipDistance4 = 0x2d0;
-        public const int ClipDistance5 = 0x2d4;
-        public const int ClipDistance6 = 0x2d8;
-        public const int ClipDistance7 = 0x2dc;
-        public const int PointCoordX   = 0x2e0;
-        public const int PointCoordY   = 0x2e4;
-        public const int TessCoordX    = 0x2f0;
-        public const int TessCoordY    = 0x2f4;
-        public const int InstanceId    = 0x2f8;
-        public const int VertexId      = 0x2fc;
-        public const int FrontFacing   = 0x3fc;
+        public const int TessLevelOuter0 = 0x000;
+        public const int TessLevelOuter1 = 0x004;
+        public const int TessLevelOuter2 = 0x008;
+        public const int TessLevelOuter3 = 0x00c;
+        public const int TessLevelInner0 = 0x010;
+        public const int TessLevelInner1 = 0x014;
+        public const int Layer           = 0x064;
+        public const int PointSize       = 0x06c;
+        public const int PositionX       = 0x070;
+        public const int PositionY       = 0x074;
+        public const int PositionZ       = 0x078;
+        public const int PositionW       = 0x07c;
+        public const int ClipDistance0   = 0x2c0;
+        public const int ClipDistance1   = 0x2c4;
+        public const int ClipDistance2   = 0x2c8;
+        public const int ClipDistance3   = 0x2cc;
+        public const int ClipDistance4   = 0x2d0;
+        public const int ClipDistance5   = 0x2d4;
+        public const int ClipDistance6   = 0x2d8;
+        public const int ClipDistance7   = 0x2dc;
+        public const int PointCoordX     = 0x2e0;
+        public const int PointCoordY     = 0x2e4;
+        public const int TessCoordX      = 0x2f0;
+        public const int TessCoordY      = 0x2f4;
+        public const int InstanceId      = 0x2f8;
+        public const int VertexId        = 0x2fc;
+        public const int FrontFacing     = 0x3fc;
 
         public const int UserAttributesCount = 32;
         public const int UserAttributeBase   = 0x80;
         public const int UserAttributeEnd    = UserAttributeBase + UserAttributesCount * 16;
 
+        public const int LoadOutputMask = 1 << 30;
+        public const int Mask = 0x3fffffff;
+
 
         // Note: Those attributes are used internally by the translator
         // only, they don't exist on Maxwell.
-        public const int SpecialMask             = 0xff << 24;
+        public const int SpecialMask             = 0xf << 24;
         public const int FragmentOutputDepth     = 0x1000000;
         public const int FragmentOutputColorBase = 0x1000010;
         public const int FragmentOutputColorEnd  = FragmentOutputColorBase + 8 * 16;
@@ -49,12 +58,16 @@ namespace Ryujinx.Graphics.Shader.Translation
 
         public const int LaneId = 0x2000020;
 
-        public const int EqMask = 0x2000024;
-        public const int GeMask = 0x2000028;
-        public const int GtMask = 0x200002c;
-        public const int LeMask = 0x2000030;
-        public const int LtMask = 0x2000034;
+        public const int InvocationId = 0x2000024;
+        public const int PrimitiveId = 0x2000028;
+        public const int PatchVerticesIn = 0x200002c;
+
+        public const int EqMask = 0x2000030;
+        public const int GeMask = 0x2000034;
+        public const int GtMask = 0x2000038;
+        public const int LeMask = 0x200003c;
+        public const int LtMask = 0x2000040;
 
-        public const int ThreadKill = 0x2000038;
+        public const int ThreadKill = 0x2000044;
     }
 }
\ No newline at end of file
diff --git a/Ryujinx.Graphics.Shader/Translation/EmitterContext.cs b/Ryujinx.Graphics.Shader/Translation/EmitterContext.cs
index bdfd9626..6f2a6c3b 100644
--- a/Ryujinx.Graphics.Shader/Translation/EmitterContext.cs
+++ b/Ryujinx.Graphics.Shader/Translation/EmitterContext.cs
@@ -216,7 +216,7 @@ namespace Ryujinx.Graphics.Shader.Translation
 
                     if (target.Enabled)
                     {
-                        Config.SetOutputUserAttribute(rtIndex);
+                        Config.SetOutputUserAttribute(rtIndex, perPatch: false);
                         regIndexBase += 4;
                     }
                 }
diff --git a/Ryujinx.Graphics.Shader/Translation/ShaderConfig.cs b/Ryujinx.Graphics.Shader/Translation/ShaderConfig.cs
index 72fa7733..ec7e8982 100644
--- a/Ryujinx.Graphics.Shader/Translation/ShaderConfig.cs
+++ b/Ryujinx.Graphics.Shader/Translation/ShaderConfig.cs
@@ -15,6 +15,8 @@ namespace Ryujinx.Graphics.Shader.Translation
 
         public bool GpPassthrough { get; }
 
+        public int ThreadsPerInputPrimitive { get; }
+
         public OutputTopology OutputTopology { get; }
 
         public int MaxOutputVertices { get; }
@@ -42,7 +44,9 @@ namespace Ryujinx.Graphics.Shader.Translation
         private readonly TranslationCounts _counts;
 
         public int UsedInputAttributes { get; private set; }
+        public int UsedInputAttributesPerPatch { get; private set; }
         public int UsedOutputAttributes { get; private set; }
+        public int UsedOutputAttributesPerPatch { get; private set; }
         public int PassthroughAttributes { get; private set; }
 
         private int _usedConstantBuffers;
@@ -111,15 +115,16 @@ namespace Ryujinx.Graphics.Shader.Translation
 
         public ShaderConfig(ShaderHeader header, IGpuAccessor gpuAccessor, TranslationOptions options, TranslationCounts counts) : this(gpuAccessor, options, counts)
         {
-            Stage             = header.Stage;
-            GpPassthrough     = header.Stage == ShaderStage.Geometry && header.GpPassthrough;
-            OutputTopology    = header.OutputTopology;
-            MaxOutputVertices = header.MaxOutputVertexCount;
-            LocalMemorySize   = header.ShaderLocalMemoryLowSize + header.ShaderLocalMemoryHighSize;
-            ImapTypes         = header.ImapTypes;
-            OmapTargets       = header.OmapTargets;
-            OmapSampleMask    = header.OmapSampleMask;
-            OmapDepth         = header.OmapDepth;
+            Stage                    = header.Stage;
+            GpPassthrough            = header.Stage == ShaderStage.Geometry && header.GpPassthrough;
+            ThreadsPerInputPrimitive = header.ThreadsPerInputPrimitive;
+            OutputTopology           = header.OutputTopology;
+            MaxOutputVertices        = header.MaxOutputVertexCount;
+            LocalMemorySize          = header.ShaderLocalMemoryLowSize + header.ShaderLocalMemoryHighSize;
+            ImapTypes                = header.ImapTypes;
+            OmapTargets              = header.OmapTargets;
+            OmapSampleMask           = header.OmapSampleMask;
+            OmapDepth                = header.OmapDepth;
         }
 
         public int GetDepthRegister()
@@ -169,7 +174,7 @@ namespace Ryujinx.Graphics.Shader.Translation
 
         public TextureFormat GetTextureFormatAtomic(int handle, int cbufSlot = -1)
         {
-            // Atomic image instructions do not support GL_EXT_shader_image_load_formatted, 
+            // Atomic image instructions do not support GL_EXT_shader_image_load_formatted,
             // and must have a type specified. Default to R32Sint if not available.
 
             var format = GpuAccessor.QueryTextureFormat(handle, cbufSlot);
@@ -219,17 +224,31 @@ namespace Ryujinx.Graphics.Shader.Translation
             }
         }
 
-        public void SetInputUserAttribute(int index)
+        public void SetInputUserAttribute(int index, bool perPatch)
         {
-            UsedInputAttributes |= 1 << index;
+            if (perPatch)
+            {
+                UsedInputAttributesPerPatch |= 1 << index;
+            }
+            else
+            {
+                UsedInputAttributes |= 1 << index;
+            }
         }
 
-        public void SetOutputUserAttribute(int index)
+        public void SetOutputUserAttribute(int index, bool perPatch)
         {
-            UsedOutputAttributes |= 1 << index;
+            if (perPatch)
+            {
+                UsedOutputAttributesPerPatch |= 1 << index;
+            }
+            else
+            {
+                UsedOutputAttributes |= 1 << index;
+            }
         }
 
-        public void MergeOutputUserAttributes(int mask)
+        public void MergeOutputUserAttributes(int mask, int maskPerPatch)
         {
             if (GpPassthrough)
             {
@@ -238,6 +257,7 @@ namespace Ryujinx.Graphics.Shader.Translation
             else
             {
                 UsedOutputAttributes |= mask;
+                UsedOutputAttributesPerPatch |= maskPerPatch;
             }
         }
 
diff --git a/Ryujinx.Graphics.Shader/Translation/Translator.cs b/Ryujinx.Graphics.Shader/Translation/Translator.cs
index 0a0ee4a7..0243eba1 100644
--- a/Ryujinx.Graphics.Shader/Translation/Translator.cs
+++ b/Ryujinx.Graphics.Shader/Translation/Translator.cs
@@ -216,27 +216,38 @@ namespace Ryujinx.Graphics.Shader.Translation
                 return;
             }
 
-            void InitializeOutput(int baseAttr)
+            if (config.Stage == ShaderStage.Vertex)
             {
-                for (int c = 0; c < 4; c++)
-                {
-                    context.Copy(Attribute(baseAttr + c * 4), ConstF(c == 3 ? 1f : 0f));
-                }
+                InitializeOutput(context, AttributeConsts.PositionX, perPatch: false);
             }
 
-            if (config.Stage == ShaderStage.Vertex)
+            int usedAttributes = context.Config.UsedOutputAttributes;
+            while (usedAttributes != 0)
             {
-                InitializeOutput(AttributeConsts.PositionX);
+                int index = BitOperations.TrailingZeroCount(usedAttributes);
+
+                InitializeOutput(context, AttributeConsts.UserAttributeBase + index * 16, perPatch: false);
+
+                usedAttributes &= ~(1 << index);
             }
 
-            int usedAttribtes = context.Config.UsedOutputAttributes;
-            while (usedAttribtes != 0)
+            int usedAttributesPerPatch = context.Config.UsedOutputAttributesPerPatch;
+            while (usedAttributesPerPatch != 0)
             {
-                int index = BitOperations.TrailingZeroCount(usedAttribtes);
+                int index = BitOperations.TrailingZeroCount(usedAttributesPerPatch);
 
-                InitializeOutput(AttributeConsts.UserAttributeBase + index * 16);
+                InitializeOutput(context, AttributeConsts.UserAttributeBase + index * 16, perPatch: true);
 
-                usedAttribtes &= ~(1 << index);
+                usedAttributesPerPatch &= ~(1 << index);
+            }
+        }
+
+        private static void InitializeOutput(EmitterContext context, int baseAttr, bool perPatch)
+        {
+            for (int c = 0; c < 4; c++)
+            {
+                int attrOffset = baseAttr + c * 4;
+                context.Copy(perPatch ? AttributePerPatch(attrOffset) : Attribute(attrOffset), ConstF(c == 3 ? 1f : 0f));
             }
         }
 
diff --git a/Ryujinx.Graphics.Shader/Translation/TranslatorContext.cs b/Ryujinx.Graphics.Shader/Translation/TranslatorContext.cs
index 3c7b3c2b..34b116d6 100644
--- a/Ryujinx.Graphics.Shader/Translation/TranslatorContext.cs
+++ b/Ryujinx.Graphics.Shader/Translation/TranslatorContext.cs
@@ -32,10 +32,13 @@ namespace Ryujinx.Graphics.Shader.Translation
 
         private static bool IsUserAttribute(Operand operand)
         {
-            return operand != null &&
-                   operand.Type == OperandType.Attribute &&
-                   operand.Value >= AttributeConsts.UserAttributeBase &&
-                   operand.Value < AttributeConsts.UserAttributeEnd;
+            if (operand != null && operand.Type.IsAttribute())
+            {
+                int value = operand.Value & AttributeConsts.Mask;
+                return value >= AttributeConsts.UserAttributeBase && value < AttributeConsts.UserAttributeEnd;
+            }
+
+            return false;
         }
 
         private static FunctionCode[] Combine(FunctionCode[] a, FunctionCode[] b, int aStart)
@@ -133,14 +136,16 @@ namespace Ryujinx.Graphics.Shader.Translation
         {
             if (nextStage != null)
             {
-                _config.MergeOutputUserAttributes(nextStage._config.UsedInputAttributes);
+                _config.MergeOutputUserAttributes(
+                    nextStage._config.UsedInputAttributes,
+                    nextStage._config.UsedInputAttributesPerPatch);
             }
 
             FunctionCode[] code = EmitShader(_cfg, _config, initializeOutputs: other == null, out _);
 
             if (other != null)
             {
-                other._config.MergeOutputUserAttributes(_config.UsedOutputAttributes);
+                other._config.MergeOutputUserAttributes(_config.UsedOutputAttributes, 0);
 
                 FunctionCode[] otherCode = EmitShader(other._cfg, other._config, initializeOutputs: true, out int aStart);