Refactor attribute handling on the shader generator (#4565)

* Refactor attribute handling on the shader generator

* Implement gl_ViewportMask[]

* Add back the Intel FrontFacing bug workaround

* Fix GLSL transform feedback outputs mistmatch with fragment stage

* Shader cache version bump

* Fix geometry shader recognition

* PR feedback

* Delete GetOperandDef and GetOperandUse

* Remove replacements that are no longer needed on GLSL compilation on Vulkan

* Fix incorrect load for per-patch outputs

* Fix build

4 files changed, 265 insertions, 62 deletions

diff --git a/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGen.cs b/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGen.cs
index 9ca4618d..01bd11e5 100644
--- a/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGen.cs
+++ b/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGen.cs
@@ -73,7 +73,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
                     // For shared memory access, the second argument is unused and should be ignored.
                     // It is there to make both storage and shared access have the same number of arguments.
                     // For storage, both inputs are consumed when the argument index is 0, so we should skip it here.
-                    if (argIndex == 1 && (atomic || (inst & Instruction.MrMask) == Instruction.MrShared))
+                    if (argIndex == 1 && (atomic || operation.StorageKind == StorageKind.SharedMemory))
                     {
                         continue;
                     }
@@ -85,14 +85,12 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
 
                     if (argIndex == 0 && atomic)
                     {
-                        Instruction memRegion = inst & Instruction.MrMask;
-
-                        switch (memRegion)
+                        switch (operation.StorageKind)
                         {
-                            case Instruction.MrShared: args += LoadShared(context, operation); break;
-                            case Instruction.MrStorage: args += LoadStorage(context, operation); break;
+                            case StorageKind.SharedMemory: args += LoadShared(context, operation); break;
+                            case StorageKind.StorageBuffer: args += LoadStorage(context, operation); break;
 
-                            default: throw new InvalidOperationException($"Invalid memory region \"{memRegion}\".");
+                            default: throw new InvalidOperationException($"Invalid storage kind \"{operation.StorageKind}\".");
                         }
                     }
                     else
@@ -166,8 +164,8 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
                     case Instruction.ImageAtomic:
                         return ImageLoadOrStore(context, operation);
 
-                    case Instruction.LoadAttribute:
-                        return LoadAttribute(context, operation);
+                    case Instruction.Load:
+                        return Load(context, operation);
 
                     case Instruction.LoadConstant:
                         return LoadConstant(context, operation);
@@ -193,8 +191,8 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
                     case Instruction.PackHalf2x16:
                         return PackHalf2x16(context, operation);
 
-                    case Instruction.StoreAttribute:
-                        return StoreAttribute(context, operation);
+                    case Instruction.Store:
+                        return Store(context, operation);
 
                     case Instruction.StoreLocal:
                         return StoreLocal(context, operation);
diff --git a/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenHelper.cs b/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenHelper.cs
index 743b695c..00478f6a 100644
--- a/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenHelper.cs
+++ b/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenHelper.cs
@@ -82,7 +82,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
             Add(Instruction.ImageStore,               InstType.Special);
             Add(Instruction.ImageAtomic,              InstType.Special);
             Add(Instruction.IsNan,                    InstType.CallUnary,      "isnan");
-            Add(Instruction.LoadAttribute,            InstType.Special);
+            Add(Instruction.Load,                     InstType.Special);
             Add(Instruction.LoadConstant,             InstType.Special);
             Add(Instruction.LoadLocal,                InstType.Special);
             Add(Instruction.LoadShared,               InstType.Special);
@@ -118,7 +118,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
             Add(Instruction.ShuffleXor,               InstType.CallQuaternary, HelperFunctionNames.ShuffleXor);
             Add(Instruction.Sine,                     InstType.CallUnary,      "sin");
             Add(Instruction.SquareRoot,               InstType.CallUnary,      "sqrt");
-            Add(Instruction.StoreAttribute,           InstType.Special);
+            Add(Instruction.Store,                    InstType.Special);
             Add(Instruction.StoreLocal,               InstType.Special);
             Add(Instruction.StoreShared,              InstType.Special);
             Add(Instruction.StoreShared16,            InstType.Special);
diff --git a/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenMemory.cs b/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenMemory.cs
index a5d2632c..99519837 100644
--- a/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenMemory.cs
+++ b/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenMemory.cs
@@ -210,30 +210,9 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
             return texCallBuilder.ToString();
         }
 
-        public static string LoadAttribute(CodeGenContext context, AstOperation operation)
+        public static string Load(CodeGenContext context, AstOperation operation)
         {
-            IAstNode src1 = operation.GetSource(0);
-            IAstNode src2 = operation.GetSource(1);
-            IAstNode src3 = operation.GetSource(2);
-
-            if (!(src1 is AstOperand baseAttr) || baseAttr.Type != OperandType.Constant)
-            {
-                throw new InvalidOperationException($"First input of {nameof(Instruction.LoadAttribute)} must be a constant operand.");
-            }
-
-            string indexExpr = GetSoureExpr(context, src3, GetSrcVarType(operation.Inst, 2));
-
-            if (src2 is AstOperand operand && operand.Type == OperandType.Constant)
-            {
-                int attrOffset = baseAttr.Value + (operand.Value << 2);
-                return OperandManager.GetAttributeName(context, attrOffset, perPatch: false, isOutAttr: false, indexExpr);
-            }
-            else
-            {
-                string attrExpr = GetSoureExpr(context, src2, GetSrcVarType(operation.Inst, 1));
-                attrExpr = Enclose(attrExpr, src2, Instruction.ShiftRightS32, isLhs: true);
-                return OperandManager.GetAttributeName(attrExpr, context.Config, isOutAttr: false, indexExpr);
-            }
+            return GenerateLoadOrStore(context, operation, isStore: false);
         }
 
         public static string LoadConstant(CodeGenContext context, AstOperation operation)
@@ -337,33 +316,9 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
             return $"textureQueryLod({samplerName}, {coordsExpr}){GetMask(texOp.Index)}";
         }
 
-        public static string StoreAttribute(CodeGenContext context, AstOperation operation)
+        public static string Store(CodeGenContext context, AstOperation operation)
         {
-            IAstNode src1 = operation.GetSource(0);
-            IAstNode src2 = operation.GetSource(1);
-            IAstNode src3 = operation.GetSource(2);
-
-            if (!(src1 is AstOperand baseAttr) || baseAttr.Type != OperandType.Constant)
-            {
-                throw new InvalidOperationException($"First input of {nameof(Instruction.StoreAttribute)} must be a constant operand.");
-            }
-
-            string attrName;
-
-            if (src2 is AstOperand operand && operand.Type == OperandType.Constant)
-            {
-                int attrOffset = baseAttr.Value + (operand.Value << 2);
-                attrName = OperandManager.GetAttributeName(context, attrOffset, perPatch: false, isOutAttr: true);
-            }
-            else
-            {
-                string attrExpr = GetSoureExpr(context, src2, GetSrcVarType(operation.Inst, 1));
-                attrExpr = Enclose(attrExpr, src2, Instruction.ShiftRightS32, isLhs: true);
-                attrName = OperandManager.GetAttributeName(attrExpr, context.Config, isOutAttr: true);
-            }
-
-            string value = GetSoureExpr(context, src3, GetSrcVarType(operation.Inst, 2));
-            return $"{attrName} = {value}";
+            return GenerateLoadOrStore(context, operation, isStore: true);
         }
 
         public static string StoreLocal(CodeGenContext context, AstOperation operation)
@@ -847,6 +802,111 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
             }
         }
 
+        private static string GenerateLoadOrStore(CodeGenContext context, AstOperation operation, bool isStore)
+        {
+            StorageKind storageKind = operation.StorageKind;
+
+            string varName;
+            AggregateType varType;
+            int srcIndex = 0;
+
+            switch (storageKind)
+            {
+                case StorageKind.Input:
+                case StorageKind.InputPerPatch:
+                case StorageKind.Output:
+                case StorageKind.OutputPerPatch:
+                    if (!(operation.GetSource(srcIndex++) is AstOperand varId) || varId.Type != OperandType.Constant)
+                    {
+                        throw new InvalidOperationException($"First input of {operation.Inst} with {storageKind} storage must be a constant operand.");
+                    }
+
+                    IoVariable ioVariable = (IoVariable)varId.Value;
+                    bool isOutput = storageKind.IsOutput();
+                    bool isPerPatch = storageKind.IsPerPatch();
+                    int location = -1;
+                    int component = 0;
+
+                    if (context.Config.HasPerLocationInputOrOutput(ioVariable, isOutput))
+                    {
+                        if (!(operation.GetSource(srcIndex++) is AstOperand vecIndex) || vecIndex.Type != OperandType.Constant)
+                        {
+                            throw new InvalidOperationException($"Second input of {operation.Inst} with {storageKind} storage must be a constant operand.");
+                        }
+
+                        location = vecIndex.Value;
+
+                        if (operation.SourcesCount > srcIndex &&
+                            operation.GetSource(srcIndex) is AstOperand elemIndex &&
+                            elemIndex.Type == OperandType.Constant &&
+                            context.Config.HasPerLocationInputOrOutputComponent(ioVariable, location, elemIndex.Value, isOutput))
+                        {
+                            component = elemIndex.Value;
+                            srcIndex++;
+                        }
+                    }
+
+                    (varName, varType) = IoMap.GetGlslVariable(context.Config, ioVariable, location, component, isOutput, isPerPatch);
+
+                    if (IoMap.IsPerVertexBuiltIn(context.Config.Stage, ioVariable, isOutput))
+                    {
+                        // Since those exist both as input and output on geometry and tessellation shaders,
+                        // we need the gl_in and gl_out prefixes to disambiguate.
+
+                        if (storageKind == StorageKind.Input)
+                        {
+                            string expr = GetSoureExpr(context, operation.GetSource(srcIndex++), AggregateType.S32);
+                            varName = $"gl_in[{expr}].{varName}";
+                        }
+                        else if (storageKind == StorageKind.Output)
+                        {
+                            string expr = GetSoureExpr(context, operation.GetSource(srcIndex++), AggregateType.S32);
+                            varName = $"gl_out[{expr}].{varName}";
+                        }
+                    }
+
+                    int firstSrcIndex = srcIndex;
+                    int inputsCount = isStore ? operation.SourcesCount - 1 : operation.SourcesCount;
+
+                    for (; srcIndex < inputsCount; srcIndex++)
+                    {
+                        IAstNode src = operation.GetSource(srcIndex);
+
+                        if ((varType & AggregateType.ElementCountMask) != 0 &&
+                            srcIndex == inputsCount - 1 &&
+                            src is AstOperand elementIndex &&
+                            elementIndex.Type == OperandType.Constant)
+                        {
+                            varName += "." + "xyzw"[elementIndex.Value & 3];
+                        }
+                        else if (srcIndex == firstSrcIndex && context.Config.Stage == ShaderStage.TessellationControl && storageKind == StorageKind.Output)
+                        {
+                            // GLSL requires that for tessellation control shader outputs,
+                            // that the index expression must be *exactly* "gl_InvocationID",
+                            // otherwise the compilation fails.
+                            // TODO: Get rid of this and use expression propagation to make sure we generate the correct code from IR.
+                            varName += "[gl_InvocationID]";
+                        }
+                        else
+                        {
+                            varName += $"[{GetSoureExpr(context, src, AggregateType.S32)}]";
+                        }
+                    }
+                    break;
+
+                default:
+                    throw new InvalidOperationException($"Invalid storage kind {storageKind}.");
+            }
+
+            if (isStore)
+            {
+                varType &= AggregateType.ElementTypeMask;
+                varName = $"{varName} = {GetSoureExpr(context, operation.GetSource(srcIndex), varType)}";
+            }
+
+            return varName;
+        }
+
         private static string GetStorageBufferAccessor(string slotExpr, string offsetExpr, ShaderStage stage)
         {
             string sbName = OperandManager.GetShaderStagePrefix(stage);
diff --git a/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/IoMap.cs b/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/IoMap.cs
new file mode 100644
index 00000000..093ee232
--- /dev/null
+++ b/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/IoMap.cs
@@ -0,0 +1,145 @@
+using Ryujinx.Graphics.Shader.IntermediateRepresentation;
+using Ryujinx.Graphics.Shader.Translation;
+using System.Globalization;
+
+namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
+{
+    static class IoMap
+    {
+        public static (string, AggregateType) GetGlslVariable(
+            ShaderConfig config,
+            IoVariable ioVariable,
+            int location,
+            int component,
+            bool isOutput,
+            bool isPerPatch)
+        {
+            return ioVariable switch
+            {
+                IoVariable.BackColorDiffuse => ("gl_BackColor", AggregateType.Vector4 | AggregateType.FP32), // Deprecated.
+                IoVariable.BackColorSpecular => ("gl_BackSecondaryColor", AggregateType.Vector4 | AggregateType.FP32), // Deprecated.
+                IoVariable.BaseInstance => ("gl_BaseInstanceARB", AggregateType.S32),
+                IoVariable.BaseVertex => ("gl_BaseVertexARB", AggregateType.S32),
+                IoVariable.ClipDistance => ("gl_ClipDistance", AggregateType.Array | AggregateType.FP32),
+                IoVariable.CtaId => ("gl_WorkGroupID", AggregateType.Vector3 | AggregateType.U32),
+                IoVariable.DrawIndex => ("gl_DrawIDARB", AggregateType.S32),
+                IoVariable.FogCoord => ("gl_FogFragCoord", AggregateType.FP32), // Deprecated.
+                IoVariable.FragmentCoord => ("gl_FragCoord", AggregateType.Vector4 | AggregateType.FP32),
+                IoVariable.FragmentOutputColor => GetFragmentOutputColorVariableName(config, location),
+                IoVariable.FragmentOutputDepth => ("gl_FragDepth", AggregateType.FP32),
+                IoVariable.FragmentOutputIsBgra => (DefaultNames.SupportBlockIsBgraName, AggregateType.Array | AggregateType.Bool),
+                IoVariable.FrontColorDiffuse => ("gl_FrontColor", AggregateType.Vector4 | AggregateType.FP32), // Deprecated.
+                IoVariable.FrontColorSpecular  => ("gl_FrontSecondaryColor", AggregateType.Vector4 | AggregateType.FP32), // Deprecated.
+                IoVariable.FrontFacing => ("gl_FrontFacing", AggregateType.Bool),
+                IoVariable.InstanceId => ("gl_InstanceID", AggregateType.S32),
+                IoVariable.InstanceIndex => ("gl_InstanceIndex", AggregateType.S32),
+                IoVariable.InvocationId => ("gl_InvocationID", AggregateType.S32),
+                IoVariable.Layer => ("gl_Layer", AggregateType.S32),
+                IoVariable.PatchVertices => ("gl_PatchVerticesIn", AggregateType.S32),
+                IoVariable.PointCoord => ("gl_PointCoord", AggregateType.Vector2 | AggregateType.FP32),
+                IoVariable.PointSize => ("gl_PointSize", AggregateType.FP32),
+                IoVariable.Position => ("gl_Position", AggregateType.Vector4 | AggregateType.FP32),
+                IoVariable.PrimitiveId => GetPrimitiveIdVariableName(config.Stage, isOutput),
+                IoVariable.SubgroupEqMask => GetSubgroupMaskVariableName(config, "Eq"),
+                IoVariable.SubgroupGeMask => GetSubgroupMaskVariableName(config, "Ge"),
+                IoVariable.SubgroupGtMask => GetSubgroupMaskVariableName(config, "Gt"),
+                IoVariable.SubgroupLaneId => GetSubgroupInvocationIdVariableName(config),
+                IoVariable.SubgroupLeMask => GetSubgroupMaskVariableName(config, "Le"),
+                IoVariable.SubgroupLtMask => GetSubgroupMaskVariableName(config, "Lt"),
+                IoVariable.SupportBlockRenderScale => (DefaultNames.SupportBlockRenderScaleName, AggregateType.Array | AggregateType.FP32),
+                IoVariable.SupportBlockViewInverse => (DefaultNames.SupportBlockViewportInverse, AggregateType.Vector2 | AggregateType.FP32),
+                IoVariable.TessellationCoord => ("gl_TessCoord", AggregateType.Vector3 | AggregateType.FP32),
+                IoVariable.TessellationLevelInner => ("gl_TessLevelInner", AggregateType.Array | AggregateType.FP32),
+                IoVariable.TessellationLevelOuter => ("gl_TessLevelOuter", AggregateType.Array | AggregateType.FP32),
+                IoVariable.TextureCoord => ("gl_TexCoord", AggregateType.Array | AggregateType.Vector4 | AggregateType.FP32), // Deprecated.
+                IoVariable.ThreadId => ("gl_LocalInvocationID", AggregateType.Vector3 | AggregateType.U32),
+                IoVariable.ThreadKill => ("gl_HelperInvocation", AggregateType.Bool),
+                IoVariable.UserDefined => GetUserDefinedVariableName(config, location, component, isOutput, isPerPatch),
+                IoVariable.VertexId => ("gl_VertexID", AggregateType.S32),
+                IoVariable.VertexIndex => ("gl_VertexIndex", AggregateType.S32),
+                IoVariable.ViewportIndex => ("gl_ViewportIndex", AggregateType.S32),
+                IoVariable.ViewportMask => ("gl_ViewportMask", AggregateType.Array | AggregateType.S32),
+                _ => (null, AggregateType.Invalid)
+            };
+        }
+
+        public static bool IsPerVertexBuiltIn(ShaderStage stage, IoVariable ioVariable, bool isOutput)
+        {
+            switch (ioVariable)
+            {
+                case IoVariable.Layer:
+                case IoVariable.ViewportIndex:
+                case IoVariable.PointSize:
+                case IoVariable.Position:
+                case IoVariable.ClipDistance:
+                case IoVariable.PointCoord:
+                case IoVariable.ViewportMask:
+                    if (isOutput)
+                    {
+                        return stage == ShaderStage.TessellationControl;
+                    }
+                    else
+                    {
+                        return stage == ShaderStage.TessellationControl ||
+                               stage == ShaderStage.TessellationEvaluation ||
+                               stage == ShaderStage.Geometry;
+                    }
+            }
+
+            return false;
+        }
+
+        private static (string, AggregateType) GetFragmentOutputColorVariableName(ShaderConfig config, int location)
+        {
+            if (location < 0)
+            {
+                return (DefaultNames.OAttributePrefix, config.GetFragmentOutputColorType(0));
+            }
+
+            string name = DefaultNames.OAttributePrefix + location.ToString(CultureInfo.InvariantCulture);
+
+            return (name, config.GetFragmentOutputColorType(location));
+        }
+
+        private static (string, AggregateType) GetPrimitiveIdVariableName(ShaderStage stage, bool isOutput)
+        {
+            // The geometry stage has an additional gl_PrimitiveIDIn variable.
+            return (isOutput || stage != ShaderStage.Geometry ? "gl_PrimitiveID" : "gl_PrimitiveIDIn", AggregateType.S32);
+        }
+
+        private static (string, AggregateType) GetSubgroupMaskVariableName(ShaderConfig config, string cc)
+        {
+            return config.GpuAccessor.QueryHostSupportsShaderBallot()
+                ? ($"unpackUint2x32(gl_SubGroup{cc}MaskARB)", AggregateType.Vector2 | AggregateType.U32)
+                : ($"gl_Subgroup{cc}Mask", AggregateType.Vector4 | AggregateType.U32);
+        }
+
+        private static (string, AggregateType) GetSubgroupInvocationIdVariableName(ShaderConfig config)
+        {
+            return config.GpuAccessor.QueryHostSupportsShaderBallot()
+                ? ("gl_SubGroupInvocationARB", AggregateType.U32)
+                : ("gl_SubgroupInvocationID", AggregateType.U32);
+        }
+
+        private static (string, AggregateType) GetUserDefinedVariableName(ShaderConfig config, int location, int component, bool isOutput, bool isPerPatch)
+        {
+            string name = isPerPatch
+                ? DefaultNames.PerPatchAttributePrefix
+                : (isOutput ? DefaultNames.OAttributePrefix : DefaultNames.IAttributePrefix);
+
+            if (location < 0)
+            {
+                return (name, config.GetUserDefinedType(0, isOutput));
+            }
+
+            name += location.ToString(CultureInfo.InvariantCulture);
+
+            if (config.HasPerLocationInputOrOutputComponent(IoVariable.UserDefined, location, component, isOutput))
+            {
+                name += "_" + "xyzw"[component & 3];
+            }
+
+            return (name, config.GetUserDefinedType(location, isOutput));
+        }
+    }
+}
\ No newline at end of file