Geometry shader emulation for macOS (#5551)

* Implement vertex and geometry shader conversion to compute

* Call InitializeReservedCounts for compute too

* PR feedback

* Set clip distance mask for geometry and tessellation shaders too

* Transform feedback emulation only for vertex

6 files changed, 25 insertions, 28 deletions

diff --git a/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/Declarations.cs b/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/Declarations.cs
index 607ff431..500de71f 100644
--- a/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/Declarations.cs
+++ b/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/Declarations.cs
@@ -100,10 +100,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl
                     else
                     {
                         string outPrimitive = context.Definitions.OutputTopology.ToGlslString();
-
-                        int maxOutputVertices = context.Definitions.GpPassthrough
-                            ? context.Definitions.InputTopology.ToInputVertices()
-                            : context.Definitions.MaxOutputVertices;
+                        int maxOutputVertices = context.Definitions.MaxOutputVertices;
 
                         context.AppendLine($"layout ({outPrimitive}, max_vertices = {maxOutputVertices}) out;");
                     }
@@ -320,15 +317,22 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl
             {
                 string typeName = GetVarTypeName(context, memory.Type & ~AggregateType.Array);
 
-                if (memory.ArrayLength > 0)
+                if (memory.Type.HasFlag(AggregateType.Array))
                 {
-                    string arraySize = memory.ArrayLength.ToString(CultureInfo.InvariantCulture);
+                    if (memory.ArrayLength > 0)
+                    {
+                        string arraySize = memory.ArrayLength.ToString(CultureInfo.InvariantCulture);
 
-                    context.AppendLine($"{prefix}{typeName} {memory.Name}[{arraySize}];");
+                        context.AppendLine($"{prefix}{typeName} {memory.Name}[{arraySize}];");
+                    }
+                    else
+                    {
+                        context.AppendLine($"{prefix}{typeName} {memory.Name}[];");
+                    }
                 }
                 else
                 {
-                    context.AppendLine($"{prefix}{typeName} {memory.Name}[];");
+                    context.AppendLine($"{prefix}{typeName} {memory.Name};");
                 }
             }
         }
diff --git a/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/IoMap.cs b/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/IoMap.cs
index b5f453ae..caa6ef64 100644
--- a/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/IoMap.cs
+++ b/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/IoMap.cs
@@ -31,6 +31,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
                 IoVariable.FrontColorDiffuse => ("gl_FrontColor", AggregateType.Vector4 | AggregateType.FP32), // Deprecated.
                 IoVariable.FrontColorSpecular => ("gl_FrontSecondaryColor", AggregateType.Vector4 | AggregateType.FP32), // Deprecated.
                 IoVariable.FrontFacing => ("gl_FrontFacing", AggregateType.Bool),
+                IoVariable.GlobalId => ("gl_GlobalInvocationID", AggregateType.Vector3 | AggregateType.U32),
                 IoVariable.InstanceId => ("gl_InstanceID", AggregateType.S32),
                 IoVariable.InstanceIndex => ("gl_InstanceIndex", AggregateType.S32),
                 IoVariable.InvocationId => ("gl_InvocationID", AggregateType.S32),
diff --git a/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/CodeGenContext.cs b/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/CodeGenContext.cs
index d385782a..9f9411a9 100644
--- a/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/CodeGenContext.cs
+++ b/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/CodeGenContext.cs
@@ -27,8 +27,6 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv
         public ILogger Logger { get; }
         public TargetApi TargetApi { get; }
 
-        public int InputVertices { get; }
-
         public Dictionary<int, Instruction> ConstantBuffers { get; } = new();
         public Dictionary<int, Instruction> StorageBuffers { get; } = new();
 
@@ -101,19 +99,6 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv
             Logger = parameters.Logger;
             TargetApi = parameters.TargetApi;
 
-            if (parameters.Definitions.Stage == ShaderStage.Geometry)
-            {
-                InputVertices = parameters.Definitions.InputTopology switch
-                {
-                    InputTopology.Points => 1,
-                    InputTopology.Lines => 2,
-                    InputTopology.LinesAdjacency => 2,
-                    InputTopology.Triangles => 3,
-                    InputTopology.TrianglesAdjacency => 3,
-                    _ => throw new InvalidOperationException($"Invalid input topology \"{parameters.Definitions.InputTopology}\"."),
-                };
-            }
-
             AddCapability(Capability.Shader);
             AddCapability(Capability.Float64);
 
diff --git a/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/Declarations.cs b/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/Declarations.cs
index b0659ba4..54767c2f 100644
--- a/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/Declarations.cs
+++ b/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/Declarations.cs
@@ -369,7 +369,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv
                 if (context.Definitions.Stage != ShaderStage.Vertex)
                 {
                     var perVertexInputStructType = CreatePerVertexStructType(context);
-                    int arraySize = context.Definitions.Stage == ShaderStage.Geometry ? context.InputVertices : 32;
+                    int arraySize = context.Definitions.Stage == ShaderStage.Geometry ? context.Definitions.InputTopology.ToInputVertices() : 32;
                     var perVertexInputArrayType = context.TypeArray(perVertexInputStructType, context.Constant(context.TypeU32(), arraySize));
                     var perVertexInputPointerType = context.TypePointer(StorageClass.Input, perVertexInputArrayType);
                     var perVertexInputVariable = context.Variable(perVertexInputPointerType, StorageClass.Input);
@@ -506,7 +506,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv
 
             if (!isPerPatch && IoMap.IsPerVertex(ioVariable, context.Definitions.Stage, isOutput))
             {
-                int arraySize = context.Definitions.Stage == ShaderStage.Geometry ? context.InputVertices : 32;
+                int arraySize = context.Definitions.Stage == ShaderStage.Geometry ? context.Definitions.InputTopology.ToInputVertices() : 32;
                 spvType = context.TypeArray(spvType, context.Constant(context.TypeU32(), arraySize));
 
                 if (context.Definitions.GpPassthrough && context.HostCapabilities.SupportsGeometryShaderPassthrough)
diff --git a/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/IoMap.cs b/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/IoMap.cs
index 08d403e2..7b4e14ff 100644
--- a/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/IoMap.cs
+++ b/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/IoMap.cs
@@ -22,6 +22,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv
                 IoVariable.FragmentCoord => (BuiltIn.FragCoord, AggregateType.Vector4 | AggregateType.FP32),
                 IoVariable.FragmentOutputDepth => (BuiltIn.FragDepth, AggregateType.FP32),
                 IoVariable.FrontFacing => (BuiltIn.FrontFacing, AggregateType.Bool),
+                IoVariable.GlobalId => (BuiltIn.GlobalInvocationId, AggregateType.Vector3 | AggregateType.U32),
                 IoVariable.InstanceId => (BuiltIn.InstanceId, AggregateType.S32),
                 IoVariable.InstanceIndex => (BuiltIn.InstanceIndex, AggregateType.S32),
                 IoVariable.InvocationId => (BuiltIn.InvocationId, AggregateType.S32),
diff --git a/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/SpirvGenerator.cs b/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/SpirvGenerator.cs
index 70f1dd3c..44d3e985 100644
--- a/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/SpirvGenerator.cs
+++ b/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/SpirvGenerator.cs
@@ -239,9 +239,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv
                         _ => throw new InvalidOperationException($"Invalid output topology \"{context.Definitions.OutputTopology}\"."),
                     });
 
-                    int maxOutputVertices = context.Definitions.GpPassthrough ? context.InputVertices : context.Definitions.MaxOutputVertices;
-
-                    context.AddExecutionMode(spvFunc, ExecutionMode.OutputVertices, (SpvLiteralInteger)maxOutputVertices);
+                    context.AddExecutionMode(spvFunc, ExecutionMode.OutputVertices, (SpvLiteralInteger)context.Definitions.MaxOutputVertices);
                 }
                 else if (context.Definitions.Stage == ShaderStage.Fragment)
                 {
@@ -279,6 +277,14 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv
                         localSizeZ);
                 }
 
+                if (context.Definitions.Stage != ShaderStage.Fragment &&
+                    context.Definitions.Stage != ShaderStage.Geometry &&
+                    context.Definitions.Stage != ShaderStage.Compute &&
+                    context.Info.IoDefinitions.Contains(new IoDefinition(StorageKind.Output, IoVariable.Layer)))
+                {
+                    context.AddCapability(Capability.ShaderLayer);
+                }
+
                 if (context.Definitions.TransformFeedbackEnabled && context.Definitions.LastInVertexPipeline)
                 {
                     context.AddExecutionMode(spvFunc, ExecutionMode.Xfb);