Geometry shader emulation for macOS (#5551)

* Implement vertex and geometry shader conversion to compute

* Call InitializeReservedCounts for compute too

* PR feedback

* Set clip distance mask for geometry and tessellation shaders too

* Transform feedback emulation only for vertex

4 files changed, 12 insertions, 20 deletions

diff --git a/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/CodeGenContext.cs b/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/CodeGenContext.cs
index d385782a..9f9411a9 100644
--- a/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/CodeGenContext.cs
+++ b/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/CodeGenContext.cs
@@ -27,8 +27,6 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv
         public ILogger Logger { get; }
         public TargetApi TargetApi { get; }
 
-        public int InputVertices { get; }
-
         public Dictionary<int, Instruction> ConstantBuffers { get; } = new();
         public Dictionary<int, Instruction> StorageBuffers { get; } = new();
 
@@ -101,19 +99,6 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv
             Logger = parameters.Logger;
             TargetApi = parameters.TargetApi;
 
-            if (parameters.Definitions.Stage == ShaderStage.Geometry)
-            {
-                InputVertices = parameters.Definitions.InputTopology switch
-                {
-                    InputTopology.Points => 1,
-                    InputTopology.Lines => 2,
-                    InputTopology.LinesAdjacency => 2,
-                    InputTopology.Triangles => 3,
-                    InputTopology.TrianglesAdjacency => 3,
-                    _ => throw new InvalidOperationException($"Invalid input topology \"{parameters.Definitions.InputTopology}\"."),
-                };
-            }
-
             AddCapability(Capability.Shader);
             AddCapability(Capability.Float64);
 
diff --git a/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/Declarations.cs b/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/Declarations.cs
index b0659ba4..54767c2f 100644
--- a/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/Declarations.cs
+++ b/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/Declarations.cs
@@ -369,7 +369,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv
                 if (context.Definitions.Stage != ShaderStage.Vertex)
                 {
                     var perVertexInputStructType = CreatePerVertexStructType(context);
-                    int arraySize = context.Definitions.Stage == ShaderStage.Geometry ? context.InputVertices : 32;
+                    int arraySize = context.Definitions.Stage == ShaderStage.Geometry ? context.Definitions.InputTopology.ToInputVertices() : 32;
                     var perVertexInputArrayType = context.TypeArray(perVertexInputStructType, context.Constant(context.TypeU32(), arraySize));
                     var perVertexInputPointerType = context.TypePointer(StorageClass.Input, perVertexInputArrayType);
                     var perVertexInputVariable = context.Variable(perVertexInputPointerType, StorageClass.Input);
@@ -506,7 +506,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv
 
             if (!isPerPatch && IoMap.IsPerVertex(ioVariable, context.Definitions.Stage, isOutput))
             {
-                int arraySize = context.Definitions.Stage == ShaderStage.Geometry ? context.InputVertices : 32;
+                int arraySize = context.Definitions.Stage == ShaderStage.Geometry ? context.Definitions.InputTopology.ToInputVertices() : 32;
                 spvType = context.TypeArray(spvType, context.Constant(context.TypeU32(), arraySize));
 
                 if (context.Definitions.GpPassthrough && context.HostCapabilities.SupportsGeometryShaderPassthrough)
diff --git a/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/IoMap.cs b/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/IoMap.cs
index 08d403e2..7b4e14ff 100644
--- a/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/IoMap.cs
+++ b/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/IoMap.cs
@@ -22,6 +22,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv
                 IoVariable.FragmentCoord => (BuiltIn.FragCoord, AggregateType.Vector4 | AggregateType.FP32),
                 IoVariable.FragmentOutputDepth => (BuiltIn.FragDepth, AggregateType.FP32),
                 IoVariable.FrontFacing => (BuiltIn.FrontFacing, AggregateType.Bool),
+                IoVariable.GlobalId => (BuiltIn.GlobalInvocationId, AggregateType.Vector3 | AggregateType.U32),
                 IoVariable.InstanceId => (BuiltIn.InstanceId, AggregateType.S32),
                 IoVariable.InstanceIndex => (BuiltIn.InstanceIndex, AggregateType.S32),
                 IoVariable.InvocationId => (BuiltIn.InvocationId, AggregateType.S32),
diff --git a/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/SpirvGenerator.cs b/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/SpirvGenerator.cs
index 70f1dd3c..44d3e985 100644
--- a/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/SpirvGenerator.cs
+++ b/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/SpirvGenerator.cs
@@ -239,9 +239,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv
                         _ => throw new InvalidOperationException($"Invalid output topology \"{context.Definitions.OutputTopology}\"."),
                     });
 
-                    int maxOutputVertices = context.Definitions.GpPassthrough ? context.InputVertices : context.Definitions.MaxOutputVertices;
-
-                    context.AddExecutionMode(spvFunc, ExecutionMode.OutputVertices, (SpvLiteralInteger)maxOutputVertices);
+                    context.AddExecutionMode(spvFunc, ExecutionMode.OutputVertices, (SpvLiteralInteger)context.Definitions.MaxOutputVertices);
                 }
                 else if (context.Definitions.Stage == ShaderStage.Fragment)
                 {
@@ -279,6 +277,14 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv
                         localSizeZ);
                 }
 
+                if (context.Definitions.Stage != ShaderStage.Fragment &&
+                    context.Definitions.Stage != ShaderStage.Geometry &&
+                    context.Definitions.Stage != ShaderStage.Compute &&
+                    context.Info.IoDefinitions.Contains(new IoDefinition(StorageKind.Output, IoVariable.Layer)))
+                {
+                    context.AddCapability(Capability.ShaderLayer);
+                }
+
                 if (context.Definitions.TransformFeedbackEnabled && context.Definitions.LastInVertexPipeline)
                 {
                     context.AddExecutionMode(spvFunc, ExecutionMode.Xfb);