Move gl_Layer to vertex shader if geometry is not supported (#4368)

* Set gl_Layer on vertex shader if it's set on the geometry shader and it does nothing else

* Shader cache version bump

* PR feedback

* Fix typo

8 files changed, 209 insertions, 2 deletions

diff --git a/Ryujinx.Graphics.Shader/IGpuAccessor.cs b/Ryujinx.Graphics.Shader/IGpuAccessor.cs
index 55df8dc3..f364437c 100644
--- a/Ryujinx.Graphics.Shader/IGpuAccessor.cs
+++ b/Ryujinx.Graphics.Shader/IGpuAccessor.cs
@@ -260,6 +260,15 @@ namespace Ryujinx.Graphics.Shader
         }
 
         /// <summary>
+        /// Queries host GPU geometry shader support.
+        /// </summary>
+        /// <returns>True if the GPU and driver supports geometry shaders, false otherwise</returns>
+        bool QueryHostSupportsGeometryShader()
+        {
+            return true;
+        }
+
+        /// <summary>
         /// Queries host GPU geometry shader passthrough support.
         /// </summary>
         /// <returns>True if the GPU and driver supports geometry shader passthrough, false otherwise</returns>
diff --git a/Ryujinx.Graphics.Shader/ShaderIdentification.cs b/Ryujinx.Graphics.Shader/ShaderIdentification.cs
new file mode 100644
index 00000000..3f015762
--- /dev/null
+++ b/Ryujinx.Graphics.Shader/ShaderIdentification.cs
@@ -0,0 +1,8 @@
+namespace Ryujinx.Graphics.Shader
+{
+    public enum ShaderIdentification
+    {
+        None,
+        GeometryLayerPassthrough
+    }
+}
\ No newline at end of file
diff --git a/Ryujinx.Graphics.Shader/ShaderProgramInfo.cs b/Ryujinx.Graphics.Shader/ShaderProgramInfo.cs
index bb75b10a..30f0ffaa 100644
--- a/Ryujinx.Graphics.Shader/ShaderProgramInfo.cs
+++ b/Ryujinx.Graphics.Shader/ShaderProgramInfo.cs
@@ -10,6 +10,8 @@ namespace Ryujinx.Graphics.Shader
         public ReadOnlyCollection<TextureDescriptor> Textures { get; }
         public ReadOnlyCollection<TextureDescriptor> Images { get; }
 
+        public ShaderIdentification Identification { get; }
+        public int GpLayerInputAttribute { get; }
         public ShaderStage Stage { get; }
         public bool UsesInstanceId { get; }
         public bool UsesDrawParameters { get; }
@@ -22,6 +24,8 @@ namespace Ryujinx.Graphics.Shader
             BufferDescriptor[] sBuffers,
             TextureDescriptor[] textures,
             TextureDescriptor[] images,
+            ShaderIdentification identification,
+            int gpLayerInputAttribute,
             ShaderStage stage,
             bool usesInstanceId,
             bool usesDrawParameters,
@@ -34,6 +38,8 @@ namespace Ryujinx.Graphics.Shader
             Textures = Array.AsReadOnly(textures);
             Images = Array.AsReadOnly(images);
 
+            Identification = identification;
+            GpLayerInputAttribute = gpLayerInputAttribute;
             Stage = stage;
             UsesInstanceId = usesInstanceId;
             UsesDrawParameters = usesDrawParameters;
diff --git a/Ryujinx.Graphics.Shader/Translation/EmitterContext.cs b/Ryujinx.Graphics.Shader/Translation/EmitterContext.cs
index ad55c010..8f33cced 100644
--- a/Ryujinx.Graphics.Shader/Translation/EmitterContext.cs
+++ b/Ryujinx.Graphics.Shader/Translation/EmitterContext.cs
@@ -241,6 +241,13 @@ namespace Ryujinx.Graphics.Shader.Translation
 
                 this.Copy(Attribute(AttributeConsts.PositionZ), this.FPFusedMultiplyAdd(z, ConstF(0.5f), halfW));
             }
+
+            if (Config.Stage != ShaderStage.Geometry && Config.HasLayerInputAttribute)
+            {
+                Config.SetUsedFeature(FeatureFlags.RtLayer);
+
+                this.Copy(Attribute(AttributeConsts.Layer), Attribute(Config.GpLayerInputAttribute | AttributeConsts.LoadOutputMask));
+            }
         }
 
         public void PrepareForVertexReturn(out Operand oldXLocal, out Operand oldYLocal, out Operand oldZLocal)
diff --git a/Ryujinx.Graphics.Shader/Translation/ShaderConfig.cs b/Ryujinx.Graphics.Shader/Translation/ShaderConfig.cs
index a79ef6f5..2caa8f63 100644
--- a/Ryujinx.Graphics.Shader/Translation/ShaderConfig.cs
+++ b/Ryujinx.Graphics.Shader/Translation/ShaderConfig.cs
@@ -20,6 +20,8 @@ namespace Ryujinx.Graphics.Shader.Translation
         public bool LastInPipeline { get; private set; }
         public bool LastInVertexPipeline { get; private set; }
 
+        public bool HasLayerInputAttribute { get; private set; }
+        public int GpLayerInputAttribute { get; private set; }
         public int ThreadsPerInputPrimitive { get; }
 
         public OutputTopology OutputTopology { get; }
@@ -245,6 +247,22 @@ namespace Ryujinx.Graphics.Shader.Translation
             LayerOutputAttribute = attr;
         }
 
+        public void SetGeometryShaderLayerInputAttribute(int attr)
+        {
+            HasLayerInputAttribute = true;
+            GpLayerInputAttribute = attr;
+        }
+
+        public void SetLastInVertexPipeline(bool hasFragment)
+        {
+            if (!hasFragment)
+            {
+                LastInPipeline = true;
+            }
+
+            LastInVertexPipeline = true;
+        }
+
         public void SetInputUserAttributeFixedFunc(int index)
         {
             UsedInputAttributes |= 1 << index;
@@ -706,13 +724,15 @@ namespace Ryujinx.Graphics.Shader.Translation
             return FindDescriptorIndex(GetImageDescriptors(), texOp);
         }
 
-        public ShaderProgramInfo CreateProgramInfo()
+        public ShaderProgramInfo CreateProgramInfo(ShaderIdentification identification = ShaderIdentification.None)
         {
             return new ShaderProgramInfo(
                 GetConstantBufferDescriptors(),
                 GetStorageBufferDescriptors(),
                 GetTextureDescriptors(),
                 GetImageDescriptors(),
+                identification,
+                GpLayerInputAttribute,
                 Stage,
                 UsedFeatures.HasFlag(FeatureFlags.InstanceId),
                 UsedFeatures.HasFlag(FeatureFlags.DrawParameters),
diff --git a/Ryujinx.Graphics.Shader/Translation/ShaderIdentifier.cs b/Ryujinx.Graphics.Shader/Translation/ShaderIdentifier.cs
new file mode 100644
index 00000000..206718f2
--- /dev/null
+++ b/Ryujinx.Graphics.Shader/Translation/ShaderIdentifier.cs
@@ -0,0 +1,145 @@
+using Ryujinx.Graphics.Shader.IntermediateRepresentation;
+using static Ryujinx.Graphics.Shader.IntermediateRepresentation.OperandHelper;
+
+namespace Ryujinx.Graphics.Shader.Translation
+{
+    static class ShaderIdentifier
+    {
+        public static ShaderIdentification Identify(Function[] functions, ShaderConfig config)
+        {
+            if (config.Stage == ShaderStage.Geometry &&
+                config.GpuAccessor.QueryPrimitiveTopology() == InputTopology.Triangles &&
+                !config.GpuAccessor.QueryHostSupportsGeometryShader() &&
+                IsLayerPassthroughGeometryShader(functions, out int layerInputAttr))
+            {
+                config.SetGeometryShaderLayerInputAttribute(layerInputAttr);
+
+                return ShaderIdentification.GeometryLayerPassthrough;
+            }
+
+            return ShaderIdentification.None;
+        }
+
+        private static bool IsLayerPassthroughGeometryShader(Function[] functions, out int layerInputAttr)
+        {
+            bool writesLayer = false;
+            layerInputAttr = 0;
+
+            if (functions.Length != 1)
+            {
+                return false;
+            }
+
+            int verticesCount = 0;
+            int totalVerticesCount = 0;
+
+            foreach (BasicBlock block in functions[0].Blocks)
+            {
+                // We are not expecting loops or any complex control flow here, so fail in those cases.
+                if (block.Branch != null && block.Branch.Index <= block.Index)
+                {
+                    return false;
+                }
+
+                foreach (INode node in block.Operations)
+                {
+                    if (!(node is Operation operation))
+                    {
+                        continue;
+                    }
+
+                    if (IsResourceWrite(operation.Inst))
+                    {
+                        return false;
+                    }
+
+                    if (operation.Inst == Instruction.StoreAttribute)
+                    {
+                        return false;
+                    }
+
+                    if (operation.Inst == Instruction.Copy && operation.Dest.Type == OperandType.Attribute)
+                    {
+                        Operand src = operation.GetSource(0);
+
+                        if (src.Type == OperandType.LocalVariable && src.AsgOp is Operation asgOp && asgOp.Inst == Instruction.LoadAttribute)
+                        {
+                            src = Attribute(asgOp.GetSource(0).Value);
+                        }
+
+                        if (src.Type == OperandType.Attribute)
+                        {
+                            if (operation.Dest.Value == AttributeConsts.Layer)
+                            {
+                                if ((src.Value & AttributeConsts.LoadOutputMask) != 0)
+                                {
+                                    return false;
+                                }
+
+                                writesLayer = true;
+                                layerInputAttr = src.Value;
+                            }
+                            else if (src.Value != operation.Dest.Value)
+                            {
+                                return false;
+                            }
+                        }
+                        else if (src.Type == OperandType.Constant)
+                        {
+                            int dstComponent = (operation.Dest.Value >> 2) & 3;
+                            float expectedValue = dstComponent == 3 ? 1f : 0f;
+
+                            if (src.AsFloat() != expectedValue)
+                            {
+                                return false;
+                            }
+                        }
+                        else
+                        {
+                            return false;
+                        }
+                    }
+                    else if (operation.Inst == Instruction.EmitVertex)
+                    {
+                        verticesCount++;
+                    }
+                    else if (operation.Inst == Instruction.EndPrimitive)
+                    {
+                        totalVerticesCount += verticesCount;
+                        verticesCount = 0;
+                    }
+                }
+            }
+
+            return totalVerticesCount + verticesCount == 3 && writesLayer;
+        }
+
+        private static bool IsResourceWrite(Instruction inst)
+        {
+            switch (inst)
+            {
+                case Instruction.AtomicAdd:
+                case Instruction.AtomicAnd:
+                case Instruction.AtomicCompareAndSwap:
+                case Instruction.AtomicMaxS32:
+                case Instruction.AtomicMaxU32:
+                case Instruction.AtomicMinS32:
+                case Instruction.AtomicMinU32:
+                case Instruction.AtomicOr:
+                case Instruction.AtomicSwap:
+                case Instruction.AtomicXor:
+                case Instruction.ImageAtomic:
+                case Instruction.ImageStore:
+                case Instruction.StoreGlobal:
+                case Instruction.StoreGlobal16:
+                case Instruction.StoreGlobal8:
+                case Instruction.StoreStorage:
+                case Instruction.StoreStorage16:
+                case Instruction.StoreStorage8:
+                    return true;
+            }
+
+            return false;
+        }
+    }
+}
diff --git a/Ryujinx.Graphics.Shader/Translation/Translator.cs b/Ryujinx.Graphics.Shader/Translation/Translator.cs
index 3fb586cb..6a123045 100644
--- a/Ryujinx.Graphics.Shader/Translation/Translator.cs
+++ b/Ryujinx.Graphics.Shader/Translation/Translator.cs
@@ -77,9 +77,11 @@ namespace Ryujinx.Graphics.Shader.Translation
                 funcs[i] = new Function(cfg.Blocks, $"fun{i}", false, inArgumentsCount, outArgumentsCount);
             }
 
+            var identification = ShaderIdentifier.Identify(funcs, config);
+
             var sInfo = StructuredProgram.MakeStructuredProgram(funcs, config);
 
-            var info = config.CreateProgramInfo();
+            var info = config.CreateProgramInfo(identification);
 
             return config.Options.TargetLanguage switch
             {
diff --git a/Ryujinx.Graphics.Shader/Translation/TranslatorContext.cs b/Ryujinx.Graphics.Shader/Translation/TranslatorContext.cs
index 127f84a6..3b88fdba 100644
--- a/Ryujinx.Graphics.Shader/Translation/TranslatorContext.cs
+++ b/Ryujinx.Graphics.Shader/Translation/TranslatorContext.cs
@@ -138,6 +138,16 @@ namespace Ryujinx.Graphics.Shader.Translation
             _config.MergeFromtNextStage(nextStage._config);
         }
 
+        public void SetGeometryShaderLayerInputAttribute(int attr)
+        {
+            _config.SetGeometryShaderLayerInputAttribute(attr);
+        }
+
+        public void SetLastInVertexPipeline(bool hasFragment)
+        {
+            _config.SetLastInVertexPipeline(hasFragment);
+        }
+
         public ShaderProgram Translate(TranslatorContext other = null)
         {
             FunctionCode[] code = EmitShader(_program, _config, initializeOutputs: other == null, out _);