Move gl_Layer to vertex shader if geometry is not supported (#4368)

* Set gl_Layer on vertex shader if it's set on the geometry shader and it does nothing else

* Shader cache version bump

* PR feedback

* Fix typo

4 files changed, 48 insertions, 1 deletions

diff --git a/Ryujinx.Graphics.Gpu/Shader/DiskCache/DiskCacheHostStorage.cs b/Ryujinx.Graphics.Gpu/Shader/DiskCache/DiskCacheHostStorage.cs
index 1f6dab89..edc5a8a0 100644
--- a/Ryujinx.Graphics.Gpu/Shader/DiskCache/DiskCacheHostStorage.cs
+++ b/Ryujinx.Graphics.Gpu/Shader/DiskCache/DiskCacheHostStorage.cs
@@ -22,7 +22,7 @@ namespace Ryujinx.Graphics.Gpu.Shader.DiskCache
         private const ushort FileFormatVersionMajor = 1;
         private const ushort FileFormatVersionMinor = 2;
         private const uint FileFormatVersionPacked = ((uint)FileFormatVersionMajor << 16) | FileFormatVersionMinor;
-        private const uint CodeGenVersion = 4369;
+        private const uint CodeGenVersion = 4368;
 
         private const string SharedTocFileName = "shared.toc";
         private const string SharedDataFileName = "shared.data";
@@ -774,6 +774,8 @@ namespace Ryujinx.Graphics.Gpu.Shader.DiskCache
                 sBuffers,
                 textures,
                 images,
+                ShaderIdentification.None,
+                0,
                 dataInfo.Stage,
                 dataInfo.UsesInstanceId,
                 dataInfo.UsesDrawParameters,
diff --git a/Ryujinx.Graphics.Gpu/Shader/DiskCache/ParallelDiskCacheLoader.cs b/Ryujinx.Graphics.Gpu/Shader/DiskCache/ParallelDiskCacheLoader.cs
index 722e66b3..77fb3ca4 100644
--- a/Ryujinx.Graphics.Gpu/Shader/DiskCache/ParallelDiskCacheLoader.cs
+++ b/Ryujinx.Graphics.Gpu/Shader/DiskCache/ParallelDiskCacheLoader.cs
@@ -633,6 +633,11 @@ namespace Ryujinx.Graphics.Gpu.Shader.DiskCache
                 }
             }
 
+            if (!_context.Capabilities.SupportsGeometryShader)
+            {
+                ShaderCache.TryRemoveGeometryStage(translatorContexts);
+            }
+
             CachedShaderStage[] shaders = new CachedShaderStage[guestShaders.Length];
             List<ShaderProgram> translatedStages = new List<ShaderProgram>();
 
diff --git a/Ryujinx.Graphics.Gpu/Shader/GpuAccessorBase.cs b/Ryujinx.Graphics.Gpu/Shader/GpuAccessorBase.cs
index d36ffd70..1402f146 100644
--- a/Ryujinx.Graphics.Gpu/Shader/GpuAccessorBase.cs
+++ b/Ryujinx.Graphics.Gpu/Shader/GpuAccessorBase.cs
@@ -126,6 +126,8 @@ namespace Ryujinx.Graphics.Gpu.Shader
 
         public bool QueryHostSupportsFragmentShaderOrderingIntel() => _context.Capabilities.SupportsFragmentShaderOrderingIntel;
 
+        public bool QueryHostSupportsGeometryShader() => _context.Capabilities.SupportsGeometryShader;
+
         public bool QueryHostSupportsGeometryShaderPassthrough() => _context.Capabilities.SupportsGeometryShaderPassthrough;
 
         public bool QueryHostSupportsImageLoadFormatted() => _context.Capabilities.SupportsImageLoadFormatted;
diff --git a/Ryujinx.Graphics.Gpu/Shader/ShaderCache.cs b/Ryujinx.Graphics.Gpu/Shader/ShaderCache.cs
index 5c045d9b..11f7085d 100644
--- a/Ryujinx.Graphics.Gpu/Shader/ShaderCache.cs
+++ b/Ryujinx.Graphics.Gpu/Shader/ShaderCache.cs
@@ -353,6 +353,11 @@ namespace Ryujinx.Graphics.Gpu.Shader
                 }
             }
 
+            if (!_context.Capabilities.SupportsGeometryShader)
+            {
+                TryRemoveGeometryStage(translatorContexts);
+            }
+
             CachedShaderStage[] shaders = new CachedShaderStage[Constants.ShaderStages + 1];
             List<ShaderSource> shaderSources = new List<ShaderSource>();
 
@@ -422,6 +427,39 @@ namespace Ryujinx.Graphics.Gpu.Shader
         }
 
         /// <summary>
+        /// Tries to eliminate the geometry stage from the array of translator contexts.
+        /// </summary>
+        /// <param name="translatorContexts">Array of translator contexts</param>
+        public static void TryRemoveGeometryStage(TranslatorContext[] translatorContexts)
+        {
+            if (translatorContexts[4] != null)
+            {
+                // We have a geometry shader, but geometry shaders are not supported.
+                // Try to eliminate the geometry shader.
+
+                ShaderProgramInfo info = translatorContexts[4].Translate().Info;
+
+                if (info.Identification == ShaderIdentification.GeometryLayerPassthrough)
+                {
+                    // We managed to identify that this geometry shader is only used to set the output Layer value,
+                    // we can set the Layer on the previous stage instead (usually the vertex stage) and eliminate it.
+
+                    for (int i = 3; i >= 1; i--)
+                    {
+                        if (translatorContexts[i] != null)
+                        {
+                            translatorContexts[i].SetGeometryShaderLayerInputAttribute(info.GpLayerInputAttribute);
+                            translatorContexts[i].SetLastInVertexPipeline(translatorContexts[5] != null);
+                            break;
+                        }
+                    }
+
+                    translatorContexts[4] = null;
+                }
+            }
+        }
+
+        /// <summary>
         /// Creates a shader source for use with the backend from a translated shader program.
         /// </summary>
         /// <param name="program">Translated shader program</param>