Salieri: Detect and avoid caching shaders using bindless textures (#2097)

* Salieri: Add blacklist system and blacklist shaders using bindless

Currently the shader cache doesn't have the right format to support
bindless textures correctly and may cache shaders that it cannot rebuild
after host invalidation.

This PR address the issue by blacklisting shaders using bindless
textures.

THis also support detection of already cached broken shader and handle removal
of those.

* Move to a feature flags design to avoid intrusive changes in the translator

This remove the auto correct behaviour

* Reduce diff on TranslationFlags

* Reduce comma on last entry of TranslationFlags

* Fix inverted logic and remove leftovers

* remove debug edits oops

4 files changed, 39 insertions, 4 deletions

diff --git a/Ryujinx.Graphics.Gpu/Shader/ShaderCache.cs b/Ryujinx.Graphics.Gpu/Shader/ShaderCache.cs
index 768a58e7..a085936a 100644
--- a/Ryujinx.Graphics.Gpu/Shader/ShaderCache.cs
+++ b/Ryujinx.Graphics.Gpu/Shader/ShaderCache.cs
@@ -195,7 +195,7 @@ namespace Ryujinx.Graphics.Gpu.Shader
 
                         if (tfd != null)
                         {
-                            flags = TranslationFlags.Feedback;
+                            flags |= TranslationFlags.Feedback;
                         }
 
                         TranslationCounts counts = new TranslationCounts();
@@ -426,6 +426,8 @@ namespace Ryujinx.Graphics.Gpu.Shader
                 // The shader isn't currently cached, translate it and compile it.
                 ShaderCodeHolder shader = TranslateShader(shaderContexts[0]);
 
+                bool isDiskShaderCacheIncompatible = shaderContexts[0].UsedFeatures.HasFlag(FeatureFlags.Bindless);
+
                 shader.HostShader = _context.Renderer.CompileShader(ShaderStage.Compute, shader.Program.Code);
 
                 IProgram hostProgram = _context.Renderer.CreateProgram(new IShader[] { shader.HostShader }, null);
@@ -434,7 +436,7 @@ namespace Ryujinx.Graphics.Gpu.Shader
 
                 cpShader = new ShaderBundle(hostProgram, shader);
 
-                if (isShaderCacheEnabled)
+                if (isShaderCacheEnabled && !isDiskShaderCacheIncompatible)
                 {
                     _cpProgramsDiskCache.Add(programCodeHash, cpShader);
 
@@ -540,6 +542,17 @@ namespace Ryujinx.Graphics.Gpu.Shader
                 shaders[3] = TranslateShader(shaderContexts[4]);
                 shaders[4] = TranslateShader(shaderContexts[5]);
 
+                bool isDiskShaderCacheIncompatible = false;
+
+                for (int i = 0; i < shaderContexts.Length; i++)
+                {
+                    if (shaderContexts[i] != null && shaderContexts[i].UsedFeatures.HasFlag(FeatureFlags.Bindless))
+                    {
+                        isDiskShaderCacheIncompatible = true;
+                        break;
+                    }
+                }
+
                 List<IShader> hostShaders = new List<IShader>();
 
                 for (int stage = 0; stage < Constants.ShaderStages; stage++)
@@ -564,7 +577,7 @@ namespace Ryujinx.Graphics.Gpu.Shader
 
                 gpShaders = new ShaderBundle(hostProgram, shaders);
 
-                if (isShaderCacheEnabled)
+                if (isShaderCacheEnabled && !isDiskShaderCacheIncompatible)
                 {
                     _gpProgramsDiskCache.Add(programCodeHash, gpShaders);
 
diff --git a/Ryujinx.Graphics.Shader/Translation/FeatureFlags.cs b/Ryujinx.Graphics.Shader/Translation/FeatureFlags.cs
index 9c65038a..d2b53f84 100644
--- a/Ryujinx.Graphics.Shader/Translation/FeatureFlags.cs
+++ b/Ryujinx.Graphics.Shader/Translation/FeatureFlags.cs
@@ -13,6 +13,8 @@ namespace Ryujinx.Graphics.Shader.Translation
 
         // Affected by resolution scaling.
         FragCoordXY     = 1 << 1,
-        IntegerSampling = 1 << 0
+        IntegerSampling = 1 << 0,
+
+        Bindless        = 1 << 2,
     }
 }
diff --git a/Ryujinx.Graphics.Shader/Translation/Translator.cs b/Ryujinx.Graphics.Shader/Translation/Translator.cs
index 0d64bccc..c7eb27e5 100644
--- a/Ryujinx.Graphics.Shader/Translation/Translator.cs
+++ b/Ryujinx.Graphics.Shader/Translation/Translator.cs
@@ -36,6 +36,22 @@ namespace Ryujinx.Graphics.Shader.Translation
             return new TranslatorContext(address, cfg, config);
         }
 
+        private static void ScanForBindless(BasicBlock[] blocks, ShaderConfig config)
+        {
+            for (int blkIndex = 0; blkIndex < blocks.Length; blkIndex++)
+            {
+                // Right now the guest shader cache cannot handle bindless textures correctly.
+                for (LinkedListNode<INode> node = blocks[blkIndex].Operations.First; node != null; node = node.Next)
+                {
+                    if (node.Value is TextureOperation texOp && (texOp.Flags & TextureFlags.Bindless) != 0)
+                    {
+                        config.SetUsedFeature(FeatureFlags.Bindless);
+                        break;
+                    }
+                }
+            }
+        }
+
         internal static ShaderProgram Translate(FunctionCode[] functions, ShaderConfig config, out ShaderProgramInfo shaderProgramInfo)
         {
             var cfgs = new ControlFlowGraph[functions.Length];
@@ -75,6 +91,8 @@ namespace Ryujinx.Graphics.Shader.Translation
                     Dominance.FindDominators(cfg);
                     Dominance.FindDominanceFrontiers(cfg.Blocks);
 
+                    ScanForBindless(cfg.Blocks, config);
+
                     Ssa.Rename(cfg.Blocks);
 
                     Optimizer.RunPass(cfg.Blocks, config);
diff --git a/Ryujinx.Graphics.Shader/Translation/TranslatorContext.cs b/Ryujinx.Graphics.Shader/Translation/TranslatorContext.cs
index 501f7880..649911a2 100644
--- a/Ryujinx.Graphics.Shader/Translation/TranslatorContext.cs
+++ b/Ryujinx.Graphics.Shader/Translation/TranslatorContext.cs
@@ -17,6 +17,8 @@ namespace Ryujinx.Graphics.Shader.Translation
         public ShaderStage Stage => _config.Stage;
         public int Size => _config.Size;
 
+        public FeatureFlags UsedFeatures => _config.UsedFeatures;
+
         public HashSet<int> TextureHandlesForCache => _config.TextureHandlesForCache;
 
         public IGpuAccessor GpuAccessor => _config.GpuAccessor;