Add support for large sampler arrays on Vulkan (#6489)

* Add support for large sampler arrays on Vulkan

* Shader cache version bump

* Format whitespace

* Move DescriptorSetManager to PipelineLayoutCacheEntry to allow different pool sizes per layout

* Handle array textures with different types on the same buffer

* Somewhat better caching system

* Avoid useless buffer data modification checks

* Move redundant bindings update checking to the backend

* Fix an issue where texture arrays would get the same bindings across stages on Vulkan

* Backport some fixes from part 2

* Fix typo

* PR feedback

* Format whitespace

* Add some missing XML docs

1 files changed, 167 insertions, 141 deletions

diff --git a/src/Ryujinx.Graphics.Shader/Translation/Optimizations/BindlessElimination.cs b/src/Ryujinx.Graphics.Shader/Translation/Optimizations/BindlessElimination.cs
index a8890327..ad955278 100644
--- a/src/Ryujinx.Graphics.Shader/Translation/Optimizations/BindlessElimination.cs
+++ b/src/Ryujinx.Graphics.Shader/Translation/Optimizations/BindlessElimination.cs
@@ -15,8 +15,12 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations
             // - The handle is a constant buffer value.
             // - The handle is the result of a bitwise OR logical operation.
             // - Both sources of the OR operation comes from a constant buffer.
-            for (LinkedListNode<INode> node = block.Operations.First; node != null; node = node.Next)
+            LinkedListNode<INode> nextNode;
+
+            for (LinkedListNode<INode> node = block.Operations.First; node != null; node = nextNode)
             {
+                nextNode = node.Next;
+
                 if (node.Value is not TextureOperation texOp)
                 {
                     continue;
@@ -27,185 +31,207 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations
                     continue;
                 }
 
-                if (texOp.Inst == Instruction.TextureSample || texOp.Inst.IsTextureQuery())
+                if (!TryConvertBindless(block, resourceManager, gpuAccessor, texOp))
                 {
-                    Operand bindlessHandle = texOp.GetSource(0);
+                    // If we can't do bindless elimination, remove the texture operation.
+                    // Set any destination variables to zero.
 
-                    // In some cases the compiler uses a shuffle operation to get the handle,
-                    // for some textureGrad implementations. In those cases, we can skip the shuffle.
-                    if (bindlessHandle.AsgOp is Operation shuffleOp && shuffleOp.Inst == Instruction.Shuffle)
+                    for (int destIndex = 0; destIndex < texOp.DestsCount; destIndex++)
                     {
-                        bindlessHandle = shuffleOp.GetSource(0);
+                        block.Operations.AddBefore(node, new Operation(Instruction.Copy, texOp.GetDest(destIndex), OperandHelper.Const(0)));
                     }
 
-                    bindlessHandle = Utils.FindLastOperation(bindlessHandle, block);
+                    Utils.DeleteNode(node, texOp);
+                }
+            }
+        }
 
-                    // Some instructions do not encode an accurate sampler type:
-                    // - Most instructions uses the same type for 1D and Buffer.
-                    // - Query instructions may not have any type.
-                    // For those cases, we need to try getting the type from current GPU state,
-                    // as long bindless elimination is successful and we know where the texture descriptor is located.
-                    bool rewriteSamplerType =
-                        texOp.Type == SamplerType.TextureBuffer ||
-                        texOp.Inst == Instruction.TextureQuerySamples ||
-                        texOp.Inst == Instruction.TextureQuerySize;
+        private static bool TryConvertBindless(BasicBlock block, ResourceManager resourceManager, IGpuAccessor gpuAccessor, TextureOperation texOp)
+        {
+            if (texOp.Inst == Instruction.TextureSample || texOp.Inst.IsTextureQuery())
+            {
+                Operand bindlessHandle = texOp.GetSource(0);
 
-                    if (bindlessHandle.Type == OperandType.ConstantBuffer)
-                    {
-                        SetHandle(
-                            resourceManager,
-                            gpuAccessor,
-                            texOp,
-                            bindlessHandle.GetCbufOffset(),
-                            bindlessHandle.GetCbufSlot(),
-                            rewriteSamplerType,
-                            isImage: false);
-
-                        continue;
-                    }
+                // In some cases the compiler uses a shuffle operation to get the handle,
+                // for some textureGrad implementations. In those cases, we can skip the shuffle.
+                if (bindlessHandle.AsgOp is Operation shuffleOp && shuffleOp.Inst == Instruction.Shuffle)
+                {
+                    bindlessHandle = shuffleOp.GetSource(0);
+                }
 
-                    if (!TryGetOperation(bindlessHandle.AsgOp, out Operation handleCombineOp))
-                    {
-                        continue;
-                    }
+                bindlessHandle = Utils.FindLastOperation(bindlessHandle, block);
 
-                    if (handleCombineOp.Inst != Instruction.BitwiseOr)
-                    {
-                        continue;
-                    }
+                // Some instructions do not encode an accurate sampler type:
+                // - Most instructions uses the same type for 1D and Buffer.
+                // - Query instructions may not have any type.
+                // For those cases, we need to try getting the type from current GPU state,
+                // as long bindless elimination is successful and we know where the texture descriptor is located.
+                bool rewriteSamplerType =
+                    texOp.Type == SamplerType.TextureBuffer ||
+                    texOp.Inst == Instruction.TextureQuerySamples ||
+                    texOp.Inst == Instruction.TextureQuerySize;
 
-                    Operand src0 = Utils.FindLastOperation(handleCombineOp.GetSource(0), block);
-                    Operand src1 = Utils.FindLastOperation(handleCombineOp.GetSource(1), block);
+                if (bindlessHandle.Type == OperandType.ConstantBuffer)
+                {
+                    SetHandle(
+                        resourceManager,
+                        gpuAccessor,
+                        texOp,
+                        bindlessHandle.GetCbufOffset(),
+                        bindlessHandle.GetCbufSlot(),
+                        rewriteSamplerType,
+                        isImage: false);
+
+                    return true;
+                }
 
-                    // For cases where we have a constant, ensure that the constant is always
-                    // the second operand.
-                    // Since this is a commutative operation, both are fine,
-                    // and having a "canonical" representation simplifies some checks below.
-                    if (src0.Type == OperandType.Constant && src1.Type != OperandType.Constant)
-                    {
-                        (src0, src1) = (src1, src0);
-                    }
+                if (!TryGetOperation(bindlessHandle.AsgOp, out Operation handleCombineOp))
+                {
+                    return false;
+                }
 
-                    TextureHandleType handleType = TextureHandleType.SeparateSamplerHandle;
-
-                    // Try to match the following patterns:
-                    // Masked pattern:
-                    //  - samplerHandle = samplerHandle & 0xFFF00000;
-                    //  - textureHandle = textureHandle & 0xFFFFF;
-                    //  - combinedHandle = samplerHandle | textureHandle;
-                    //  Where samplerHandle and textureHandle comes from a constant buffer.
-                    // Shifted pattern:
-                    //  - samplerHandle = samplerId << 20;
-                    //  - combinedHandle = samplerHandle | textureHandle;
-                    //  Where samplerId and textureHandle comes from a constant buffer.
-                    // Constant pattern:
-                    //  - combinedHandle = samplerHandleConstant | textureHandle;
-                    //  Where samplerHandleConstant is a constant value, and textureHandle comes from a constant buffer.
-                    if (src0.AsgOp is Operation src0AsgOp)
+                if (handleCombineOp.Inst != Instruction.BitwiseOr)
+                {
+                    return false;
+                }
+
+                Operand src0 = Utils.FindLastOperation(handleCombineOp.GetSource(0), block);
+                Operand src1 = Utils.FindLastOperation(handleCombineOp.GetSource(1), block);
+
+                // For cases where we have a constant, ensure that the constant is always
+                // the second operand.
+                // Since this is a commutative operation, both are fine,
+                // and having a "canonical" representation simplifies some checks below.
+                if (src0.Type == OperandType.Constant && src1.Type != OperandType.Constant)
+                {
+                    (src0, src1) = (src1, src0);
+                }
+
+                TextureHandleType handleType = TextureHandleType.SeparateSamplerHandle;
+
+                // Try to match the following patterns:
+                // Masked pattern:
+                //  - samplerHandle = samplerHandle & 0xFFF00000;
+                //  - textureHandle = textureHandle & 0xFFFFF;
+                //  - combinedHandle = samplerHandle | textureHandle;
+                //  Where samplerHandle and textureHandle comes from a constant buffer.
+                // Shifted pattern:
+                //  - samplerHandle = samplerId << 20;
+                //  - combinedHandle = samplerHandle | textureHandle;
+                //  Where samplerId and textureHandle comes from a constant buffer.
+                // Constant pattern:
+                //  - combinedHandle = samplerHandleConstant | textureHandle;
+                //  Where samplerHandleConstant is a constant value, and textureHandle comes from a constant buffer.
+                if (src0.AsgOp is Operation src0AsgOp)
+                {
+                    if (src1.AsgOp is Operation src1AsgOp &&
+                        src0AsgOp.Inst == Instruction.BitwiseAnd &&
+                        src1AsgOp.Inst == Instruction.BitwiseAnd)
                     {
-                        if (src1.AsgOp is Operation src1AsgOp &&
-                            src0AsgOp.Inst == Instruction.BitwiseAnd &&
-                            src1AsgOp.Inst == Instruction.BitwiseAnd)
+                        src0 = GetSourceForMaskedHandle(src0AsgOp, 0xFFFFF);
+                        src1 = GetSourceForMaskedHandle(src1AsgOp, 0xFFF00000);
+
+                        // The OR operation is commutative, so we can also try to swap the operands to get a match.
+                        if (src0 == null || src1 == null)
                         {
-                            src0 = GetSourceForMaskedHandle(src0AsgOp, 0xFFFFF);
-                            src1 = GetSourceForMaskedHandle(src1AsgOp, 0xFFF00000);
-
-                            // The OR operation is commutative, so we can also try to swap the operands to get a match.
-                            if (src0 == null || src1 == null)
-                            {
-                                src0 = GetSourceForMaskedHandle(src1AsgOp, 0xFFFFF);
-                                src1 = GetSourceForMaskedHandle(src0AsgOp, 0xFFF00000);
-                            }
-
-                            if (src0 == null || src1 == null)
-                            {
-                                continue;
-                            }
+                            src0 = GetSourceForMaskedHandle(src1AsgOp, 0xFFFFF);
+                            src1 = GetSourceForMaskedHandle(src0AsgOp, 0xFFF00000);
                         }
-                        else if (src0AsgOp.Inst == Instruction.ShiftLeft)
+
+                        if (src0 == null || src1 == null)
                         {
-                            Operand shift = src0AsgOp.GetSource(1);
-
-                            if (shift.Type == OperandType.Constant && shift.Value == 20)
-                            {
-                                src0 = src1;
-                                src1 = src0AsgOp.GetSource(0);
-                                handleType = TextureHandleType.SeparateSamplerId;
-                            }
+                            return false;
                         }
                     }
-                    else if (src1.AsgOp is Operation src1AsgOp && src1AsgOp.Inst == Instruction.ShiftLeft)
+                    else if (src0AsgOp.Inst == Instruction.ShiftLeft)
                     {
-                        Operand shift = src1AsgOp.GetSource(1);
+                        Operand shift = src0AsgOp.GetSource(1);
 
                         if (shift.Type == OperandType.Constant && shift.Value == 20)
                         {
-                            src1 = src1AsgOp.GetSource(0);
+                            src0 = src1;
+                            src1 = src0AsgOp.GetSource(0);
                             handleType = TextureHandleType.SeparateSamplerId;
                         }
                     }
-                    else if (src1.Type == OperandType.Constant && (src1.Value & 0xfffff) == 0)
-                    {
-                        handleType = TextureHandleType.SeparateConstantSamplerHandle;
-                    }
+                }
+                else if (src1.AsgOp is Operation src1AsgOp && src1AsgOp.Inst == Instruction.ShiftLeft)
+                {
+                    Operand shift = src1AsgOp.GetSource(1);
 
-                    if (src0.Type != OperandType.ConstantBuffer)
+                    if (shift.Type == OperandType.Constant && shift.Value == 20)
                     {
-                        continue;
+                        src1 = src1AsgOp.GetSource(0);
+                        handleType = TextureHandleType.SeparateSamplerId;
                     }
+                }
+                else if (src1.Type == OperandType.Constant && (src1.Value & 0xfffff) == 0)
+                {
+                    handleType = TextureHandleType.SeparateConstantSamplerHandle;
+                }
 
-                    if (handleType == TextureHandleType.SeparateConstantSamplerHandle)
-                    {
-                        SetHandle(
-                            resourceManager,
-                            gpuAccessor,
-                            texOp,
-                            TextureHandle.PackOffsets(src0.GetCbufOffset(), ((src1.Value >> 20) & 0xfff), handleType),
-                            TextureHandle.PackSlots(src0.GetCbufSlot(), 0),
-                            rewriteSamplerType,
-                            isImage: false);
-                    }
-                    else if (src1.Type == OperandType.ConstantBuffer)
-                    {
-                        SetHandle(
-                            resourceManager,
-                            gpuAccessor,
-                            texOp,
-                            TextureHandle.PackOffsets(src0.GetCbufOffset(), src1.GetCbufOffset(), handleType),
-                            TextureHandle.PackSlots(src0.GetCbufSlot(), src1.GetCbufSlot()),
-                            rewriteSamplerType,
-                            isImage: false);
-                    }
+                if (src0.Type != OperandType.ConstantBuffer)
+                {
+                    return false;
                 }
-                else if (texOp.Inst == Instruction.ImageLoad ||
-                         texOp.Inst == Instruction.ImageStore ||
-                         texOp.Inst == Instruction.ImageAtomic)
+
+                if (handleType == TextureHandleType.SeparateConstantSamplerHandle)
                 {
-                    Operand src0 = Utils.FindLastOperation(texOp.GetSource(0), block);
+                    SetHandle(
+                        resourceManager,
+                        gpuAccessor,
+                        texOp,
+                        TextureHandle.PackOffsets(src0.GetCbufOffset(), ((src1.Value >> 20) & 0xfff), handleType),
+                        TextureHandle.PackSlots(src0.GetCbufSlot(), 0),
+                        rewriteSamplerType,
+                        isImage: false);
+
+                    return true;
+                }
+                else if (src1.Type == OperandType.ConstantBuffer)
+                {
+                    SetHandle(
+                        resourceManager,
+                        gpuAccessor,
+                        texOp,
+                        TextureHandle.PackOffsets(src0.GetCbufOffset(), src1.GetCbufOffset(), handleType),
+                        TextureHandle.PackSlots(src0.GetCbufSlot(), src1.GetCbufSlot()),
+                        rewriteSamplerType,
+                        isImage: false);
+
+                    return true;
+                }
+            }
+            else if (texOp.Inst.IsImage())
+            {
+                Operand src0 = Utils.FindLastOperation(texOp.GetSource(0), block);
 
-                    if (src0.Type == OperandType.ConstantBuffer)
-                    {
-                        int cbufOffset = src0.GetCbufOffset();
-                        int cbufSlot = src0.GetCbufSlot();
+                if (src0.Type == OperandType.ConstantBuffer)
+                {
+                    int cbufOffset = src0.GetCbufOffset();
+                    int cbufSlot = src0.GetCbufSlot();
 
-                        if (texOp.Format == TextureFormat.Unknown)
+                    if (texOp.Format == TextureFormat.Unknown)
+                    {
+                        if (texOp.Inst == Instruction.ImageAtomic)
                         {
-                            if (texOp.Inst == Instruction.ImageAtomic)
-                            {
-                                texOp.Format = ShaderProperties.GetTextureFormatAtomic(gpuAccessor, cbufOffset, cbufSlot);
-                            }
-                            else
-                            {
-                                texOp.Format = ShaderProperties.GetTextureFormat(gpuAccessor, cbufOffset, cbufSlot);
-                            }
+                            texOp.Format = ShaderProperties.GetTextureFormatAtomic(gpuAccessor, cbufOffset, cbufSlot);
                         }
+                        else
+                        {
+                            texOp.Format = ShaderProperties.GetTextureFormat(gpuAccessor, cbufOffset, cbufSlot);
+                        }
+                    }
 
-                        bool rewriteSamplerType = texOp.Type == SamplerType.TextureBuffer;
+                    bool rewriteSamplerType = texOp.Type == SamplerType.TextureBuffer;
 
-                        SetHandle(resourceManager, gpuAccessor, texOp, cbufOffset, cbufSlot, rewriteSamplerType, isImage: true);
-                    }
+                    SetHandle(resourceManager, gpuAccessor, texOp, cbufOffset, cbufSlot, rewriteSamplerType, isImage: true);
+
+                    return true;
                 }
             }
+
+            return false;
         }
 
         private static bool TryGetOperation(INode asgOp, out Operation outOperation)