Fix vote and shuffle shader instructions on AMD GPUs (#5540)

* Move shuffle handling out of the backend to a transform pass

* Handle subgroup sizes higher than 32

* Stop using the subgroup size control extension

* Make GenerateShuffleFunction static

* Shader cache version bump

2 files changed, 53 insertions, 0 deletions

diff --git a/src/Ryujinx.Graphics.Shader/Translation/Transforms/ShufflePass.cs b/src/Ryujinx.Graphics.Shader/Translation/Transforms/ShufflePass.cs
new file mode 100644
index 00000000..839d4f81
--- /dev/null
+++ b/src/Ryujinx.Graphics.Shader/Translation/Transforms/ShufflePass.cs
@@ -0,0 +1,52 @@
+using Ryujinx.Graphics.Shader.IntermediateRepresentation;
+using Ryujinx.Graphics.Shader.Translation.Optimizations;
+using System.Collections.Generic;
+using static Ryujinx.Graphics.Shader.IntermediateRepresentation.OperandHelper;
+
+namespace Ryujinx.Graphics.Shader.Translation.Transforms
+{
+    class ShufflePass : ITransformPass
+    {
+        public static bool IsEnabled(IGpuAccessor gpuAccessor, ShaderStage stage, TargetLanguage targetLanguage, FeatureFlags usedFeatures)
+        {
+            return usedFeatures.HasFlag(FeatureFlags.Shuffle);
+        }
+
+        public static LinkedListNode<INode> RunPass(TransformContext context, LinkedListNode<INode> node)
+        {
+            Operation operation = (Operation)node.Value;
+
+            HelperFunctionName functionName = operation.Inst switch
+            {
+                Instruction.Shuffle => HelperFunctionName.Shuffle,
+                Instruction.ShuffleDown => HelperFunctionName.ShuffleDown,
+                Instruction.ShuffleUp => HelperFunctionName.ShuffleUp,
+                Instruction.ShuffleXor => HelperFunctionName.ShuffleXor,
+                _ => HelperFunctionName.Invalid,
+            };
+
+            if (functionName == HelperFunctionName.Invalid || operation.SourcesCount != 3 || operation.DestsCount != 2)
+            {
+                return node;
+            }
+
+            int functionId = context.Hfm.GetOrCreateShuffleFunctionId(functionName, context.GpuAccessor.QueryHostSubgroupSize());
+
+            Operand result = operation.GetDest(0);
+            Operand valid = operation.GetDest(1);
+            Operand value = operation.GetSource(0);
+            Operand index = operation.GetSource(1);
+            Operand mask = operation.GetSource(2);
+
+            operation.Dest = null;
+
+            Operand[] callArgs = new Operand[] { Const(functionId), value, index, mask, valid };
+
+            LinkedListNode<INode> newNode = node.List.AddBefore(node, new Operation(Instruction.Call, 0, result, callArgs));
+
+            Utils.DeleteNode(node, operation);
+
+            return newNode;
+        }
+    }
+}
diff --git a/src/Ryujinx.Graphics.Shader/Translation/Transforms/TransformPasses.cs b/src/Ryujinx.Graphics.Shader/Translation/Transforms/TransformPasses.cs
index c3bbe7dd..29393880 100644
--- a/src/Ryujinx.Graphics.Shader/Translation/Transforms/TransformPasses.cs
+++ b/src/Ryujinx.Graphics.Shader/Translation/Transforms/TransformPasses.cs
@@ -13,6 +13,7 @@ namespace Ryujinx.Graphics.Shader.Translation.Transforms
             RunPass<TexturePass>(context);
             RunPass<SharedStoreSmallIntCas>(context);
             RunPass<SharedAtomicSignedCas>(context);
+            RunPass<ShufflePass>(context);
         }
 
         private static void RunPass<T>(TransformContext context) where T : ITransformPass