Add ATOMS, LDS, POPC, RED, STS and VOTE shader instructions, start changing the way how global memory is handled

10 files changed, 252 insertions, 86 deletions

diff --git a/Ryujinx.Graphics.Shader/CodeGen/Glsl/Declarations.cs b/Ryujinx.Graphics.Shader/CodeGen/Glsl/Declarations.cs
index 6c4ba949..a8cabaaf 100644
--- a/Ryujinx.Graphics.Shader/CodeGen/Glsl/Declarations.cs
+++ b/Ryujinx.Graphics.Shader/CodeGen/Glsl/Declarations.cs
@@ -16,7 +16,9 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl
         public static void Declare(CodeGenContext context, StructuredProgramInfo info)
         {
             context.AppendLine("#version 420 core");
+            context.AppendLine("#extension GL_ARB_gpu_shader_int64 : enable");
             context.AppendLine("#extension GL_ARB_shader_ballot : enable");
+            context.AppendLine("#extension GL_ARB_shader_group_vote : enable");
             context.AppendLine("#extension GL_ARB_shader_storage_buffer_object : enable");
 
             if (context.Config.Stage == ShaderStage.Compute)
@@ -66,9 +68,15 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl
 
             context.AppendLine();
 
-            context.AppendLine($"precise float {DefaultNames.LocalMemoryName}[0x100];");
+            context.AppendLine($"uint {DefaultNames.LocalMemoryName}[0x100];");
             context.AppendLine();
 
+            if (context.Config.Stage == ShaderStage.Compute)
+            {
+                context.AppendLine($"shared uint {DefaultNames.SharedMemoryName}[0x100];");
+                context.AppendLine();
+            }
+
             if (info.CBuffers.Count != 0)
             {
                 DeclareUniforms(context, info);
@@ -78,7 +86,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl
 
             if (info.SBuffers.Count != 0)
             {
-                DeclareStorage(context, info);
+                DeclareUsedStorage(context, info);
 
                 context.AppendLine();
             }
@@ -168,6 +176,11 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl
 
                 context.AppendLine(GetVarTypeName(decl.VarType) + " " + name + ";");
             }
+
+            if ((info.HelperFunctionsMask & HelperFunctionsMask.GlobalMemory) != 0)
+            {
+                context.AppendLine($"ivec2 {DefaultNames.GmemOffsetName};");
+            }
         }
 
         private static string GetVarTypeName(VariableType type)
@@ -205,24 +218,59 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl
             }
         }
 
-        private static void DeclareStorage(CodeGenContext context, StructuredProgramInfo info)
+        private static void DeclareAllStorage(CodeGenContext context, StructuredProgramInfo info)
         {
-            foreach (int sbufSlot in info.SBuffers.OrderBy(x => x))
+            string sbName = OperandManager.GetShaderStagePrefix(context.Config.Stage);
+
+            sbName += "_" + DefaultNames.StorageNamePrefix;
+
+            string blockName = $"{sbName}_{DefaultNames.BlockSuffix}";
+
+            context.AppendLine("layout (std430) buffer " + blockName);
+
+            context.EnterScope();
+
+            context.AppendLine("uint " + DefaultNames.DataName + "[];");
+
+            string arraySize = NumberFormatter.FormatInt(Constants.MaxShaderStorageBuffers);
+
+            context.LeaveScope($" {sbName}[{arraySize}];");
+
+            for (int sbufSlot = 0; sbufSlot < Constants.MaxShaderStorageBuffers; sbufSlot++)
             {
-                string sbName = OperandManager.GetShaderStagePrefix(context.Config.Stage);
+                context.SBufferDescriptors.Add(new BufferDescriptor($"{blockName}[{sbufSlot}]", sbufSlot));
+            }
+        }
 
-                sbName += "_" + DefaultNames.StorageNamePrefix + sbufSlot;
+        private static void DeclareUsedStorage(CodeGenContext context, StructuredProgramInfo info)
+        {
+            string sbName = OperandManager.GetShaderStagePrefix(context.Config.Stage);
 
-                context.SBufferDescriptors.Add(new BufferDescriptor(sbName, sbufSlot));
+            sbName += "_" + DefaultNames.StorageNamePrefix;
 
-                context.AppendLine("layout (std430) buffer " + sbName);
+            string blockName = $"{sbName}_{DefaultNames.BlockSuffix}";
 
-                context.EnterScope();
+            int maxSlot = 0;
 
-                context.AppendLine("precise float " + OperandManager.GetSbName(context.Config.Stage, sbufSlot) + "[];");
+            foreach (int sbufSlot in info.SBuffers)
+            {
+                context.SBufferDescriptors.Add(new BufferDescriptor($"{blockName}[{sbufSlot}]", sbufSlot));
 
-                context.LeaveScope(";");
+                if (maxSlot < sbufSlot)
+                {
+                    maxSlot = sbufSlot;
+                }
             }
+
+            context.AppendLine("layout (std430) buffer " + blockName);
+
+            context.EnterScope();
+
+            context.AppendLine("uint " + DefaultNames.DataName + "[];");
+
+            string arraySize = NumberFormatter.FormatInt(maxSlot + 1);
+
+            context.LeaveScope($" {sbName}[{arraySize}];");
         }
 
         private static void DeclareSamplers(CodeGenContext context, StructuredProgramInfo info)
diff --git a/Ryujinx.Graphics.Shader/CodeGen/Glsl/DefaultNames.cs b/Ryujinx.Graphics.Shader/CodeGen/Glsl/DefaultNames.cs
index a06b0cc8..f1abc949 100644
--- a/Ryujinx.Graphics.Shader/CodeGen/Glsl/DefaultNames.cs
+++ b/Ryujinx.Graphics.Shader/CodeGen/Glsl/DefaultNames.cs
@@ -11,12 +11,18 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl
         public const string OAttributePrefix = "out_attr";
 
         public const string StorageNamePrefix = "s";
-        public const string StorageNameSuffix = "data";
+
+        public const string DataName = "data";
+
+        public const string BlockSuffix = "block";
 
         public const string UniformNamePrefix = "c";
         public const string UniformNameSuffix = "data";
 
-        public const string LocalMemoryName = "local_mem";
+        public const string LocalMemoryName  = "local_mem";
+        public const string SharedMemoryName = "shared_mem";
+
+        public const string GmemOffsetName  = "gmemOffset";
 
         public const string UndefinedName = "undef";
     }
diff --git a/Ryujinx.Graphics.Shader/CodeGen/Glsl/HelperFunctions/GlobalMemory.glsl b/Ryujinx.Graphics.Shader/CodeGen/Glsl/HelperFunctions/GlobalMemory.glsl
new file mode 100644
index 00000000..b8544ae2
--- /dev/null
+++ b/Ryujinx.Graphics.Shader/CodeGen/Glsl/HelperFunctions/GlobalMemory.glsl
@@ -0,0 +1,18 @@
+ivec2 Helper_GetStorageBuffer(uint aLow, uint aHigh)
+{
+    uint64_t address = packUint2x32(uvec2(aLow, aHigh));
+    int i;
+    for (i = 0; i < 16; i++)
+    {
+        int offset = 0x40 + i * 4;
+        uint baseLow  = fp_c0_data[offset];
+        uint baseHigh = fp_c0_data[offset + 1];
+        uint size     = fp_c0_data[offset + 2];
+        uint64_t baseAddr = packUint2x32(uvec2(baseLow, baseHigh));
+        if (address >= baseAddr && address < baseAddr + packUint2x32(uvec2(size, 0)))
+        {
+            return ivec2(i, int(unpackUint2x32(address - (baseAddr & ~63ul)).x) >> 2);
+        }
+    }
+    return ivec2(0);
+}
\ No newline at end of file
diff --git a/Ryujinx.Graphics.Shader/CodeGen/Glsl/HelperFunctions/HelperFunctionNames.cs b/Ryujinx.Graphics.Shader/CodeGen/Glsl/HelperFunctions/HelperFunctionNames.cs
index f1540fbf..302b56ad 100644
--- a/Ryujinx.Graphics.Shader/CodeGen/Glsl/HelperFunctions/HelperFunctionNames.cs
+++ b/Ryujinx.Graphics.Shader/CodeGen/Glsl/HelperFunctions/HelperFunctionNames.cs
@@ -2,6 +2,8 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl
 {
     static class HelperFunctionNames
     {
+        public static string GetStorageBuffer = "Helper_GetStorageBuffer";
+
         public static string Shuffle     = "Helper_Shuffle";
         public static string ShuffleDown = "Helper_ShuffleDown";
         public static string ShuffleUp   = "Helper_ShuffleUp";
diff --git a/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGen.cs b/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGen.cs
index 3bf31c16..b5cab54e 100644
--- a/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGen.cs
+++ b/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGen.cs
@@ -3,6 +3,7 @@ using Ryujinx.Graphics.Shader.StructuredIr;
 using System;
 
 using static Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions.InstGenHelper;
+using static Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions.InstGenMemory;
 using static Ryujinx.Graphics.Shader.StructuredIr.InstructionInfo;
 
 namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
@@ -31,6 +32,8 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
 
             if ((info.Type & InstType.Call) != 0)
             {
+                bool atomic = (info.Type & InstType.Atomic) != 0;
+
                 int arity = (int)(info.Type & InstType.ArityMask);
 
                 string args = string.Empty;
@@ -44,10 +47,29 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
 
                     VariableType dstType = GetSrcVarType(inst, argIndex);
 
-                    args += GetSoureExpr(context, operation.GetSource(argIndex), dstType);
+                    if (argIndex == 0 && atomic)
+                    {
+                        switch (inst & Instruction.MrMask)
+                        {
+                            // TODO: Global.
+                            case Instruction.MrShared:  args += LoadShared (context, operation); break;
+                            case Instruction.MrStorage: args += LoadStorage(context, operation); break;
+                        }
+                    }
+                    else
+                    {
+                        args += GetSoureExpr(context, operation.GetSource(argIndex), dstType);
+                    }
                 }
 
-                return info.OpName + "(" + args + ")";
+                if (inst == Instruction.Ballot)
+                {
+                    return $"unpackUint2x32({info.OpName}({args})).x";
+                }
+                else
+                {
+                    return info.OpName + "(" + args + ")";
+                }
             }
             else if ((info.Type & InstType.Op) != 0)
             {
@@ -99,6 +121,9 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
                     case Instruction.LoadLocal:
                         return InstGenMemory.LoadLocal(context, operation);
 
+                    case Instruction.LoadShared:
+                        return InstGenMemory.LoadShared(context, operation);
+
                     case Instruction.LoadStorage:
                         return InstGenMemory.LoadStorage(context, operation);
 
@@ -108,6 +133,9 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
                     case Instruction.StoreLocal:
                         return InstGenMemory.StoreLocal(context, operation);
 
+                    case Instruction.StoreShared:
+                        return InstGenMemory.StoreShared(context, operation);
+
                     case Instruction.StoreStorage:
                         return InstGenMemory.StoreStorage(context, operation);
 
diff --git a/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenHelper.cs b/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenHelper.cs
index 2aaae71c..ec133272 100644
--- a/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenHelper.cs
+++ b/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenHelper.cs
@@ -13,8 +13,19 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
         {
             _infoTbl = new InstInfo[(int)Instruction.Count];
 
+            Add(Instruction.AtomicAdd,                InstType.AtomicBinary,   "atomicAdd");
+            Add(Instruction.AtomicAnd,                InstType.AtomicBinary,   "atomicAnd");
+            Add(Instruction.AtomicCompareAndSwap,     InstType.AtomicTernary,  "atomicCompSwap");
+            Add(Instruction.AtomicMaxS32,             InstType.AtomicBinary,   "atomicMax");
+            Add(Instruction.AtomicMaxU32,             InstType.AtomicBinary,   "atomicMax");
+            Add(Instruction.AtomicMinS32,             InstType.AtomicBinary,   "atomicMin");
+            Add(Instruction.AtomicMinU32,             InstType.AtomicBinary,   "atomicMin");
+            Add(Instruction.AtomicOr,                 InstType.AtomicBinary,   "atomicOr");
+            Add(Instruction.AtomicSwap,               InstType.AtomicBinary,   "atomicExchange");
+            Add(Instruction.AtomicXor,                InstType.AtomicBinary,   "atomicXor");
             Add(Instruction.Absolute,                 InstType.CallUnary,      "abs");
             Add(Instruction.Add,                      InstType.OpBinaryCom,    "+",               2);
+            Add(Instruction.Ballot,                   InstType.CallUnary,      "ballotARB");
             Add(Instruction.BitCount,                 InstType.CallUnary,      "bitCount");
             Add(Instruction.BitfieldExtractS32,       InstType.CallTernary,    "bitfieldExtract");
             Add(Instruction.BitfieldExtractU32,       InstType.CallTernary,    "bitfieldExtract");
@@ -59,6 +70,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
             Add(Instruction.LoadAttribute,            InstType.Special);
             Add(Instruction.LoadConstant,             InstType.Special);
             Add(Instruction.LoadLocal,                InstType.Special);
+            Add(Instruction.LoadShared,               InstType.Special);
             Add(Instruction.LoadStorage,              InstType.Special);
             Add(Instruction.LogarithmB2,              InstType.CallUnary,      "log2");
             Add(Instruction.LogicalAnd,               InstType.OpBinaryCom,    "&&",              9);
@@ -87,6 +99,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
             Add(Instruction.Sine,                     InstType.CallUnary,      "sin");
             Add(Instruction.SquareRoot,               InstType.CallUnary,      "sqrt");
             Add(Instruction.StoreLocal,               InstType.Special);
+            Add(Instruction.StoreShared,              InstType.Special);
             Add(Instruction.StoreStorage,             InstType.Special);
             Add(Instruction.Subtract,                 InstType.OpBinary,       "-",               2);
             Add(Instruction.SwizzleAdd,               InstType.CallTernary,    HelperFunctionNames.SwizzleAdd);
@@ -94,6 +107,9 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
             Add(Instruction.TextureSize,              InstType.Special);
             Add(Instruction.Truncate,                 InstType.CallUnary,      "trunc");
             Add(Instruction.UnpackHalf2x16,           InstType.Special);
+            Add(Instruction.VoteAll,                  InstType.CallUnary,      "allInvocationsARB");
+            Add(Instruction.VoteAllEqual,             InstType.CallUnary,      "allInvocationsEqualARB");
+            Add(Instruction.VoteAny,                  InstType.CallUnary,      "anyInvocationARB");
         }
 
         private static void Add(Instruction inst, InstType flags, string opName = null, int precedence = 0)
diff --git a/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenMemory.cs b/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenMemory.cs
index 21e39fcf..c535d8fc 100644
--- a/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenMemory.cs
+++ b/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenMemory.cs
@@ -1,5 +1,6 @@
 using Ryujinx.Graphics.Shader.IntermediateRepresentation;
 using Ryujinx.Graphics.Shader.StructuredIr;
+using Ryujinx.Graphics.Shader.Translation.Optimizations;
 using System;
 
 using static Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions.InstGenHelper;
@@ -118,27 +119,76 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
             return OperandManager.GetConstantBufferName(src1, offsetExpr, context.Config.Stage);
         }
 
+        public static string LoadGlobal(CodeGenContext context, AstOperation operation)
+        {
+            IAstNode src1 = operation.GetSource(0);
+            IAstNode src2 = operation.GetSource(1);
+
+            string addrLowExpr  = GetSoureExpr(context, src1, GetSrcVarType(operation.Inst, 0));
+            string addrHighExpr = GetSoureExpr(context, src2, GetSrcVarType(operation.Inst, 1));
+
+            context.AppendLine($"{DefaultNames.GmemOffsetName} = {HelperFunctionNames.GetStorageBuffer}({addrLowExpr}, {addrHighExpr});");
+
+            return GetStorageBufferAccessor($"{DefaultNames.GmemOffsetName}.x", $"{DefaultNames.GmemOffsetName}.y", context.Config.Stage);
+        }
+
         public static string LoadLocal(CodeGenContext context, AstOperation operation)
         {
+            return LoadLocalOrShared(context, operation, DefaultNames.LocalMemoryName);
+        }
+
+        public static string LoadShared(CodeGenContext context, AstOperation operation)
+        {
+            return LoadLocalOrShared(context, operation, DefaultNames.SharedMemoryName);
+        }
+
+        private static string LoadLocalOrShared(CodeGenContext context, AstOperation operation, string arrayName)
+        {
             IAstNode src1 = operation.GetSource(0);
 
             string offsetExpr = GetSoureExpr(context, src1, GetSrcVarType(operation.Inst, 0));
 
-            return $"{DefaultNames.LocalMemoryName}[{offsetExpr}]";
+            return $"{arrayName}[{offsetExpr}]";
         }
 
         public static string LoadStorage(CodeGenContext context, AstOperation operation)
         {
             IAstNode src1 = operation.GetSource(0);
+
+            string offsetExpr = GetSoureExpr(context, src1, GetSrcVarType(operation.Inst, 0));
+
+            return GetStorageBufferAccessor(operation.Index, offsetExpr, context.Config.Stage);
+        }
+
+        public static string StoreGlobal(CodeGenContext context, AstOperation operation)
+        {
+            IAstNode src1 = operation.GetSource(0);
             IAstNode src2 = operation.GetSource(1);
+            IAstNode src3 = operation.GetSource(2);
 
-            string offsetExpr = GetSoureExpr(context, src2, GetSrcVarType(operation.Inst, 1));
+            string addrLowExpr  = GetSoureExpr(context, src1, GetSrcVarType(operation.Inst, 0));
+            string addrHighExpr = GetSoureExpr(context, src2, GetSrcVarType(operation.Inst, 1));
+            string valueExpr    = GetSoureExpr(context, src3, GetSrcVarType(operation.Inst, 2));
 
-            return OperandManager.GetStorageBufferName(src1, offsetExpr, context.Config.Stage);
+            context.AppendLine($"{DefaultNames.GmemOffsetName} = {HelperFunctionNames.GetStorageBuffer}({addrLowExpr}, {addrHighExpr});");
+
+            string sb = GetStorageBufferAccessor($"{DefaultNames.GmemOffsetName}.x", $"{DefaultNames.GmemOffsetName}.y", context.Config.Stage);
+
+            return $"{sb} = {valueExpr}";
         }
 
         public static string StoreLocal(CodeGenContext context, AstOperation operation)
         {
+            return StoreLocalOrShared(context, operation, DefaultNames.LocalMemoryName);
+        }
+
+        public static string StoreShared(CodeGenContext context, AstOperation operation)
+        {
+            return StoreLocalOrShared(context, operation, DefaultNames.SharedMemoryName);
+        }
+
+        private static string StoreLocalOrShared(CodeGenContext context, AstOperation operation, string arrayName)
+        {
             IAstNode src1 = operation.GetSource(0);
             IAstNode src2 = operation.GetSource(1);
 
@@ -146,26 +196,25 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
 
             VariableType srcType = OperandManager.GetNodeDestType(src2);
 
-            string src = TypeConversion.ReinterpretCast(context, src2, srcType, VariableType.F32);
+            string src = TypeConversion.ReinterpretCast(context, src2, srcType, VariableType.U32);
 
-            return $"{DefaultNames.LocalMemoryName}[{offsetExpr}] = {src}";
+            return $"{arrayName}[{offsetExpr}] = {src}";
         }
 
         public static string StoreStorage(CodeGenContext context, AstOperation operation)
         {
             IAstNode src1 = operation.GetSource(0);
             IAstNode src2 = operation.GetSource(1);
-            IAstNode src3 = operation.GetSource(2);
 
-            string offsetExpr = GetSoureExpr(context, src2, GetSrcVarType(operation.Inst, 1));
+            string offsetExpr = GetSoureExpr(context, src1, GetSrcVarType(operation.Inst, 0));
 
-            VariableType srcType = OperandManager.GetNodeDestType(src3);
+            VariableType srcType = OperandManager.GetNodeDestType(src2);
 
-            string src = TypeConversion.ReinterpretCast(context, src3, srcType, VariableType.F32);
+            string src = TypeConversion.ReinterpretCast(context, src2, srcType, VariableType.U32);
 
-            string sbName = OperandManager.GetStorageBufferName(src1, offsetExpr, context.Config.Stage);
+            string sb = GetStorageBufferAccessor(operation.Index, offsetExpr, context.Config.Stage);
 
-            return $"{sbName} = {src}";
+            return $"{sb} = {src}";
         }
 
         public static string TextureSample(CodeGenContext context, AstOperation operation)
@@ -402,7 +451,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
                Append(Src(VariableType.S32));
             }
 
-            texCall += ")" + (isGather || !isShadow ? GetMask(texOp.ComponentMask) : "");
+            texCall += ")" + (isGather || !isShadow ? GetMask(texOp.Index) : "");
 
             return texCall;
         }
@@ -428,22 +477,42 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
 
             string src0Expr = GetSoureExpr(context, src0, GetSrcVarType(operation.Inst, 0));
 
-            return $"textureSize({samplerName}, {src0Expr}){GetMask(texOp.ComponentMask)}";
+            return $"textureSize({samplerName}, {src0Expr}){GetMask(texOp.Index)}";
         }
 
-        private static string GetMask(int compMask)
+        private static string GetStorageBufferAccessor(string slotExpr, string offsetExpr, ShaderStage stage)
         {
-            string mask = ".";
+            string sbName = OperandManager.GetShaderStagePrefix(stage);
 
-            for (int index = 0; index < 4; index++)
-            {
-                if ((compMask & (1 << index)) != 0)
-                {
-                    mask += "rgba".Substring(index, 1);
-                }
-            }
+            sbName += "_" + DefaultNames.StorageNamePrefix;
+
+            return $"{sbName}[{slotExpr}].{DefaultNames.DataName}[{offsetExpr}]";
+        }
+
+        private static string GetStorageBufferAccessor(int slot, string offsetExpr, ShaderStage stage)
+        {
+            string sbName = OperandManager.GetShaderStagePrefix(stage);
+
+            sbName += "_" + DefaultNames.StorageNamePrefix;
+
+            string mask = NumberFormatter.FormatUint(~(64u - 1));
 
-            return mask;
+            // Subtract the base address of the global memory, to get the
+            // storage buffer offset. The mask is used to keep the lower bits,
+            // since the bound storage buffer must match the host alignment
+            // restrictions.
+            int ubOffset = GlobalToStorage.GetStorageCbOffset(stage, slot);
+
+            string ubName = OperandManager.GetConstantBufferName(0, ubOffset, stage);
+
+            offsetExpr = $"{offsetExpr} - int((floatBitsToUint({ubName}) & {mask}) >> 2)";
+
+            return $"{sbName}[{NumberFormatter.FormatInt(slot)}].{DefaultNames.DataName}[{offsetExpr}]";
+        }
+
+        private static string GetMask(int index)
+        {
+            return '.' + "rgba".Substring(index, 1);
         }
     }
 }
\ No newline at end of file
diff --git a/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenPacking.cs b/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenPacking.cs
index 4a40032c..e5167f93 100644
--- a/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenPacking.cs
+++ b/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenPacking.cs
@@ -24,22 +24,12 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
 
             string srcExpr = GetSoureExpr(context, src, GetSrcVarType(operation.Inst, 0));
 
-            return $"unpackHalf2x16({srcExpr}){GetMask(operation.ComponentMask)}";
+            return $"unpackHalf2x16({srcExpr}){GetMask(operation.Index)}";
         }
 
-        private static string GetMask(int compMask)
+        private static string GetMask(int index)
         {
-            string mask = ".";
-
-            for (int index = 0; index < 2; index++)
-            {
-                if ((compMask & (1 << index)) != 0)
-                {
-                    mask += "xy".Substring(index, 1);
-                }
-            }
-
-            return mask;
+            return '.' + "xy".Substring(index, 1);
         }
     }
 }
\ No newline at end of file
diff --git a/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstType.cs b/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstType.cs
index 121cd079..5836e981 100644
--- a/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstType.cs
+++ b/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstType.cs
@@ -8,8 +8,11 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
         OpNullary   = Op | 0,
         OpUnary     = Op | 1,
         OpBinary    = Op | 2,
+        OpBinaryCom = Op | 2 | Commutative,
         OpTernary   = Op | 3,
-        OpBinaryCom = OpBinary | Commutative,
+
+        AtomicBinary  = CallBinary  | Atomic,
+        AtomicTernary = CallTernary | Atomic,
 
         CallNullary    = Call | 0,
         CallUnary      = Call | 1,
@@ -20,7 +23,8 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
         Commutative = 1 << 8,
         Op          = 1 << 9,
         Call        = 1 << 10,
-        Special     = 1 << 11,
+        Atomic      = 1 << 11,
+        Special     = 1 << 12,
 
         ArityMask = 0xff
     }
diff --git a/Ryujinx.Graphics.Shader/CodeGen/Glsl/OperandManager.cs b/Ryujinx.Graphics.Shader/CodeGen/Glsl/OperandManager.cs
index 36f76ec5..4c9d5b55 100644
--- a/Ryujinx.Graphics.Shader/CodeGen/Glsl/OperandManager.cs
+++ b/Ryujinx.Graphics.Shader/CodeGen/Glsl/OperandManager.cs
@@ -51,13 +51,19 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl
             { AttributeConsts.FrontFacing,         new BuiltInAttribute("gl_FrontFacing",     VariableType.Bool) },
 
             // Special.
-            { AttributeConsts.FragmentOutputDepth, new BuiltInAttribute("gl_FragDepth",           VariableType.F32) },
-            { AttributeConsts.ThreadIdX,           new BuiltInAttribute("gl_LocalInvocationID.x", VariableType.U32) },
-            { AttributeConsts.ThreadIdY,           new BuiltInAttribute("gl_LocalInvocationID.y", VariableType.U32) },
-            { AttributeConsts.ThreadIdZ,           new BuiltInAttribute("gl_LocalInvocationID.z", VariableType.U32) },
-            { AttributeConsts.CtaIdX,              new BuiltInAttribute("gl_WorkGroupID.x",       VariableType.U32) },
-            { AttributeConsts.CtaIdY,              new BuiltInAttribute("gl_WorkGroupID.y",       VariableType.U32) },
-            { AttributeConsts.CtaIdZ,              new BuiltInAttribute("gl_WorkGroupID.z",       VariableType.U32) },
+            { AttributeConsts.FragmentOutputDepth, new BuiltInAttribute("gl_FragDepth",                           VariableType.F32) },
+            { AttributeConsts.ThreadIdX,           new BuiltInAttribute("gl_LocalInvocationID.x",                 VariableType.U32) },
+            { AttributeConsts.ThreadIdY,           new BuiltInAttribute("gl_LocalInvocationID.y",                 VariableType.U32) },
+            { AttributeConsts.ThreadIdZ,           new BuiltInAttribute("gl_LocalInvocationID.z",                 VariableType.U32) },
+            { AttributeConsts.CtaIdX,              new BuiltInAttribute("gl_WorkGroupID.x",                       VariableType.U32) },
+            { AttributeConsts.CtaIdY,              new BuiltInAttribute("gl_WorkGroupID.y",                       VariableType.U32) },
+            { AttributeConsts.CtaIdZ,              new BuiltInAttribute("gl_WorkGroupID.z",                       VariableType.U32) },
+            { AttributeConsts.LaneId,              new BuiltInAttribute("gl_SubGroupInvocationARB",               VariableType.U32) },
+            { AttributeConsts.EqMask,              new BuiltInAttribute("unpackUint2x32(gl_SubGroupEqMaskARB).x", VariableType.U32) },
+            { AttributeConsts.GeMask,              new BuiltInAttribute("unpackUint2x32(gl_SubGroupGeMaskARB).x", VariableType.U32) },
+            { AttributeConsts.GtMask,              new BuiltInAttribute("unpackUint2x32(gl_SubGroupGtMaskARB).x", VariableType.U32) },
+            { AttributeConsts.LeMask,              new BuiltInAttribute("unpackUint2x32(gl_SubGroupLeMaskARB).x", VariableType.U32) },
+            { AttributeConsts.LtMask,              new BuiltInAttribute("unpackUint2x32(gl_SubGroupLtMaskARB).x", VariableType.U32) },
         };
 
         private Dictionary<AstOperand, string> _locals;
@@ -87,7 +93,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl
                     return NumberFormatter.FormatInt(operand.Value);
 
                 case OperandType.ConstantBuffer:
-                    return GetConstantBufferName(operand, stage);
+                    return GetConstantBufferName(operand.CbufSlot, operand.CbufOffset, stage);
 
                 case OperandType.LocalVariable:
                     return _locals[operand];
@@ -99,25 +105,13 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl
             throw new ArgumentException($"Invalid operand type \"{operand.Type}\".");
         }
 
-        public static string GetConstantBufferName(AstOperand cbuf, ShaderStage stage)
+        public static string GetConstantBufferName(int slot, int offset, ShaderStage stage)
         {
-            string ubName = GetUbName(stage, cbuf.CbufSlot);
+            string ubName = GetUbName(stage, slot);
 
-            ubName += "[" + (cbuf.CbufOffset >> 2) + "]";
+            ubName += "[" + (offset >> 2) + "]";
 
-            return ubName + "." + GetSwizzleMask(cbuf.CbufOffset & 3);
-        }
-
-        public static string GetStorageBufferName(IAstNode slot, string offsetExpr, ShaderStage stage)
-        {
-            // Non-constant slots are not supported.
-            // It is expected that upstream stages are never going to generate non-constant
-            // slot access.
-            AstOperand operand = (AstOperand)slot;
-
-            string sbName = GetSbName(stage, operand.Value);
-
-            return $"{sbName}[{offsetExpr}]";
+            return ubName + "." + GetSwizzleMask(offset & 3);
         }
 
         public static string GetConstantBufferName(IAstNode slot, string offsetExpr, ShaderStage stage)
@@ -205,15 +199,6 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl
             return isOutAttr ? "// bad_attr0x" + value.ToString("X") : "0.0";
         }
 
-        public static string GetSbName(ShaderStage stage, int slot)
-        {
-            string sbName = GetShaderStagePrefix(stage);
-
-            sbName += "_" + DefaultNames.StorageNamePrefix + slot;
-
-            return sbName + "_" + DefaultNames.StorageNameSuffix;
-        }
-
         public static string GetUbName(ShaderStage stage, int slot)
         {
             string ubName = GetShaderStagePrefix(stage);