Add a pass to turn global memory access into storage access, and do all storage related transformations on IR

7 files changed, 24 insertions, 116 deletions

diff --git a/Ryujinx.Graphics.Shader/CodeGen/Glsl/Declarations.cs b/Ryujinx.Graphics.Shader/CodeGen/Glsl/Declarations.cs
index e8b44961..a5c8cc9a 100644
--- a/Ryujinx.Graphics.Shader/CodeGen/Glsl/Declarations.cs
+++ b/Ryujinx.Graphics.Shader/CodeGen/Glsl/Declarations.cs
@@ -86,7 +86,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl
 
             if (info.SBuffers.Count != 0)
             {
-                DeclareUsedStorage(context, info);
+                DeclareStorages(context, info);
 
                 context.AppendLine();
             }
@@ -176,11 +176,6 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl
 
                 context.AppendLine(GetVarTypeName(decl.VarType) + " " + name + ";");
             }
-
-            if ((info.HelperFunctionsMask & HelperFunctionsMask.GlobalMemory) != 0)
-            {
-                context.AppendLine($"ivec2 {DefaultNames.GmemOffsetName};");
-            }
         }
 
         private static string GetVarTypeName(VariableType type)
@@ -218,31 +213,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl
             }
         }
 
-        private static void DeclareAllStorage(CodeGenContext context, StructuredProgramInfo info)
-        {
-            string sbName = OperandManager.GetShaderStagePrefix(context.Config.Stage);
-
-            sbName += "_" + DefaultNames.StorageNamePrefix;
-
-            string blockName = $"{sbName}_{DefaultNames.BlockSuffix}";
-
-            context.AppendLine("layout (std430) buffer " + blockName);
-
-            context.EnterScope();
-
-            context.AppendLine("uint " + DefaultNames.DataName + "[];");
-
-            string arraySize = NumberFormatter.FormatInt(Constants.MaxShaderStorageBuffers);
-
-            context.LeaveScope($" {sbName}[{arraySize}];");
-
-            for (int sbufSlot = 0; sbufSlot < Constants.MaxShaderStorageBuffers; sbufSlot++)
-            {
-                context.SBufferDescriptors.Add(new BufferDescriptor($"{blockName}[{sbufSlot}]", sbufSlot));
-            }
-        }
-
-        private static void DeclareUsedStorage(CodeGenContext context, StructuredProgramInfo info)
+        private static void DeclareStorages(CodeGenContext context, StructuredProgramInfo info)
         {
             string sbName = OperandManager.GetShaderStagePrefix(context.Config.Stage);
 
diff --git a/Ryujinx.Graphics.Shader/CodeGen/Glsl/DefaultNames.cs b/Ryujinx.Graphics.Shader/CodeGen/Glsl/DefaultNames.cs
index f1abc949..4da38b2d 100644
--- a/Ryujinx.Graphics.Shader/CodeGen/Glsl/DefaultNames.cs
+++ b/Ryujinx.Graphics.Shader/CodeGen/Glsl/DefaultNames.cs
@@ -22,8 +22,6 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl
         public const string LocalMemoryName  = "local_mem";
         public const string SharedMemoryName = "shared_mem";
 
-        public const string GmemOffsetName  = "gmemOffset";
-
         public const string UndefinedName = "undef";
     }
 }
\ No newline at end of file
diff --git a/Ryujinx.Graphics.Shader/CodeGen/Glsl/HelperFunctions/GlobalMemory.glsl b/Ryujinx.Graphics.Shader/CodeGen/Glsl/HelperFunctions/GlobalMemory.glsl
deleted file mode 100644
index b8544ae2..00000000
--- a/Ryujinx.Graphics.Shader/CodeGen/Glsl/HelperFunctions/GlobalMemory.glsl
+++ /dev/null
@@ -1,18 +0,0 @@
-ivec2 Helper_GetStorageBuffer(uint aLow, uint aHigh)
-{
-    uint64_t address = packUint2x32(uvec2(aLow, aHigh));
-    int i;
-    for (i = 0; i < 16; i++)
-    {
-        int offset = 0x40 + i * 4;
-        uint baseLow  = fp_c0_data[offset];
-        uint baseHigh = fp_c0_data[offset + 1];
-        uint size     = fp_c0_data[offset + 2];
-        uint64_t baseAddr = packUint2x32(uvec2(baseLow, baseHigh));
-        if (address >= baseAddr && address < baseAddr + packUint2x32(uvec2(size, 0)))
-        {
-            return ivec2(i, int(unpackUint2x32(address - (baseAddr & ~63ul)).x) >> 2);
-        }
-    }
-    return ivec2(0);
-}
\ No newline at end of file
diff --git a/Ryujinx.Graphics.Shader/CodeGen/Glsl/HelperFunctions/HelperFunctionNames.cs b/Ryujinx.Graphics.Shader/CodeGen/Glsl/HelperFunctions/HelperFunctionNames.cs
index 302b56ad..f1540fbf 100644
--- a/Ryujinx.Graphics.Shader/CodeGen/Glsl/HelperFunctions/HelperFunctionNames.cs
+++ b/Ryujinx.Graphics.Shader/CodeGen/Glsl/HelperFunctions/HelperFunctionNames.cs
@@ -2,8 +2,6 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl
 {
     static class HelperFunctionNames
     {
-        public static string GetStorageBuffer = "Helper_GetStorageBuffer";
-
         public static string Shuffle     = "Helper_Shuffle";
         public static string ShuffleDown = "Helper_ShuffleDown";
         public static string ShuffleUp   = "Helper_ShuffleUp";
diff --git a/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGen.cs b/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGen.cs
index b5cab54e..b6cdd7f6 100644
--- a/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGen.cs
+++ b/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGen.cs
@@ -49,12 +49,18 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
 
                     if (argIndex == 0 && atomic)
                     {
-                        switch (inst & Instruction.MrMask)
+                        Instruction memRegion = inst & Instruction.MrMask;
+
+                        switch (memRegion)
                         {
-                            // TODO: Global.
                             case Instruction.MrShared:  args += LoadShared (context, operation); break;
                             case Instruction.MrStorage: args += LoadStorage(context, operation); break;
+
+                            default: throw new InvalidOperationException($"Invalid memory region \"{memRegion}\".");
                         }
+
+                        // We use the first 2 operands above.
+                        argIndex++;
                     }
                     else
                     {
@@ -150,8 +156,6 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
                 }
             }
 
-            return "0";
-
             throw new InvalidOperationException($"Unexpected instruction type \"{info.Type}\".");
         }
     }
diff --git a/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenMemory.cs b/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenMemory.cs
index c535d8fc..5c2ea85e 100644
--- a/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenMemory.cs
+++ b/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenMemory.cs
@@ -119,19 +119,6 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
             return OperandManager.GetConstantBufferName(src1, offsetExpr, context.Config.Stage);
         }
 
-        public static string LoadGlobal(CodeGenContext context, AstOperation operation)
-        {
-            IAstNode src1 = operation.GetSource(0);
-            IAstNode src2 = operation.GetSource(1);
-
-            string addrLowExpr  = GetSoureExpr(context, src1, GetSrcVarType(operation.Inst, 0));
-            string addrHighExpr = GetSoureExpr(context, src2, GetSrcVarType(operation.Inst, 1));
-
-            context.AppendLine($"{DefaultNames.GmemOffsetName} = {HelperFunctionNames.GetStorageBuffer}({addrLowExpr}, {addrHighExpr});");
-
-            return GetStorageBufferAccessor($"{DefaultNames.GmemOffsetName}.x", $"{DefaultNames.GmemOffsetName}.y", context.Config.Stage);
-        }
-
         public static string LoadLocal(CodeGenContext context, AstOperation operation)
         {
             return LoadLocalOrShared(context, operation, DefaultNames.LocalMemoryName);
@@ -154,27 +141,12 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
         public static string LoadStorage(CodeGenContext context, AstOperation operation)
         {
             IAstNode src1 = operation.GetSource(0);
-
-            string offsetExpr = GetSoureExpr(context, src1, GetSrcVarType(operation.Inst, 0));
-
-            return GetStorageBufferAccessor(operation.Index, offsetExpr, context.Config.Stage);
-        }
-
-        public static string StoreGlobal(CodeGenContext context, AstOperation operation)
-        {
-            IAstNode src1 = operation.GetSource(0);
             IAstNode src2 = operation.GetSource(1);
-            IAstNode src3 = operation.GetSource(2);
-
-            string addrLowExpr  = GetSoureExpr(context, src1, GetSrcVarType(operation.Inst, 0));
-            string addrHighExpr = GetSoureExpr(context, src2, GetSrcVarType(operation.Inst, 1));
-            string valueExpr    = GetSoureExpr(context, src3, GetSrcVarType(operation.Inst, 2));
-
-            context.AppendLine($"{DefaultNames.GmemOffsetName} = {HelperFunctionNames.GetStorageBuffer}({addrLowExpr}, {addrHighExpr});");
 
-            string sb = GetStorageBufferAccessor($"{DefaultNames.GmemOffsetName}.x", $"{DefaultNames.GmemOffsetName}.y", context.Config.Stage);
+            string indexExpr  = GetSoureExpr(context, src1, GetSrcVarType(operation.Inst, 0));
+            string offsetExpr = GetSoureExpr(context, src2, GetSrcVarType(operation.Inst, 1));
 
-            return $"{sb} = {valueExpr}";
+            return GetStorageBufferAccessor(indexExpr, offsetExpr, context.Config.Stage);
         }
 
         public static string StoreLocal(CodeGenContext context, AstOperation operation)
@@ -205,14 +177,16 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
         {
             IAstNode src1 = operation.GetSource(0);
             IAstNode src2 = operation.GetSource(1);
+            IAstNode src3 = operation.GetSource(2);
 
-            string offsetExpr = GetSoureExpr(context, src1, GetSrcVarType(operation.Inst, 0));
+            string indexExpr  = GetSoureExpr(context, src1, GetSrcVarType(operation.Inst, 0));
+            string offsetExpr = GetSoureExpr(context, src2, GetSrcVarType(operation.Inst, 1));
 
-            VariableType srcType = OperandManager.GetNodeDestType(src2);
+            VariableType srcType = OperandManager.GetNodeDestType(src3);
 
-            string src = TypeConversion.ReinterpretCast(context, src2, srcType, VariableType.U32);
+            string src = TypeConversion.ReinterpretCast(context, src3, srcType, VariableType.U32);
 
-            string sb = GetStorageBufferAccessor(operation.Index, offsetExpr, context.Config.Stage);
+            string sb = GetStorageBufferAccessor(indexExpr, offsetExpr, context.Config.Stage);
 
             return $"{sb} = {src}";
         }
@@ -489,27 +463,6 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
             return $"{sbName}[{slotExpr}].{DefaultNames.DataName}[{offsetExpr}]";
         }
 
-        private static string GetStorageBufferAccessor(int slot, string offsetExpr, ShaderStage stage)
-        {
-            string sbName = OperandManager.GetShaderStagePrefix(stage);
-
-            sbName += "_" + DefaultNames.StorageNamePrefix;
-
-            string mask = NumberFormatter.FormatUint(~(64u - 1));
-
-            // Subtract the base address of the global memory, to get the
-            // storage buffer offset. The mask is used to keep the lower bits,
-            // since the bound storage buffer must match the host alignment
-            // restrictions.
-            int ubOffset = GlobalToStorage.GetStorageCbOffset(stage, slot);
-
-            string ubName = OperandManager.GetConstantBufferName(0, ubOffset, stage);
-
-            offsetExpr = $"{offsetExpr} - int((floatBitsToUint({ubName}) & {mask}) >> 2)";
-
-            return $"{sbName}[{NumberFormatter.FormatInt(slot)}].{DefaultNames.DataName}[{offsetExpr}]";
-        }
-
         private static string GetMask(int index)
         {
             return '.' + "rgba".Substring(index, 1);
diff --git a/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstType.cs b/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstType.cs
index 5836e981..84e36cdd 100644
--- a/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstType.cs
+++ b/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstType.cs
@@ -11,15 +11,17 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
         OpBinaryCom = Op | 2 | Commutative,
         OpTernary   = Op | 3,
 
-        AtomicBinary  = CallBinary  | Atomic,
-        AtomicTernary = CallTernary | Atomic,
-
         CallNullary    = Call | 0,
         CallUnary      = Call | 1,
         CallBinary     = Call | 2,
         CallTernary    = Call | 3,
         CallQuaternary = Call | 4,
 
+        // The atomic instructions have one extra operand,
+        // for the storage slot and offset pair.
+        AtomicBinary  = Call | Atomic | 3,
+        AtomicTernary = Call | Atomic | 4,
+
         Commutative = 1 << 8,
         Op          = 1 << 9,
         Call        = 1 << 10,