Add ATOMS, LDS, POPC, RED, STS and VOTE shader instructions, start changing the way how global memory is handled

5 files changed, 147 insertions, 40 deletions

diff --git a/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGen.cs b/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGen.cs
index 3bf31c16..b5cab54e 100644
--- a/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGen.cs
+++ b/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGen.cs
@@ -3,6 +3,7 @@ using Ryujinx.Graphics.Shader.StructuredIr;
 using System;
 
 using static Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions.InstGenHelper;
+using static Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions.InstGenMemory;
 using static Ryujinx.Graphics.Shader.StructuredIr.InstructionInfo;
 
 namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
@@ -31,6 +32,8 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
 
             if ((info.Type & InstType.Call) != 0)
             {
+                bool atomic = (info.Type & InstType.Atomic) != 0;
+
                 int arity = (int)(info.Type & InstType.ArityMask);
 
                 string args = string.Empty;
@@ -44,10 +47,29 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
 
                     VariableType dstType = GetSrcVarType(inst, argIndex);
 
-                    args += GetSoureExpr(context, operation.GetSource(argIndex), dstType);
+                    if (argIndex == 0 && atomic)
+                    {
+                        switch (inst & Instruction.MrMask)
+                        {
+                            // TODO: Global.
+                            case Instruction.MrShared:  args += LoadShared (context, operation); break;
+                            case Instruction.MrStorage: args += LoadStorage(context, operation); break;
+                        }
+                    }
+                    else
+                    {
+                        args += GetSoureExpr(context, operation.GetSource(argIndex), dstType);
+                    }
                 }
 
-                return info.OpName + "(" + args + ")";
+                if (inst == Instruction.Ballot)
+                {
+                    return $"unpackUint2x32({info.OpName}({args})).x";
+                }
+                else
+                {
+                    return info.OpName + "(" + args + ")";
+                }
             }
             else if ((info.Type & InstType.Op) != 0)
             {
@@ -99,6 +121,9 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
                     case Instruction.LoadLocal:
                         return InstGenMemory.LoadLocal(context, operation);
 
+                    case Instruction.LoadShared:
+                        return InstGenMemory.LoadShared(context, operation);
+
                     case Instruction.LoadStorage:
                         return InstGenMemory.LoadStorage(context, operation);
 
@@ -108,6 +133,9 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
                     case Instruction.StoreLocal:
                         return InstGenMemory.StoreLocal(context, operation);
 
+                    case Instruction.StoreShared:
+                        return InstGenMemory.StoreShared(context, operation);
+
                     case Instruction.StoreStorage:
                         return InstGenMemory.StoreStorage(context, operation);
 
diff --git a/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenHelper.cs b/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenHelper.cs
index 2aaae71c..ec133272 100644
--- a/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenHelper.cs
+++ b/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenHelper.cs
@@ -13,8 +13,19 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
         {
             _infoTbl = new InstInfo[(int)Instruction.Count];
 
+            Add(Instruction.AtomicAdd,                InstType.AtomicBinary,   "atomicAdd");
+            Add(Instruction.AtomicAnd,                InstType.AtomicBinary,   "atomicAnd");
+            Add(Instruction.AtomicCompareAndSwap,     InstType.AtomicTernary,  "atomicCompSwap");
+            Add(Instruction.AtomicMaxS32,             InstType.AtomicBinary,   "atomicMax");
+            Add(Instruction.AtomicMaxU32,             InstType.AtomicBinary,   "atomicMax");
+            Add(Instruction.AtomicMinS32,             InstType.AtomicBinary,   "atomicMin");
+            Add(Instruction.AtomicMinU32,             InstType.AtomicBinary,   "atomicMin");
+            Add(Instruction.AtomicOr,                 InstType.AtomicBinary,   "atomicOr");
+            Add(Instruction.AtomicSwap,               InstType.AtomicBinary,   "atomicExchange");
+            Add(Instruction.AtomicXor,                InstType.AtomicBinary,   "atomicXor");
             Add(Instruction.Absolute,                 InstType.CallUnary,      "abs");
             Add(Instruction.Add,                      InstType.OpBinaryCom,    "+",               2);
+            Add(Instruction.Ballot,                   InstType.CallUnary,      "ballotARB");
             Add(Instruction.BitCount,                 InstType.CallUnary,      "bitCount");
             Add(Instruction.BitfieldExtractS32,       InstType.CallTernary,    "bitfieldExtract");
             Add(Instruction.BitfieldExtractU32,       InstType.CallTernary,    "bitfieldExtract");
@@ -59,6 +70,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
             Add(Instruction.LoadAttribute,            InstType.Special);
             Add(Instruction.LoadConstant,             InstType.Special);
             Add(Instruction.LoadLocal,                InstType.Special);
+            Add(Instruction.LoadShared,               InstType.Special);
             Add(Instruction.LoadStorage,              InstType.Special);
             Add(Instruction.LogarithmB2,              InstType.CallUnary,      "log2");
             Add(Instruction.LogicalAnd,               InstType.OpBinaryCom,    "&&",              9);
@@ -87,6 +99,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
             Add(Instruction.Sine,                     InstType.CallUnary,      "sin");
             Add(Instruction.SquareRoot,               InstType.CallUnary,      "sqrt");
             Add(Instruction.StoreLocal,               InstType.Special);
+            Add(Instruction.StoreShared,              InstType.Special);
             Add(Instruction.StoreStorage,             InstType.Special);
             Add(Instruction.Subtract,                 InstType.OpBinary,       "-",               2);
             Add(Instruction.SwizzleAdd,               InstType.CallTernary,    HelperFunctionNames.SwizzleAdd);
@@ -94,6 +107,9 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
             Add(Instruction.TextureSize,              InstType.Special);
             Add(Instruction.Truncate,                 InstType.CallUnary,      "trunc");
             Add(Instruction.UnpackHalf2x16,           InstType.Special);
+            Add(Instruction.VoteAll,                  InstType.CallUnary,      "allInvocationsARB");
+            Add(Instruction.VoteAllEqual,             InstType.CallUnary,      "allInvocationsEqualARB");
+            Add(Instruction.VoteAny,                  InstType.CallUnary,      "anyInvocationARB");
         }
 
         private static void Add(Instruction inst, InstType flags, string opName = null, int precedence = 0)
diff --git a/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenMemory.cs b/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenMemory.cs
index 21e39fcf..c535d8fc 100644
--- a/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenMemory.cs
+++ b/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenMemory.cs
@@ -1,5 +1,6 @@
 using Ryujinx.Graphics.Shader.IntermediateRepresentation;
 using Ryujinx.Graphics.Shader.StructuredIr;
+using Ryujinx.Graphics.Shader.Translation.Optimizations;
 using System;
 
 using static Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions.InstGenHelper;
@@ -118,27 +119,76 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
             return OperandManager.GetConstantBufferName(src1, offsetExpr, context.Config.Stage);
         }
 
+        public static string LoadGlobal(CodeGenContext context, AstOperation operation)
+        {
+            IAstNode src1 = operation.GetSource(0);
+            IAstNode src2 = operation.GetSource(1);
+
+            string addrLowExpr  = GetSoureExpr(context, src1, GetSrcVarType(operation.Inst, 0));
+            string addrHighExpr = GetSoureExpr(context, src2, GetSrcVarType(operation.Inst, 1));
+
+            context.AppendLine($"{DefaultNames.GmemOffsetName} = {HelperFunctionNames.GetStorageBuffer}({addrLowExpr}, {addrHighExpr});");
+
+            return GetStorageBufferAccessor($"{DefaultNames.GmemOffsetName}.x", $"{DefaultNames.GmemOffsetName}.y", context.Config.Stage);
+        }
+
         public static string LoadLocal(CodeGenContext context, AstOperation operation)
         {
+            return LoadLocalOrShared(context, operation, DefaultNames.LocalMemoryName);
+        }
+
+        public static string LoadShared(CodeGenContext context, AstOperation operation)
+        {
+            return LoadLocalOrShared(context, operation, DefaultNames.SharedMemoryName);
+        }
+
+        private static string LoadLocalOrShared(CodeGenContext context, AstOperation operation, string arrayName)
+        {
             IAstNode src1 = operation.GetSource(0);
 
             string offsetExpr = GetSoureExpr(context, src1, GetSrcVarType(operation.Inst, 0));
 
-            return $"{DefaultNames.LocalMemoryName}[{offsetExpr}]";
+            return $"{arrayName}[{offsetExpr}]";
         }
 
         public static string LoadStorage(CodeGenContext context, AstOperation operation)
         {
             IAstNode src1 = operation.GetSource(0);
+
+            string offsetExpr = GetSoureExpr(context, src1, GetSrcVarType(operation.Inst, 0));
+
+            return GetStorageBufferAccessor(operation.Index, offsetExpr, context.Config.Stage);
+        }
+
+        public static string StoreGlobal(CodeGenContext context, AstOperation operation)
+        {
+            IAstNode src1 = operation.GetSource(0);
             IAstNode src2 = operation.GetSource(1);
+            IAstNode src3 = operation.GetSource(2);
 
-            string offsetExpr = GetSoureExpr(context, src2, GetSrcVarType(operation.Inst, 1));
+            string addrLowExpr  = GetSoureExpr(context, src1, GetSrcVarType(operation.Inst, 0));
+            string addrHighExpr = GetSoureExpr(context, src2, GetSrcVarType(operation.Inst, 1));
+            string valueExpr    = GetSoureExpr(context, src3, GetSrcVarType(operation.Inst, 2));
 
-            return OperandManager.GetStorageBufferName(src1, offsetExpr, context.Config.Stage);
+            context.AppendLine($"{DefaultNames.GmemOffsetName} = {HelperFunctionNames.GetStorageBuffer}({addrLowExpr}, {addrHighExpr});");
+
+            string sb = GetStorageBufferAccessor($"{DefaultNames.GmemOffsetName}.x", $"{DefaultNames.GmemOffsetName}.y", context.Config.Stage);
+
+            return $"{sb} = {valueExpr}";
         }
 
         public static string StoreLocal(CodeGenContext context, AstOperation operation)
         {
+            return StoreLocalOrShared(context, operation, DefaultNames.LocalMemoryName);
+        }
+
+        public static string StoreShared(CodeGenContext context, AstOperation operation)
+        {
+            return StoreLocalOrShared(context, operation, DefaultNames.SharedMemoryName);
+        }
+
+        private static string StoreLocalOrShared(CodeGenContext context, AstOperation operation, string arrayName)
+        {
             IAstNode src1 = operation.GetSource(0);
             IAstNode src2 = operation.GetSource(1);
 
@@ -146,26 +196,25 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
 
             VariableType srcType = OperandManager.GetNodeDestType(src2);
 
-            string src = TypeConversion.ReinterpretCast(context, src2, srcType, VariableType.F32);
+            string src = TypeConversion.ReinterpretCast(context, src2, srcType, VariableType.U32);
 
-            return $"{DefaultNames.LocalMemoryName}[{offsetExpr}] = {src}";
+            return $"{arrayName}[{offsetExpr}] = {src}";
         }
 
         public static string StoreStorage(CodeGenContext context, AstOperation operation)
         {
             IAstNode src1 = operation.GetSource(0);
             IAstNode src2 = operation.GetSource(1);
-            IAstNode src3 = operation.GetSource(2);
 
-            string offsetExpr = GetSoureExpr(context, src2, GetSrcVarType(operation.Inst, 1));
+            string offsetExpr = GetSoureExpr(context, src1, GetSrcVarType(operation.Inst, 0));
 
-            VariableType srcType = OperandManager.GetNodeDestType(src3);
+            VariableType srcType = OperandManager.GetNodeDestType(src2);
 
-            string src = TypeConversion.ReinterpretCast(context, src3, srcType, VariableType.F32);
+            string src = TypeConversion.ReinterpretCast(context, src2, srcType, VariableType.U32);
 
-            string sbName = OperandManager.GetStorageBufferName(src1, offsetExpr, context.Config.Stage);
+            string sb = GetStorageBufferAccessor(operation.Index, offsetExpr, context.Config.Stage);
 
-            return $"{sbName} = {src}";
+            return $"{sb} = {src}";
         }
 
         public static string TextureSample(CodeGenContext context, AstOperation operation)
@@ -402,7 +451,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
                Append(Src(VariableType.S32));
             }
 
-            texCall += ")" + (isGather || !isShadow ? GetMask(texOp.ComponentMask) : "");
+            texCall += ")" + (isGather || !isShadow ? GetMask(texOp.Index) : "");
 
             return texCall;
         }
@@ -428,22 +477,42 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
 
             string src0Expr = GetSoureExpr(context, src0, GetSrcVarType(operation.Inst, 0));
 
-            return $"textureSize({samplerName}, {src0Expr}){GetMask(texOp.ComponentMask)}";
+            return $"textureSize({samplerName}, {src0Expr}){GetMask(texOp.Index)}";
         }
 
-        private static string GetMask(int compMask)
+        private static string GetStorageBufferAccessor(string slotExpr, string offsetExpr, ShaderStage stage)
         {
-            string mask = ".";
+            string sbName = OperandManager.GetShaderStagePrefix(stage);
 
-            for (int index = 0; index < 4; index++)
-            {
-                if ((compMask & (1 << index)) != 0)
-                {
-                    mask += "rgba".Substring(index, 1);
-                }
-            }
+            sbName += "_" + DefaultNames.StorageNamePrefix;
+
+            return $"{sbName}[{slotExpr}].{DefaultNames.DataName}[{offsetExpr}]";
+        }
+
+        private static string GetStorageBufferAccessor(int slot, string offsetExpr, ShaderStage stage)
+        {
+            string sbName = OperandManager.GetShaderStagePrefix(stage);
+
+            sbName += "_" + DefaultNames.StorageNamePrefix;
+
+            string mask = NumberFormatter.FormatUint(~(64u - 1));
 
-            return mask;
+            // Subtract the base address of the global memory, to get the
+            // storage buffer offset. The mask is used to keep the lower bits,
+            // since the bound storage buffer must match the host alignment
+            // restrictions.
+            int ubOffset = GlobalToStorage.GetStorageCbOffset(stage, slot);
+
+            string ubName = OperandManager.GetConstantBufferName(0, ubOffset, stage);
+
+            offsetExpr = $"{offsetExpr} - int((floatBitsToUint({ubName}) & {mask}) >> 2)";
+
+            return $"{sbName}[{NumberFormatter.FormatInt(slot)}].{DefaultNames.DataName}[{offsetExpr}]";
+        }
+
+        private static string GetMask(int index)
+        {
+            return '.' + "rgba".Substring(index, 1);
         }
     }
 }
\ No newline at end of file
diff --git a/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenPacking.cs b/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenPacking.cs
index 4a40032c..e5167f93 100644
--- a/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenPacking.cs
+++ b/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenPacking.cs
@@ -24,22 +24,12 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
 
             string srcExpr = GetSoureExpr(context, src, GetSrcVarType(operation.Inst, 0));
 
-            return $"unpackHalf2x16({srcExpr}){GetMask(operation.ComponentMask)}";
+            return $"unpackHalf2x16({srcExpr}){GetMask(operation.Index)}";
         }
 
-        private static string GetMask(int compMask)
+        private static string GetMask(int index)
         {
-            string mask = ".";
-
-            for (int index = 0; index < 2; index++)
-            {
-                if ((compMask & (1 << index)) != 0)
-                {
-                    mask += "xy".Substring(index, 1);
-                }
-            }
-
-            return mask;
+            return '.' + "xy".Substring(index, 1);
         }
     }
 }
\ No newline at end of file
diff --git a/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstType.cs b/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstType.cs
index 121cd079..5836e981 100644
--- a/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstType.cs
+++ b/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstType.cs
@@ -8,8 +8,11 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
         OpNullary   = Op | 0,
         OpUnary     = Op | 1,
         OpBinary    = Op | 2,
+        OpBinaryCom = Op | 2 | Commutative,
         OpTernary   = Op | 3,
-        OpBinaryCom = OpBinary | Commutative,
+
+        AtomicBinary  = CallBinary  | Atomic,
+        AtomicTernary = CallTernary | Atomic,
 
         CallNullary    = Call | 0,
         CallUnary      = Call | 1,
@@ -20,7 +23,8 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions
         Commutative = 1 << 8,
         Op          = 1 << 9,
         Call        = 1 << 10,
-        Special     = 1 << 11,
+        Atomic      = 1 << 11,
+        Special     = 1 << 12,
 
         ArityMask = 0xff
     }