Do not allow render targets not explicitly written by the fragment shader to be modified (#3063)

* Do not allow render targets not explicitly written by the fragment shader to be modified

* Shader cache version bump

* Remove blank lines

* Avoid redundant color mask updates

* HostShaderCacheEntry can be null

* Avoid more redundant glColorMask calls

* nit: Mask -> Masks

* Fix currentComponentMask

* More efficient way to update _currentComponentMasks

4 files changed, 20 insertions, 73 deletions

diff --git a/Ryujinx.Graphics.Shader/Translation/EmitterContext.cs b/Ryujinx.Graphics.Shader/Translation/EmitterContext.cs
index 3dcb04ad..775f1217 100644
--- a/Ryujinx.Graphics.Shader/Translation/EmitterContext.cs
+++ b/Ryujinx.Graphics.Shader/Translation/EmitterContext.cs
@@ -172,11 +172,10 @@ namespace Ryujinx.Graphics.Shader.Translation
 
                 for (int rtIndex = 0; rtIndex < 8; rtIndex++)
                 {
-                    OmapTarget target = Config.OmapTargets[rtIndex];
-
                     for (int component = 0; component < 4; component++)
                     {
-                        if (!target.ComponentEnabled(component))
+                        bool componentEnabled = (Config.OmapTargets & (1 << (rtIndex * 4 + component))) != 0;
+                        if (!componentEnabled)
                         {
                             continue;
                         }
@@ -210,7 +209,8 @@ namespace Ryujinx.Graphics.Shader.Translation
                         }
                     }
 
-                    if (target.Enabled)
+                    bool targetEnabled = (Config.OmapTargets & (0xf << (rtIndex * 4))) != 0;
+                    if (targetEnabled)
                     {
                         Config.SetOutputUserAttribute(rtIndex, perPatch: false);
                         regIndexBase += 4;
diff --git a/Ryujinx.Graphics.Shader/Translation/ShaderConfig.cs b/Ryujinx.Graphics.Shader/Translation/ShaderConfig.cs
index 21f17041..996c2814 100644
--- a/Ryujinx.Graphics.Shader/Translation/ShaderConfig.cs
+++ b/Ryujinx.Graphics.Shader/Translation/ShaderConfig.cs
@@ -25,9 +25,9 @@ namespace Ryujinx.Graphics.Shader.Translation
 
         public ImapPixelType[] ImapTypes { get; }
 
-        public OmapTarget[] OmapTargets    { get; }
-        public bool         OmapSampleMask { get; }
-        public bool         OmapDepth      { get; }
+        public int OmapTargets { get; }
+        public bool OmapSampleMask { get; }
+        public bool OmapDepth { get; }
 
         public IGpuAccessor GpuAccessor { get; }
 
@@ -135,21 +135,8 @@ namespace Ryujinx.Graphics.Shader.Translation
 
         public int GetDepthRegister()
         {
-            int count = 0;
-
-            for (int index = 0; index < OmapTargets.Length; index++)
-            {
-                for (int component = 0; component < 4; component++)
-                {
-                    if (OmapTargets[index].ComponentEnabled(component))
-                    {
-                        count++;
-                    }
-                }
-            }
-
             // The depth register is always two registers after the last color output.
-            return count + 1;
+            return BitOperations.PopCount((uint)OmapTargets) + 1;
         }
 
         public TextureFormat GetTextureFormat(int handle, int cbufSlot = -1)
diff --git a/Ryujinx.Graphics.Shader/Translation/ShaderHeader.cs b/Ryujinx.Graphics.Shader/Translation/ShaderHeader.cs
index 0ad172da..e53c77af 100644
--- a/Ryujinx.Graphics.Shader/Translation/ShaderHeader.cs
+++ b/Ryujinx.Graphics.Shader/Translation/ShaderHeader.cs
@@ -36,37 +36,6 @@ namespace Ryujinx.Graphics.Shader.Translation
         }
     }
 
-    struct OmapTarget
-    {
-        public bool Red   { get; }
-        public bool Green { get; }
-        public bool Blue  { get; }
-        public bool Alpha { get; }
-
-        public bool Enabled => Red || Green || Blue || Alpha;
-
-        public OmapTarget(bool red, bool green, bool blue, bool alpha)
-        {
-            Red   = red;
-            Green = green;
-            Blue  = blue;
-            Alpha = alpha;
-        }
-
-        public bool ComponentEnabled(int component)
-        {
-            switch (component)
-            {
-                case 0: return Red;
-                case 1: return Green;
-                case 2: return Blue;
-                case 3: return Alpha;
-            }
-
-            throw new ArgumentOutOfRangeException(nameof(component));
-        }
-    }
-
     class ShaderHeader
     {
         public int SphType { get; }
@@ -85,7 +54,7 @@ namespace Ryujinx.Graphics.Shader.Translation
         public bool GpPassthrough { get; }
 
         public bool DoesLoadOrStore { get; }
-        public bool DoesFp64        { get; }
+        public bool DoesFp64 { get; }
 
         public int StreamOutMask { get; }
 
@@ -104,13 +73,13 @@ namespace Ryujinx.Graphics.Shader.Translation
         public int MaxOutputVertexCount { get; }
 
         public int StoreReqStart { get; }
-        public int StoreReqEnd   { get; }
+        public int StoreReqEnd { get; }
 
         public ImapPixelType[] ImapTypes { get; }
 
-        public OmapTarget[] OmapTargets    { get; }
-        public bool         OmapSampleMask { get; }
-        public bool         OmapDepth      { get; }
+        public int OmapTargets { get; }
+        public bool OmapSampleMask { get; }
+        public bool OmapDepth { get; }
 
         public ShaderHeader(IGpuAccessor gpuAccessor, ulong address)
         {
@@ -144,7 +113,7 @@ namespace Ryujinx.Graphics.Shader.Translation
             GpPassthrough = commonWord0.Extract(24);
 
             DoesLoadOrStore = commonWord0.Extract(26);
-            DoesFp64        = commonWord0.Extract(27);
+            DoesFp64 = commonWord0.Extract(27);
 
             StreamOutMask = commonWord0.Extract(28, 4);
 
@@ -163,7 +132,7 @@ namespace Ryujinx.Graphics.Shader.Translation
             MaxOutputVertexCount = commonWord4.Extract(0, 12);
 
             StoreReqStart = commonWord4.Extract(12, 8);
-            StoreReqEnd   = commonWord4.Extract(24, 8);
+            StoreReqEnd = commonWord4.Extract(24, 8);
 
             ImapTypes = new ImapPixelType[32];
 
@@ -179,21 +148,11 @@ namespace Ryujinx.Graphics.Shader.Translation
             }
 
             int type2OmapTarget = header[18];
-            int type2Omap       = header[19];
-
-            OmapTargets = new OmapTarget[8];
-
-            for (int offset = 0; offset < OmapTargets.Length * 4; offset += 4)
-            {
-                OmapTargets[offset >> 2] = new OmapTarget(
-                    type2OmapTarget.Extract(offset + 0),
-                    type2OmapTarget.Extract(offset + 1),
-                    type2OmapTarget.Extract(offset + 2),
-                    type2OmapTarget.Extract(offset + 3));
-            }
+            int type2Omap = header[19];
 
+            OmapTargets = type2OmapTarget;
             OmapSampleMask = type2Omap.Extract(0);
-            OmapDepth      = type2Omap.Extract(1);
+            OmapDepth = type2Omap.Extract(1);
         }
     }
 }
\ No newline at end of file
diff --git a/Ryujinx.Graphics.Shader/Translation/Translator.cs b/Ryujinx.Graphics.Shader/Translation/Translator.cs
index 709b16db..603b20d6 100644
--- a/Ryujinx.Graphics.Shader/Translation/Translator.cs
+++ b/Ryujinx.Graphics.Shader/Translation/Translator.cs
@@ -105,7 +105,8 @@ namespace Ryujinx.Graphics.Shader.Translation
                 config.GetImageDescriptors(),
                 config.UsedFeatures.HasFlag(FeatureFlags.InstanceId),
                 config.UsedFeatures.HasFlag(FeatureFlags.RtLayer),
-                config.ClipDistancesWritten);
+                config.ClipDistancesWritten,
+                config.OmapTargets);
 
             return program;
         }