Vulkan: Add workarounds for MoltenVK (#4202)

* Add MVK basics.

* Use appropriate output attribute types

* 4kb vertex alignment, bunch of fixes

* Add reduced shader precision mode for mvk.

* Disable ASTC on MVK for now

* Only request robustnes2 when it is available.

* It's just the one feature actually

* Add triangle fan conversion

* Allow NullDescriptor on MVK for some reason.

* Force safe blit on MoltenVK

* Use ASTC only when formats are all available.

* Disable multilevel 3d texture views

* Filter duplicate render targets (on backend)

* Add Automatic MoltenVK Configuration

* Do not create color attachment views with formats that are not RT compatible

* Make sure that the host format matches the vertex shader input types for invalid/unknown guest formats

* FIx rebase for Vertex Attrib State

* Fix 4b alignment for vertex

* Use asynchronous queue submits for MVK

* Ensure color clear shader has correct output type

* Update MoltenVK config

* Always use MoltenVK workarounds on MacOS

* Make MVK supersede all vendors

* Fix rebase

* Various fixes on rebase

* Get portability flags from extension

* Fix some minor rebasing issues

* Style change

* Use LibraryImport for MVKConfiguration

* Rename MoltenVK vendor to Apple

Intel and AMD GPUs on moltenvk report with the those vendors - only apple silicon reports with vendor 0x106B.

* Fix features2 rebase conflict

* Rename fragment output type

* Add missing check for fragment output types

Might have caused the crash in MK8

* Only do fragment output specialization on MoltenVK

* Avoid copy when passing capabilities

* Self feedback

* Address feedback

Co-authored-by: gdk <gab.dark.100@gmail.com>
Co-authored-by: nastys <nastys@users.noreply.github.com>

4 files changed, 79 insertions, 3 deletions

diff --git a/Ryujinx.Graphics.Gpu/Engine/Threed/SpecializationStateUpdater.cs b/Ryujinx.Graphics.Gpu/Engine/Threed/SpecializationStateUpdater.cs
index 13b332f4..62df15e7 100644
--- a/Ryujinx.Graphics.Gpu/Engine/Threed/SpecializationStateUpdater.cs
+++ b/Ryujinx.Graphics.Gpu/Engine/Threed/SpecializationStateUpdater.cs
@@ -1,5 +1,6 @@
 using Ryujinx.Common.Memory;
 using Ryujinx.Graphics.GAL;
+using Ryujinx.Graphics.Gpu.Engine.Types;
 using Ryujinx.Graphics.Gpu.Shader;
 using Ryujinx.Graphics.Shader;
 
@@ -10,6 +11,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
     /// </summary>
     internal class SpecializationStateUpdater
     {
+        private readonly GpuContext _context;
         private GpuChannelGraphicsState _graphics;
         private GpuChannelPoolState _pool;
 
@@ -19,6 +21,15 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
         private bool _changed;
 
         /// <summary>
+        /// Creates a new instance of the specialization state updater class.
+        /// </summary>
+        /// <param name="context">GPU context</param>
+        public SpecializationStateUpdater(GpuContext context)
+        {
+            _context = context;
+        }
+
+        /// <summary>
         /// Signal that the specialization state has changed.
         /// </summary>
         private void Signal()
@@ -233,6 +244,42 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
         }
 
         /// <summary>
+        /// Updates the type of the outputs produced by the fragment shader based on the current render target state.
+        /// </summary>
+        /// <param name="rtControl">The render target control register</param>
+        /// <param name="state">The color attachment state</param>
+        public void SetFragmentOutputTypes(RtControl rtControl, ref Array8<RtColorState> state)
+        {
+            bool changed = false;
+            int count = rtControl.UnpackCount();
+
+            for (int index = 0; index < Constants.TotalRenderTargets; index++)
+            {
+                int rtIndex = rtControl.UnpackPermutationIndex(index);
+
+                var colorState = state[rtIndex];
+
+                if (index < count && StateUpdater.IsRtEnabled(colorState))
+                {
+                    Format format = colorState.Format.Convert().Format;
+
+                    AttributeType type = format.IsInteger() ? (format.IsSint() ? AttributeType.Sint : AttributeType.Uint) : AttributeType.Float;
+
+                    if (type != _graphics.FragmentOutputTypes[index])
+                    {
+                        _graphics.FragmentOutputTypes[index] = type;
+                        changed = true;
+                    }
+                }
+            }
+
+            if (changed && _context.Capabilities.NeedsFragmentOutputSpecialization)
+            {
+                Signal();
+            }
+        }
+
+        /// <summary>
         /// Indicates that the draw is writing the base vertex, base instance and draw index to Constant Buffer 0.
         /// </summary>
         /// <param name="value">The new value</param>
diff --git a/Ryujinx.Graphics.Gpu/Engine/Threed/StateUpdateTracker.cs b/Ryujinx.Graphics.Gpu/Engine/Threed/StateUpdateTracker.cs
index 3ed5607a..7c730967 100644
--- a/Ryujinx.Graphics.Gpu/Engine/Threed/StateUpdateTracker.cs
+++ b/Ryujinx.Graphics.Gpu/Engine/Threed/StateUpdateTracker.cs
@@ -139,6 +139,16 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
         }
 
         /// <summary>
+        /// Check if the given register group is dirty without clearing it.
+        /// </summary>
+        /// <param name="groupIndex">Index of the group to check</param>
+        /// <returns>True if dirty, false otherwise</returns>
+        public bool IsDirty(int groupIndex)
+        {
+            return (_dirtyMask & (1UL << groupIndex)) != 0;
+        }
+
+        /// <summary>
         /// Check all the groups specified by <paramref name="checkMask"/> for modification, and update if modified.
         /// </summary>
         /// <param name="checkMask">Mask, where each bit set corresponds to a group index that should be checked</param>
diff --git a/Ryujinx.Graphics.Gpu/Engine/Threed/StateUpdater.cs b/Ryujinx.Graphics.Gpu/Engine/Threed/StateUpdater.cs
index 64fa1735..9b59009c 100644
--- a/Ryujinx.Graphics.Gpu/Engine/Threed/StateUpdater.cs
+++ b/Ryujinx.Graphics.Gpu/Engine/Threed/StateUpdater.cs
@@ -20,6 +20,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
         public const int ScissorStateIndex = 16;
         public const int VertexBufferStateIndex = 0;
         public const int PrimitiveRestartStateIndex = 12;
+        public const int RenderTargetStateIndex = 27;
 
         private readonly GpuContext _context;
         private readonly GpuChannel _channel;
@@ -264,6 +265,11 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
                 _prevTfEnable = false;
             }
 
+            if (_updateTracker.IsDirty(RenderTargetStateIndex))
+            {
+                UpdateRenderTargetSpecialization();
+            }
+
             _updateTracker.Update(ulong.MaxValue);
 
             // If any state that the shader depends on changed,
@@ -527,11 +533,19 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
         }
 
         /// <summary>
+        /// Updates specialization state based on render target state.
+        /// </summary>
+        public void UpdateRenderTargetSpecialization()
+        {
+            _currentSpecState.SetFragmentOutputTypes(_state.State.RtControl, ref _state.State.RtColorState);
+        }
+
+        /// <summary>
         /// Checks if a render target color buffer is used.
         /// </summary>
         /// <param name="colorState">Color buffer information</param>
         /// <returns>True if the specified buffer is enabled/used, false otherwise</returns>
-        private static bool IsRtEnabled(RtColorState colorState)
+        internal static bool IsRtEnabled(RtColorState colorState)
         {
             // Colors are disabled by writing 0 to the format.
             return colorState.Format != 0 && colorState.WidthOrStride != 0;
@@ -893,7 +907,12 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
                 {
                     Logger.Debug?.Print(LogClass.Gpu, $"Invalid attribute format 0x{vertexAttrib.UnpackFormat():X}.");
 
-                    format = Format.R32G32B32A32Float;
+                    format = vertexAttrib.UnpackType() switch
+                    {
+                        VertexAttribType.Sint => Format.R32G32B32A32Sint,
+                        VertexAttribType.Uint => Format.R32G32B32A32Uint,
+                        _ => Format.R32G32B32A32Float
+                    };
                 }
 
                 vertexAttribs[index] = new VertexAttribDescriptor(
diff --git a/Ryujinx.Graphics.Gpu/Engine/Threed/ThreedClass.cs b/Ryujinx.Graphics.Gpu/Engine/Threed/ThreedClass.cs
index a38c0987..19eb8b46 100644
--- a/Ryujinx.Graphics.Gpu/Engine/Threed/ThreedClass.cs
+++ b/Ryujinx.Graphics.Gpu/Engine/Threed/ThreedClass.cs
@@ -71,7 +71,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
 
             _i2mClass = new InlineToMemoryClass(context, channel, initializeState: false);
 
-            var spec = new SpecializationStateUpdater();
+            var spec = new SpecializationStateUpdater(context);
             var drawState = new DrawState();
 
             _drawManager = new DrawManager(context, channel, _state, drawState, spec);