Vulkan: Add workarounds for MoltenVK (#4202)

* Add MVK basics.

* Use appropriate output attribute types

* 4kb vertex alignment, bunch of fixes

* Add reduced shader precision mode for mvk.

* Disable ASTC on MVK for now

* Only request robustnes2 when it is available.

* It's just the one feature actually

* Add triangle fan conversion

* Allow NullDescriptor on MVK for some reason.

* Force safe blit on MoltenVK

* Use ASTC only when formats are all available.

* Disable multilevel 3d texture views

* Filter duplicate render targets (on backend)

* Add Automatic MoltenVK Configuration

* Do not create color attachment views with formats that are not RT compatible

* Make sure that the host format matches the vertex shader input types for invalid/unknown guest formats

* FIx rebase for Vertex Attrib State

* Fix 4b alignment for vertex

* Use asynchronous queue submits for MVK

* Ensure color clear shader has correct output type

* Update MoltenVK config

* Always use MoltenVK workarounds on MacOS

* Make MVK supersede all vendors

* Fix rebase

* Various fixes on rebase

* Get portability flags from extension

* Fix some minor rebasing issues

* Style change

* Use LibraryImport for MVKConfiguration

* Rename MoltenVK vendor to Apple

Intel and AMD GPUs on moltenvk report with the those vendors - only apple silicon reports with vendor 0x106B.

* Fix features2 rebase conflict

* Rename fragment output type

* Add missing check for fragment output types

Might have caused the crash in MK8

* Only do fragment output specialization on MoltenVK

* Avoid copy when passing capabilities

* Self feedback

* Address feedback

Co-authored-by: gdk <gab.dark.100@gmail.com>
Co-authored-by: nastys <nastys@users.noreply.github.com>

10 files changed, 114 insertions, 5 deletions

diff --git a/Ryujinx.Graphics.Gpu/Engine/Threed/SpecializationStateUpdater.cs b/Ryujinx.Graphics.Gpu/Engine/Threed/SpecializationStateUpdater.cs
index 13b332f4..62df15e7 100644
--- a/Ryujinx.Graphics.Gpu/Engine/Threed/SpecializationStateUpdater.cs
+++ b/Ryujinx.Graphics.Gpu/Engine/Threed/SpecializationStateUpdater.cs
@@ -1,5 +1,6 @@
 using Ryujinx.Common.Memory;
 using Ryujinx.Graphics.GAL;
+using Ryujinx.Graphics.Gpu.Engine.Types;
 using Ryujinx.Graphics.Gpu.Shader;
 using Ryujinx.Graphics.Shader;
 
@@ -10,6 +11,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
     /// </summary>
     internal class SpecializationStateUpdater
     {
+        private readonly GpuContext _context;
         private GpuChannelGraphicsState _graphics;
         private GpuChannelPoolState _pool;
 
@@ -19,6 +21,15 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
         private bool _changed;
 
         /// <summary>
+        /// Creates a new instance of the specialization state updater class.
+        /// </summary>
+        /// <param name="context">GPU context</param>
+        public SpecializationStateUpdater(GpuContext context)
+        {
+            _context = context;
+        }
+
+        /// <summary>
         /// Signal that the specialization state has changed.
         /// </summary>
         private void Signal()
@@ -233,6 +244,42 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
         }
 
         /// <summary>
+        /// Updates the type of the outputs produced by the fragment shader based on the current render target state.
+        /// </summary>
+        /// <param name="rtControl">The render target control register</param>
+        /// <param name="state">The color attachment state</param>
+        public void SetFragmentOutputTypes(RtControl rtControl, ref Array8<RtColorState> state)
+        {
+            bool changed = false;
+            int count = rtControl.UnpackCount();
+
+            for (int index = 0; index < Constants.TotalRenderTargets; index++)
+            {
+                int rtIndex = rtControl.UnpackPermutationIndex(index);
+
+                var colorState = state[rtIndex];
+
+                if (index < count && StateUpdater.IsRtEnabled(colorState))
+                {
+                    Format format = colorState.Format.Convert().Format;
+
+                    AttributeType type = format.IsInteger() ? (format.IsSint() ? AttributeType.Sint : AttributeType.Uint) : AttributeType.Float;
+
+                    if (type != _graphics.FragmentOutputTypes[index])
+                    {
+                        _graphics.FragmentOutputTypes[index] = type;
+                        changed = true;
+                    }
+                }
+            }
+
+            if (changed && _context.Capabilities.NeedsFragmentOutputSpecialization)
+            {
+                Signal();
+            }
+        }
+
+        /// <summary>
         /// Indicates that the draw is writing the base vertex, base instance and draw index to Constant Buffer 0.
         /// </summary>
         /// <param name="value">The new value</param>
diff --git a/Ryujinx.Graphics.Gpu/Engine/Threed/StateUpdateTracker.cs b/Ryujinx.Graphics.Gpu/Engine/Threed/StateUpdateTracker.cs
index 3ed5607a..7c730967 100644
--- a/Ryujinx.Graphics.Gpu/Engine/Threed/StateUpdateTracker.cs
+++ b/Ryujinx.Graphics.Gpu/Engine/Threed/StateUpdateTracker.cs
@@ -139,6 +139,16 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
         }
 
         /// <summary>
+        /// Check if the given register group is dirty without clearing it.
+        /// </summary>
+        /// <param name="groupIndex">Index of the group to check</param>
+        /// <returns>True if dirty, false otherwise</returns>
+        public bool IsDirty(int groupIndex)
+        {
+            return (_dirtyMask & (1UL << groupIndex)) != 0;
+        }
+
+        /// <summary>
         /// Check all the groups specified by <paramref name="checkMask"/> for modification, and update if modified.
         /// </summary>
         /// <param name="checkMask">Mask, where each bit set corresponds to a group index that should be checked</param>
diff --git a/Ryujinx.Graphics.Gpu/Engine/Threed/StateUpdater.cs b/Ryujinx.Graphics.Gpu/Engine/Threed/StateUpdater.cs
index 64fa1735..9b59009c 100644
--- a/Ryujinx.Graphics.Gpu/Engine/Threed/StateUpdater.cs
+++ b/Ryujinx.Graphics.Gpu/Engine/Threed/StateUpdater.cs
@@ -20,6 +20,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
         public const int ScissorStateIndex = 16;
         public const int VertexBufferStateIndex = 0;
         public const int PrimitiveRestartStateIndex = 12;
+        public const int RenderTargetStateIndex = 27;
 
         private readonly GpuContext _context;
         private readonly GpuChannel _channel;
@@ -264,6 +265,11 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
                 _prevTfEnable = false;
             }
 
+            if (_updateTracker.IsDirty(RenderTargetStateIndex))
+            {
+                UpdateRenderTargetSpecialization();
+            }
+
             _updateTracker.Update(ulong.MaxValue);
 
             // If any state that the shader depends on changed,
@@ -527,11 +533,19 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
         }
 
         /// <summary>
+        /// Updates specialization state based on render target state.
+        /// </summary>
+        public void UpdateRenderTargetSpecialization()
+        {
+            _currentSpecState.SetFragmentOutputTypes(_state.State.RtControl, ref _state.State.RtColorState);
+        }
+
+        /// <summary>
         /// Checks if a render target color buffer is used.
         /// </summary>
         /// <param name="colorState">Color buffer information</param>
         /// <returns>True if the specified buffer is enabled/used, false otherwise</returns>
-        private static bool IsRtEnabled(RtColorState colorState)
+        internal static bool IsRtEnabled(RtColorState colorState)
         {
             // Colors are disabled by writing 0 to the format.
             return colorState.Format != 0 && colorState.WidthOrStride != 0;
@@ -893,7 +907,12 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
                 {
                     Logger.Debug?.Print(LogClass.Gpu, $"Invalid attribute format 0x{vertexAttrib.UnpackFormat():X}.");
 
-                    format = Format.R32G32B32A32Float;
+                    format = vertexAttrib.UnpackType() switch
+                    {
+                        VertexAttribType.Sint => Format.R32G32B32A32Sint,
+                        VertexAttribType.Uint => Format.R32G32B32A32Uint,
+                        _ => Format.R32G32B32A32Float
+                    };
                 }
 
                 vertexAttribs[index] = new VertexAttribDescriptor(
diff --git a/Ryujinx.Graphics.Gpu/Engine/Threed/ThreedClass.cs b/Ryujinx.Graphics.Gpu/Engine/Threed/ThreedClass.cs
index a38c0987..19eb8b46 100644
--- a/Ryujinx.Graphics.Gpu/Engine/Threed/ThreedClass.cs
+++ b/Ryujinx.Graphics.Gpu/Engine/Threed/ThreedClass.cs
@@ -71,7 +71,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
 
             _i2mClass = new InlineToMemoryClass(context, channel, initializeState: false);
 
-            var spec = new SpecializationStateUpdater();
+            var spec = new SpecializationStateUpdater(context);
             var drawState = new DrawState();
 
             _drawManager = new DrawManager(context, channel, _state, drawState, spec);
diff --git a/Ryujinx.Graphics.Gpu/GpuChannel.cs b/Ryujinx.Graphics.Gpu/GpuChannel.cs
index 57c632da..fe858762 100644
--- a/Ryujinx.Graphics.Gpu/GpuChannel.cs
+++ b/Ryujinx.Graphics.Gpu/GpuChannel.cs
@@ -1,4 +1,5 @@
-using Ryujinx.Graphics.Gpu.Engine.GPFifo;
+using Ryujinx.Graphics.GAL;
+using Ryujinx.Graphics.Gpu.Engine.GPFifo;
 using Ryujinx.Graphics.Gpu.Image;
 using Ryujinx.Graphics.Gpu.Memory;
 using System;
@@ -32,6 +33,11 @@ namespace Ryujinx.Graphics.Gpu
         internal MemoryManager MemoryManager => _memoryManager;
 
         /// <summary>
+        /// Host hardware capabilities from the GPU context.
+        /// </summary>
+        internal ref Capabilities Capabilities => ref _context.Capabilities;
+
+        /// <summary>
         /// Creates a new instance of a GPU channel.
         /// </summary>
         /// <param name="context">GPU context that the channel belongs to</param>
diff --git a/Ryujinx.Graphics.Gpu/Shader/DiskCache/DiskCacheGpuAccessor.cs b/Ryujinx.Graphics.Gpu/Shader/DiskCache/DiskCacheGpuAccessor.cs
index c567c2c0..97173c96 100644
--- a/Ryujinx.Graphics.Gpu/Shader/DiskCache/DiskCacheGpuAccessor.cs
+++ b/Ryujinx.Graphics.Gpu/Shader/DiskCache/DiskCacheGpuAccessor.cs
@@ -108,6 +108,12 @@ namespace Ryujinx.Graphics.Gpu.Shader.DiskCache
         }
 
         /// <inheritdoc/>
+        public AttributeType QueryFragmentOutputType(int location)
+        {
+            return _oldSpecState.GraphicsState.FragmentOutputTypes[location];
+        }
+
+        /// <inheritdoc/>
         public int QueryComputeLocalSizeX() => _oldSpecState.ComputeState.LocalSizeX;
 
         /// <inheritdoc/>
diff --git a/Ryujinx.Graphics.Gpu/Shader/GpuAccessor.cs b/Ryujinx.Graphics.Gpu/Shader/GpuAccessor.cs
index 28ea430c..05631a21 100644
--- a/Ryujinx.Graphics.Gpu/Shader/GpuAccessor.cs
+++ b/Ryujinx.Graphics.Gpu/Shader/GpuAccessor.cs
@@ -114,6 +114,12 @@ namespace Ryujinx.Graphics.Gpu.Shader
         }
 
         /// <inheritdoc/>
+        public AttributeType QueryFragmentOutputType(int location)
+        {
+            return _state.GraphicsState.FragmentOutputTypes[location];
+        }
+
+        /// <inheritdoc/>
         public int QueryComputeLocalSizeX() => _state.ComputeState.LocalSizeX;
 
         /// <inheritdoc/>
diff --git a/Ryujinx.Graphics.Gpu/Shader/GpuAccessorBase.cs b/Ryujinx.Graphics.Gpu/Shader/GpuAccessorBase.cs
index 33f06b6e..d36ffd70 100644
--- a/Ryujinx.Graphics.Gpu/Shader/GpuAccessorBase.cs
+++ b/Ryujinx.Graphics.Gpu/Shader/GpuAccessorBase.cs
@@ -112,6 +112,8 @@ namespace Ryujinx.Graphics.Gpu.Shader
             };
         }
 
+        public bool QueryHostReducedPrecision() => _context.Capabilities.ReduceShaderPrecision;
+
         public bool QueryHostHasFrontFacingBug() => _context.Capabilities.HasFrontFacingBug;
 
         public bool QueryHostHasVectorIndexingBug() => _context.Capabilities.HasVectorIndexingBug;
diff --git a/Ryujinx.Graphics.Gpu/Shader/GpuChannelGraphicsState.cs b/Ryujinx.Graphics.Gpu/Shader/GpuChannelGraphicsState.cs
index e5e48626..70ac5017 100644
--- a/Ryujinx.Graphics.Gpu/Shader/GpuChannelGraphicsState.cs
+++ b/Ryujinx.Graphics.Gpu/Shader/GpuChannelGraphicsState.cs
@@ -88,6 +88,11 @@ namespace Ryujinx.Graphics.Gpu.Shader
         public bool HasUnalignedStorageBuffer;
 
         /// <summary>
+        /// Type of the fragment shader outputs.
+        /// </summary>
+        public Array8<AttributeType> FragmentOutputTypes;
+
+        /// <summary>
         /// Creates a new GPU graphics state.
         /// </summary>
         /// <param name="earlyZForce">Early Z force enable</param>
@@ -105,6 +110,7 @@ namespace Ryujinx.Graphics.Gpu.Shader
         /// <param name="attributeTypes">Type of the vertex attributes consumed by the shader</param>
         /// <param name="hasConstantBufferDrawParameters">Indicates that the draw is writing the base vertex, base instance and draw index to Constant Buffer 0</param>
         /// <param name="hasUnalignedStorageBuffer">Indicates that any storage buffer use is unaligned</param>
+        /// <param name="fragmentOutputTypes">Type of the fragment shader outputs</param>
         public GpuChannelGraphicsState(
             bool earlyZForce,
             PrimitiveTopology topology,
@@ -120,7 +126,8 @@ namespace Ryujinx.Graphics.Gpu.Shader
             float alphaTestReference,
             ref Array32<AttributeType> attributeTypes,
             bool hasConstantBufferDrawParameters,
-            bool hasUnalignedStorageBuffer)
+            bool hasUnalignedStorageBuffer,
+            ref Array8<AttributeType> fragmentOutputTypes)
         {
             EarlyZForce = earlyZForce;
             Topology = topology;
@@ -137,6 +144,7 @@ namespace Ryujinx.Graphics.Gpu.Shader
             AttributeTypes = attributeTypes;
             HasConstantBufferDrawParameters = hasConstantBufferDrawParameters;
             HasUnalignedStorageBuffer = hasUnalignedStorageBuffer;
+            FragmentOutputTypes = fragmentOutputTypes;
         }
     }
 }
\ No newline at end of file
diff --git a/Ryujinx.Graphics.Gpu/Shader/ShaderSpecializationState.cs b/Ryujinx.Graphics.Gpu/Shader/ShaderSpecializationState.cs
index b0d77d8a..a4bf8136 100644
--- a/Ryujinx.Graphics.Gpu/Shader/ShaderSpecializationState.cs
+++ b/Ryujinx.Graphics.Gpu/Shader/ShaderSpecializationState.cs
@@ -530,6 +530,11 @@ namespace Ryujinx.Graphics.Gpu.Shader
                 return false;
             }
 
+            if (channel.Capabilities.NeedsFragmentOutputSpecialization && !graphicsState.FragmentOutputTypes.AsSpan().SequenceEqual(GraphicsState.FragmentOutputTypes.AsSpan()))
+            {
+                return false;
+            }
+
             return Matches(channel, ref poolState, checkTextures, isCompute: false);
         }