Vulkan: Add workarounds for MoltenVK (#4202)

* Add MVK basics.

* Use appropriate output attribute types

* 4kb vertex alignment, bunch of fixes

* Add reduced shader precision mode for mvk.

* Disable ASTC on MVK for now

* Only request robustnes2 when it is available.

* It's just the one feature actually

* Add triangle fan conversion

* Allow NullDescriptor on MVK for some reason.

* Force safe blit on MoltenVK

* Use ASTC only when formats are all available.

* Disable multilevel 3d texture views

* Filter duplicate render targets (on backend)

* Add Automatic MoltenVK Configuration

* Do not create color attachment views with formats that are not RT compatible

* Make sure that the host format matches the vertex shader input types for invalid/unknown guest formats

* FIx rebase for Vertex Attrib State

* Fix 4b alignment for vertex

* Use asynchronous queue submits for MVK

* Ensure color clear shader has correct output type

* Update MoltenVK config

* Always use MoltenVK workarounds on MacOS

* Make MVK supersede all vendors

* Fix rebase

* Various fixes on rebase

* Get portability flags from extension

* Fix some minor rebasing issues

* Style change

* Use LibraryImport for MVKConfiguration

* Rename MoltenVK vendor to Apple

Intel and AMD GPUs on moltenvk report with the those vendors - only apple silicon reports with vendor 0x106B.

* Fix features2 rebase conflict

* Rename fragment output type

* Add missing check for fragment output types

Might have caused the crash in MK8

* Only do fragment output specialization on MoltenVK

* Avoid copy when passing capabilities

* Self feedback

* Address feedback

Co-authored-by: gdk <gab.dark.100@gmail.com>
Co-authored-by: nastys <nastys@users.noreply.github.com>

1 files changed, 84 insertions, 10 deletions

diff --git a/Ryujinx.Graphics.Vulkan/PipelineBase.cs b/Ryujinx.Graphics.Vulkan/PipelineBase.cs
index dfcb32d4..1c0c836b 100644
--- a/Ryujinx.Graphics.Vulkan/PipelineBase.cs
+++ b/Ryujinx.Graphics.Vulkan/PipelineBase.cs
@@ -2,6 +2,7 @@
 using Ryujinx.Graphics.Shader;
 using Silk.NET.Vulkan;
 using System;
+using System.Linq;
 using System.Numerics;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
@@ -50,6 +51,11 @@ namespace Ryujinx.Graphics.Vulkan
         private Auto<DisposableRenderPass> _renderPass;
         private int _writtenAttachmentCount;
 
+        private bool _framebufferUsingColorWriteMask;
+
+        private ITexture[] _preMaskColors;
+        private ITexture _preMaskDepthStencil;
+
         private readonly DescriptorSetUpdater _descriptorSetUpdater;
 
         private IndexBufferState _indexBuffer;
@@ -905,24 +911,37 @@ namespace Ryujinx.Graphics.Vulkan
                 }
             }
 
-            SignalStateChange();
-
-            if (writtenAttachments != _writtenAttachmentCount)
+            if (_framebufferUsingColorWriteMask)
             {
-                SignalAttachmentChange();
-                _writtenAttachmentCount = writtenAttachments;
+                SetRenderTargetsInternal(_preMaskColors, _preMaskDepthStencil, true);
+            }
+            else
+            {
+                SignalStateChange();
+
+                if (writtenAttachments != _writtenAttachmentCount)
+                {
+                    SignalAttachmentChange();
+                    _writtenAttachmentCount = writtenAttachments;
+                }
             }
         }
 
-        public void SetRenderTargets(ITexture[] colors, ITexture depthStencil)
+        private void SetRenderTargetsInternal(ITexture[] colors, ITexture depthStencil, bool filterWriteMasked)
         {
             FramebufferParams?.UpdateModifications();
-            CreateFramebuffer(colors, depthStencil);
+            CreateFramebuffer(colors, depthStencil, filterWriteMasked);
             CreateRenderPass();
             SignalStateChange();
             SignalAttachmentChange();
         }
 
+        public void SetRenderTargets(ITexture[] colors, ITexture depthStencil)
+        {
+            _framebufferUsingColorWriteMask = false;
+            SetRenderTargetsInternal(colors, depthStencil, Gd.IsTBDR);
+        }
+
         public void SetRenderTargetScale(float scale)
         {
             _renderScale[0].X = scale;
@@ -1102,7 +1121,7 @@ namespace Ryujinx.Graphics.Vulkan
 
                         int vbSize = vertexBuffer.Buffer.Size;
 
-                        if (Gd.Vendor == Vendor.Amd && vertexBuffer.Stride > 0)
+                        if (Gd.Vendor == Vendor.Amd && !Gd.IsMoltenVk && vertexBuffer.Stride > 0)
                         {
                             // AMD has a bug where if offset + stride * count is greater than
                             // the size, then the last attribute will have the wrong value.
@@ -1119,7 +1138,8 @@ namespace Ryujinx.Graphics.Vulkan
 
                         buffer.Dispose();
 
-                        if ((vertexBuffer.Stride % FormatExtensions.MaxBufferFormatScalarSize) == 0)
+                        if (!Gd.Capabilities.PortabilitySubset.HasFlag(PortabilitySubsetFlags.VertexBufferAlignment4B) &&
+                            (vertexBuffer.Stride % FormatExtensions.MaxBufferFormatScalarSize) == 0)
                         {
                             buffer = new VertexBufferState(
                                 vb,
@@ -1259,8 +1279,62 @@ namespace Ryujinx.Graphics.Vulkan
             _currentPipelineHandle = 0;
         }
 
-        private void CreateFramebuffer(ITexture[] colors, ITexture depthStencil)
+        private void CreateFramebuffer(ITexture[] colors, ITexture depthStencil, bool filterWriteMasked)
         {
+            if (filterWriteMasked)
+            {
+                // TBDR GPUs don't work properly if the same attachment is bound to multiple targets,
+                // due to each attachment being a copy of the real attachment, rather than a direct write.
+
+                // Just try to remove duplicate attachments.
+                // Save a copy of the array to rebind when mask changes.
+
+                void maskOut()
+                {
+                    if (!_framebufferUsingColorWriteMask)
+                    {
+                        _preMaskColors = colors.ToArray();
+                        _preMaskDepthStencil = depthStencil;
+                    }
+
+                    // If true, then the framebuffer must be recreated when the mask changes.
+                    _framebufferUsingColorWriteMask = true;
+                }
+
+                // Look for textures that are masked out.
+
+                for (int i = 0; i < colors.Length; i++)
+                {
+                    if (colors[i] == null)
+                    {
+                        continue;
+                    }
+
+                    ref var vkBlend = ref _newState.Internal.ColorBlendAttachmentState[i];
+
+                    for (int j = 0; j < i; j++)
+                    {
+                        // Check each binding for a duplicate binding before it.
+
+                        if (colors[i] == colors[j])
+                        {
+                            // Prefer the binding with no write mask.
+                            ref var vkBlend2 = ref _newState.Internal.ColorBlendAttachmentState[j];
+                            if (vkBlend.ColorWriteMask == 0)
+                            {
+                                colors[i] = null;
+                                maskOut();
+                            }
+                            else if (vkBlend2.ColorWriteMask == 0)
+                            {
+                                colors[j] = null;
+                                maskOut();
+                            }
+                        }
+                    }
+                }
+            }
+
             FramebufferParams = new FramebufferParams(Device, colors, depthStencil);
             UpdatePipelineAttachmentFormats();
         }