Vulkan: Feedback loop detection and barriers (#7226)

* Vulkan: Feedback loop improvements

This PR allows the Vulkan backend to detect attachment feedback loops. These are currently used in the following ways:

- Partial use of VK_EXT_attachment_feedback_loop_layout
  - All renderable textures have AttachmentFeedbackLoopBitExt
  - Compile pipelines with Color/DepthStencil feedback loop flags when present
- Support using FragmentBarrier for feedback loops (fixes regressions from https://github.com/Ryujinx/Ryujinx/pull/7012 )

TODO:
- AMD GPUs may need layout transitions for it to properly allow textures to be used in feedback loops.
- Use dynamic state for feedback loops. The background pipeline will always miss since feedback loop state isn't known on the GPU project.
- How is the barrier dependency flag used? (DXVK just ignores it, there's no vulkan validation...)
- Improve subpass dependencies to fix validation errors

* Mark field readonly

* Add feedback loop dynamic state

* fix: add MoltenVK resolver workaround

fix: add MoltenVK resolver workaround

* Formatting

* Fix more complaints

* RADV dcc workaround

* Use dynamic state properly, cleanup.

* Use aspects flags in more places

1 files changed, 32 insertions, 2 deletions

diff --git a/src/Ryujinx.Graphics.Vulkan/PipelineDynamicState.cs b/src/Ryujinx.Graphics.Vulkan/PipelineDynamicState.cs
index 1cc33f72..ad26ff7b 100644
--- a/src/Ryujinx.Graphics.Vulkan/PipelineDynamicState.cs
+++ b/src/Ryujinx.Graphics.Vulkan/PipelineDynamicState.cs
@@ -1,5 +1,6 @@
 using Ryujinx.Common.Memory;
 using Silk.NET.Vulkan;
+using Silk.NET.Vulkan.Extensions.EXT;
 
 namespace Ryujinx.Graphics.Vulkan
 {
@@ -21,6 +22,8 @@ namespace Ryujinx.Graphics.Vulkan
 
         private Array4<float> _blendConstants;
 
+        private FeedbackLoopAspects _feedbackLoopAspects;
+
         public uint ViewportsCount;
         public Array16<Viewport> Viewports;
 
@@ -32,7 +35,8 @@ namespace Ryujinx.Graphics.Vulkan
             Scissor = 1 << 2,
             Stencil = 1 << 3,
             Viewport = 1 << 4,
-            All = Blend | DepthBias | Scissor | Stencil | Viewport,
+            FeedbackLoop = 1 << 5,
+            All = Blend | DepthBias | Scissor | Stencil | Viewport | FeedbackLoop,
         }
 
         private DirtyFlags _dirty;
@@ -99,13 +103,22 @@ namespace Ryujinx.Graphics.Vulkan
             }
         }
 
+        public void SetFeedbackLoop(FeedbackLoopAspects aspects)
+        {
+            _feedbackLoopAspects = aspects;
+
+            _dirty |= DirtyFlags.FeedbackLoop;
+        }
+
         public void ForceAllDirty()
         {
             _dirty = DirtyFlags.All;
         }
 
-        public void ReplayIfDirty(Vk api, CommandBuffer commandBuffer)
+        public void ReplayIfDirty(VulkanRenderer gd, CommandBuffer commandBuffer)
         {
+            Vk api = gd.Api;
+
             if (_dirty.HasFlag(DirtyFlags.Blend))
             {
                 RecordBlend(api, commandBuffer);
@@ -131,6 +144,11 @@ namespace Ryujinx.Graphics.Vulkan
                 RecordViewport(api, commandBuffer);
             }
 
+            if (_dirty.HasFlag(DirtyFlags.FeedbackLoop) && gd.Capabilities.SupportsDynamicAttachmentFeedbackLoop)
+            {
+                RecordFeedbackLoop(gd.DynamicFeedbackLoopApi, commandBuffer);
+            }
+
             _dirty = DirtyFlags.None;
         }
 
@@ -169,5 +187,17 @@ namespace Ryujinx.Graphics.Vulkan
                 api.CmdSetViewport(commandBuffer, 0, ViewportsCount, Viewports.AsSpan());
             }
         }
+
+        private readonly void RecordFeedbackLoop(ExtAttachmentFeedbackLoopDynamicState api, CommandBuffer commandBuffer)
+        {
+            ImageAspectFlags aspects = (_feedbackLoopAspects & FeedbackLoopAspects.Color) != 0 ? ImageAspectFlags.ColorBit : 0;
+
+            if ((_feedbackLoopAspects & FeedbackLoopAspects.Depth) != 0)
+            {
+                aspects |= ImageAspectFlags.DepthBit | ImageAspectFlags.StencilBit;
+            }
+
+            api.CmdSetAttachmentFeedbackLoopEnable(commandBuffer, aspects);
+        }
     }
 }