Vulkan: Feedback loop detection and barriers (#7226)

* Vulkan: Feedback loop improvements

This PR allows the Vulkan backend to detect attachment feedback loops. These are currently used in the following ways:

- Partial use of VK_EXT_attachment_feedback_loop_layout
  - All renderable textures have AttachmentFeedbackLoopBitExt
  - Compile pipelines with Color/DepthStencil feedback loop flags when present
- Support using FragmentBarrier for feedback loops (fixes regressions from https://github.com/Ryujinx/Ryujinx/pull/7012 )

TODO:
- AMD GPUs may need layout transitions for it to properly allow textures to be used in feedback loops.
- Use dynamic state for feedback loops. The background pipeline will always miss since feedback loop state isn't known on the GPU project.
- How is the barrier dependency flag used? (DXVK just ignores it, there's no vulkan validation...)
- Improve subpass dependencies to fix validation errors

* Mark field readonly

* Add feedback loop dynamic state

* fix: add MoltenVK resolver workaround

fix: add MoltenVK resolver workaround

* Formatting

* Fix more complaints

* RADV dcc workaround

* Use dynamic state properly, cleanup.

* Use aspects flags in more places

1 files changed, 35 insertions, 3 deletions

diff --git a/src/Ryujinx.Graphics.Vulkan/PipelineState.cs b/src/Ryujinx.Graphics.Vulkan/PipelineState.cs
index 6b6b46a9..a726b9ed 100644
--- a/src/Ryujinx.Graphics.Vulkan/PipelineState.cs
+++ b/src/Ryujinx.Graphics.Vulkan/PipelineState.cs
@@ -8,6 +8,7 @@ namespace Ryujinx.Graphics.Vulkan
     struct PipelineState : IDisposable
     {
         private const int RequiredSubgroupSize = 32;
+        private const int MaxDynamicStatesCount = 9;
 
         public PipelineUid Internal;
 
@@ -299,6 +300,12 @@ namespace Ryujinx.Graphics.Vulkan
             set => Internal.Id8 = (Internal.Id8 & 0xFFFFFFFFFFFFFFBF) | ((value ? 1UL : 0UL) << 6);
         }
 
+        public FeedbackLoopAspects FeedbackLoopAspects
+        {
+            readonly get => (FeedbackLoopAspects)((Internal.Id8 >> 7) & 0x3);
+            set => Internal.Id8 = (Internal.Id8 & 0xFFFFFFFFFFFFFE7F) | (((ulong)value) << 7);
+        }
+
         public bool HasTessellationControlShader;
         public NativeArray<PipelineShaderStageCreateInfo> Stages;
         public PipelineLayout PipelineLayout;
@@ -564,9 +571,11 @@ namespace Ryujinx.Graphics.Vulkan
                 }
 
                 bool supportsExtDynamicState = gd.Capabilities.SupportsExtendedDynamicState;
-                int dynamicStatesCount = supportsExtDynamicState ? 8 : 7;
+                bool supportsFeedbackLoopDynamicState = gd.Capabilities.SupportsDynamicAttachmentFeedbackLoop;
 
-                DynamicState* dynamicStates = stackalloc DynamicState[dynamicStatesCount];
+                DynamicState* dynamicStates = stackalloc DynamicState[MaxDynamicStatesCount];
+
+                int dynamicStatesCount = 7;
 
                 dynamicStates[0] = DynamicState.Viewport;
                 dynamicStates[1] = DynamicState.Scissor;
@@ -578,7 +587,12 @@ namespace Ryujinx.Graphics.Vulkan
 
                 if (supportsExtDynamicState)
                 {
-                    dynamicStates[7] = DynamicState.VertexInputBindingStrideExt;
+                    dynamicStates[dynamicStatesCount++] = DynamicState.VertexInputBindingStrideExt;
+                }
+
+                if (supportsFeedbackLoopDynamicState)
+                {
+                    dynamicStates[dynamicStatesCount++] = DynamicState.AttachmentFeedbackLoopEnableExt;
                 }
 
                 var pipelineDynamicStateCreateInfo = new PipelineDynamicStateCreateInfo
@@ -588,9 +602,27 @@ namespace Ryujinx.Graphics.Vulkan
                     PDynamicStates = dynamicStates,
                 };
 
+                PipelineCreateFlags flags = 0;
+
+                if (gd.Capabilities.SupportsAttachmentFeedbackLoop)
+                {
+                    FeedbackLoopAspects aspects = FeedbackLoopAspects;
+
+                    if ((aspects & FeedbackLoopAspects.Color) != 0)
+                    {
+                        flags |= PipelineCreateFlags.CreateColorAttachmentFeedbackLoopBitExt;
+                    }
+
+                    if ((aspects & FeedbackLoopAspects.Depth) != 0)
+                    {
+                        flags |= PipelineCreateFlags.CreateDepthStencilAttachmentFeedbackLoopBitExt;
+                    }
+                }
+
                 var pipelineCreateInfo = new GraphicsPipelineCreateInfo
                 {
                     SType = StructureType.GraphicsPipelineCreateInfo,
+                    Flags = flags,
                     StageCount = StagesCount,
                     PStages = Stages.Pointer,
                     PVertexInputState = &vertexInputState,