From 9f1cf6458c78a42256b1f390f5b3b9159b00a7cb Mon Sep 17 00:00:00 2001 From: riperiperi Date: Sun, 19 Mar 2023 20:56:48 +0000 Subject: Vulkan: Migrate buffers between memory types to improve GPU performance (#4540) * Initial implementation of migration between memory heaps - Missing OOM handling - Missing `_map` data safety when remapping - Copy may not have completed yet (needs some kind of fence) - Map may be unmapped before it is done being used. (needs scoped access) - SSBO accesses are all "writes" - maybe pass info in another way. - Missing keeping map type when resizing buffers (should this be done?) * Ensure migrated data is in place before flushing. * Fix issue where old waitable would be signalled. - There is a real issue where existing Auto<> references need to be replaced. * Swap bound Auto<> instances when swapping buffer backing * Fix conversion buffers * Don't try move buffers if the host has shared memory. * Make GPU methods return PinnedSpan with scope * Storage Hint * Fix stupidity * Fix rebase * Tweak rules Attempt to sidestep BOTW slowdown * Remove line * Migrate only when command buffers flush * Change backing swap log to debug * Address some feedback * Disallow backing swap when the flush lock is held by the current thread * Make PinnedSpan from ReadOnlySpan explicitly unsafe * Fix some small issues - Index buffer swap fixed - Allocate DeviceLocal buffers using a separate block list to images. * Remove alternative flags * Address feedback --- Ryujinx.Graphics.Vulkan/BufferManager.cs | 122 +++++++++++++++++++++++-------- 1 file changed, 93 insertions(+), 29 deletions(-) (limited to 'Ryujinx.Graphics.Vulkan/BufferManager.cs') diff --git a/Ryujinx.Graphics.Vulkan/BufferManager.cs b/Ryujinx.Graphics.Vulkan/BufferManager.cs index 49fdd75d..f8f41e5b 100644 --- a/Ryujinx.Graphics.Vulkan/BufferManager.cs +++ b/Ryujinx.Graphics.Vulkan/BufferManager.cs @@ -4,6 +4,7 @@ using System; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using VkFormat = Silk.NET.Vulkan.Format; +using VkBuffer = Silk.NET.Vulkan.Buffer; namespace Ryujinx.Graphics.Vulkan { @@ -16,17 +17,17 @@ namespace Ryujinx.Graphics.Vulkan // Some drivers don't expose a "HostCached" memory type, // so we need those alternative flags for the allocation to succeed there. - private const MemoryPropertyFlags DefaultBufferMemoryAltFlags = + private const MemoryPropertyFlags DefaultBufferMemoryNoCacheFlags = MemoryPropertyFlags.HostVisibleBit | MemoryPropertyFlags.HostCoherentBit; private const MemoryPropertyFlags DeviceLocalBufferMemoryFlags = MemoryPropertyFlags.DeviceLocalBit; - private const MemoryPropertyFlags FlushableDeviceLocalBufferMemoryFlags = + private const MemoryPropertyFlags DeviceLocalMappedBufferMemoryFlags = + MemoryPropertyFlags.DeviceLocalBit | MemoryPropertyFlags.HostVisibleBit | - MemoryPropertyFlags.HostCoherentBit | - MemoryPropertyFlags.DeviceLocalBit; + MemoryPropertyFlags.HostCoherentBit; private const BufferUsageFlags DefaultBufferUsageFlags = BufferUsageFlags.TransferSrcBit | @@ -54,14 +55,14 @@ namespace Ryujinx.Graphics.Vulkan StagingBuffer = new StagingBuffer(gd, this); } - public BufferHandle CreateWithHandle(VulkanRenderer gd, int size, bool deviceLocal) + public BufferHandle CreateWithHandle(VulkanRenderer gd, int size, BufferAllocationType baseType = BufferAllocationType.HostMapped, BufferHandle storageHint = default) { - return CreateWithHandle(gd, size, deviceLocal, out _); + return CreateWithHandle(gd, size, out _, baseType, storageHint); } - public BufferHandle CreateWithHandle(VulkanRenderer gd, int size, bool deviceLocal, out BufferHolder holder) + public BufferHandle CreateWithHandle(VulkanRenderer gd, int size, out BufferHolder holder, BufferAllocationType baseType = BufferAllocationType.HostMapped, BufferHandle storageHint = default) { - holder = Create(gd, size, deviceLocal: deviceLocal); + holder = Create(gd, size, baseType: baseType, storageHint: storageHint); if (holder == null) { return BufferHandle.Null; @@ -74,7 +75,12 @@ namespace Ryujinx.Graphics.Vulkan return Unsafe.As(ref handle64); } - public unsafe BufferHolder Create(VulkanRenderer gd, int size, bool forConditionalRendering = false, bool deviceLocal = false) + public unsafe (VkBuffer buffer, MemoryAllocation allocation, BufferAllocationType resultType) CreateBacking( + VulkanRenderer gd, + int size, + BufferAllocationType type, + bool forConditionalRendering = false, + BufferAllocationType fallbackType = BufferAllocationType.Auto) { var usage = DefaultBufferUsageFlags; @@ -98,48 +104,106 @@ namespace Ryujinx.Graphics.Vulkan gd.Api.CreateBuffer(_device, in bufferCreateInfo, null, out var buffer).ThrowOnError(); gd.Api.GetBufferMemoryRequirements(_device, buffer, out var requirements); - MemoryPropertyFlags allocateFlags; - MemoryPropertyFlags allocateFlagsAlt; + MemoryAllocation allocation; - if (deviceLocal) - { - allocateFlags = DeviceLocalBufferMemoryFlags; - allocateFlagsAlt = DeviceLocalBufferMemoryFlags; - } - else + do { - allocateFlags = DefaultBufferMemoryFlags; - allocateFlagsAlt = DefaultBufferMemoryAltFlags; + var allocateFlags = type switch + { + BufferAllocationType.HostMappedNoCache => DefaultBufferMemoryNoCacheFlags, + BufferAllocationType.HostMapped => DefaultBufferMemoryFlags, + BufferAllocationType.DeviceLocal => DeviceLocalBufferMemoryFlags, + BufferAllocationType.DeviceLocalMapped => DeviceLocalMappedBufferMemoryFlags, + _ => DefaultBufferMemoryFlags + }; + + // If an allocation with this memory type fails, fall back to the previous one. + try + { + allocation = gd.MemoryAllocator.AllocateDeviceMemory(requirements, allocateFlags, true); + } + catch (VulkanException) + { + allocation = default; + } } - - var allocation = gd.MemoryAllocator.AllocateDeviceMemory(requirements, allocateFlags, allocateFlagsAlt); + while (allocation.Memory.Handle == 0 && (--type != fallbackType)); if (allocation.Memory.Handle == 0UL) { gd.Api.DestroyBuffer(_device, buffer, null); - return null; + return default; } gd.Api.BindBufferMemory(_device, buffer, allocation.Memory, allocation.Offset); - return new BufferHolder(gd, _device, buffer, allocation, size); + return (buffer, allocation, type); } - public Auto CreateView(BufferHandle handle, VkFormat format, int offset, int size) + public unsafe BufferHolder Create( + VulkanRenderer gd, + int size, + bool forConditionalRendering = false, + BufferAllocationType baseType = BufferAllocationType.HostMapped, + BufferHandle storageHint = default) + { + BufferAllocationType type = baseType; + BufferHolder storageHintHolder = null; + + if (baseType == BufferAllocationType.Auto) + { + if (gd.IsSharedMemory) + { + baseType = BufferAllocationType.HostMapped; + type = baseType; + } + else + { + type = size >= BufferHolder.DeviceLocalSizeThreshold ? BufferAllocationType.DeviceLocal : BufferAllocationType.HostMapped; + } + + if (storageHint != BufferHandle.Null) + { + if (TryGetBuffer(storageHint, out storageHintHolder)) + { + type = storageHintHolder.DesiredType; + } + } + } + + (VkBuffer buffer, MemoryAllocation allocation, BufferAllocationType resultType) = + CreateBacking(gd, size, type, forConditionalRendering); + + if (buffer.Handle != 0) + { + var holder = new BufferHolder(gd, _device, buffer, allocation, size, baseType, resultType); + + if (storageHintHolder != null) + { + holder.InheritMetrics(storageHintHolder); + } + + return holder; + } + + return null; + } + + public Auto CreateView(BufferHandle handle, VkFormat format, int offset, int size, Action invalidateView) { if (TryGetBuffer(handle, out var holder)) { - return holder.CreateView(format, offset, size); + return holder.CreateView(format, offset, size, invalidateView); } return null; } - public Auto GetBuffer(CommandBuffer commandBuffer, BufferHandle handle, bool isWrite) + public Auto GetBuffer(CommandBuffer commandBuffer, BufferHandle handle, bool isWrite, bool isSSBO = false) { if (TryGetBuffer(handle, out var holder)) { - return holder.GetBuffer(commandBuffer, isWrite); + return holder.GetBuffer(commandBuffer, isWrite, isSSBO); } return null; @@ -332,14 +396,14 @@ namespace Ryujinx.Graphics.Vulkan return null; } - public ReadOnlySpan GetData(BufferHandle handle, int offset, int size) + public PinnedSpan GetData(BufferHandle handle, int offset, int size) { if (TryGetBuffer(handle, out var holder)) { return holder.GetData(offset, size); } - return ReadOnlySpan.Empty; + return new PinnedSpan(); } public void SetData(BufferHandle handle, int offset, ReadOnlySpan data) where T : unmanaged -- cgit v1.2.3