From eb1ce41b00e415fe84537bc872ddbf13996055d5 Mon Sep 17 00:00:00 2001 From: riperiperi Date: Sun, 19 May 2024 20:53:37 +0100 Subject: GPU: Migrate buffers on GPU project, pre-emptively flush device local mappings (#6794) * GPU: Migrate buffers on GPU project, pre-emptively flush device local mappings Essentially retreading #4540, but it's on the GPU project now instead of the backend. This allows us to have a lot more control + knowledge of where the buffer backing has been changed and allows us to pre-emptively flush pages to host memory for quicker readback. It will allow us to do other stuff in the future, but we'll get there when we get there. Performance greatly improved in Hyrule Warriors: Age of Calamity. Performance notably improved in TOTK (average). Performance for BOTW restored to how it was before #4911, perhaps a bit better. - Rewrites a bunch of buffer migration stuff. Might want to tighten up how dispose stuff works. - Fixed an issue where the copy for texture pre-flush would happen _after_ the syncpoint. TODO: remove a page from pre-flush if it isn't flushed after a certain number of copies. * Add copy deactivation * Fix dependent virtual buffers * Remove logging * Fix format issues (maybe) * Vulkan: Remove backing swap * Add explicit memory access types for most buffers * Fix typo * Add device local force expiry, change buffer inheritance behaviour * General cleanup, OGL fix * BufferPreFlush comments * BufferBackingState comments * Add an extra precaution to BufferMigration This is very unlikely, but it's important to cover loose ends like this. * Address some feedback * Docs --- src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs | 174 ++++++++++++++++++++++++++++-- 1 file changed, 168 insertions(+), 6 deletions(-) (limited to 'src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs') diff --git a/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs b/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs index d293060b..e060e0b4 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs @@ -10,6 +10,8 @@ using System.Threading; namespace Ryujinx.Graphics.Gpu.Memory { + delegate void BufferFlushAction(ulong address, ulong size, ulong syncNumber); + /// /// Buffer, used to store vertex and index data, uniform and storage buffers, and others. /// @@ -23,7 +25,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// /// Host buffer handle. /// - public BufferHandle Handle { get; } + public BufferHandle Handle { get; private set; } /// /// Start address of the buffer in guest memory. @@ -60,6 +62,17 @@ namespace Ryujinx.Graphics.Gpu.Memory /// private BufferModifiedRangeList _modifiedRanges = null; + /// + /// A structure that is used to flush buffer data back to a host mapped buffer for cached readback. + /// Only used if the buffer data is explicitly owned by device local memory. + /// + private BufferPreFlush _preFlush = null; + + /// + /// Usage tracking state that determines what type of backing the buffer should use. + /// + public BufferBackingState BackingState; + private readonly MultiRegionHandle _memoryTrackingGranular; private readonly RegionHandle _memoryTracking; @@ -87,6 +100,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Physical memory where the buffer is mapped /// Start address of the buffer /// Size of the buffer in bytes + /// The type of usage that created the buffer /// Indicates if the buffer can be used in a sparse buffer mapping /// Buffers which this buffer contains, and will inherit tracking handles from public Buffer( @@ -94,6 +108,7 @@ namespace Ryujinx.Graphics.Gpu.Memory PhysicalMemory physicalMemory, ulong address, ulong size, + BufferStage stage, bool sparseCompatible, IEnumerable baseBuffers = null) { @@ -103,9 +118,11 @@ namespace Ryujinx.Graphics.Gpu.Memory Size = size; SparseCompatible = sparseCompatible; - BufferAccess access = sparseCompatible ? BufferAccess.SparseCompatible : BufferAccess.Default; + BackingState = new BufferBackingState(_context, this, stage, baseBuffers); + + BufferAccess access = BackingState.SwitchAccess(this); - Handle = context.Renderer.CreateBuffer((int)size, access, baseBuffers?.MaxBy(x => x.Size).Handle ?? BufferHandle.Null); + Handle = context.Renderer.CreateBuffer((int)size, access); _useGranular = size > GranularBufferThreshold; @@ -161,6 +178,29 @@ namespace Ryujinx.Graphics.Gpu.Memory _virtualDependenciesLock = new ReaderWriterLockSlim(); } + /// + /// Recreates the backing buffer based on the desired access type + /// reported by the backing state struct. + /// + private void ChangeBacking() + { + BufferAccess access = BackingState.SwitchAccess(this); + + BufferHandle newHandle = _context.Renderer.CreateBuffer((int)Size, access); + + _context.Renderer.Pipeline.CopyBuffer(Handle, newHandle, 0, 0, (int)Size); + + _modifiedRanges?.SelfMigration(); + + // If swtiching from device local to host mapped, pre-flushing data no longer makes sense. + // This is set to null and disposed when the migration fully completes. + _preFlush = null; + + Handle = newHandle; + + _physicalMemory.BufferCache.BufferBackingChanged(this); + } + /// /// Gets a sub-range from the buffer, from a start address til a page boundary after the given size. /// @@ -246,6 +286,7 @@ namespace Ryujinx.Graphics.Gpu.Memory } else { + BackingState.RecordSet(); _context.Renderer.SetBufferData(Handle, 0, _physicalMemory.GetSpan(Address, (int)Size)); CopyToDependantVirtualBuffers(); } @@ -283,15 +324,35 @@ namespace Ryujinx.Graphics.Gpu.Memory _modifiedRanges ??= new BufferModifiedRangeList(_context, this, Flush); } + /// + /// Checks if a backing change is deemed necessary from the given usage. + /// If it is, queues a backing change to happen on the next sync action. + /// + /// Buffer stage that can change backing type + private void TryQueueBackingChange(BufferStage stage) + { + if (BackingState.ShouldChangeBacking(stage)) + { + if (!_syncActionRegistered) + { + _context.RegisterSyncAction(this); + _syncActionRegistered = true; + } + } + } + /// /// Signal that the given region of the buffer has been modified. /// /// The start address of the modified region /// The size of the modified region - public void SignalModified(ulong address, ulong size) + /// Buffer stage that triggered the modification + public void SignalModified(ulong address, ulong size, BufferStage stage) { EnsureRangeList(); + TryQueueBackingChange(stage); + _modifiedRanges.SignalModified(address, size); if (!_syncActionRegistered) @@ -311,6 +372,37 @@ namespace Ryujinx.Graphics.Gpu.Memory _modifiedRanges?.Clear(address, size); } + /// + /// Action to be performed immediately before sync is created. + /// This will copy any buffer ranges designated for pre-flushing. + /// + /// True if the action is a guest syncpoint + public void SyncPreAction(bool syncpoint) + { + if (_referenceCount == 0) + { + return; + } + + if (BackingState.ShouldChangeBacking()) + { + ChangeBacking(); + } + + if (BackingState.IsDeviceLocal) + { + _preFlush ??= new BufferPreFlush(_context, this, FlushImpl); + + if (_preFlush.ShouldCopy) + { + _modifiedRanges?.GetRangesAtSync(Address, Size, _context.SyncNumber, (address, size) => + { + _preFlush.CopyModified(address, size); + }); + } + } + } + /// /// Action to be performed when a syncpoint is reached after modification. /// This will register read/write tracking to flush the buffer from GPU when its memory is used. @@ -466,6 +558,8 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Size of the modified region private void LoadRegion(ulong mAddress, ulong mSize) { + BackingState.RecordSet(); + int offset = (int)(mAddress - Address); _context.Renderer.SetBufferData(Handle, offset, _physicalMemory.GetSpan(mAddress, (int)mSize)); @@ -539,18 +633,84 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Flushes a range of the buffer. /// This writes the range data back into guest memory. /// + /// Buffer handle to flush data from /// Start address of the range /// Size in bytes of the range - public void Flush(ulong address, ulong size) + private void FlushImpl(BufferHandle handle, ulong address, ulong size) { int offset = (int)(address - Address); - using PinnedSpan data = _context.Renderer.GetBufferData(Handle, offset, (int)size); + using PinnedSpan data = _context.Renderer.GetBufferData(handle, offset, (int)size); // TODO: When write tracking shaders, they will need to be aware of changes in overlapping buffers. _physicalMemory.WriteUntracked(address, CopyFromDependantVirtualBuffers(data.Get(), address, size)); } + /// + /// Flushes a range of the buffer. + /// This writes the range data back into guest memory. + /// + /// Start address of the range + /// Size in bytes of the range + private void FlushImpl(ulong address, ulong size) + { + FlushImpl(Handle, address, size); + } + + /// + /// Flushes a range of the buffer from the most optimal source. + /// This writes the range data back into guest memory. + /// + /// Start address of the range + /// Size in bytes of the range + /// Sync number waited for before flushing the data + public void Flush(ulong address, ulong size, ulong syncNumber) + { + BackingState.RecordFlush(); + + BufferPreFlush preFlush = _preFlush; + + if (preFlush != null) + { + preFlush.FlushWithAction(address, size, syncNumber); + } + else + { + FlushImpl(address, size); + } + } + /// + /// Gets an action that disposes the backing buffer using its current handle. + /// Useful for deleting an old copy of the buffer after the handle changes. + /// + /// An action that flushes data from the specified range, using the buffer handle at the time the method is generated + public Action GetSnapshotDisposeAction() + { + BufferHandle handle = Handle; + BufferPreFlush preFlush = _preFlush; + + return () => + { + _context.Renderer.DeleteBuffer(handle); + preFlush?.Dispose(); + }; + } + + /// + /// Gets an action that flushes a range of the buffer using its current handle. + /// Useful for flushing data from old copies of the buffer after the handle changes. + /// + /// An action that flushes data from the specified range, using the buffer handle at the time the method is generated + public BufferFlushAction GetSnapshotFlushAction() + { + BufferHandle handle = Handle; + + return (ulong address, ulong size, ulong _) => + { + FlushImpl(handle, address, size); + }; + } + /// /// Align a given address and size region to page boundaries. /// @@ -857,6 +1017,8 @@ namespace Ryujinx.Graphics.Gpu.Memory _modifiedRanges?.Clear(); _context.Renderer.DeleteBuffer(Handle); + _preFlush?.Dispose(); + _preFlush = null; UnmappedSequence++; } -- cgit v1.2.3