From 167f50bbcd5b2378a038e540877be4d9b71a12f6 Mon Sep 17 00:00:00 2001 From: gdkchan Date: Thu, 22 Feb 2024 11:03:07 -0300 Subject: Implement virtual buffer dependencies (#6190) * Implement virtual buffer copies * Introduce TranslateAndCreateMultiBuffersPhysicalOnly, use it for copy and clear * Rename VirtualBufferCache to VirtualRangeCache * Fix potential issue where virtual range could exist in the cache, without a physical buffer * Fix bug that could cause copy with negative size on CopyToDependantVirtualBuffer * Remove virtual copy back for SyncAction * GetData XML docs * Make field readonly * Fix virtual buffer modification tracking * Remove CopyFromDependantVirtualBuffers from ExternalFlush * Move things around a little to avoid perf impact - Inline null check for CopyFromDependantVirtualBuffers - Remove extra method call for SynchronizeMemoryWithVirtualCopyBack, prefer calling CopyFromDependantVirtualBuffers separately * Fix up XML doc --------- Co-authored-by: riperiperi --- src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs | 184 +++++++++++++++++++++---- 1 file changed, 156 insertions(+), 28 deletions(-) (limited to 'src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs') diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs index bd9aa39c..c6284780 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs @@ -3,6 +3,7 @@ using Ryujinx.Memory.Range; using System; using System.Collections.Generic; using System.Linq; +using System.Runtime.CompilerServices; namespace Ryujinx.Graphics.Gpu.Memory { @@ -46,6 +47,7 @@ namespace Ryujinx.Graphics.Gpu.Memory private readonly Dictionary _dirtyCache; private readonly Dictionary _modifiedCache; private bool _pruneCaches; + private int _virtualModifiedSequenceNumber; public event Action NotifyBuffersModified; @@ -125,7 +127,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// /// Performs address translation of the GPU virtual address, and creates - /// new buffers, if needed, for the specified range. + /// new physical and virtual buffers, if needed, for the specified range. /// /// GPU memory manager where the buffer is mapped /// Start GPU virtual address of the buffer @@ -138,12 +140,10 @@ namespace Ryujinx.Graphics.Gpu.Memory return new MultiRange(MemoryManager.PteUnmapped, size); } - bool supportsSparse = _context.Capabilities.SupportsSparseBuffer; - // Fast path not taken for non-contiguous ranges, // since multi-range buffers are not coalesced, so a buffer that covers // the entire cached range might not actually exist. - if (memoryManager.VirtualBufferCache.TryGetOrAddRange(gpuVa, size, supportsSparse, out MultiRange range) && + if (memoryManager.VirtualRangeCache.TryGetOrAddRange(gpuVa, size, out MultiRange range) && range.Count == 1) { return range; @@ -154,6 +154,50 @@ namespace Ryujinx.Graphics.Gpu.Memory return range; } + /// + /// Performs address translation of the GPU virtual address, and creates + /// new physical buffers, if needed, for the specified range. + /// + /// GPU memory manager where the buffer is mapped + /// Start GPU virtual address of the buffer + /// Size in bytes of the buffer + /// Physical ranges of the buffer, after address translation + public MultiRange TranslateAndCreateMultiBuffersPhysicalOnly(MemoryManager memoryManager, ulong gpuVa, ulong size) + { + if (gpuVa == 0) + { + return new MultiRange(MemoryManager.PteUnmapped, size); + } + + // Fast path not taken for non-contiguous ranges, + // since multi-range buffers are not coalesced, so a buffer that covers + // the entire cached range might not actually exist. + if (memoryManager.VirtualRangeCache.TryGetOrAddRange(gpuVa, size, out MultiRange range) && + range.Count == 1) + { + return range; + } + + for (int i = 0; i < range.Count; i++) + { + MemoryRange subRange = range.GetSubRange(i); + + if (subRange.Address != MemoryManager.PteUnmapped) + { + if (range.Count > 1) + { + CreateBuffer(subRange.Address, subRange.Size, SparseBufferAlignmentSize); + } + else + { + CreateBuffer(subRange.Address, subRange.Size); + } + } + } + + return range; + } + /// /// Creates a new buffer for the specified range, if it does not yet exist. /// This can be used to ensure the existance of a buffer. @@ -263,43 +307,110 @@ namespace Ryujinx.Graphics.Gpu.Memory } } - BufferRange[] storages = new BufferRange[range.Count]; + MultiRangeBuffer multiRangeBuffer; + MemoryRange[] alignedSubRanges = new MemoryRange[range.Count]; ulong alignmentMask = SparseBufferAlignmentSize - 1; - for (int i = 0; i < range.Count; i++) + if (_context.Capabilities.SupportsSparseBuffer) { - MemoryRange subRange = range.GetSubRange(i); + BufferRange[] storages = new BufferRange[range.Count]; - if (subRange.Address != MemoryManager.PteUnmapped) + for (int i = 0; i < range.Count; i++) { - ulong endAddress = subRange.Address + subRange.Size; + MemoryRange subRange = range.GetSubRange(i); - ulong alignedAddress = subRange.Address & ~alignmentMask; - ulong alignedEndAddress = (endAddress + alignmentMask) & ~alignmentMask; - ulong alignedSize = alignedEndAddress - alignedAddress; + if (subRange.Address != MemoryManager.PteUnmapped) + { + ulong endAddress = subRange.Address + subRange.Size; - Buffer buffer = _buffers.FindFirstOverlap(alignedAddress, alignedSize); - BufferRange bufferRange = buffer.GetRange(alignedAddress, alignedSize, false); + ulong alignedAddress = subRange.Address & ~alignmentMask; + ulong alignedEndAddress = (endAddress + alignmentMask) & ~alignmentMask; + ulong alignedSize = alignedEndAddress - alignedAddress; - storages[i] = bufferRange; - alignedSubRanges[i] = new MemoryRange(alignedAddress, alignedSize); + Buffer buffer = _buffers.FindFirstOverlap(alignedAddress, alignedSize); + BufferRange bufferRange = buffer.GetRange(alignedAddress, alignedSize, false); + + alignedSubRanges[i] = new MemoryRange(alignedAddress, alignedSize); + storages[i] = bufferRange; + } + else + { + ulong alignedSize = (subRange.Size + alignmentMask) & ~alignmentMask; + + alignedSubRanges[i] = new MemoryRange(MemoryManager.PteUnmapped, alignedSize); + storages[i] = new BufferRange(BufferHandle.Null, 0, (int)alignedSize); + } } - else + + multiRangeBuffer = new(_context, new MultiRange(alignedSubRanges), storages); + } + else + { + for (int i = 0; i < range.Count; i++) { - ulong alignedSize = (subRange.Size + alignmentMask) & ~alignmentMask; + MemoryRange subRange = range.GetSubRange(i); + + if (subRange.Address != MemoryManager.PteUnmapped) + { + ulong endAddress = subRange.Address + subRange.Size; + + ulong alignedAddress = subRange.Address & ~alignmentMask; + ulong alignedEndAddress = (endAddress + alignmentMask) & ~alignmentMask; + ulong alignedSize = alignedEndAddress - alignedAddress; + + alignedSubRanges[i] = new MemoryRange(alignedAddress, alignedSize); + } + else + { + ulong alignedSize = (subRange.Size + alignmentMask) & ~alignmentMask; - storages[i] = new BufferRange(BufferHandle.Null, 0, (int)alignedSize); - alignedSubRanges[i] = new MemoryRange(MemoryManager.PteUnmapped, alignedSize); + alignedSubRanges[i] = new MemoryRange(MemoryManager.PteUnmapped, alignedSize); + } } - } - MultiRangeBuffer multiRangeBuffer = new(_context, new MultiRange(alignedSubRanges), storages); + multiRangeBuffer = new(_context, new MultiRange(alignedSubRanges)); + + UpdateVirtualBufferDependencies(multiRangeBuffer); + } _multiRangeBuffers.Add(multiRangeBuffer); } + /// + /// Adds two-way dependencies to all physical buffers contained within a given virtual buffer. + /// + /// Virtual buffer to have dependencies added + private void UpdateVirtualBufferDependencies(MultiRangeBuffer virtualBuffer) + { + virtualBuffer.ClearPhysicalDependencies(); + + ulong dstOffset = 0; + + HashSet physicalBuffers = new(); + + for (int i = 0; i < virtualBuffer.Range.Count; i++) + { + MemoryRange subRange = virtualBuffer.Range.GetSubRange(i); + + if (subRange.Address != MemoryManager.PteUnmapped) + { + Buffer buffer = _buffers.FindFirstOverlap(subRange.Address, subRange.Size); + + virtualBuffer.AddPhysicalDependency(buffer, subRange.Address, dstOffset, subRange.Size); + physicalBuffers.Add(buffer); + } + + dstOffset += subRange.Size; + } + + foreach (var buffer in physicalBuffers) + { + buffer.CopyToDependantVirtualBuffer(virtualBuffer); + } + } + /// /// Performs address translation of the GPU virtual address, and attempts to force /// the buffer in the region as dirty. @@ -620,8 +731,8 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Size in bytes of the copy public void CopyBuffer(MemoryManager memoryManager, ulong srcVa, ulong dstVa, ulong size) { - MultiRange srcRange = TranslateAndCreateMultiBuffers(memoryManager, srcVa, size); - MultiRange dstRange = TranslateAndCreateMultiBuffers(memoryManager, dstVa, size); + MultiRange srcRange = TranslateAndCreateMultiBuffersPhysicalOnly(memoryManager, srcVa, size); + MultiRange dstRange = TranslateAndCreateMultiBuffersPhysicalOnly(memoryManager, dstVa, size); if (srcRange.Count == 1 && dstRange.Count == 1) { @@ -701,6 +812,8 @@ namespace Ryujinx.Graphics.Gpu.Memory dstBuffer.ClearModified(dstAddress, size); memoryManager.Physical.WriteTrackedResource(dstAddress, memoryManager.Physical.GetSpan(srcAddress, (int)size), ResourceKind.Buffer); } + + dstBuffer.CopyToDependantVirtualBuffers(dstAddress, size); } /// @@ -715,7 +828,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Value to be written into the buffer public void ClearBuffer(MemoryManager memoryManager, ulong gpuVa, ulong size, uint value) { - MultiRange range = TranslateAndCreateMultiBuffers(memoryManager, gpuVa, size); + MultiRange range = TranslateAndCreateMultiBuffersPhysicalOnly(memoryManager, gpuVa, size); for (int index = 0; index < range.Count; index++) { @@ -727,6 +840,8 @@ namespace Ryujinx.Graphics.Gpu.Memory _context.Renderer.Pipeline.ClearBuffer(buffer.Handle, offset, (int)subRange.Size, value); memoryManager.Physical.FillTrackedResource(subRange.Address, subRange.Size, value, ResourceKind.Buffer); + + buffer.CopyToDependantVirtualBuffers(subRange.Address, subRange.Size); } } @@ -806,6 +921,11 @@ namespace Ryujinx.Graphics.Gpu.Memory } } + if (write && buffer != null && !_context.Capabilities.SupportsSparseBuffer) + { + buffer.AddModifiedRegion(range, ++_virtualModifiedSequenceNumber); + } + return buffer; } @@ -825,6 +945,7 @@ namespace Ryujinx.Graphics.Gpu.Memory { buffer = _buffers.FindFirstOverlap(address, size); + buffer.CopyFromDependantVirtualBuffers(); buffer.SynchronizeMemory(address, size); if (write) @@ -849,14 +970,14 @@ namespace Ryujinx.Graphics.Gpu.Memory if (range.Count == 1) { MemoryRange subRange = range.GetSubRange(0); - SynchronizeBufferRange(subRange.Address, subRange.Size); + SynchronizeBufferRange(subRange.Address, subRange.Size, copyBackVirtual: true); } else { for (int index = 0; index < range.Count; index++) { MemoryRange subRange = range.GetSubRange(index); - SynchronizeBufferRange(subRange.Address, subRange.Size); + SynchronizeBufferRange(subRange.Address, subRange.Size, copyBackVirtual: false); } } } @@ -866,12 +987,19 @@ namespace Ryujinx.Graphics.Gpu.Memory /// /// Start address of the memory range /// Size in bytes of the memory range - private void SynchronizeBufferRange(ulong address, ulong size) + /// Whether virtual buffers that uses this buffer as backing memory should have its data copied back if modified + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void SynchronizeBufferRange(ulong address, ulong size, bool copyBackVirtual) { if (size != 0) { Buffer buffer = _buffers.FindFirstOverlap(address, size); + if (copyBackVirtual) + { + buffer.CopyFromDependantVirtualBuffers(); + } + buffer.SynchronizeMemory(address, size); } } -- cgit v1.2.3