From 458452279cee03bfe1bbf2c3daf3fc9722b03a74 Mon Sep 17 00:00:00 2001 From: riperiperi Date: Thu, 1 Dec 2022 15:30:13 +0000 Subject: GPU: Track buffer migrations and flush source on incomplete copy (#3952) * Track buffer migrations and flush source on incomplete copy Makes sure that the modified range list is always from the latest iteration of the buffer, and flushes earlier iterations of a buffer if the data has not been migrated yet. * Cleanup 1 * Reduce cost for redundant signal checks on Vulkan * Only inherit the range list if there are pending ranges. * Fix OpenGL * Address Feedback * Whoops --- .../Memory/BufferModifiedRangeList.cs | 242 +++++++++++++++------ 1 file changed, 173 insertions(+), 69 deletions(-) (limited to 'Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs') diff --git a/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs b/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs index 07dbd209..d0230b62 100644 --- a/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs +++ b/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs @@ -1,6 +1,8 @@ -using Ryujinx.Common.Pools; +using Ryujinx.Common.Logging; +using Ryujinx.Common.Pools; using Ryujinx.Memory.Range; using System; +using System.Collections.Generic; using System.Linq; namespace Ryujinx.Graphics.Gpu.Memory @@ -30,17 +32,24 @@ namespace Ryujinx.Graphics.Gpu.Memory /// public ulong SyncNumber { get; internal set; } + /// + /// The range list that originally owned this range. + /// + public BufferModifiedRangeList Parent { get; internal set; } + /// /// Creates a new instance of a modified range. /// /// Start address of the range /// Size of the range in bytes /// The GPU sync number at the time of creation - public BufferModifiedRange(ulong address, ulong size, ulong syncNumber) + /// The range list that owns this range + public BufferModifiedRange(ulong address, ulong size, ulong syncNumber, BufferModifiedRangeList parent) { Address = address; Size = size; SyncNumber = syncNumber; + Parent = parent; } /// @@ -63,16 +72,39 @@ namespace Ryujinx.Graphics.Gpu.Memory private const int BackingInitialSize = 8; private GpuContext _context; + private Buffer _parent; + private Action _flushAction; + + private List _sources; + private BufferMigration _migrationTarget; private object _lock = new object(); + /// + /// Whether the modified range list has any entries or not. + /// + public bool HasRanges + { + get + { + lock (_lock) + { + return Count > 0; + } + } + } + /// /// Creates a new instance of a modified range list. /// /// GPU context that the buffer range list belongs to - public BufferModifiedRangeList(GpuContext context) : base(BackingInitialSize) + /// The parent buffer that owns this range list + /// The flush action for the parent buffer + public BufferModifiedRangeList(GpuContext context, Buffer parent, Action flushAction) : base(BackingInitialSize) { _context = context; + _parent = parent; + _flushAction = flushAction; } /// @@ -142,6 +174,7 @@ namespace Ryujinx.Graphics.Gpu.Memory { // Region already exists. Just update the existing sync number. overlap.SyncNumber = syncNumber; + overlap.Parent = this; return; } @@ -152,18 +185,18 @@ namespace Ryujinx.Graphics.Gpu.Memory { // A split item must be created behind this overlap. - Add(new BufferModifiedRange(overlap.Address, address - overlap.Address, overlap.SyncNumber)); + Add(new BufferModifiedRange(overlap.Address, address - overlap.Address, overlap.SyncNumber, overlap.Parent)); } if (overlap.Address < endAddress && overlap.EndAddress > endAddress) { // A split item must be created after this overlap. - Add(new BufferModifiedRange(endAddress, overlap.EndAddress - endAddress, overlap.SyncNumber)); + Add(new BufferModifiedRange(endAddress, overlap.EndAddress - endAddress, overlap.SyncNumber, overlap.Parent)); } } - Add(new BufferModifiedRange(address, size, syncNumber)); + Add(new BufferModifiedRange(address, size, syncNumber, this)); } } @@ -207,9 +240,102 @@ namespace Ryujinx.Graphics.Gpu.Memory } } + /// + /// Performs the given range action, or one from a migration that overlaps and has not synced yet. + /// + /// The offset to pass to the action + /// The size to pass to the action + /// The sync number that has been reached + /// The modified range list that originally owned this range + /// The action to perform + public void RangeActionWithMigration(ulong offset, ulong size, ulong syncNumber, BufferModifiedRangeList parent, Action rangeAction) + { + bool firstSource = true; + + if (parent != this) + { + lock (_lock) + { + if (_sources != null) + { + foreach (BufferMigration source in _sources) + { + if (source.Overlaps(offset, size, syncNumber)) + { + if (firstSource && !source.FullyMatches(offset, size)) + { + // Perform this buffer's action first. The migrations will run after. + rangeAction(offset, size); + } + + source.RangeActionWithMigration(offset, size, syncNumber, parent); + + firstSource = false; + } + } + } + } + } + + if (firstSource) + { + // No overlapping migrations, or they are not meant for this range, flush the data using the given action. + rangeAction(offset, size); + } + } + + /// + /// Removes modified ranges ready by the sync number from the list, and flushes their buffer data within a given address range. + /// + /// Overlapping ranges to check + /// Number of overlapping ranges + /// The highest difference between an overlapping range's sync number and the current one + /// The current sync number + /// The start address of the flush range + /// The end address of the flush range + private void RemoveRangesAndFlush( + BufferModifiedRange[] overlaps, + int rangeCount, + long highestDiff, + ulong currentSync, + ulong address, + ulong endAddress) + { + lock (_lock) + { + if (_migrationTarget == null) + { + ulong waitSync = currentSync + (ulong)highestDiff; + + for (int i = 0; i < rangeCount; i++) + { + BufferModifiedRange overlap = overlaps[i]; + + long diff = (long)(overlap.SyncNumber - currentSync); + + if (diff <= highestDiff) + { + ulong clampAddress = Math.Max(address, overlap.Address); + ulong clampEnd = Math.Min(endAddress, overlap.EndAddress); + + ClearPart(overlap, clampAddress, clampEnd); + + RangeActionWithMigration(clampAddress, clampEnd - clampAddress, waitSync, overlap.Parent, _flushAction); + } + } + + return; + } + } + + // There is a migration target to call instead. This can't be changed after set so accessing it outside the lock is fine. + + _migrationTarget.Destination.RemoveRangesAndFlush(overlaps, rangeCount, highestDiff, currentSync, address, endAddress); + } + /// /// Gets modified ranges within the specified region, waits on ones from a previous sync number, - /// and then fires the given action for each range individually. + /// and then fires the flush action for each range individually. /// /// /// This function assumes it is called from the background thread. @@ -218,8 +344,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// /// Start address to query /// Size to query - /// The action to call for each modified range - public void WaitForAndGetRanges(ulong address, ulong size, Action rangeAction) + public void WaitForAndFlushRanges(ulong address, ulong size) { ulong endAddress = address + size; ulong currentSync = _context.SyncNumber; @@ -231,10 +356,23 @@ namespace Ryujinx.Graphics.Gpu.Memory // Range list must be consistent for this operation lock (_lock) { - rangeCount = FindOverlapsNonOverlapping(address, size, ref overlaps); + if (_migrationTarget != null) + { + rangeCount = -1; + } + else + { + rangeCount = FindOverlapsNonOverlapping(address, size, ref overlaps); + } } - if (rangeCount == 0) + if (rangeCount == -1) + { + _migrationTarget.Destination.WaitForAndFlushRanges(address, size); + + return; + } + else if (rangeCount == 0) { return; } @@ -264,47 +402,37 @@ namespace Ryujinx.Graphics.Gpu.Memory // Wait for the syncpoint. _context.Renderer.WaitSync(currentSync + (ulong)highestDiff); - // Flush and remove all regions with the older syncpoint. - lock (_lock) - { - for (int i = 0; i < rangeCount; i++) - { - BufferModifiedRange overlap = overlaps[i]; - - long diff = (long)(overlap.SyncNumber - currentSync); - - if (diff <= highestDiff) - { - ulong clampAddress = Math.Max(address, overlap.Address); - ulong clampEnd = Math.Min(endAddress, overlap.EndAddress); - - ClearPart(overlap, clampAddress, clampEnd); - - rangeAction(clampAddress, clampEnd - clampAddress); - } - } - } + RemoveRangesAndFlush(overlaps, rangeCount, highestDiff, currentSync, address, endAddress); } /// /// Inherit ranges from another modified range list. /// /// The range list to inherit from - /// The action to call for each modified range - public void InheritRanges(BufferModifiedRangeList ranges, Action rangeAction) + /// The action to call for each modified range + public void InheritRanges(BufferModifiedRangeList ranges, Action registerRangeAction) { BufferModifiedRange[] inheritRanges; lock (ranges._lock) { + BufferMigration migration = new(ranges._parent, ranges._flushAction, ranges, this, _context.SyncNumber); + + ranges._parent.IncrementReferenceCount(); + ranges._migrationTarget = migration; + + _context.RegisterBufferMigration(migration); + inheritRanges = ranges.ToArray(); - } - lock (_lock) - { - foreach (BufferModifiedRange range in inheritRanges) + lock (_lock) { - Add(range); + (_sources ??= new List()).Add(migration); + + foreach (BufferModifiedRange range in inheritRanges) + { + Add(range); + } } } @@ -313,44 +441,20 @@ namespace Ryujinx.Graphics.Gpu.Memory { if (range.SyncNumber != currentSync) { - rangeAction(range.Address, range.Size); + registerRangeAction(range.Address, range.Size); } } } /// - /// Calls the given action for modified ranges that aren't from the current sync number. + /// Removes a source buffer migration, indicating its copy has completed. /// - /// The action to call for each modified range - public void ReregisterRanges(Action rangeAction) + /// The migration to remove + public void RemoveMigration(BufferMigration migration) { - ref var ranges = ref ThreadStaticArray.Get(); - int count; - - // Range list must be consistent for this operation. lock (_lock) { - count = Count; - if (ranges.Length < count) - { - Array.Resize(ref ranges, count); - } - - int i = 0; - foreach (BufferModifiedRange range in this) - { - ranges[i++] = range; - } - } - - ulong currentSync = _context.SyncNumber; - for (int i = 0; i < count; i++) - { - BufferModifiedRange range = ranges[i]; - if (range.SyncNumber != currentSync) - { - rangeAction(range.Address, range.Size); - } + _sources.Remove(migration); } } @@ -362,12 +466,12 @@ namespace Ryujinx.Graphics.Gpu.Memory if (overlap.Address < address) { - Add(new BufferModifiedRange(overlap.Address, address - overlap.Address, overlap.SyncNumber)); + Add(new BufferModifiedRange(overlap.Address, address - overlap.Address, overlap.SyncNumber, overlap.Parent)); } if (overlap.EndAddress > endAddress) { - Add(new BufferModifiedRange(endAddress, overlap.EndAddress - endAddress, overlap.SyncNumber)); + Add(new BufferModifiedRange(endAddress, overlap.EndAddress - endAddress, overlap.SyncNumber, overlap.Parent)); } } -- cgit v1.2.3