Inherit buffer tracking handles rather than recreating on resize (#2330)

This greatly speeds up games that constantly resize buffers, and removes stuttering on games that resize large buffers occasionally:

- Large improvement on Super Mario 3D All-Stars (#1663 needed for best performance)
- Improvement to Hyrule Warriors: AoC, and UE4 games. These games can still stutter due to texture creation/loading.
- Small improvement to other games, potential 1-frame stutters avoided.

`ForceSynchronizeMemory`, which was added with POWER, is no longer needed. Some tests have been added for the MultiRegionHandle.

3 files changed, 59 insertions, 47 deletions

diff --git a/Ryujinx.Graphics.Gpu/Memory/Buffer.cs b/Ryujinx.Graphics.Gpu/Memory/Buffer.cs
index c567e30c..b4854d81 100644
--- a/Ryujinx.Graphics.Gpu/Memory/Buffer.cs
+++ b/Ryujinx.Graphics.Gpu/Memory/Buffer.cs
@@ -3,6 +3,8 @@ using Ryujinx.Graphics.GAL;
 using Ryujinx.Memory.Range;
 using Ryujinx.Memory.Tracking;
 using System;
+using System.Collections.Generic;
+using System.Linq;
 
 namespace Ryujinx.Graphics.Gpu.Memory
 {
@@ -68,7 +70,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
         /// <param name="context">GPU context that the buffer belongs to</param>
         /// <param name="address">Start address of the buffer</param>
         /// <param name="size">Size of the buffer in bytes</param>
-        public Buffer(GpuContext context, ulong address, ulong size)
+        /// <param name="baseBuffers">Buffers which this buffer contains, and will inherit tracking handles from</param>
+        public Buffer(GpuContext context, ulong address, ulong size, IEnumerable<Buffer> baseBuffers = null)
         {
             _context = context;
             Address  = address;
@@ -78,13 +81,45 @@ namespace Ryujinx.Graphics.Gpu.Memory
 
             _useGranular = size > GranularBufferThreshold;
 
+            IEnumerable<IRegionHandle> baseHandles = null;
+
+            if (baseBuffers != null)
+            {
+                baseHandles = baseBuffers.SelectMany(buffer =>
+                {
+                    if (buffer._useGranular)
+                    {
+                        return buffer._memoryTrackingGranular.GetHandles();
+                    }
+                    else
+                    {
+                        return Enumerable.Repeat(buffer._memoryTracking.GetHandle(), 1);
+                    }
+                });
+            }
+
             if (_useGranular)
             {
-                _memoryTrackingGranular = context.PhysicalMemory.BeginGranularTracking(address, size);
+                _memoryTrackingGranular = context.PhysicalMemory.BeginGranularTracking(address, size, baseHandles);
             }
             else
             {
                 _memoryTracking = context.PhysicalMemory.BeginTracking(address, size);
+
+                if (baseHandles != null)
+                {
+                    _memoryTracking.Reprotect(false);
+
+                    foreach (IRegionHandle handle in baseHandles)
+                    {
+                        if (handle.Dirty)
+                        {
+                            _memoryTracking.Reprotect(true);
+                        }
+
+                        handle.Dispose();
+                    }
+                }
             }
 
             _externalFlushDelegate = new RegionSignal(ExternalFlush);
@@ -181,39 +216,6 @@ namespace Ryujinx.Graphics.Gpu.Memory
         }
 
         /// <summary>
-        /// Performs guest to host memory synchronization of the buffer data, regardless of sequence number.
-        /// </summary>
-        /// <remarks>
-        /// This causes the buffer data to be overwritten if a write was detected from the CPU,
-        /// since the last call to this method.
-        /// </remarks>
-        /// <param name="address">Start address of the range to synchronize</param>
-        /// <param name="size">Size in bytes of the range to synchronize</param>
-        public void ForceSynchronizeMemory(ulong address, ulong size)
-        {
-            if (_useGranular)
-            {
-                _memoryTrackingGranular.QueryModified(address, size, _modifiedDelegate);
-            }
-            else
-            {
-                if (_memoryTracking.DirtyOrVolatile())
-                {
-                    _memoryTracking.Reprotect();
-
-                    if (_modifiedRanges != null)
-                    {
-                        _modifiedRanges.ExcludeModifiedRegions(Address, Size, _loadDelegate);
-                    }
-                    else
-                    {
-                        _context.Renderer.SetBufferData(Handle, 0, _context.PhysicalMemory.GetSpan(Address, (int)Size));
-                    }
-                }
-            }
-        }
-
-        /// <summary>
         /// Ensure that the modified range list exists.
         /// </summary>
         private void EnsureRangeList()
@@ -461,18 +463,26 @@ namespace Ryujinx.Graphics.Gpu.Memory
         }
 
         /// <summary>
-        /// Disposes the host buffer.
+        /// Disposes the host buffer's data, not its tracking handles.
         /// </summary>
-        public void Dispose()
+        public void DisposeData()
         {
             _modifiedRanges?.Clear();
 
-            _memoryTrackingGranular?.Dispose();
-            _memoryTracking?.Dispose();
-
             _context.Renderer.DeleteBuffer(Handle);
 
             UnmappedSequence++;
         }
+
+        /// <summary>
+        /// Disposes the host buffer.
+        /// </summary>
+        public void Dispose()
+        {
+            _memoryTrackingGranular?.Dispose();
+            _memoryTracking?.Dispose();
+
+            DisposeData();
+        }
     }
 }
\ No newline at end of file
diff --git a/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs b/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs
index 4a794b19..20fa1f3a 100644
--- a/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs
+++ b/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs
@@ -533,8 +533,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
                         }
                     }
 
-                    Buffer newBuffer = new Buffer(_context, address, endAddress - address);
-                    newBuffer.SynchronizeMemory(address, endAddress - address);
+                    Buffer newBuffer = new Buffer(_context, address, endAddress - address, _bufferOverlaps.Take(overlapsCount));
 
                     lock (_buffers)
                     {
@@ -547,14 +546,14 @@ namespace Ryujinx.Graphics.Gpu.Memory
 
                         int dstOffset = (int)(buffer.Address - newBuffer.Address);
 
-                        buffer.ForceSynchronizeMemory(buffer.Address, buffer.Size);
-
                         buffer.CopyTo(newBuffer, dstOffset);
                         newBuffer.InheritModifiedRanges(buffer);
 
-                        buffer.Dispose();
+                        buffer.DisposeData();
                     }
 
+                    newBuffer.SynchronizeMemory(address, endAddress - address);
+
                     // Existing buffers were modified, we need to rebind everything.
                     _rebind = true;
                 }
diff --git a/Ryujinx.Graphics.Gpu/Memory/PhysicalMemory.cs b/Ryujinx.Graphics.Gpu/Memory/PhysicalMemory.cs
index 3d2af532..6463b932 100644
--- a/Ryujinx.Graphics.Gpu/Memory/PhysicalMemory.cs
+++ b/Ryujinx.Graphics.Gpu/Memory/PhysicalMemory.cs
@@ -2,7 +2,9 @@ using Ryujinx.Cpu;
 using Ryujinx.Cpu.Tracking;
 using Ryujinx.Memory;
 using Ryujinx.Memory.Range;
+using Ryujinx.Memory.Tracking;
 using System;
+using System.Collections.Generic;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 
@@ -200,11 +202,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
         /// </summary>
         /// <param name="address">CPU virtual address of the region</param>
         /// <param name="size">Size of the region</param>
+        /// <param name="handles">Handles to inherit state from or reuse</param>
         /// <param name="granularity">Desired granularity of write tracking</param>
         /// <returns>The memory tracking handle</returns>
-        public CpuMultiRegionHandle BeginGranularTracking(ulong address, ulong size, ulong granularity = 4096)
+        public CpuMultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable<IRegionHandle> handles = null, ulong granularity = 4096)
         {
-            return _cpuMemory.BeginGranularTracking(address, size, granularity);
+            return _cpuMemory.BeginGranularTracking(address, size, handles, granularity);
         }
 
         /// <summary>