GPU: Pre-emptively flush textures that are flushed often (to imported memory when available) (#4711)

* WIP texture pre-flush

Improve performance of TextureView GetData to buffer

Fix copy/sync ordering

Fix minor bug

Make this actually work

WIP host mapping stuff

* Fix usage flags

* message

* Cleanup 1

* Fix rebase

* Fix

* Improve pre-flush rules

* Fix pre-flush

* A lot of cleanup

* Use the host memory bits

* Select the correct memory type

* Cleanup TextureGroupHandle

* Missing comment

* Remove debugging logs

* Revert BufferHandle _value access modifier

* One interrupt action at a time.

* Support D32S8 to D24S8 conversion, safeguards

* Interrupt cannot happen in sync handle's lock

Waitable needs to be checked twice now, but this should stop it from deadlocking.

* Remove unused using

* Address some feedback

* Address feedback

* Address more feedback

* Address more feedback

* Improve sync rules

Should allow for faster sync in some cases.

3 files changed, 212 insertions, 22 deletions

diff --git a/src/Ryujinx.Graphics.Gpu/Image/Texture.cs b/src/Ryujinx.Graphics.Gpu/Image/Texture.cs
index f0df55e6..3b257988 100644
--- a/src/Ryujinx.Graphics.Gpu/Image/Texture.cs
+++ b/src/Ryujinx.Graphics.Gpu/Image/Texture.cs
@@ -1423,7 +1423,7 @@ namespace Ryujinx.Graphics.Gpu.Image
                 _scaledSetScore = Math.Max(0, _scaledSetScore - 1);
             }
 
-            if (_modifiedStale || Group.HasCopyDependencies)
+            if (_modifiedStale || Group.HasCopyDependencies || Group.HasFlushBuffer)
             {
                 _modifiedStale = false;
                 Group.SignalModifying(this, bound);
@@ -1685,6 +1685,8 @@ namespace Ryujinx.Graphics.Gpu.Image
 
             if (Group.Storage == this)
             {
+                Group.Unmapped();
+
                 Group.ClearModified(unmapRange);
             }
         }
diff --git a/src/Ryujinx.Graphics.Gpu/Image/TextureGroup.cs b/src/Ryujinx.Graphics.Gpu/Image/TextureGroup.cs
index 234e7e8c..14ab5d1e 100644
--- a/src/Ryujinx.Graphics.Gpu/Image/TextureGroup.cs
+++ b/src/Ryujinx.Graphics.Gpu/Image/TextureGroup.cs
@@ -58,6 +58,12 @@ namespace Ryujinx.Graphics.Gpu.Image
         public bool HasCopyDependencies { get; set; }
 
         /// <summary>
+        /// Indicates if the texture group has a pre-emptive flush buffer.
+        /// When one is present, the group must always be notified on unbind.
+        /// </summary>
+        public bool HasFlushBuffer => _flushBuffer != BufferHandle.Null;
+
+        /// <summary>
         /// Indicates if this texture has any incompatible overlaps alive.
         /// </summary>
         public bool HasIncompatibleOverlaps => _incompatibleOverlaps.Count > 0;
@@ -89,6 +95,10 @@ namespace Ryujinx.Graphics.Gpu.Image
         private bool _incompatibleOverlapsDirty = true;
         private bool _flushIncompatibleOverlaps;
 
+        private BufferHandle _flushBuffer;
+        private bool _flushBufferImported;
+        private bool _flushBufferInvalid;
+
         /// <summary>
         /// Create a new texture group.
         /// </summary>
@@ -464,8 +474,9 @@ namespace Ryujinx.Graphics.Gpu.Image
         /// </remarks>
         /// <param name="tracked">True if writing the texture data is tracked, false otherwise</param>
         /// <param name="sliceIndex">The index of the slice to flush</param>
+        /// <param name="inBuffer">Whether the flushed texture data is up to date in the flush buffer</param>
         /// <param name="texture">The specific host texture to flush. Defaults to the storage texture</param>
-        private void FlushTextureDataSliceToGuest(bool tracked, int sliceIndex, ITexture texture = null)
+        private void FlushTextureDataSliceToGuest(bool tracked, int sliceIndex, bool inBuffer, ITexture texture = null)
         {
             (int layer, int level) = GetLayerLevelForView(sliceIndex);
 
@@ -475,7 +486,16 @@ namespace Ryujinx.Graphics.Gpu.Image
 
             using WritableRegion region = _physicalMemory.GetWritableRegion(Storage.Range.Slice((ulong)offset, (ulong)size), tracked);
 
-            Storage.GetTextureDataSliceFromGpu(region.Memory.Span, layer, level, tracked, texture);
+            if (inBuffer)
+            {
+                using PinnedSpan<byte> data = _context.Renderer.GetBufferData(_flushBuffer, offset, size);
+
+                Storage.ConvertFromHostCompatibleFormat(region.Memory.Span, data.Get(), level, true);
+            }
+            else
+            {
+                Storage.GetTextureDataSliceFromGpu(region.Memory.Span, layer, level, tracked, texture);
+            }
         }
 
         /// <summary>
@@ -484,12 +504,13 @@ namespace Ryujinx.Graphics.Gpu.Image
         /// <param name="tracked">True if writing the texture data is tracked, false otherwise</param>
         /// <param name="sliceStart">The first slice to flush</param>
         /// <param name="sliceEnd">The slice to finish flushing on (exclusive)</param>
+        /// <param name="inBuffer">Whether the flushed texture data is up to date in the flush buffer</param>
         /// <param name="texture">The specific host texture to flush. Defaults to the storage texture</param>
-        private void FlushSliceRange(bool tracked, int sliceStart, int sliceEnd, ITexture texture = null)
+        private void FlushSliceRange(bool tracked, int sliceStart, int sliceEnd, bool inBuffer, ITexture texture = null)
         {
             for (int i = sliceStart; i < sliceEnd; i++)
             {
-                FlushTextureDataSliceToGuest(tracked, i, texture);
+                FlushTextureDataSliceToGuest(tracked, i, inBuffer, texture);
             }
         }
 
@@ -520,7 +541,7 @@ namespace Ryujinx.Graphics.Gpu.Image
                         {
                             if (endSlice > startSlice)
                             {
-                                FlushSliceRange(tracked, startSlice, endSlice);
+                                FlushSliceRange(tracked, startSlice, endSlice, false);
                                 flushed = true;
                             }
 
@@ -553,7 +574,7 @@ namespace Ryujinx.Graphics.Gpu.Image
                     }
                     else
                     {
-                        FlushSliceRange(tracked, startSlice, endSlice);
+                        FlushSliceRange(tracked, startSlice, endSlice, false);
                     }
 
                     flushed = true;
@@ -566,6 +587,58 @@ namespace Ryujinx.Graphics.Gpu.Image
         }
 
         /// <summary>
+        /// Flush the texture data into a persistently mapped buffer.
+        /// If the buffer does not exist, this method will create it.
+        /// </summary>
+        /// <param name="handle">Handle of the texture group to flush slices of</param>
+        public void FlushIntoBuffer(TextureGroupHandle handle)
+        {
+            // Ensure that the buffer exists.
+
+            if (_flushBufferInvalid && _flushBuffer != BufferHandle.Null)
+            {
+                _flushBufferInvalid = false;
+                _context.Renderer.DeleteBuffer(_flushBuffer);
+                _flushBuffer = BufferHandle.Null;
+            }
+
+            if (_flushBuffer == BufferHandle.Null)
+            {
+                if (!TextureCompatibility.CanTextureFlush(Storage.Info, _context.Capabilities))
+                {
+                    return;
+                }
+
+                bool canImport = Storage.Info.IsLinear && Storage.Info.Stride >= Storage.Info.Width * Storage.Info.FormatInfo.BytesPerPixel;
+
+                var hostPointer = canImport ? _physicalMemory.GetHostPointer(Storage.Range) : 0;
+
+                if (hostPointer != 0 && _context.Renderer.PrepareHostMapping(hostPointer, Storage.Size))
+                {
+                    _flushBuffer = _context.Renderer.CreateBuffer(hostPointer, (int)Storage.Size);
+                    _flushBufferImported = true;
+                }
+                else
+                {
+                    _flushBuffer = _context.Renderer.CreateBuffer((int)Storage.Size, BufferAccess.FlushPersistent);
+                    _flushBufferImported = false;
+                }
+
+                Storage.BlacklistScale();
+            }
+
+            int sliceStart = handle.BaseSlice;
+            int sliceEnd = sliceStart + handle.SliceCount;
+
+            for (int i = sliceStart; i < sliceEnd; i++)
+            {
+                (int layer, int level) = GetLayerLevelForView(i);
+
+                Storage.GetFlushTexture().CopyTo(new BufferRange(_flushBuffer, _allOffsets[i], _sliceSizes[level]), layer, level, _flushBufferImported ? Storage.Info.Stride : 0);
+            }
+        }
+
+        /// <summary>
         /// Clears competing modified flags for all incompatible ranges, if they have possibly been modified.
         /// </summary>
         /// <param name="texture">The texture that has been modified</param>
@@ -1570,10 +1643,7 @@ namespace Ryujinx.Graphics.Gpu.Image
 
             _context.Renderer.BackgroundContextAction(() =>
             {
-                if (!isGpuThread)
-                {
-                    handle.Sync(_context);
-                }
+                bool inBuffer = !isGpuThread && handle.Sync(_context);
 
                 Storage.SignalModifiedDirty();
 
@@ -1585,14 +1655,25 @@ namespace Ryujinx.Graphics.Gpu.Image
                     }
                 }
 
-                if (TextureCompatibility.CanTextureFlush(Storage.Info, _context.Capabilities))
+                if (TextureCompatibility.CanTextureFlush(Storage.Info, _context.Capabilities) && !(inBuffer && _flushBufferImported))
                 {
-                    FlushSliceRange(false, handle.BaseSlice, handle.BaseSlice + handle.SliceCount, Storage.GetFlushTexture());
+                    FlushSliceRange(false, handle.BaseSlice, handle.BaseSlice + handle.SliceCount, inBuffer, Storage.GetFlushTexture());
                 }
             });
         }
 
         /// <summary>
+        /// Called if any part of the storage texture is unmapped.
+        /// </summary>
+        public void Unmapped()
+        {
+            if (_flushBufferImported)
+            {
+                _flushBufferInvalid = true;
+            }
+        }
+
+        /// <summary>
         /// Dispose this texture group, disposing all related memory tracking handles.
         /// </summary>
         public void Dispose()
@@ -1606,6 +1687,11 @@ namespace Ryujinx.Graphics.Gpu.Image
             {
                 incompatible.Group._incompatibleOverlaps.RemoveAll(overlap => overlap.Group == this);
             }
+
+            if (_flushBuffer != BufferHandle.Null)
+            {
+                _context.Renderer.DeleteBuffer(_flushBuffer);
+            }
         }
     }
 }
diff --git a/src/Ryujinx.Graphics.Gpu/Image/TextureGroupHandle.cs b/src/Ryujinx.Graphics.Gpu/Image/TextureGroupHandle.cs
index ebb4e9ae..9f66744b 100644
--- a/src/Ryujinx.Graphics.Gpu/Image/TextureGroupHandle.cs
+++ b/src/Ryujinx.Graphics.Gpu/Image/TextureGroupHandle.cs
@@ -1,4 +1,5 @@
 using Ryujinx.Cpu.Tracking;
+using Ryujinx.Graphics.Gpu.Synchronization;
 using System;
 using System.Collections.Generic;
 using System.Linq;
@@ -13,8 +14,14 @@ namespace Ryujinx.Graphics.Gpu.Image
     /// Also tracks copy dependencies for the handle - references to other handles that must be kept
     /// in sync with this one before use.
     /// </summary>
-    class TextureGroupHandle : IDisposable
+    class TextureGroupHandle : ISyncActionHandler, IDisposable
     {
+        private const int FlushBalanceIncrement = 6;
+        private const int FlushBalanceWriteCost = 1;
+        private const int FlushBalanceThreshold = 7;
+        private const int FlushBalanceMax = 60;
+        private const int FlushBalanceMin = -10;
+
         private TextureGroup _group;
         private int _bindCount;
         private int _firstLevel;
@@ -26,6 +33,8 @@ namespace Ryujinx.Graphics.Gpu.Image
         /// The sync number last registered.
         /// </summary>
         private ulong _registeredSync;
+        private ulong _registeredBufferSync = ulong.MaxValue;
+        private ulong _registeredBufferGuestSync = ulong.MaxValue;
 
         /// <summary>
         /// The sync number when the texture was last modified by GPU.
@@ -43,6 +52,12 @@ namespace Ryujinx.Graphics.Gpu.Image
         private bool _syncActionRegistered;
 
         /// <summary>
+        /// Determines the balance of synced writes to flushes.
+        /// Used to determine if the texture should always write data to a persistent buffer for flush.
+        /// </summary>
+        private int _flushBalance;
+
+        /// <summary>
         /// The byte offset from the start of the storage of this handle.
         /// </summary>
         public int Offset { get; }
@@ -132,6 +147,12 @@ namespace Ryujinx.Graphics.Gpu.Image
             }
 
             Handles = handles;
+
+            if (group.Storage.Info.IsLinear)
+            {
+                // Linear textures are presumed to be used for readback initially.
+                _flushBalance = FlushBalanceThreshold + FlushBalanceIncrement;
+            }
         }
 
         /// <summary>
@@ -160,6 +181,35 @@ namespace Ryujinx.Graphics.Gpu.Image
         }
 
         /// <summary>
+        /// Determine if the next sync will copy into the flush buffer.
+        /// </summary>
+        /// <returns>True if it will copy, false otherwise</returns>
+        private bool NextSyncCopies()
+        {
+            return _flushBalance - FlushBalanceWriteCost > FlushBalanceThreshold;
+        }
+
+        /// <summary>
+        /// Alters the flush balance by the given value. Should increase significantly with each sync, decrease with each write.
+        /// A flush balance higher than the threshold will cause a texture to repeatedly copy to a flush buffer on each use.
+        /// </summary>
+        /// <param name="modifier">Value to add to the existing flush balance</param>
+        /// <returns>True if the new balance is over the threshold, false otherwise</returns>
+        private bool ModifyFlushBalance(int modifier)
+        {
+            int result;
+            int existingBalance;
+            do
+            {
+                existingBalance = _flushBalance;
+                result = Math.Max(FlushBalanceMin, Math.Min(FlushBalanceMax, existingBalance + modifier));
+            }
+            while (Interlocked.CompareExchange(ref _flushBalance, result, existingBalance) != existingBalance);
+
+            return result > FlushBalanceThreshold;
+        }
+
+        /// <summary>
         /// Adds a single texture view as an overlap if its range overlaps.
         /// </summary>
         /// <param name="offset">The offset of the view in the group</param>
@@ -204,7 +254,7 @@ namespace Ryujinx.Graphics.Gpu.Image
             if (!_syncActionRegistered)
             {
                 _modifiedSync = context.SyncNumber;
-                context.RegisterSyncAction(SyncAction, true);
+                context.RegisterSyncAction(this, true);
                 _syncActionRegistered = true;
             }
 
@@ -241,6 +291,13 @@ namespace Ryujinx.Graphics.Gpu.Image
         {
             SignalModified(context);
 
+            if (!bound && _syncActionRegistered && NextSyncCopies())
+            {
+                // On unbind, textures that flush often should immediately create sync so their result can be obtained as soon as possible.
+
+                context.CreateHostSyncIfNeeded(HostSyncFlags.Force);
+            }
+
             // Note: Bind count currently resets to 0 on inherit for safety, as the handle <-> view relationship can change.
             _bindCount = Math.Max(0, _bindCount + (bound ? 1 : -1));
         }
@@ -266,25 +323,35 @@ namespace Ryujinx.Graphics.Gpu.Image
         /// removing the modified flag if it was reached, or leaving it set if it has not yet been created.
         /// </summary>
         /// <param name="context">The GPU context used to wait for sync</param>
-        public void Sync(GpuContext context)
+        /// <returns>True if the texture data can be read from the flush buffer</returns>
+        public bool Sync(GpuContext context)
         {
-            ulong registeredSync = _registeredSync;
-            long diff = (long)(context.SyncNumber - registeredSync);
+            // Currently assumes the calling thread is a guest thread.
+
+            bool inBuffer = _registeredBufferGuestSync != ulong.MaxValue;
+            ulong sync = inBuffer ? _registeredBufferGuestSync : _registeredSync;
+
+            long diff = (long)(context.SyncNumber - sync);
+
+            ModifyFlushBalance(FlushBalanceIncrement);
 
             if (diff > 0)
             {
-                context.Renderer.WaitSync(registeredSync);
+                context.Renderer.WaitSync(sync);
 
-                if ((long)(_modifiedSync - registeredSync) > 0)
+                if ((long)(_modifiedSync - sync) > 0)
                 {
                     // Flush the data in a previous state. Do not remove the modified flag - it will be removed to ignore following writes.
-                    return;
+                    return inBuffer;
                 }
 
                 Modified = false;
+
+                return inBuffer;
             }
 
             // If the difference is <= 0, no data is not ready yet. Flush any data we can without waiting or removing modified flag.
+            return false;
         }
 
         /// <summary>
@@ -297,14 +364,40 @@ namespace Ryujinx.Graphics.Gpu.Image
         }
 
         /// <summary>
+        /// Action to perform before a sync number is registered after modification.
+        /// This action will copy the texture data to the flush buffer if this texture
+        /// flushes often enough, which is determined by the flush balance.
+        /// </summary>
+        /// <inheritdoc/>
+        public void SyncPreAction(bool syncpoint)
+        {
+            if (syncpoint || NextSyncCopies())
+            {
+                if (ModifyFlushBalance(0) && _registeredBufferSync != _modifiedSync)
+                {
+                    _group.FlushIntoBuffer(this);
+                    _registeredBufferSync = _modifiedSync;
+                }
+            }
+        }
+
+        /// <summary>
         /// Action to perform when a sync number is registered after modification.
         /// This action will register a read tracking action on the memory tracking handle so that a flush from CPU can happen.
         /// </summary>
-        private void SyncAction()
+        /// <inheritdoc/>
+        public bool SyncAction(bool syncpoint)
         {
             // The storage will need to signal modified again to update the sync number in future.
             _group.Storage.SignalModifiedDirty();
 
+            bool lastInBuffer = _registeredBufferSync == _modifiedSync;
+
+            if (!lastInBuffer)
+            {
+                _registeredBufferSync = ulong.MaxValue;
+            }
+
             lock (Overlaps)
             {
                 foreach (Texture texture in Overlaps)
@@ -314,6 +407,7 @@ namespace Ryujinx.Graphics.Gpu.Image
             }
 
             // Register region tracking for CPU? (again)
+
             _registeredSync = _modifiedSync;
             _syncActionRegistered = false;
 
@@ -321,6 +415,14 @@ namespace Ryujinx.Graphics.Gpu.Image
             {
                 _group.RegisterAction(this);
             }
+
+            if (syncpoint)
+            {
+                _registeredBufferGuestSync = _registeredBufferSync;
+            }
+
+            // If the last modification is in the buffer, keep this sync action alive until it sees a syncpoint.
+            return syncpoint || !lastInBuffer;
         }
 
         /// <summary>