Add support to IVirtualMemoryManager for zero-copy reads (#6251)

* - WritableRegion: enable wrapping IMemoryOwner<byte>
- IVirtualMemoryManager impls of GetWritableRegion() use pooled memory when region is non-contiguous.
- IVirtualMemoryManager: add GetReadOnlySequence() and impls
- ByteMemoryPool: add new method RentCopy()
- ByteMemoryPool: make class static, remove ctor and singleton field from earlier impl

* - BytesReadOnlySequenceSegment: move from Ryujinx.Common.Memory to Ryujinx.Memory
- BytesReadOnlySequenceSegment: add IsContiguousWith() and Replace() methods
- VirtualMemoryManagerBase:
  - remove generic type parameters, instead use ulong for virtual addresses and nuint for host/physical addresses
  - implement IWritableBlock
  - add virtual GetReadOnlySequence() with coalescing of contiguous segments
  - add virtual GetSpan()
  - add virtual GetWritableRegion()
  - add abstract IsMapped()
  - add virtual MapForeign(ulong, nuint, ulong)
  - add virtual Read<T>()
  - add virtual Read(ulong, Span<byte>)
  - add virtual ReadTracked<T>()
  - add virtual SignalMemoryTracking()
  - add virtual Write()
  - add virtual Write<T>()
  - add virtual WriteUntracked()
  - add virtual WriteWithRedundancyCheck()
- VirtualMemoryManagerRefCountedBase: remove generic type parameters
- AddressSpaceManager: remove redundant methods, add required overrides
- HvMemoryManager: remove redundant methods, add required overrides, add overrides for _invalidAccessHandler handling
- MemoryManager: remove redundant methods, add required overrides, add overrides for _invalidAccessHandler handling
- MemoryManagerHostMapped: remove redundant methods, add required overrides, add overrides for _invalidAccessHandler handling
- NativeMemoryManager: add get properties for Pointer and Length
- throughout: removed invalid <inheritdoc/> comments

* make HvMemoryManager class sealed

* remove unused method

* adjust MemoryManagerHostTracked

* let MemoryManagerHostTracked override WriteImpl()

1 files changed, 67 insertions, 126 deletions

diff --git a/src/Ryujinx.Cpu/Jit/MemoryManagerHostTracked.cs b/src/Ryujinx.Cpu/Jit/MemoryManagerHostTracked.cs
index 18404bcc..b2964cd2 100644
--- a/src/Ryujinx.Cpu/Jit/MemoryManagerHostTracked.cs
+++ b/src/Ryujinx.Cpu/Jit/MemoryManagerHostTracked.cs
@@ -8,14 +8,13 @@ using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Runtime.CompilerServices;
-using System.Runtime.InteropServices;
 
 namespace Ryujinx.Cpu.Jit
 {
     /// <summary>
     /// Represents a CPU memory manager which maps guest virtual memory directly onto a host virtual region.
     /// </summary>
-    public sealed class MemoryManagerHostTracked : VirtualMemoryManagerRefCountedBase<ulong, ulong>, IWritableBlock, IMemoryManager, IVirtualMemoryManagerTracked
+    public sealed class MemoryManagerHostTracked : VirtualMemoryManagerRefCountedBase, IMemoryManager, IVirtualMemoryManagerTracked
     {
         private readonly InvalidAccessHandler _invalidAccessHandler;
         private readonly bool _unsafeMode;
@@ -101,12 +100,6 @@ namespace Ryujinx.Cpu.Jit
         }
 
         /// <inheritdoc/>
-        public void MapForeign(ulong va, nuint hostPointer, ulong size)
-        {
-            throw new NotSupportedException();
-        }
-
-        /// <inheritdoc/>
         public void Unmap(ulong va, ulong size)
         {
             AssertValidAddressAndSize(va, size);
@@ -120,18 +113,11 @@ namespace Ryujinx.Cpu.Jit
             _nativePageTable.Unmap(va, size);
         }
 
-        public T Read<T>(ulong va) where T : unmanaged
-        {
-            return MemoryMarshal.Cast<byte, T>(GetSpan(va, Unsafe.SizeOf<T>()))[0];
-        }
-
-        public T ReadTracked<T>(ulong va) where T : unmanaged
+        public override T ReadTracked<T>(ulong va)
         {
             try
             {
-                SignalMemoryTracking(va, (ulong)Unsafe.SizeOf<T>(), false);
-
-                return Read<T>(va);
+                return base.ReadTracked<T>(va);
             }
             catch (InvalidMemoryRegionException)
             {
@@ -146,37 +132,38 @@ namespace Ryujinx.Cpu.Jit
 
         public override void Read(ulong va, Span<byte> data)
         {
-            ReadImpl(va, data);
-        }
-
-        public void Write<T>(ulong va, T value) where T : unmanaged
-        {
-            Write(va, MemoryMarshal.Cast<T, byte>(MemoryMarshal.CreateSpan(ref value, 1)));
-        }
-
-        public void Write(ulong va, ReadOnlySpan<byte> data)
-        {
             if (data.Length == 0)
             {
                 return;
             }
 
-            SignalMemoryTracking(va, (ulong)data.Length, true);
+            try
+            {
+                AssertValidAddressAndSize(va, (ulong)data.Length);
+
+                ulong endVa = va + (ulong)data.Length;
+                int offset = 0;
 
-            WriteImpl(va, data);
-        }
+                while (va < endVa)
+                {
+                    (MemoryBlock memory, ulong rangeOffset, ulong copySize) = GetMemoryOffsetAndSize(va, (ulong)(data.Length - offset));
 
-        public void WriteUntracked(ulong va, ReadOnlySpan<byte> data)
-        {
-            if (data.Length == 0)
+                    memory.GetSpan(rangeOffset, (int)copySize).CopyTo(data.Slice(offset, (int)copySize));
+
+                    va += copySize;
+                    offset += (int)copySize;
+                }
+            }
+            catch (InvalidMemoryRegionException)
             {
-                return;
+                if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
+                {
+                    throw;
+                }
             }
-
-            WriteImpl(va, data);
         }
 
-        public bool WriteWithRedundancyCheck(ulong va, ReadOnlySpan<byte> data)
+        public override bool WriteWithRedundancyCheck(ulong va, ReadOnlySpan<byte> data)
         {
             if (data.Length == 0)
             {
@@ -206,35 +193,7 @@ namespace Ryujinx.Cpu.Jit
             }
         }
 
-        private void WriteImpl(ulong va, ReadOnlySpan<byte> data)
-        {
-            try
-            {
-                AssertValidAddressAndSize(va, (ulong)data.Length);
-
-                ulong endVa = va + (ulong)data.Length;
-                int offset = 0;
-
-                while (va < endVa)
-                {
-                    (MemoryBlock memory, ulong rangeOffset, ulong copySize) = GetMemoryOffsetAndSize(va, (ulong)(data.Length - offset));
-
-                    data.Slice(offset, (int)copySize).CopyTo(memory.GetSpan(rangeOffset, (int)copySize));
-
-                    va += copySize;
-                    offset += (int)copySize;
-                }
-            }
-            catch (InvalidMemoryRegionException)
-            {
-                if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
-                {
-                    throw;
-                }
-            }
-        }
-
-        public ReadOnlySpan<byte> GetSpan(ulong va, int size, bool tracked = false)
+        public override ReadOnlySpan<byte> GetSpan(ulong va, int size, bool tracked = false)
         {
             if (size == 0)
             {
@@ -254,13 +213,13 @@ namespace Ryujinx.Cpu.Jit
             {
                 Span<byte> data = new byte[size];
 
-                ReadImpl(va, data);
+                Read(va, data);
 
                 return data;
             }
         }
 
-        public WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false)
+        public override WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false)
         {
             if (size == 0)
             {
@@ -280,7 +239,7 @@ namespace Ryujinx.Cpu.Jit
             {
                 Memory<byte> memory = new byte[size];
 
-                ReadImpl(va, memory.Span);
+                Read(va, memory.Span);
 
                 return new WritableRegion(this, va, memory);
             }
@@ -299,7 +258,7 @@ namespace Ryujinx.Cpu.Jit
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public bool IsMapped(ulong va)
+        public override bool IsMapped(ulong va)
         {
             return ValidateAddress(va) && _pages.IsMapped(va);
         }
@@ -311,8 +270,6 @@ namespace Ryujinx.Cpu.Jit
             return _pages.IsRangeMapped(va, size);
         }
 
-        private static void ThrowMemoryNotContiguous() => throw new MemoryNotContiguousException();
-
         private bool TryGetVirtualContiguous(ulong va, int size, out MemoryBlock memory, out ulong offset)
         {
             if (_addressSpace.HasAnyPrivateAllocation(va, (ulong)size, out PrivateRange range))
@@ -491,44 +448,11 @@ namespace Ryujinx.Cpu.Jit
             return regions;
         }
 
-        private void ReadImpl(ulong va, Span<byte> data)
-        {
-            if (data.Length == 0)
-            {
-                return;
-            }
-
-            try
-            {
-                AssertValidAddressAndSize(va, (ulong)data.Length);
-
-                ulong endVa = va + (ulong)data.Length;
-                int offset = 0;
-
-                while (va < endVa)
-                {
-                    (MemoryBlock memory, ulong rangeOffset, ulong copySize) = GetMemoryOffsetAndSize(va, (ulong)(data.Length - offset));
-
-                    memory.GetSpan(rangeOffset, (int)copySize).CopyTo(data.Slice(offset, (int)copySize));
-
-                    va += copySize;
-                    offset += (int)copySize;
-                }
-            }
-            catch (InvalidMemoryRegionException)
-            {
-                if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
-                {
-                    throw;
-                }
-            }
-        }
-
         /// <inheritdoc/>
         /// <remarks>
         /// This function also validates that the given range is both valid and mapped, and will throw if it is not.
         /// </remarks>
-        public void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null)
+        public override void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null)
         {
             AssertValidAddressAndSize(va, size);
 
@@ -543,23 +467,6 @@ namespace Ryujinx.Cpu.Jit
             _pages.SignalMemoryTracking(Tracking, va, size, write, exemptId);
         }
 
-        /// <summary>
-        /// Computes the number of pages in a virtual address range.
-        /// </summary>
-        /// <param name="va">Virtual address of the range</param>
-        /// <param name="size">Size of the range</param>
-        /// <param name="startVa">The virtual address of the beginning of the first page</param>
-        /// <remarks>This function does not differentiate between allocated and unallocated pages.</remarks>
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private int GetPagesCount(ulong va, ulong size, out ulong startVa)
-        {
-            // WARNING: Always check if ulong does not overflow during the operations.
-            startVa = va & ~(ulong)PageMask;
-            ulong vaSpan = (va - startVa + size + PageMask) & ~(ulong)PageMask;
-
-            return (int)(vaSpan / PageSize);
-        }
-
         public RegionHandle BeginTracking(ulong address, ulong size, int id, RegionFlags flags = RegionFlags.None)
         {
             return Tracking.BeginTracking(address, size, id, flags);
@@ -618,10 +525,44 @@ namespace Ryujinx.Cpu.Jit
             _nativePageTable.Dispose();
         }
 
-        protected override Span<byte> GetPhysicalAddressSpan(ulong pa, int size)
+        protected override Memory<byte> GetPhysicalAddressMemory(nuint pa, int size)
+            => _backingMemory.GetMemory(pa, size);
+
+        protected override Span<byte> GetPhysicalAddressSpan(nuint pa, int size)
             => _backingMemory.GetSpan(pa, size);
 
-        protected override ulong TranslateVirtualAddressForRead(ulong va)
-            => GetPhysicalAddressInternal(va);
+        protected override void WriteImpl(ulong va, ReadOnlySpan<byte> data)
+        {
+            try
+            {
+                AssertValidAddressAndSize(va, (ulong)data.Length);
+
+                ulong endVa = va + (ulong)data.Length;
+                int offset = 0;
+
+                while (va < endVa)
+                {
+                    (MemoryBlock memory, ulong rangeOffset, ulong copySize) = GetMemoryOffsetAndSize(va, (ulong)(data.Length - offset));
+
+                    data.Slice(offset, (int)copySize).CopyTo(memory.GetSpan(rangeOffset, (int)copySize));
+
+                    va += copySize;
+                    offset += (int)copySize;
+                }
+            }
+            catch (InvalidMemoryRegionException)
+            {
+                if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
+                {
+                    throw;
+                }
+            }
+        }
+
+        protected override nuint TranslateVirtualAddressChecked(ulong va)
+            => (nuint)GetPhysicalAddressChecked(va);
+
+        protected override nuint TranslateVirtualAddressUnchecked(ulong va)
+            => (nuint)GetPhysicalAddressInternal(va);
     }
 }