From 5def0429f82c795d820b2307a0301b78dfb1e6b7 Mon Sep 17 00:00:00 2001 From: jhorv <38920027+jhorv@users.noreply.github.com> Date: Thu, 4 Apr 2024 21:23:03 -0400 Subject: Add support to IVirtualMemoryManager for zero-copy reads (#6251) * - WritableRegion: enable wrapping IMemoryOwner - IVirtualMemoryManager impls of GetWritableRegion() use pooled memory when region is non-contiguous. - IVirtualMemoryManager: add GetReadOnlySequence() and impls - ByteMemoryPool: add new method RentCopy() - ByteMemoryPool: make class static, remove ctor and singleton field from earlier impl * - BytesReadOnlySequenceSegment: move from Ryujinx.Common.Memory to Ryujinx.Memory - BytesReadOnlySequenceSegment: add IsContiguousWith() and Replace() methods - VirtualMemoryManagerBase: - remove generic type parameters, instead use ulong for virtual addresses and nuint for host/physical addresses - implement IWritableBlock - add virtual GetReadOnlySequence() with coalescing of contiguous segments - add virtual GetSpan() - add virtual GetWritableRegion() - add abstract IsMapped() - add virtual MapForeign(ulong, nuint, ulong) - add virtual Read() - add virtual Read(ulong, Span) - add virtual ReadTracked() - add virtual SignalMemoryTracking() - add virtual Write() - add virtual Write() - add virtual WriteUntracked() - add virtual WriteWithRedundancyCheck() - VirtualMemoryManagerRefCountedBase: remove generic type parameters - AddressSpaceManager: remove redundant methods, add required overrides - HvMemoryManager: remove redundant methods, add required overrides, add overrides for _invalidAccessHandler handling - MemoryManager: remove redundant methods, add required overrides, add overrides for _invalidAccessHandler handling - MemoryManagerHostMapped: remove redundant methods, add required overrides, add overrides for _invalidAccessHandler handling - NativeMemoryManager: add get properties for Pointer and Length - throughout: removed invalid comments * make HvMemoryManager class sealed * remove unused method * adjust MemoryManagerHostTracked * let MemoryManagerHostTracked override WriteImpl() --- src/Ryujinx.Memory/VirtualMemoryManagerBase.cs | 348 +++++++++++++++++++++++-- 1 file changed, 331 insertions(+), 17 deletions(-) (limited to 'src/Ryujinx.Memory/VirtualMemoryManagerBase.cs') diff --git a/src/Ryujinx.Memory/VirtualMemoryManagerBase.cs b/src/Ryujinx.Memory/VirtualMemoryManagerBase.cs index cbec88cc..506e25f6 100644 --- a/src/Ryujinx.Memory/VirtualMemoryManagerBase.cs +++ b/src/Ryujinx.Memory/VirtualMemoryManagerBase.cs @@ -1,34 +1,171 @@ +using Ryujinx.Common.Memory; using System; -using System.Numerics; +using System.Buffers; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; namespace Ryujinx.Memory { - public abstract class VirtualMemoryManagerBase - where TVirtual : IBinaryInteger - where TPhysical : IBinaryInteger + public abstract class VirtualMemoryManagerBase : IWritableBlock { public const int PageBits = 12; public const int PageSize = 1 << PageBits; public const int PageMask = PageSize - 1; - protected abstract TVirtual AddressSpaceSize { get; } + protected abstract ulong AddressSpaceSize { get; } - public virtual void Read(TVirtual va, Span data) + public virtual ReadOnlySequence GetReadOnlySequence(ulong va, int size, bool tracked = false) + { + if (size == 0) + { + return ReadOnlySequence.Empty; + } + + if (tracked) + { + SignalMemoryTracking(va, (ulong)size, false); + } + + if (IsContiguousAndMapped(va, size)) + { + nuint pa = TranslateVirtualAddressUnchecked(va); + + return new ReadOnlySequence(GetPhysicalAddressMemory(pa, size)); + } + else + { + AssertValidAddressAndSize(va, size); + + int offset = 0, segmentSize; + + BytesReadOnlySequenceSegment first = null, last = null; + + if ((va & PageMask) != 0) + { + nuint pa = TranslateVirtualAddressChecked(va); + + segmentSize = Math.Min(size, PageSize - (int)(va & PageMask)); + + Memory memory = GetPhysicalAddressMemory(pa, segmentSize); + + first = last = new BytesReadOnlySequenceSegment(memory); + + offset += segmentSize; + } + + for (; offset < size; offset += segmentSize) + { + nuint pa = TranslateVirtualAddressChecked(va + (ulong)offset); + + segmentSize = Math.Min(size - offset, PageSize); + + Memory memory = GetPhysicalAddressMemory(pa, segmentSize); + + if (first is null) + { + first = last = new BytesReadOnlySequenceSegment(memory); + } + else + { + if (last.IsContiguousWith(memory, out nuint contiguousStart, out int contiguousSize)) + { + last.Replace(GetPhysicalAddressMemory(contiguousStart, contiguousSize)); + } + else + { + last = last.Append(memory); + } + } + } + + return new ReadOnlySequence(first, 0, last, (int)(size - last.RunningIndex)); + } + } + + public virtual ReadOnlySpan GetSpan(ulong va, int size, bool tracked = false) + { + if (size == 0) + { + return ReadOnlySpan.Empty; + } + + if (tracked) + { + SignalMemoryTracking(va, (ulong)size, false); + } + + if (IsContiguousAndMapped(va, size)) + { + nuint pa = TranslateVirtualAddressUnchecked(va); + + return GetPhysicalAddressSpan(pa, size); + } + else + { + Span data = new byte[size]; + + Read(va, data); + + return data; + } + } + + public virtual WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false) + { + if (size == 0) + { + return new WritableRegion(null, va, Memory.Empty); + } + + if (tracked) + { + SignalMemoryTracking(va, (ulong)size, true); + } + + if (IsContiguousAndMapped(va, size)) + { + nuint pa = TranslateVirtualAddressUnchecked(va); + + return new WritableRegion(null, va, GetPhysicalAddressMemory(pa, size)); + } + else + { + IMemoryOwner memoryOwner = ByteMemoryPool.Rent(size); + + Read(va, memoryOwner.Memory.Span); + + return new WritableRegion(this, va, memoryOwner); + } + } + + public abstract bool IsMapped(ulong va); + + public virtual void MapForeign(ulong va, nuint hostPointer, ulong size) + { + throw new NotSupportedException(); + } + + public virtual T Read(ulong va) where T : unmanaged + { + return MemoryMarshal.Cast(GetSpan(va, Unsafe.SizeOf()))[0]; + } + + public virtual void Read(ulong va, Span data) { if (data.Length == 0) { return; } - AssertValidAddressAndSize(va, TVirtual.CreateChecked(data.Length)); + AssertValidAddressAndSize(va, data.Length); int offset = 0, size; - if ((int.CreateTruncating(va) & PageMask) != 0) + if ((va & PageMask) != 0) { - TPhysical pa = TranslateVirtualAddressForRead(va); + nuint pa = TranslateVirtualAddressChecked(va); - size = Math.Min(data.Length, PageSize - ((int.CreateTruncating(va) & PageMask))); + size = Math.Min(data.Length, PageSize - (int)(va & PageMask)); GetPhysicalAddressSpan(pa, size).CopyTo(data[..size]); @@ -37,7 +174,7 @@ namespace Ryujinx.Memory for (; offset < data.Length; offset += size) { - TPhysical pa = TranslateVirtualAddressForRead(va + TVirtual.CreateChecked(offset)); + nuint pa = TranslateVirtualAddressChecked(va + (ulong)offset); size = Math.Min(data.Length - offset, PageSize); @@ -45,13 +182,84 @@ namespace Ryujinx.Memory } } + public virtual T ReadTracked(ulong va) where T : unmanaged + { + SignalMemoryTracking(va, (ulong)Unsafe.SizeOf(), false); + + return Read(va); + } + + public virtual void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null) + { + // No default implementation + } + + public virtual void Write(ulong va, ReadOnlySpan data) + { + if (data.Length == 0) + { + return; + } + + SignalMemoryTracking(va, (ulong)data.Length, true); + + WriteImpl(va, data); + } + + public virtual void Write(ulong va, T value) where T : unmanaged + { + Write(va, MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref value, 1))); + } + + public virtual void WriteUntracked(ulong va, ReadOnlySpan data) + { + if (data.Length == 0) + { + return; + } + + WriteImpl(va, data); + } + + public virtual bool WriteWithRedundancyCheck(ulong va, ReadOnlySpan data) + { + if (data.Length == 0) + { + return false; + } + + if (IsContiguousAndMapped(va, data.Length)) + { + SignalMemoryTracking(va, (ulong)data.Length, false); + + nuint pa = TranslateVirtualAddressChecked(va); + + var target = GetPhysicalAddressSpan(pa, data.Length); + + bool changed = !data.SequenceEqual(target); + + if (changed) + { + data.CopyTo(target); + } + + return changed; + } + else + { + Write(va, data); + + return true; + } + } + /// /// Ensures the combination of virtual address and size is part of the addressable space. /// /// Virtual address of the range /// Size of the range in bytes /// Throw when the memory region specified outside the addressable space - protected void AssertValidAddressAndSize(TVirtual va, TVirtual size) + protected void AssertValidAddressAndSize(ulong va, ulong size) { if (!ValidateAddressAndSize(va, size)) { @@ -59,16 +267,82 @@ namespace Ryujinx.Memory } } - protected abstract Span GetPhysicalAddressSpan(TPhysical pa, int size); + /// + /// Ensures the combination of virtual address and size is part of the addressable space. + /// + /// Virtual address of the range + /// Size of the range in bytes + /// Throw when the memory region specified outside the addressable space + [MethodImpl(MethodImplOptions.AggressiveInlining)] + protected void AssertValidAddressAndSize(ulong va, int size) + => AssertValidAddressAndSize(va, (ulong)size); + + /// + /// Computes the number of pages in a virtual address range. + /// + /// Virtual address of the range + /// Size of the range + /// The virtual address of the beginning of the first page + /// This function does not differentiate between allocated and unallocated pages. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + protected static int GetPagesCount(ulong va, ulong size, out ulong startVa) + { + // WARNING: Always check if ulong does not overflow during the operations. + startVa = va & ~(ulong)PageMask; + ulong vaSpan = (va - startVa + size + PageMask) & ~(ulong)PageMask; + + return (int)(vaSpan / PageSize); + } + + protected abstract Memory GetPhysicalAddressMemory(nuint pa, int size); + + protected abstract Span GetPhysicalAddressSpan(nuint pa, int size); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + protected bool IsContiguous(ulong va, int size) => IsContiguous(va, (ulong)size); + + protected virtual bool IsContiguous(ulong va, ulong size) + { + if (!ValidateAddress(va) || !ValidateAddressAndSize(va, size)) + { + return false; + } + + int pages = GetPagesCount(va, size, out va); + + for (int page = 0; page < pages - 1; page++) + { + if (!ValidateAddress(va + PageSize)) + { + return false; + } + + if (TranslateVirtualAddressUnchecked(va) + PageSize != TranslateVirtualAddressUnchecked(va + PageSize)) + { + return false; + } + + va += PageSize; + } + + return true; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + protected bool IsContiguousAndMapped(ulong va, int size) + => IsContiguous(va, size) && IsMapped(va); - protected abstract TPhysical TranslateVirtualAddressForRead(TVirtual va); + protected abstract nuint TranslateVirtualAddressChecked(ulong va); + + protected abstract nuint TranslateVirtualAddressUnchecked(ulong va); /// /// Checks if the virtual address is part of the addressable space. /// /// Virtual address /// True if the virtual address is part of the addressable space - protected bool ValidateAddress(TVirtual va) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + protected bool ValidateAddress(ulong va) { return va < AddressSpaceSize; } @@ -79,13 +353,53 @@ namespace Ryujinx.Memory /// Virtual address of the range /// Size of the range in bytes /// True if the combination of virtual address and size is part of the addressable space - protected bool ValidateAddressAndSize(TVirtual va, TVirtual size) + protected bool ValidateAddressAndSize(ulong va, ulong size) { - TVirtual endVa = va + size; + ulong endVa = va + size; return endVa >= va && endVa >= size && endVa <= AddressSpaceSize; } protected static void ThrowInvalidMemoryRegionException(string message) => throw new InvalidMemoryRegionException(message); + + protected static void ThrowMemoryNotContiguous() + => throw new MemoryNotContiguousException(); + + protected virtual void WriteImpl(ulong va, ReadOnlySpan data) + { + AssertValidAddressAndSize(va, data.Length); + + if (IsContiguousAndMapped(va, data.Length)) + { + nuint pa = TranslateVirtualAddressUnchecked(va); + + data.CopyTo(GetPhysicalAddressSpan(pa, data.Length)); + } + else + { + int offset = 0, size; + + if ((va & PageMask) != 0) + { + nuint pa = TranslateVirtualAddressChecked(va); + + size = Math.Min(data.Length, PageSize - (int)(va & PageMask)); + + data[..size].CopyTo(GetPhysicalAddressSpan(pa, size)); + + offset += size; + } + + for (; offset < data.Length; offset += size) + { + nuint pa = TranslateVirtualAddressChecked(va + (ulong)offset); + + size = Math.Min(data.Length - offset, PageSize); + + data.Slice(offset, size).CopyTo(GetPhysicalAddressSpan(pa, size)); + } + } + } + } } -- cgit v1.2.3