From 5def0429f82c795d820b2307a0301b78dfb1e6b7 Mon Sep 17 00:00:00 2001 From: jhorv <38920027+jhorv@users.noreply.github.com> Date: Thu, 4 Apr 2024 21:23:03 -0400 Subject: Add support to IVirtualMemoryManager for zero-copy reads (#6251) * - WritableRegion: enable wrapping IMemoryOwner - IVirtualMemoryManager impls of GetWritableRegion() use pooled memory when region is non-contiguous. - IVirtualMemoryManager: add GetReadOnlySequence() and impls - ByteMemoryPool: add new method RentCopy() - ByteMemoryPool: make class static, remove ctor and singleton field from earlier impl * - BytesReadOnlySequenceSegment: move from Ryujinx.Common.Memory to Ryujinx.Memory - BytesReadOnlySequenceSegment: add IsContiguousWith() and Replace() methods - VirtualMemoryManagerBase: - remove generic type parameters, instead use ulong for virtual addresses and nuint for host/physical addresses - implement IWritableBlock - add virtual GetReadOnlySequence() with coalescing of contiguous segments - add virtual GetSpan() - add virtual GetWritableRegion() - add abstract IsMapped() - add virtual MapForeign(ulong, nuint, ulong) - add virtual Read() - add virtual Read(ulong, Span) - add virtual ReadTracked() - add virtual SignalMemoryTracking() - add virtual Write() - add virtual Write() - add virtual WriteUntracked() - add virtual WriteWithRedundancyCheck() - VirtualMemoryManagerRefCountedBase: remove generic type parameters - AddressSpaceManager: remove redundant methods, add required overrides - HvMemoryManager: remove redundant methods, add required overrides, add overrides for _invalidAccessHandler handling - MemoryManager: remove redundant methods, add required overrides, add overrides for _invalidAccessHandler handling - MemoryManagerHostMapped: remove redundant methods, add required overrides, add overrides for _invalidAccessHandler handling - NativeMemoryManager: add get properties for Pointer and Length - throughout: removed invalid comments * make HvMemoryManager class sealed * remove unused method * adjust MemoryManagerHostTracked * let MemoryManagerHostTracked override WriteImpl() --- src/Ryujinx.Cpu/Jit/MemoryManagerHostTracked.cs | 193 ++++++++---------------- 1 file changed, 67 insertions(+), 126 deletions(-) (limited to 'src/Ryujinx.Cpu/Jit/MemoryManagerHostTracked.cs') diff --git a/src/Ryujinx.Cpu/Jit/MemoryManagerHostTracked.cs b/src/Ryujinx.Cpu/Jit/MemoryManagerHostTracked.cs index 18404bcc..b2964cd2 100644 --- a/src/Ryujinx.Cpu/Jit/MemoryManagerHostTracked.cs +++ b/src/Ryujinx.Cpu/Jit/MemoryManagerHostTracked.cs @@ -8,14 +8,13 @@ using System; using System.Collections.Generic; using System.Linq; using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; namespace Ryujinx.Cpu.Jit { /// /// Represents a CPU memory manager which maps guest virtual memory directly onto a host virtual region. /// - public sealed class MemoryManagerHostTracked : VirtualMemoryManagerRefCountedBase, IWritableBlock, IMemoryManager, IVirtualMemoryManagerTracked + public sealed class MemoryManagerHostTracked : VirtualMemoryManagerRefCountedBase, IMemoryManager, IVirtualMemoryManagerTracked { private readonly InvalidAccessHandler _invalidAccessHandler; private readonly bool _unsafeMode; @@ -100,12 +99,6 @@ namespace Ryujinx.Cpu.Jit Tracking.Map(va, size); } - /// - public void MapForeign(ulong va, nuint hostPointer, ulong size) - { - throw new NotSupportedException(); - } - /// public void Unmap(ulong va, ulong size) { @@ -120,18 +113,11 @@ namespace Ryujinx.Cpu.Jit _nativePageTable.Unmap(va, size); } - public T Read(ulong va) where T : unmanaged - { - return MemoryMarshal.Cast(GetSpan(va, Unsafe.SizeOf()))[0]; - } - - public T ReadTracked(ulong va) where T : unmanaged + public override T ReadTracked(ulong va) { try { - SignalMemoryTracking(va, (ulong)Unsafe.SizeOf(), false); - - return Read(va); + return base.ReadTracked(va); } catch (InvalidMemoryRegionException) { @@ -145,38 +131,39 @@ namespace Ryujinx.Cpu.Jit } public override void Read(ulong va, Span data) - { - ReadImpl(va, data); - } - - public void Write(ulong va, T value) where T : unmanaged - { - Write(va, MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref value, 1))); - } - - public void Write(ulong va, ReadOnlySpan data) { if (data.Length == 0) { return; } - SignalMemoryTracking(va, (ulong)data.Length, true); + try + { + AssertValidAddressAndSize(va, (ulong)data.Length); + + ulong endVa = va + (ulong)data.Length; + int offset = 0; - WriteImpl(va, data); - } + while (va < endVa) + { + (MemoryBlock memory, ulong rangeOffset, ulong copySize) = GetMemoryOffsetAndSize(va, (ulong)(data.Length - offset)); - public void WriteUntracked(ulong va, ReadOnlySpan data) - { - if (data.Length == 0) + memory.GetSpan(rangeOffset, (int)copySize).CopyTo(data.Slice(offset, (int)copySize)); + + va += copySize; + offset += (int)copySize; + } + } + catch (InvalidMemoryRegionException) { - return; + if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) + { + throw; + } } - - WriteImpl(va, data); } - public bool WriteWithRedundancyCheck(ulong va, ReadOnlySpan data) + public override bool WriteWithRedundancyCheck(ulong va, ReadOnlySpan data) { if (data.Length == 0) { @@ -206,35 +193,7 @@ namespace Ryujinx.Cpu.Jit } } - private void WriteImpl(ulong va, ReadOnlySpan data) - { - try - { - AssertValidAddressAndSize(va, (ulong)data.Length); - - ulong endVa = va + (ulong)data.Length; - int offset = 0; - - while (va < endVa) - { - (MemoryBlock memory, ulong rangeOffset, ulong copySize) = GetMemoryOffsetAndSize(va, (ulong)(data.Length - offset)); - - data.Slice(offset, (int)copySize).CopyTo(memory.GetSpan(rangeOffset, (int)copySize)); - - va += copySize; - offset += (int)copySize; - } - } - catch (InvalidMemoryRegionException) - { - if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) - { - throw; - } - } - } - - public ReadOnlySpan GetSpan(ulong va, int size, bool tracked = false) + public override ReadOnlySpan GetSpan(ulong va, int size, bool tracked = false) { if (size == 0) { @@ -254,13 +213,13 @@ namespace Ryujinx.Cpu.Jit { Span data = new byte[size]; - ReadImpl(va, data); + Read(va, data); return data; } } - public WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false) + public override WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false) { if (size == 0) { @@ -280,7 +239,7 @@ namespace Ryujinx.Cpu.Jit { Memory memory = new byte[size]; - ReadImpl(va, memory.Span); + Read(va, memory.Span); return new WritableRegion(this, va, memory); } @@ -299,7 +258,7 @@ namespace Ryujinx.Cpu.Jit } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool IsMapped(ulong va) + public override bool IsMapped(ulong va) { return ValidateAddress(va) && _pages.IsMapped(va); } @@ -311,8 +270,6 @@ namespace Ryujinx.Cpu.Jit return _pages.IsRangeMapped(va, size); } - private static void ThrowMemoryNotContiguous() => throw new MemoryNotContiguousException(); - private bool TryGetVirtualContiguous(ulong va, int size, out MemoryBlock memory, out ulong offset) { if (_addressSpace.HasAnyPrivateAllocation(va, (ulong)size, out PrivateRange range)) @@ -491,44 +448,11 @@ namespace Ryujinx.Cpu.Jit return regions; } - private void ReadImpl(ulong va, Span data) - { - if (data.Length == 0) - { - return; - } - - try - { - AssertValidAddressAndSize(va, (ulong)data.Length); - - ulong endVa = va + (ulong)data.Length; - int offset = 0; - - while (va < endVa) - { - (MemoryBlock memory, ulong rangeOffset, ulong copySize) = GetMemoryOffsetAndSize(va, (ulong)(data.Length - offset)); - - memory.GetSpan(rangeOffset, (int)copySize).CopyTo(data.Slice(offset, (int)copySize)); - - va += copySize; - offset += (int)copySize; - } - } - catch (InvalidMemoryRegionException) - { - if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) - { - throw; - } - } - } - /// /// /// This function also validates that the given range is both valid and mapped, and will throw if it is not. /// - public void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null) + public override void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null) { AssertValidAddressAndSize(va, size); @@ -543,23 +467,6 @@ namespace Ryujinx.Cpu.Jit _pages.SignalMemoryTracking(Tracking, va, size, write, exemptId); } - /// - /// Computes the number of pages in a virtual address range. - /// - /// Virtual address of the range - /// Size of the range - /// The virtual address of the beginning of the first page - /// This function does not differentiate between allocated and unallocated pages. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private int GetPagesCount(ulong va, ulong size, out ulong startVa) - { - // WARNING: Always check if ulong does not overflow during the operations. - startVa = va & ~(ulong)PageMask; - ulong vaSpan = (va - startVa + size + PageMask) & ~(ulong)PageMask; - - return (int)(vaSpan / PageSize); - } - public RegionHandle BeginTracking(ulong address, ulong size, int id, RegionFlags flags = RegionFlags.None) { return Tracking.BeginTracking(address, size, id, flags); @@ -618,10 +525,44 @@ namespace Ryujinx.Cpu.Jit _nativePageTable.Dispose(); } - protected override Span GetPhysicalAddressSpan(ulong pa, int size) + protected override Memory GetPhysicalAddressMemory(nuint pa, int size) + => _backingMemory.GetMemory(pa, size); + + protected override Span GetPhysicalAddressSpan(nuint pa, int size) => _backingMemory.GetSpan(pa, size); - protected override ulong TranslateVirtualAddressForRead(ulong va) - => GetPhysicalAddressInternal(va); + protected override void WriteImpl(ulong va, ReadOnlySpan data) + { + try + { + AssertValidAddressAndSize(va, (ulong)data.Length); + + ulong endVa = va + (ulong)data.Length; + int offset = 0; + + while (va < endVa) + { + (MemoryBlock memory, ulong rangeOffset, ulong copySize) = GetMemoryOffsetAndSize(va, (ulong)(data.Length - offset)); + + data.Slice(offset, (int)copySize).CopyTo(memory.GetSpan(rangeOffset, (int)copySize)); + + va += copySize; + offset += (int)copySize; + } + } + catch (InvalidMemoryRegionException) + { + if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) + { + throw; + } + } + } + + protected override nuint TranslateVirtualAddressChecked(ulong va) + => (nuint)GetPhysicalAddressChecked(va); + + protected override nuint TranslateVirtualAddressUnchecked(ulong va) + => (nuint)GetPhysicalAddressInternal(va); } } -- cgit v1.2.3