aboutsummaryrefslogtreecommitdiff
path: root/src/Ryujinx.Cpu/Jit
diff options
context:
space:
mode:
authorTSR Berry <20988865+TSRBerry@users.noreply.github.com>2023-04-08 01:22:00 +0200
committerMary <thog@protonmail.com>2023-04-27 23:51:14 +0200
commitcee712105850ac3385cd0091a923438167433f9f (patch)
tree4a5274b21d8b7f938c0d0ce18736d3f2993b11b1 /src/Ryujinx.Cpu/Jit
parentcd124bda587ef09668a971fa1cac1c3f0cfc9f21 (diff)
Move solution and projects to src
Diffstat (limited to 'src/Ryujinx.Cpu/Jit')
-rw-r--r--src/Ryujinx.Cpu/Jit/JitCpuContext.cs53
-rw-r--r--src/Ryujinx.Cpu/Jit/JitDiskCacheLoadState.cs38
-rw-r--r--src/Ryujinx.Cpu/Jit/JitEngine.cs20
-rw-r--r--src/Ryujinx.Cpu/Jit/JitExecutionContext.cs123
-rw-r--r--src/Ryujinx.Cpu/Jit/JitMemoryAllocator.cs13
-rw-r--r--src/Ryujinx.Cpu/Jit/JitMemoryBlock.cs24
-rw-r--r--src/Ryujinx.Cpu/Jit/MemoryManager.cs704
-rw-r--r--src/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs817
8 files changed, 1792 insertions, 0 deletions
diff --git a/src/Ryujinx.Cpu/Jit/JitCpuContext.cs b/src/Ryujinx.Cpu/Jit/JitCpuContext.cs
new file mode 100644
index 00000000..02465a0b
--- /dev/null
+++ b/src/Ryujinx.Cpu/Jit/JitCpuContext.cs
@@ -0,0 +1,53 @@
+using ARMeilleure.Memory;
+using ARMeilleure.Translation;
+
+namespace Ryujinx.Cpu.Jit
+{
+ class JitCpuContext : ICpuContext
+ {
+ private readonly ITickSource _tickSource;
+ private readonly Translator _translator;
+
+ public JitCpuContext(ITickSource tickSource, IMemoryManager memory, bool for64Bit)
+ {
+ _tickSource = tickSource;
+ _translator = new Translator(new JitMemoryAllocator(), memory, for64Bit);
+ memory.UnmapEvent += UnmapHandler;
+ }
+
+ private void UnmapHandler(ulong address, ulong size)
+ {
+ _translator.InvalidateJitCacheRegion(address, size);
+ }
+
+ /// <inheritdoc/>
+ public IExecutionContext CreateExecutionContext(ExceptionCallbacks exceptionCallbacks)
+ {
+ return new JitExecutionContext(new JitMemoryAllocator(), _tickSource, exceptionCallbacks);
+ }
+
+ /// <inheritdoc/>
+ public void Execute(IExecutionContext context, ulong address)
+ {
+ _translator.Execute(((JitExecutionContext)context).Impl, address);
+ }
+
+ /// <inheritdoc/>
+ public void InvalidateCacheRegion(ulong address, ulong size)
+ {
+ _translator.InvalidateJitCacheRegion(address, size);
+ }
+
+ /// <inheritdoc/>
+ public IDiskCacheLoadState LoadDiskCache(string titleIdText, string displayVersion, bool enabled)
+ {
+ return new JitDiskCacheLoadState(_translator.LoadDiskCache(titleIdText, displayVersion, enabled));
+ }
+
+ /// <inheritdoc/>
+ public void PrepareCodeRange(ulong address, ulong size)
+ {
+ _translator.PrepareCodeRange(address, size);
+ }
+ }
+}
diff --git a/src/Ryujinx.Cpu/Jit/JitDiskCacheLoadState.cs b/src/Ryujinx.Cpu/Jit/JitDiskCacheLoadState.cs
new file mode 100644
index 00000000..7a4b670b
--- /dev/null
+++ b/src/Ryujinx.Cpu/Jit/JitDiskCacheLoadState.cs
@@ -0,0 +1,38 @@
+using ARMeilleure.Translation.PTC;
+using System;
+
+namespace Ryujinx.Cpu.Jit
+{
+ public class JitDiskCacheLoadState : IDiskCacheLoadState
+ {
+ /// <inheritdoc/>
+ public event Action<LoadState, int, int> StateChanged;
+
+ private readonly IPtcLoadState _loadState;
+
+ public JitDiskCacheLoadState(IPtcLoadState loadState)
+ {
+ loadState.PtcStateChanged += LoadStateChanged;
+ _loadState = loadState;
+ }
+
+ private void LoadStateChanged(PtcLoadingState newState, int current, int total)
+ {
+ LoadState state = newState switch
+ {
+ PtcLoadingState.Start => LoadState.Unloaded,
+ PtcLoadingState.Loading => LoadState.Loading,
+ PtcLoadingState.Loaded => LoadState.Loaded,
+ _ => throw new ArgumentException($"Invalid load state \"{newState}\".")
+ };
+
+ StateChanged?.Invoke(state, current, total);
+ }
+
+ /// <inheritdoc/>
+ public void Cancel()
+ {
+ _loadState.Continue();
+ }
+ }
+} \ No newline at end of file
diff --git a/src/Ryujinx.Cpu/Jit/JitEngine.cs b/src/Ryujinx.Cpu/Jit/JitEngine.cs
new file mode 100644
index 00000000..b158074f
--- /dev/null
+++ b/src/Ryujinx.Cpu/Jit/JitEngine.cs
@@ -0,0 +1,20 @@
+using ARMeilleure.Memory;
+
+namespace Ryujinx.Cpu.Jit
+{
+ public class JitEngine : ICpuEngine
+ {
+ private readonly ITickSource _tickSource;
+
+ public JitEngine(ITickSource tickSource)
+ {
+ _tickSource = tickSource;
+ }
+
+ /// <inheritdoc/>
+ public ICpuContext CreateCpuContext(IMemoryManager memoryManager, bool for64Bit)
+ {
+ return new JitCpuContext(_tickSource, memoryManager, for64Bit);
+ }
+ }
+} \ No newline at end of file
diff --git a/src/Ryujinx.Cpu/Jit/JitExecutionContext.cs b/src/Ryujinx.Cpu/Jit/JitExecutionContext.cs
new file mode 100644
index 00000000..e1a527b1
--- /dev/null
+++ b/src/Ryujinx.Cpu/Jit/JitExecutionContext.cs
@@ -0,0 +1,123 @@
+using ARMeilleure.Memory;
+using ARMeilleure.State;
+
+namespace Ryujinx.Cpu.Jit
+{
+ class JitExecutionContext : IExecutionContext
+ {
+ private readonly ExecutionContext _impl;
+ internal ExecutionContext Impl => _impl;
+
+ /// <inheritdoc/>
+ public ulong Pc => _impl.Pc;
+
+ /// <inheritdoc/>
+ public long TpidrEl0
+ {
+ get => _impl.TpidrEl0;
+ set => _impl.TpidrEl0 = value;
+ }
+
+ /// <inheritdoc/>
+ public long TpidrroEl0
+ {
+ get => _impl.TpidrroEl0;
+ set => _impl.TpidrroEl0 = value;
+ }
+
+ /// <inheritdoc/>
+ public uint Pstate
+ {
+ get => _impl.Pstate;
+ set => _impl.Pstate = value;
+ }
+
+ /// <inheritdoc/>
+ public uint Fpcr
+ {
+ get => (uint)_impl.Fpcr;
+ set => _impl.Fpcr = (FPCR)value;
+ }
+
+ /// <inheritdoc/>
+ public uint Fpsr
+ {
+ get => (uint)_impl.Fpsr;
+ set => _impl.Fpsr = (FPSR)value;
+ }
+
+ /// <inheritdoc/>
+ public bool IsAarch32
+ {
+ get => _impl.IsAarch32;
+ set => _impl.IsAarch32 = value;
+ }
+
+ /// <inheritdoc/>
+ public bool Running => _impl.Running;
+
+ private readonly ExceptionCallbacks _exceptionCallbacks;
+
+ public JitExecutionContext(IJitMemoryAllocator allocator, ICounter counter, ExceptionCallbacks exceptionCallbacks)
+ {
+ _impl = new ExecutionContext(
+ allocator,
+ counter,
+ InterruptHandler,
+ BreakHandler,
+ SupervisorCallHandler,
+ UndefinedHandler);
+
+ _exceptionCallbacks = exceptionCallbacks;
+ }
+
+ /// <inheritdoc/>
+ public ulong GetX(int index) => _impl.GetX(index);
+
+ /// <inheritdoc/>
+ public void SetX(int index, ulong value) => _impl.SetX(index, value);
+
+ /// <inheritdoc/>
+ public V128 GetV(int index) => _impl.GetV(index);
+
+ /// <inheritdoc/>
+ public void SetV(int index, V128 value) => _impl.SetV(index, value);
+
+ private void InterruptHandler(ExecutionContext context)
+ {
+ _exceptionCallbacks.InterruptCallback?.Invoke(this);
+ }
+
+ private void BreakHandler(ExecutionContext context, ulong address, int imm)
+ {
+ _exceptionCallbacks.BreakCallback?.Invoke(this, address, imm);
+ }
+
+ private void SupervisorCallHandler(ExecutionContext context, ulong address, int imm)
+ {
+ _exceptionCallbacks.SupervisorCallback?.Invoke(this, address, imm);
+ }
+
+ private void UndefinedHandler(ExecutionContext context, ulong address, int opCode)
+ {
+ _exceptionCallbacks.UndefinedCallback?.Invoke(this, address, opCode);
+ }
+
+ /// <inheritdoc/>
+ public void RequestInterrupt()
+ {
+ _impl.RequestInterrupt();
+ }
+
+ /// <inheritdoc/>
+ public void StopRunning()
+ {
+ _impl.StopRunning();
+ }
+
+ public void Dispose()
+ {
+ _impl.Dispose();
+ }
+ }
+} \ No newline at end of file
diff --git a/src/Ryujinx.Cpu/Jit/JitMemoryAllocator.cs b/src/Ryujinx.Cpu/Jit/JitMemoryAllocator.cs
new file mode 100644
index 00000000..4aa78d06
--- /dev/null
+++ b/src/Ryujinx.Cpu/Jit/JitMemoryAllocator.cs
@@ -0,0 +1,13 @@
+using ARMeilleure.Memory;
+using Ryujinx.Memory;
+
+namespace Ryujinx.Cpu.Jit
+{
+ public class JitMemoryAllocator : IJitMemoryAllocator
+ {
+ public IJitMemoryBlock Allocate(ulong size) => new JitMemoryBlock(size, MemoryAllocationFlags.None);
+ public IJitMemoryBlock Reserve(ulong size) => new JitMemoryBlock(size, MemoryAllocationFlags.Reserve | MemoryAllocationFlags.Jit);
+
+ public ulong GetPageSize() => MemoryBlock.GetPageSize();
+ }
+}
diff --git a/src/Ryujinx.Cpu/Jit/JitMemoryBlock.cs b/src/Ryujinx.Cpu/Jit/JitMemoryBlock.cs
new file mode 100644
index 00000000..327fb303
--- /dev/null
+++ b/src/Ryujinx.Cpu/Jit/JitMemoryBlock.cs
@@ -0,0 +1,24 @@
+using ARMeilleure.Memory;
+using Ryujinx.Memory;
+using System;
+
+namespace Ryujinx.Cpu.Jit
+{
+ public class JitMemoryBlock : IJitMemoryBlock
+ {
+ private readonly MemoryBlock _impl;
+
+ public IntPtr Pointer => _impl.Pointer;
+
+ public JitMemoryBlock(ulong size, MemoryAllocationFlags flags)
+ {
+ _impl = new MemoryBlock(size, flags);
+ }
+
+ public bool Commit(ulong offset, ulong size) => _impl.Commit(offset, size);
+ public void MapAsRx(ulong offset, ulong size) => _impl.Reprotect(offset, size, MemoryPermission.ReadAndExecute);
+ public void MapAsRwx(ulong offset, ulong size) => _impl.Reprotect(offset, size, MemoryPermission.ReadWriteExecute);
+
+ public void Dispose() => _impl.Dispose();
+ }
+}
diff --git a/src/Ryujinx.Cpu/Jit/MemoryManager.cs b/src/Ryujinx.Cpu/Jit/MemoryManager.cs
new file mode 100644
index 00000000..8542d53e
--- /dev/null
+++ b/src/Ryujinx.Cpu/Jit/MemoryManager.cs
@@ -0,0 +1,704 @@
+using ARMeilleure.Memory;
+using Ryujinx.Cpu.Tracking;
+using Ryujinx.Memory;
+using Ryujinx.Memory.Range;
+using Ryujinx.Memory.Tracking;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using System.Threading;
+
+namespace Ryujinx.Cpu.Jit
+{
+ /// <summary>
+ /// Represents a CPU memory manager.
+ /// </summary>
+ public sealed class MemoryManager : MemoryManagerBase, IMemoryManager, IVirtualMemoryManagerTracked, IWritableBlock
+ {
+ public const int PageBits = 12;
+ public const int PageSize = 1 << PageBits;
+ public const int PageMask = PageSize - 1;
+
+ private const int PteSize = 8;
+
+ private const int PointerTagBit = 62;
+
+ private readonly MemoryBlock _backingMemory;
+ private readonly InvalidAccessHandler _invalidAccessHandler;
+
+ /// <inheritdoc/>
+ public bool Supports4KBPages => true;
+
+ /// <summary>
+ /// Address space width in bits.
+ /// </summary>
+ public int AddressSpaceBits { get; }
+
+ private readonly ulong _addressSpaceSize;
+
+ private readonly MemoryBlock _pageTable;
+
+ /// <summary>
+ /// Page table base pointer.
+ /// </summary>
+ public IntPtr PageTablePointer => _pageTable.Pointer;
+
+ public MemoryManagerType Type => MemoryManagerType.SoftwarePageTable;
+
+ public MemoryTracking Tracking { get; }
+
+ public event Action<ulong, ulong> UnmapEvent;
+
+ /// <summary>
+ /// Creates a new instance of the memory manager.
+ /// </summary>
+ /// <param name="backingMemory">Physical backing memory where virtual memory will be mapped to</param>
+ /// <param name="addressSpaceSize">Size of the address space</param>
+ /// <param name="invalidAccessHandler">Optional function to handle invalid memory accesses</param>
+ public MemoryManager(MemoryBlock backingMemory, ulong addressSpaceSize, InvalidAccessHandler invalidAccessHandler = null)
+ {
+ _backingMemory = backingMemory;
+ _invalidAccessHandler = invalidAccessHandler;
+
+ ulong asSize = PageSize;
+ int asBits = PageBits;
+
+ while (asSize < addressSpaceSize)
+ {
+ asSize <<= 1;
+ asBits++;
+ }
+
+ AddressSpaceBits = asBits;
+ _addressSpaceSize = asSize;
+ _pageTable = new MemoryBlock((asSize / PageSize) * PteSize);
+
+ Tracking = new MemoryTracking(this, PageSize);
+ }
+
+ /// <inheritdoc/>
+ public void Map(ulong va, ulong pa, ulong size, MemoryMapFlags flags)
+ {
+ AssertValidAddressAndSize(va, size);
+
+ ulong remainingSize = size;
+ ulong oVa = va;
+ ulong oPa = pa;
+ while (remainingSize != 0)
+ {
+ _pageTable.Write((va / PageSize) * PteSize, PaToPte(pa));
+
+ va += PageSize;
+ pa += PageSize;
+ remainingSize -= PageSize;
+ }
+
+ Tracking.Map(oVa, size);
+ }
+
+ /// <inheritdoc/>
+ public void MapForeign(ulong va, nuint hostPointer, ulong size)
+ {
+ throw new NotSupportedException();
+ }
+
+ /// <inheritdoc/>
+ public void Unmap(ulong va, ulong size)
+ {
+ // If size is 0, there's nothing to unmap, just exit early.
+ if (size == 0)
+ {
+ return;
+ }
+
+ AssertValidAddressAndSize(va, size);
+
+ UnmapEvent?.Invoke(va, size);
+ Tracking.Unmap(va, size);
+
+ ulong remainingSize = size;
+ while (remainingSize != 0)
+ {
+ _pageTable.Write((va / PageSize) * PteSize, 0UL);
+
+ va += PageSize;
+ remainingSize -= PageSize;
+ }
+ }
+
+ /// <inheritdoc/>
+ public T Read<T>(ulong va) where T : unmanaged
+ {
+ return MemoryMarshal.Cast<byte, T>(GetSpan(va, Unsafe.SizeOf<T>()))[0];
+ }
+
+ /// <inheritdoc/>
+ public T ReadTracked<T>(ulong va) where T : unmanaged
+ {
+ try
+ {
+ SignalMemoryTracking(va, (ulong)Unsafe.SizeOf<T>(), false);
+
+ return Read<T>(va);
+ }
+ catch (InvalidMemoryRegionException)
+ {
+ if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
+ {
+ throw;
+ }
+
+ return default;
+ }
+ }
+
+ /// <inheritdoc/>
+ public void Read(ulong va, Span<byte> data)
+ {
+ ReadImpl(va, data);
+ }
+
+ /// <inheritdoc/>
+ public void Write<T>(ulong va, T value) where T : unmanaged
+ {
+ Write(va, MemoryMarshal.Cast<T, byte>(MemoryMarshal.CreateSpan(ref value, 1)));
+ }
+
+ /// <inheritdoc/>
+ public void Write(ulong va, ReadOnlySpan<byte> data)
+ {
+ if (data.Length == 0)
+ {
+ return;
+ }
+
+ SignalMemoryTracking(va, (ulong)data.Length, true);
+
+ WriteImpl(va, data);
+ }
+
+ /// <inheritdoc/>
+ public void WriteUntracked(ulong va, ReadOnlySpan<byte> data)
+ {
+ if (data.Length == 0)
+ {
+ return;
+ }
+
+ WriteImpl(va, data);
+ }
+
+ /// <inheritdoc/>
+ public bool WriteWithRedundancyCheck(ulong va, ReadOnlySpan<byte> data)
+ {
+ if (data.Length == 0)
+ {
+ return false;
+ }
+
+ SignalMemoryTracking(va, (ulong)data.Length, false);
+
+ if (IsContiguousAndMapped(va, data.Length))
+ {
+ var target = _backingMemory.GetSpan(GetPhysicalAddressInternal(va), data.Length);
+
+ bool changed = !data.SequenceEqual(target);
+
+ if (changed)
+ {
+ data.CopyTo(target);
+ }
+
+ return changed;
+ }
+ else
+ {
+ WriteImpl(va, data);
+
+ return true;
+ }
+ }
+
+ /// <summary>
+ /// Writes data to CPU mapped memory.
+ /// </summary>
+ /// <param name="va">Virtual address to write the data into</param>
+ /// <param name="data">Data to be written</param>
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private void WriteImpl(ulong va, ReadOnlySpan<byte> data)
+ {
+ try
+ {
+ AssertValidAddressAndSize(va, (ulong)data.Length);
+
+ if (IsContiguousAndMapped(va, data.Length))
+ {
+ data.CopyTo(_backingMemory.GetSpan(GetPhysicalAddressInternal(va), data.Length));
+ }
+ else
+ {
+ int offset = 0, size;
+
+ if ((va & PageMask) != 0)
+ {
+ ulong pa = GetPhysicalAddressInternal(va);
+
+ size = Math.Min(data.Length, PageSize - (int)(va & PageMask));
+
+ data.Slice(0, size).CopyTo(_backingMemory.GetSpan(pa, size));
+
+ offset += size;
+ }
+
+ for (; offset < data.Length; offset += size)
+ {
+ ulong pa = GetPhysicalAddressInternal(va + (ulong)offset);
+
+ size = Math.Min(data.Length - offset, PageSize);
+
+ data.Slice(offset, size).CopyTo(_backingMemory.GetSpan(pa, size));
+ }
+ }
+ }
+ catch (InvalidMemoryRegionException)
+ {
+ if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
+ {
+ throw;
+ }
+ }
+ }
+
+ /// <inheritdoc/>
+ public ReadOnlySpan<byte> GetSpan(ulong va, int size, bool tracked = false)
+ {
+ if (size == 0)
+ {
+ return ReadOnlySpan<byte>.Empty;
+ }
+
+ if (tracked)
+ {
+ SignalMemoryTracking(va, (ulong)size, false);
+ }
+
+ if (IsContiguousAndMapped(va, size))
+ {
+ return _backingMemory.GetSpan(GetPhysicalAddressInternal(va), size);
+ }
+ else
+ {
+ Span<byte> data = new byte[size];
+
+ ReadImpl(va, data);
+
+ return data;
+ }
+ }
+
+ /// <inheritdoc/>
+ public unsafe WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false)
+ {
+ if (size == 0)
+ {
+ return new WritableRegion(null, va, Memory<byte>.Empty);
+ }
+
+ if (IsContiguousAndMapped(va, size))
+ {
+ if (tracked)
+ {
+ SignalMemoryTracking(va, (ulong)size, true);
+ }
+
+ return new WritableRegion(null, va, _backingMemory.GetMemory(GetPhysicalAddressInternal(va), size));
+ }
+ else
+ {
+ Memory<byte> memory = new byte[size];
+
+ GetSpan(va, size).CopyTo(memory.Span);
+
+ return new WritableRegion(this, va, memory, tracked);
+ }
+ }
+
+ /// <inheritdoc/>
+ public ref T GetRef<T>(ulong va) where T : unmanaged
+ {
+ if (!IsContiguous(va, Unsafe.SizeOf<T>()))
+ {
+ ThrowMemoryNotContiguous();
+ }
+
+ SignalMemoryTracking(va, (ulong)Unsafe.SizeOf<T>(), true);
+
+ return ref _backingMemory.GetRef<T>(GetPhysicalAddressInternal(va));
+ }
+
+ /// <summary>
+ /// Computes the number of pages in a virtual address range.
+ /// </summary>
+ /// <param name="va">Virtual address of the range</param>
+ /// <param name="size">Size of the range</param>
+ /// <param name="startVa">The virtual address of the beginning of the first page</param>
+ /// <remarks>This function does not differentiate between allocated and unallocated pages.</remarks>
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private int GetPagesCount(ulong va, uint size, out ulong startVa)
+ {
+ // WARNING: Always check if ulong does not overflow during the operations.
+ startVa = va & ~(ulong)PageMask;
+ ulong vaSpan = (va - startVa + size + PageMask) & ~(ulong)PageMask;
+
+ return (int)(vaSpan / PageSize);
+ }
+
+ private static void ThrowMemoryNotContiguous() => throw new MemoryNotContiguousException();
+
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private bool IsContiguousAndMapped(ulong va, int size) => IsContiguous(va, size) && IsMapped(va);
+
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private bool IsContiguous(ulong va, int size)
+ {
+ if (!ValidateAddress(va) || !ValidateAddressAndSize(va, (ulong)size))
+ {
+ return false;
+ }
+
+ int pages = GetPagesCount(va, (uint)size, out va);
+
+ for (int page = 0; page < pages - 1; page++)
+ {
+ if (!ValidateAddress(va + PageSize))
+ {
+ return false;
+ }
+
+ if (GetPhysicalAddressInternal(va) + PageSize != GetPhysicalAddressInternal(va + PageSize))
+ {
+ return false;
+ }
+
+ va += PageSize;
+ }
+
+ return true;
+ }
+
+ /// <inheritdoc/>
+ public IEnumerable<HostMemoryRange> GetHostRegions(ulong va, ulong size)
+ {
+ if (size == 0)
+ {
+ return Enumerable.Empty<HostMemoryRange>();
+ }
+
+ var guestRegions = GetPhysicalRegionsImpl(va, size);
+ if (guestRegions == null)
+ {
+ return null;
+ }
+
+ var regions = new HostMemoryRange[guestRegions.Count];
+
+ for (int i = 0; i < regions.Length; i++)
+ {
+ var guestRegion = guestRegions[i];
+ IntPtr pointer = _backingMemory.GetPointer(guestRegion.Address, guestRegion.Size);
+ regions[i] = new HostMemoryRange((nuint)(ulong)pointer, guestRegion.Size);
+ }
+
+ return regions;
+ }
+
+ /// <inheritdoc/>
+ public IEnumerable<MemoryRange> GetPhysicalRegions(ulong va, ulong size)
+ {
+ if (size == 0)
+ {
+ return Enumerable.Empty<MemoryRange>();
+ }
+
+ return GetPhysicalRegionsImpl(va, size);
+ }
+
+ private List<MemoryRange> GetPhysicalRegionsImpl(ulong va, ulong size)
+ {
+ if (!ValidateAddress(va) || !ValidateAddressAndSize(va, size))
+ {
+ return null;
+ }
+
+ int pages = GetPagesCount(va, (uint)size, out va);
+
+ var regions = new List<MemoryRange>();
+
+ ulong regionStart = GetPhysicalAddressInternal(va);
+ ulong regionSize = PageSize;
+
+ for (int page = 0; page < pages - 1; page++)
+ {
+ if (!ValidateAddress(va + PageSize))
+ {
+ return null;
+ }
+
+ ulong newPa = GetPhysicalAddressInternal(va + PageSize);
+
+ if (GetPhysicalAddressInternal(va) + PageSize != newPa)
+ {
+ regions.Add(new MemoryRange(regionStart, regionSize));
+ regionStart = newPa;
+ regionSize = 0;
+ }
+
+ va += PageSize;
+ regionSize += PageSize;
+ }
+
+ regions.Add(new MemoryRange(regionStart, regionSize));
+
+ return regions;
+ }
+
+ private void ReadImpl(ulong va, Span<byte> data)
+ {
+ if (data.Length == 0)
+ {
+ return;
+ }
+
+ try
+ {
+ AssertValidAddressAndSize(va, (ulong)data.Length);
+
+ int offset = 0, size;
+
+ if ((va & PageMask) != 0)
+ {
+ ulong pa = GetPhysicalAddressInternal(va);
+
+ size = Math.Min(data.Length, PageSize - (int)(va & PageMask));
+
+ _backingMemory.GetSpan(pa, size).CopyTo(data.Slice(0, size));
+
+ offset += size;
+ }
+
+ for (; offset < data.Length; offset += size)
+ {
+ ulong pa = GetPhysicalAddressInternal(va + (ulong)offset);
+
+ size = Math.Min(data.Length - offset, PageSize);
+
+ _backingMemory.GetSpan(pa, size).CopyTo(data.Slice(offset, size));
+ }
+ }
+ catch (InvalidMemoryRegionException)
+ {
+ if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
+ {
+ throw;
+ }
+ }
+ }
+
+ /// <inheritdoc/>
+ public bool IsRangeMapped(ulong va, ulong size)
+ {
+ if (size == 0UL)
+ {
+ return true;
+ }
+
+ if (!ValidateAddressAndSize(va, size))
+ {
+ return false;
+ }
+
+ int pages = GetPagesCount(va, (uint)size, out va);
+
+ for (int page = 0; page < pages; page++)
+ {
+ if (!IsMapped(va))
+ {
+ return false;
+ }
+
+ va += PageSize;
+ }
+
+ return true;
+ }
+
+ /// <inheritdoc/>
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public bool IsMapped(ulong va)
+ {
+ if (!ValidateAddress(va))
+ {
+ return false;
+ }
+
+ return _pageTable.Read<ulong>((va / PageSize) * PteSize) != 0;
+ }
+
+ private bool ValidateAddress(ulong va)
+ {
+ return va < _addressSpaceSize;
+ }
+
+ /// <summary>
+ /// Checks if the combination of virtual address and size is part of the addressable space.
+ /// </summary>
+ /// <param name="va">Virtual address of the range</param>
+ /// <param name="size">Size of the range in bytes</param>
+ /// <returns>True if the combination of virtual address and size is part of the addressable space</returns>
+ private bool ValidateAddressAndSize(ulong va, ulong size)
+ {
+ ulong endVa = va + size;
+ return endVa >= va && endVa >= size && endVa <= _addressSpaceSize;
+ }
+
+ /// <summary>
+ /// Ensures the combination of virtual address and size is part of the addressable space.
+ /// </summary>
+ /// <param name="va">Virtual address of the range</param>
+ /// <param name="size">Size of the range in bytes</param>
+ /// <exception cref="InvalidMemoryRegionException">Throw when the memory region specified outside the addressable space</exception>
+ private void AssertValidAddressAndSize(ulong va, ulong size)
+ {
+ if (!ValidateAddressAndSize(va, size))
+ {
+ throw new InvalidMemoryRegionException($"va=0x{va:X16}, size=0x{size:X16}");
+ }
+ }
+
+ private ulong GetPhysicalAddress(ulong va)
+ {
+ // We return -1L if the virtual address is invalid or unmapped.
+ if (!ValidateAddress(va) || !IsMapped(va))
+ {
+ return ulong.MaxValue;
+ }
+
+ return GetPhysicalAddressInternal(va);
+ }
+
+ private ulong GetPhysicalAddressInternal(ulong va)
+ {
+ return PteToPa(_pageTable.Read<ulong>((va / PageSize) * PteSize) & ~(0xffffUL << 48)) + (va & PageMask);
+ }
+
+ /// <inheritdoc/>
+ public void TrackingReprotect(ulong va, ulong size, MemoryPermission protection)
+ {
+ AssertValidAddressAndSize(va, size);
+
+ // Protection is inverted on software pages, since the default value is 0.
+ protection = (~protection) & MemoryPermission.ReadAndWrite;
+
+ long tag = protection switch
+ {
+ MemoryPermission.None => 0L,
+ MemoryPermission.Write => 2L << PointerTagBit,
+ _ => 3L << PointerTagBit
+ };
+
+ int pages = GetPagesCount(va, (uint)size, out va);
+ ulong pageStart = va >> PageBits;
+ long invTagMask = ~(0xffffL << 48);
+
+ for (int page = 0; page < pages; page++)
+ {
+ ref long pageRef = ref _pageTable.GetRef<long>(pageStart * PteSize);
+
+ long pte;
+
+ do
+ {
+ pte = Volatile.Read(ref pageRef);
+ }
+ while (pte != 0 && Interlocked.CompareExchange(ref pageRef, (pte & invTagMask) | tag, pte) != pte);
+
+ pageStart++;
+ }
+ }
+
+ /// <inheritdoc/>
+ public CpuRegionHandle BeginTracking(ulong address, ulong size, int id)
+ {
+ return new CpuRegionHandle(Tracking.BeginTracking(address, size, id));
+ }
+
+ /// <inheritdoc/>
+ public CpuMultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable<IRegionHandle> handles, ulong granularity, int id)
+ {
+ return new CpuMultiRegionHandle(Tracking.BeginGranularTracking(address, size, handles, granularity, id));
+ }
+
+ /// <inheritdoc/>
+ public CpuSmartMultiRegionHandle BeginSmartGranularTracking(ulong address, ulong size, ulong granularity, int id)
+ {
+ return new CpuSmartMultiRegionHandle(Tracking.BeginSmartGranularTracking(address, size, granularity, id));
+ }
+
+ /// <inheritdoc/>
+ public void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null)
+ {
+ AssertValidAddressAndSize(va, size);
+
+ if (precise)
+ {
+ Tracking.VirtualMemoryEvent(va, size, write, precise: true, exemptId);
+ return;
+ }
+
+ // We emulate guard pages for software memory access. This makes for an easy transition to
+ // tracking using host guard pages in future, but also supporting platforms where this is not possible.
+
+ // Write tag includes read protection, since we don't have any read actions that aren't performed before write too.
+ long tag = (write ? 3L : 1L) << PointerTagBit;
+
+ int pages = GetPagesCount(va, (uint)size, out _);
+ ulong pageStart = va >> PageBits;
+
+ for (int page = 0; page < pages; page++)
+ {
+ ref long pageRef = ref _pageTable.GetRef<long>(pageStart * PteSize);
+
+ long pte;
+
+ pte = Volatile.Read(ref pageRef);
+
+ if ((pte & tag) != 0)
+ {
+ Tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId);
+ break;
+ }
+
+ pageStart++;
+ }
+ }
+
+ private ulong PaToPte(ulong pa)
+ {
+ return (ulong)_backingMemory.GetPointer(pa, PageSize);
+ }
+
+ private ulong PteToPa(ulong pte)
+ {
+ return (ulong)((long)pte - _backingMemory.Pointer.ToInt64());
+ }
+
+ /// <summary>
+ /// Disposes of resources used by the memory manager.
+ /// </summary>
+ protected override void Destroy() => _pageTable.Dispose();
+
+ private void ThrowInvalidMemoryRegionException(string message) => throw new InvalidMemoryRegionException(message);
+ }
+}
diff --git a/src/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs b/src/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs
new file mode 100644
index 00000000..090740ab
--- /dev/null
+++ b/src/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs
@@ -0,0 +1,817 @@
+using ARMeilleure.Memory;
+using Ryujinx.Cpu.Tracking;
+using Ryujinx.Memory;
+using Ryujinx.Memory.Range;
+using Ryujinx.Memory.Tracking;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Threading;
+
+namespace Ryujinx.Cpu.Jit
+{
+ /// <summary>
+ /// Represents a CPU memory manager which maps guest virtual memory directly onto a host virtual region.
+ /// </summary>
+ public sealed class MemoryManagerHostMapped : MemoryManagerBase, IMemoryManager, IVirtualMemoryManagerTracked, IWritableBlock
+ {
+ public const int PageBits = 12;
+ public const int PageSize = 1 << PageBits;
+ public const int PageMask = PageSize - 1;
+
+ public const int PageToPteShift = 5; // 32 pages (2 bits each) in one ulong page table entry.
+ public const ulong BlockMappedMask = 0x5555555555555555; // First bit of each table entry set.
+
+ private enum HostMappedPtBits : ulong
+ {
+ Unmapped = 0,
+ Mapped,
+ WriteTracked,
+ ReadWriteTracked,
+
+ MappedReplicated = 0x5555555555555555,
+ WriteTrackedReplicated = 0xaaaaaaaaaaaaaaaa,
+ ReadWriteTrackedReplicated = ulong.MaxValue
+ }
+
+ private readonly InvalidAccessHandler _invalidAccessHandler;
+ private readonly bool _unsafeMode;
+
+ private readonly AddressSpace _addressSpace;
+ private readonly ulong _addressSpaceSize;
+
+ private readonly PageTable<ulong> _pageTable;
+
+ private readonly MemoryEhMeilleure _memoryEh;
+
+ private readonly ulong[] _pageBitmap;
+
+ /// <inheritdoc/>
+ public bool Supports4KBPages => MemoryBlock.GetPageSize() == PageSize;
+
+ public int AddressSpaceBits { get; }
+
+ public IntPtr PageTablePointer => _addressSpace.Base.Pointer;
+
+ public MemoryManagerType Type => _unsafeMode ? MemoryManagerType.HostMappedUnsafe : MemoryManagerType.HostMapped;
+
+ public MemoryTracking Tracking { get; }
+
+ public event Action<ulong, ulong> UnmapEvent;
+
+ /// <summary>
+ /// Creates a new instance of the host mapped memory manager.
+ /// </summary>
+ /// <param name="backingMemory">Physical backing memory where virtual memory will be mapped to</param>
+ /// <param name="addressSpaceSize">Size of the address space</param>
+ /// <param name="unsafeMode">True if unmanaged access should not be masked (unsafe), false otherwise.</param>
+ /// <param name="invalidAccessHandler">Optional function to handle invalid memory accesses</param>
+ public MemoryManagerHostMapped(MemoryBlock backingMemory, ulong addressSpaceSize, bool unsafeMode, InvalidAccessHandler invalidAccessHandler = null)
+ {
+ _pageTable = new PageTable<ulong>();
+ _invalidAccessHandler = invalidAccessHandler;
+ _unsafeMode = unsafeMode;
+ _addressSpaceSize = addressSpaceSize;
+
+ ulong asSize = PageSize;
+ int asBits = PageBits;
+
+ while (asSize < addressSpaceSize)
+ {
+ asSize <<= 1;
+ asBits++;
+ }
+
+ AddressSpaceBits = asBits;
+
+ _pageBitmap = new ulong[1 << (AddressSpaceBits - (PageBits + PageToPteShift))];
+
+ _addressSpace = new AddressSpace(backingMemory, asSize, Supports4KBPages);
+
+ Tracking = new MemoryTracking(this, (int)MemoryBlock.GetPageSize(), invalidAccessHandler);
+ _memoryEh = new MemoryEhMeilleure(_addressSpace.Base, _addressSpace.Mirror, Tracking);
+ }
+
+ /// <summary>
+ /// Checks if the virtual address is part of the addressable space.
+ /// </summary>
+ /// <param name="va">Virtual address</param>
+ /// <returns>True if the virtual address is part of the addressable space</returns>
+ private bool ValidateAddress(ulong va)
+ {
+ return va < _addressSpaceSize;
+ }
+
+ /// <summary>
+ /// Checks if the combination of virtual address and size is part of the addressable space.
+ /// </summary>
+ /// <param name="va">Virtual address of the range</param>
+ /// <param name="size">Size of the range in bytes</param>
+ /// <returns>True if the combination of virtual address and size is part of the addressable space</returns>
+ private bool ValidateAddressAndSize(ulong va, ulong size)
+ {
+ ulong endVa = va + size;
+ return endVa >= va && endVa >= size && endVa <= _addressSpaceSize;
+ }
+
+ /// <summary>
+ /// Ensures the combination of virtual address and size is part of the addressable space.
+ /// </summary>
+ /// <param name="va">Virtual address of the range</param>
+ /// <param name="size">Size of the range in bytes</param>
+ /// <exception cref="InvalidMemoryRegionException">Throw when the memory region specified outside the addressable space</exception>
+ private void AssertValidAddressAndSize(ulong va, ulong size)
+ {
+ if (!ValidateAddressAndSize(va, size))
+ {
+ throw new InvalidMemoryRegionException($"va=0x{va:X16}, size=0x{size:X16}");
+ }
+ }
+
+ /// <summary>
+ /// Ensures the combination of virtual address and size is part of the addressable space and fully mapped.
+ /// </summary>
+ /// <param name="va">Virtual address of the range</param>
+ /// <param name="size">Size of the range in bytes</param>
+ private void AssertMapped(ulong va, ulong size)
+ {
+ if (!ValidateAddressAndSize(va, size) || !IsRangeMappedImpl(va, size))
+ {
+ throw new InvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}");
+ }
+ }
+
+ /// <inheritdoc/>
+ public void Map(ulong va, ulong pa, ulong size, MemoryMapFlags flags)
+ {
+ AssertValidAddressAndSize(va, size);
+
+ _addressSpace.Map(va, pa, size, flags);
+ AddMapping(va, size);
+ PtMap(va, pa, size);
+
+ Tracking.Map(va, size);
+ }
+
+ /// <inheritdoc/>
+ public void MapForeign(ulong va, nuint hostPointer, ulong size)
+ {
+ throw new NotSupportedException();
+ }
+
+ /// <inheritdoc/>
+ public void Unmap(ulong va, ulong size)
+ {
+ AssertValidAddressAndSize(va, size);
+
+ UnmapEvent?.Invoke(va, size);
+ Tracking.Unmap(va, size);
+
+ RemoveMapping(va, size);
+ PtUnmap(va, size);
+ _addressSpace.Unmap(va, size);
+ }
+
+ private void PtMap(ulong va, ulong pa, ulong size)
+ {
+ while (size != 0)
+ {
+ _pageTable.Map(va, pa);
+
+ va += PageSize;
+ pa += PageSize;
+ size -= PageSize;
+ }
+ }
+
+ private void PtUnmap(ulong va, ulong size)
+ {
+ while (size != 0)
+ {
+ _pageTable.Unmap(va);
+
+ va += PageSize;
+ size -= PageSize;
+ }
+ }
+
+ /// <inheritdoc/>
+ public T Read<T>(ulong va) where T : unmanaged
+ {
+ try
+ {
+ AssertMapped(va, (ulong)Unsafe.SizeOf<T>());
+
+ return _addressSpace.Mirror.Read<T>(va);
+ }
+ catch (InvalidMemoryRegionException)
+ {
+ if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
+ {
+ throw;
+ }
+
+ return default;
+ }
+ }
+
+ /// <inheritdoc/>
+ public T ReadTracked<T>(ulong va) where T : unmanaged
+ {
+ try
+ {
+ SignalMemoryTracking(va, (ulong)Unsafe.SizeOf<T>(), false);
+
+ return Read<T>(va);
+ }
+ catch (InvalidMemoryRegionException)
+ {
+ if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
+ {
+ throw;
+ }
+
+ return default;
+ }
+ }
+
+ /// <inheritdoc/>
+ public void Read(ulong va, Span<byte> data)
+ {
+ try
+ {
+ AssertMapped(va, (ulong)data.Length);
+
+ _addressSpace.Mirror.Read(va, data);
+ }
+ catch (InvalidMemoryRegionException)
+ {
+ if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
+ {
+ throw;
+ }
+ }
+ }
+
+
+ /// <inheritdoc/>
+ public void Write<T>(ulong va, T value) where T : unmanaged
+ {
+ try
+ {
+ SignalMemoryTracking(va, (ulong)Unsafe.SizeOf<T>(), write: true);
+
+ _addressSpace.Mirror.Write(va, value);
+ }
+ catch (InvalidMemoryRegionException)
+ {
+ if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
+ {
+ throw;
+ }
+ }
+ }
+
+ /// <inheritdoc/>
+ public void Write(ulong va, ReadOnlySpan<byte> data)
+ {
+ try
+ {
+ SignalMemoryTracking(va, (ulong)data.Length, write: true);
+
+ _addressSpace.Mirror.Write(va, data);
+ }
+ catch (InvalidMemoryRegionException)
+ {
+ if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
+ {
+ throw;
+ }
+ }
+ }
+
+ /// <inheritdoc/>
+ public void WriteUntracked(ulong va, ReadOnlySpan<byte> data)
+ {
+ try
+ {
+ AssertMapped(va, (ulong)data.Length);
+
+ _addressSpace.Mirror.Write(va, data);
+ }
+ catch (InvalidMemoryRegionException)
+ {
+ if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
+ {
+ throw;
+ }
+ }
+ }
+
+ /// <inheritdoc/>
+ public bool WriteWithRedundancyCheck(ulong va, ReadOnlySpan<byte> data)
+ {
+ try
+ {
+ SignalMemoryTracking(va, (ulong)data.Length, false);
+
+ Span<byte> target = _addressSpace.Mirror.GetSpan(va, data.Length);
+ bool changed = !data.SequenceEqual(target);
+
+ if (changed)
+ {
+ data.CopyTo(target);
+ }
+
+ return changed;
+ }
+ catch (InvalidMemoryRegionException)
+ {
+ if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
+ {
+ throw;
+ }
+
+ return true;
+ }
+ }
+
+ /// <inheritdoc/>
+ public ReadOnlySpan<byte> GetSpan(ulong va, int size, bool tracked = false)
+ {
+ if (tracked)
+ {
+ SignalMemoryTracking(va, (ulong)size, write: false);
+ }
+ else
+ {
+ AssertMapped(va, (ulong)size);
+ }
+
+ return _addressSpace.Mirror.GetSpan(va, size);
+ }
+
+ /// <inheritdoc/>
+ public WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false)
+ {
+ if (tracked)
+ {
+ SignalMemoryTracking(va, (ulong)size, true);
+ }
+ else
+ {
+ AssertMapped(va, (ulong)size);
+ }
+
+ return _addressSpace.Mirror.GetWritableRegion(va, size);
+ }
+
+ /// <inheritdoc/>
+ public ref T GetRef<T>(ulong va) where T : unmanaged
+ {
+ SignalMemoryTracking(va, (ulong)Unsafe.SizeOf<T>(), true);
+
+ return ref _addressSpace.Mirror.GetRef<T>(va);
+ }
+
+ /// <inheritdoc/>
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public bool IsMapped(ulong va)
+ {
+ return ValidateAddress(va) && IsMappedImpl(va);
+ }
+
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private bool IsMappedImpl(ulong va)
+ {
+ ulong page = va >> PageBits;
+
+ int bit = (int)((page & 31) << 1);
+
+ int pageIndex = (int)(page >> PageToPteShift);
+ ref ulong pageRef = ref _pageBitmap[pageIndex];
+
+ ulong pte = Volatile.Read(ref pageRef);
+
+ return ((pte >> bit) & 3) != 0;
+ }
+
+ /// <inheritdoc/>
+ public bool IsRangeMapped(ulong va, ulong size)
+ {
+ AssertValidAddressAndSize(va, size);
+
+ return IsRangeMappedImpl(va, size);
+ }
+
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private void GetPageBlockRange(ulong pageStart, ulong pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex)
+ {
+ startMask = ulong.MaxValue << ((int)(pageStart & 31) << 1);
+ endMask = ulong.MaxValue >> (64 - ((int)(pageEnd & 31) << 1));
+
+ pageIndex = (int)(pageStart >> PageToPteShift);
+ pageEndIndex = (int)((pageEnd - 1) >> PageToPteShift);
+ }
+
+ private bool IsRangeMappedImpl(ulong va, ulong size)
+ {
+ int pages = GetPagesCount(va, size, out _);
+
+ if (pages == 1)
+ {
+ return IsMappedImpl(va);
+ }
+
+ ulong pageStart = va >> PageBits;
+ ulong pageEnd = pageStart + (ulong)pages;
+
+ GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex);
+
+ // Check if either bit in each 2 bit page entry is set.
+ // OR the block with itself shifted down by 1, and check the first bit of each entry.
+
+ ulong mask = BlockMappedMask & startMask;
+
+ while (pageIndex <= pageEndIndex)
+ {
+ if (pageIndex == pageEndIndex)
+ {
+ mask &= endMask;
+ }
+
+ ref ulong pageRef = ref _pageBitmap[pageIndex++];
+ ulong pte = Volatile.Read(ref pageRef);
+
+ pte |= pte >> 1;
+ if ((pte & mask) != mask)
+ {
+ return false;
+ }
+
+ mask = BlockMappedMask;
+ }
+
+ return true;
+ }
+
+ /// <inheritdoc/>
+ public IEnumerable<HostMemoryRange> GetHostRegions(ulong va, ulong size)
+ {
+ AssertValidAddressAndSize(va, size);
+
+ return Enumerable.Repeat(new HostMemoryRange((nuint)(ulong)_addressSpace.Mirror.GetPointer(va, size), size), 1);
+ }
+
+ /// <inheritdoc/>
+ public IEnumerable<MemoryRange> GetPhysicalRegions(ulong va, ulong size)
+ {
+ int pages = GetPagesCount(va, (uint)size, out va);
+
+ var regions = new List<MemoryRange>();
+
+ ulong regionStart = GetPhysicalAddressChecked(va);
+ ulong regionSize = PageSize;
+
+ for (int page = 0; page < pages - 1; page++)
+ {
+ if (!ValidateAddress(va + PageSize))
+ {
+ return null;
+ }
+
+ ulong newPa = GetPhysicalAddressChecked(va + PageSize);
+
+ if (GetPhysicalAddressChecked(va) + PageSize != newPa)
+ {
+ regions.Add(new MemoryRange(regionStart, regionSize));
+ regionStart = newPa;
+ regionSize = 0;
+ }
+
+ va += PageSize;
+ regionSize += PageSize;
+ }
+
+ regions.Add(new MemoryRange(regionStart, regionSize));
+
+ return regions;
+ }
+
+ private ulong GetPhysicalAddressChecked(ulong va)
+ {
+ if (!IsMapped(va))
+ {
+ ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}");
+ }
+
+ return GetPhysicalAddressInternal(va);
+ }
+
+ private ulong GetPhysicalAddressInternal(ulong va)
+ {
+ return _pageTable.Read(va) + (va & PageMask);
+ }
+
+ /// <inheritdoc/>
+ /// <remarks>
+ /// This function also validates that the given range is both valid and mapped, and will throw if it is not.
+ /// </remarks>
+ public void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null)
+ {
+ AssertValidAddressAndSize(va, size);
+
+ if (precise)
+ {
+ Tracking.VirtualMemoryEvent(va, size, write, precise: true, exemptId);
+ return;
+ }
+
+ // Software table, used for managed memory tracking.
+
+ int pages = GetPagesCount(va, size, out _);
+ ulong pageStart = va >> PageBits;
+
+ if (pages == 1)
+ {
+ ulong tag = (ulong)(write ? HostMappedPtBits.WriteTracked : HostMappedPtBits.ReadWriteTracked);
+
+ int bit = (int)((pageStart & 31) << 1);
+
+ int pageIndex = (int)(pageStart >> PageToPteShift);
+ ref ulong pageRef = ref _pageBitmap[pageIndex];
+
+ ulong pte = Volatile.Read(ref pageRef);
+ ulong state = ((pte >> bit) & 3);
+
+ if (state >= tag)
+ {
+ Tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId);
+ return;
+ }
+ else if (state == 0)
+ {
+ ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}");
+ }
+ }
+ else
+ {
+ ulong pageEnd = pageStart + (ulong)pages;
+
+ GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex);
+
+ ulong mask = startMask;
+
+ ulong anyTrackingTag = (ulong)HostMappedPtBits.WriteTrackedReplicated;
+
+ while (pageIndex <= pageEndIndex)
+ {
+ if (pageIndex == pageEndIndex)
+ {
+ mask &= endMask;
+ }
+
+ ref ulong pageRef = ref _pageBitmap[pageIndex++];
+
+ ulong pte = Volatile.Read(ref pageRef);
+ ulong mappedMask = mask & BlockMappedMask;
+
+ ulong mappedPte = pte | (pte >> 1);
+ if ((mappedPte & mappedMask) != mappedMask)
+ {
+ ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}");
+ }
+
+ pte &= mask;
+ if ((pte & anyTrackingTag) != 0) // Search for any tracking.
+ {
+ // Writes trigger any tracking.
+ // Only trigger tracking from reads if both bits are set on any page.
+ if (write || (pte & (pte >> 1) & BlockMappedMask) != 0)
+ {
+ Tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId);
+ break;
+ }
+ }
+
+ mask = ulong.MaxValue;
+ }
+ }
+ }
+
+ /// <summary>
+ /// Computes the number of pages in a virtual address range.
+ /// </summary>
+ /// <param name="va">Virtual address of the range</param>
+ /// <param name="size">Size of the range</param>
+ /// <param name="startVa">The virtual address of the beginning of the first page</param>
+ /// <remarks>This function does not differentiate between allocated and unallocated pages.</remarks>
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private int GetPagesCount(ulong va, ulong size, out ulong startVa)
+ {
+ // WARNING: Always check if ulong does not overflow during the operations.
+ startVa = va & ~(ulong)PageMask;
+ ulong vaSpan = (va - startVa + size + PageMask) & ~(ulong)PageMask;
+
+ return (int)(vaSpan / PageSize);
+ }
+
+ /// <inheritdoc/>
+ public void TrackingReprotect(ulong va, ulong size, MemoryPermission protection)
+ {
+ // Protection is inverted on software pages, since the default value is 0.
+ protection = (~protection) & MemoryPermission.ReadAndWrite;
+
+ int pages = GetPagesCount(va, size, out va);
+ ulong pageStart = va >> PageBits;
+
+ if (pages == 1)
+ {
+ ulong protTag = protection switch
+ {
+ MemoryPermission.None => (ulong)HostMappedPtBits.Mapped,
+ MemoryPermission.Write => (ulong)HostMappedPtBits.WriteTracked,
+ _ => (ulong)HostMappedPtBits.ReadWriteTracked,
+ };
+
+ int bit = (int)((pageStart & 31) << 1);
+
+ ulong tagMask = 3UL << bit;
+ ulong invTagMask = ~tagMask;
+
+ ulong tag = protTag << bit;
+
+ int pageIndex = (int)(pageStart >> PageToPteShift);
+ ref ulong pageRef = ref _pageBitmap[pageIndex];
+
+ ulong pte;
+
+ do
+ {
+ pte = Volatile.Read(ref pageRef);
+ }
+ while ((pte & tagMask) != 0 && Interlocked.CompareExchange(ref pageRef, (pte & invTagMask) | tag, pte) != pte);
+ }
+ else
+ {
+ ulong pageEnd = pageStart + (ulong)pages;
+
+ GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex);
+
+ ulong mask = startMask;
+
+ ulong protTag = protection switch
+ {
+ MemoryPermission.None => (ulong)HostMappedPtBits.MappedReplicated,
+ MemoryPermission.Write => (ulong)HostMappedPtBits.WriteTrackedReplicated,
+ _ => (ulong)HostMappedPtBits.ReadWriteTrackedReplicated,
+ };
+
+ while (pageIndex <= pageEndIndex)
+ {
+ if (pageIndex == pageEndIndex)
+ {
+ mask &= endMask;
+ }
+
+ ref ulong pageRef = ref _pageBitmap[pageIndex++];
+
+ ulong pte;
+ ulong mappedMask;
+
+ // Change the protection of all 2 bit entries that are mapped.
+ do
+ {
+ pte = Volatile.Read(ref pageRef);
+
+ mappedMask = pte | (pte >> 1);
+ mappedMask |= (mappedMask & BlockMappedMask) << 1;
+ mappedMask &= mask; // Only update mapped pages within the given range.
+ }
+ while (Interlocked.CompareExchange(ref pageRef, (pte & (~mappedMask)) | (protTag & mappedMask), pte) != pte);
+
+ mask = ulong.MaxValue;
+ }
+ }
+
+ protection = protection switch
+ {
+ MemoryPermission.None => MemoryPermission.ReadAndWrite,
+ MemoryPermission.Write => MemoryPermission.Read,
+ _ => MemoryPermission.None
+ };
+
+ _addressSpace.Base.Reprotect(va, size, protection, false);
+ }
+
+ /// <inheritdoc/>
+ public CpuRegionHandle BeginTracking(ulong address, ulong size, int id)
+ {
+ return new CpuRegionHandle(Tracking.BeginTracking(address, size, id));
+ }
+
+ /// <inheritdoc/>
+ public CpuMultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable<IRegionHandle> handles, ulong granularity, int id)
+ {
+ return new CpuMultiRegionHandle(Tracking.BeginGranularTracking(address, size, handles, granularity, id));
+ }
+
+ /// <inheritdoc/>
+ public CpuSmartMultiRegionHandle BeginSmartGranularTracking(ulong address, ulong size, ulong granularity, int id)
+ {
+ return new CpuSmartMultiRegionHandle(Tracking.BeginSmartGranularTracking(address, size, granularity, id));
+ }
+
+ /// <summary>
+ /// Adds the given address mapping to the page table.
+ /// </summary>
+ /// <param name="va">Virtual memory address</param>
+ /// <param name="size">Size to be mapped</param>
+ private void AddMapping(ulong va, ulong size)
+ {
+ int pages = GetPagesCount(va, size, out _);
+ ulong pageStart = va >> PageBits;
+ ulong pageEnd = pageStart + (ulong)pages;
+
+ GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex);
+
+ ulong mask = startMask;
+
+ while (pageIndex <= pageEndIndex)
+ {
+ if (pageIndex == pageEndIndex)
+ {
+ mask &= endMask;
+ }
+
+ ref ulong pageRef = ref _pageBitmap[pageIndex++];
+
+ ulong pte;
+ ulong mappedMask;
+
+ // Map all 2-bit entries that are unmapped.
+ do
+ {
+ pte = Volatile.Read(ref pageRef);
+
+ mappedMask = pte | (pte >> 1);
+ mappedMask |= (mappedMask & BlockMappedMask) << 1;
+ mappedMask |= ~mask; // Treat everything outside the range as mapped, thus unchanged.
+ }
+ while (Interlocked.CompareExchange(ref pageRef, (pte & mappedMask) | (BlockMappedMask & (~mappedMask)), pte) != pte);
+
+ mask = ulong.MaxValue;
+ }
+ }
+
+ /// <summary>
+ /// Removes the given address mapping from the page table.
+ /// </summary>
+ /// <param name="va">Virtual memory address</param>
+ /// <param name="size">Size to be unmapped</param>
+ private void RemoveMapping(ulong va, ulong size)
+ {
+ int pages = GetPagesCount(va, size, out _);
+ ulong pageStart = va >> PageBits;
+ ulong pageEnd = pageStart + (ulong)pages;
+
+ GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex);
+
+ startMask = ~startMask;
+ endMask = ~endMask;
+
+ ulong mask = startMask;
+
+ while (pageIndex <= pageEndIndex)
+ {
+ if (pageIndex == pageEndIndex)
+ {
+ mask |= endMask;
+ }
+
+ ref ulong pageRef = ref _pageBitmap[pageIndex++];
+ ulong pte;
+
+ do
+ {
+ pte = Volatile.Read(ref pageRef);
+ }
+ while (Interlocked.CompareExchange(ref pageRef, pte & mask, pte) != pte);
+
+ mask = 0;
+ }
+ }
+
+ /// <summary>
+ /// Disposes of resources used by the memory manager.
+ /// </summary>
+ protected override void Destroy()
+ {
+ _addressSpace.Dispose();
+ _memoryEh.Dispose();
+ }
+
+ private static void ThrowInvalidMemoryRegionException(string message) => throw new InvalidMemoryRegionException(message);
+ }
+}