diff options
Diffstat (limited to 'src/Ryujinx.Cpu')
| -rw-r--r-- | src/Ryujinx.Cpu/AppleHv/HvMemoryManager.cs | 336 | ||||
| -rw-r--r-- | src/Ryujinx.Cpu/IVirtualMemoryManagerTracked.cs | 6 | ||||
| -rw-r--r-- | src/Ryujinx.Cpu/Jit/MemoryManager.cs | 148 | ||||
| -rw-r--r-- | src/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs | 338 | ||||
| -rw-r--r-- | src/Ryujinx.Cpu/ManagedPageFlags.cs | 389 |
5 files changed, 527 insertions, 690 deletions
diff --git a/src/Ryujinx.Cpu/AppleHv/HvMemoryManager.cs b/src/Ryujinx.Cpu/AppleHv/HvMemoryManager.cs index 6e864f4c..80f7c8a1 100644 --- a/src/Ryujinx.Cpu/AppleHv/HvMemoryManager.cs +++ b/src/Ryujinx.Cpu/AppleHv/HvMemoryManager.cs @@ -8,7 +8,6 @@ using System.Linq; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Versioning; -using System.Threading; namespace Ryujinx.Cpu.AppleHv { @@ -18,21 +17,6 @@ namespace Ryujinx.Cpu.AppleHv [SupportedOSPlatform("macos")] public class HvMemoryManager : VirtualMemoryManagerRefCountedBase<ulong, ulong>, IMemoryManager, IVirtualMemoryManagerTracked, IWritableBlock { - public const int PageToPteShift = 5; // 32 pages (2 bits each) in one ulong page table entry. - public const ulong BlockMappedMask = 0x5555555555555555; // First bit of each table entry set. - - private enum HostMappedPtBits : ulong - { - Unmapped = 0, - Mapped, - WriteTracked, - ReadWriteTracked, - - MappedReplicated = 0x5555555555555555, - WriteTrackedReplicated = 0xaaaaaaaaaaaaaaaa, - ReadWriteTrackedReplicated = ulong.MaxValue, - } - private readonly InvalidAccessHandler _invalidAccessHandler; private readonly HvAddressSpace _addressSpace; @@ -42,7 +26,7 @@ namespace Ryujinx.Cpu.AppleHv private readonly MemoryBlock _backingMemory; private readonly PageTable<ulong> _pageTable; - private readonly ulong[] _pageBitmap; + private readonly ManagedPageFlags _pages; public bool Supports4KBPages => true; @@ -84,7 +68,7 @@ namespace Ryujinx.Cpu.AppleHv AddressSpaceBits = asBits; - _pageBitmap = new ulong[1 << (AddressSpaceBits - (PageBits + PageToPteShift))]; + _pages = new ManagedPageFlags(AddressSpaceBits); Tracking = new MemoryTracking(this, PageSize, invalidAccessHandler); } @@ -95,7 +79,7 @@ namespace Ryujinx.Cpu.AppleHv PtMap(va, pa, size); _addressSpace.MapUser(va, pa, size, MemoryPermission.ReadWriteExecute); - AddMapping(va, size); + _pages.AddMapping(va, size); Tracking.Map(va, size); } @@ -126,7 +110,7 @@ namespace Ryujinx.Cpu.AppleHv UnmapEvent?.Invoke(va, size); Tracking.Unmap(va, size); - RemoveMapping(va, size); + _pages.RemoveMapping(va, size); _addressSpace.UnmapUser(va, size); PtUnmap(va, size); } @@ -360,22 +344,7 @@ namespace Ryujinx.Cpu.AppleHv [MethodImpl(MethodImplOptions.AggressiveInlining)] public bool IsMapped(ulong va) { - return ValidateAddress(va) && IsMappedImpl(va); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool IsMappedImpl(ulong va) - { - ulong page = va >> PageBits; - - int bit = (int)((page & 31) << 1); - - int pageIndex = (int)(page >> PageToPteShift); - ref ulong pageRef = ref _pageBitmap[pageIndex]; - - ulong pte = Volatile.Read(ref pageRef); - - return ((pte >> bit) & 3) != 0; + return ValidateAddress(va) && _pages.IsMapped(va); } /// <inheritdoc/> @@ -383,58 +352,7 @@ namespace Ryujinx.Cpu.AppleHv { AssertValidAddressAndSize(va, size); - return IsRangeMappedImpl(va, size); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void GetPageBlockRange(ulong pageStart, ulong pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex) - { - startMask = ulong.MaxValue << ((int)(pageStart & 31) << 1); - endMask = ulong.MaxValue >> (64 - ((int)(pageEnd & 31) << 1)); - - pageIndex = (int)(pageStart >> PageToPteShift); - pageEndIndex = (int)((pageEnd - 1) >> PageToPteShift); - } - - private bool IsRangeMappedImpl(ulong va, ulong size) - { - int pages = GetPagesCount(va, size, out _); - - if (pages == 1) - { - return IsMappedImpl(va); - } - - ulong pageStart = va >> PageBits; - ulong pageEnd = pageStart + (ulong)pages; - - GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); - - // Check if either bit in each 2 bit page entry is set. - // OR the block with itself shifted down by 1, and check the first bit of each entry. - - ulong mask = BlockMappedMask & startMask; - - while (pageIndex <= pageEndIndex) - { - if (pageIndex == pageEndIndex) - { - mask &= endMask; - } - - ref ulong pageRef = ref _pageBitmap[pageIndex++]; - ulong pte = Volatile.Read(ref pageRef); - - pte |= pte >> 1; - if ((pte & mask) != mask) - { - return false; - } - - mask = BlockMappedMask; - } - - return true; + return _pages.IsRangeMapped(va, size); } private static void ThrowMemoryNotContiguous() => throw new MemoryNotContiguousException(); @@ -560,76 +478,7 @@ namespace Ryujinx.Cpu.AppleHv return; } - // Software table, used for managed memory tracking. - - int pages = GetPagesCount(va, size, out _); - ulong pageStart = va >> PageBits; - - if (pages == 1) - { - ulong tag = (ulong)(write ? HostMappedPtBits.WriteTracked : HostMappedPtBits.ReadWriteTracked); - - int bit = (int)((pageStart & 31) << 1); - - int pageIndex = (int)(pageStart >> PageToPteShift); - ref ulong pageRef = ref _pageBitmap[pageIndex]; - - ulong pte = Volatile.Read(ref pageRef); - ulong state = ((pte >> bit) & 3); - - if (state >= tag) - { - Tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId); - return; - } - else if (state == 0) - { - ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}"); - } - } - else - { - ulong pageEnd = pageStart + (ulong)pages; - - GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); - - ulong mask = startMask; - - ulong anyTrackingTag = (ulong)HostMappedPtBits.WriteTrackedReplicated; - - while (pageIndex <= pageEndIndex) - { - if (pageIndex == pageEndIndex) - { - mask &= endMask; - } - - ref ulong pageRef = ref _pageBitmap[pageIndex++]; - - ulong pte = Volatile.Read(ref pageRef); - ulong mappedMask = mask & BlockMappedMask; - - ulong mappedPte = pte | (pte >> 1); - if ((mappedPte & mappedMask) != mappedMask) - { - ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}"); - } - - pte &= mask; - if ((pte & anyTrackingTag) != 0) // Search for any tracking. - { - // Writes trigger any tracking. - // Only trigger tracking from reads if both bits are set on any page. - if (write || (pte & (pte >> 1) & BlockMappedMask) != 0) - { - Tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId); - break; - } - } - - mask = ulong.MaxValue; - } - } + _pages.SignalMemoryTracking(Tracking, va, size, write, exemptId); } /// <summary> @@ -656,103 +505,28 @@ namespace Ryujinx.Cpu.AppleHv } /// <inheritdoc/> - public void TrackingReprotect(ulong va, ulong size, MemoryPermission protection) + public void TrackingReprotect(ulong va, ulong size, MemoryPermission protection, bool guest) { - // Protection is inverted on software pages, since the default value is 0. - protection = (~protection) & MemoryPermission.ReadAndWrite; - - int pages = GetPagesCount(va, size, out va); - ulong pageStart = va >> PageBits; - - if (pages == 1) + if (guest) { - ulong protTag = protection switch - { - MemoryPermission.None => (ulong)HostMappedPtBits.Mapped, - MemoryPermission.Write => (ulong)HostMappedPtBits.WriteTracked, - _ => (ulong)HostMappedPtBits.ReadWriteTracked, - }; - - int bit = (int)((pageStart & 31) << 1); - - ulong tagMask = 3UL << bit; - ulong invTagMask = ~tagMask; - - ulong tag = protTag << bit; - - int pageIndex = (int)(pageStart >> PageToPteShift); - ref ulong pageRef = ref _pageBitmap[pageIndex]; - - ulong pte; - - do - { - pte = Volatile.Read(ref pageRef); - } - while ((pte & tagMask) != 0 && Interlocked.CompareExchange(ref pageRef, (pte & invTagMask) | tag, pte) != pte); + _addressSpace.ReprotectUser(va, size, protection); } else { - ulong pageEnd = pageStart + (ulong)pages; - - GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); - - ulong mask = startMask; - - ulong protTag = protection switch - { - MemoryPermission.None => (ulong)HostMappedPtBits.MappedReplicated, - MemoryPermission.Write => (ulong)HostMappedPtBits.WriteTrackedReplicated, - _ => (ulong)HostMappedPtBits.ReadWriteTrackedReplicated, - }; - - while (pageIndex <= pageEndIndex) - { - if (pageIndex == pageEndIndex) - { - mask &= endMask; - } - - ref ulong pageRef = ref _pageBitmap[pageIndex++]; - - ulong pte; - ulong mappedMask; - - // Change the protection of all 2 bit entries that are mapped. - do - { - pte = Volatile.Read(ref pageRef); - - mappedMask = pte | (pte >> 1); - mappedMask |= (mappedMask & BlockMappedMask) << 1; - mappedMask &= mask; // Only update mapped pages within the given range. - } - while (Interlocked.CompareExchange(ref pageRef, (pte & (~mappedMask)) | (protTag & mappedMask), pte) != pte); - - mask = ulong.MaxValue; - } + _pages.TrackingReprotect(va, size, protection); } - - protection = protection switch - { - MemoryPermission.None => MemoryPermission.ReadAndWrite, - MemoryPermission.Write => MemoryPermission.Read, - _ => MemoryPermission.None, - }; - - _addressSpace.ReprotectUser(va, size, protection); } /// <inheritdoc/> - public RegionHandle BeginTracking(ulong address, ulong size, int id) + public RegionHandle BeginTracking(ulong address, ulong size, int id, RegionFlags flags = RegionFlags.None) { - return Tracking.BeginTracking(address, size, id); + return Tracking.BeginTracking(address, size, id, flags); } /// <inheritdoc/> - public MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable<IRegionHandle> handles, ulong granularity, int id) + public MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable<IRegionHandle> handles, ulong granularity, int id, RegionFlags flags = RegionFlags.None) { - return Tracking.BeginGranularTracking(address, size, handles, granularity, id); + return Tracking.BeginGranularTracking(address, size, handles, granularity, id, flags); } /// <inheritdoc/> @@ -761,86 +535,6 @@ namespace Ryujinx.Cpu.AppleHv return Tracking.BeginSmartGranularTracking(address, size, granularity, id); } - /// <summary> - /// Adds the given address mapping to the page table. - /// </summary> - /// <param name="va">Virtual memory address</param> - /// <param name="size">Size to be mapped</param> - private void AddMapping(ulong va, ulong size) - { - int pages = GetPagesCount(va, size, out _); - ulong pageStart = va >> PageBits; - ulong pageEnd = pageStart + (ulong)pages; - - GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); - - ulong mask = startMask; - - while (pageIndex <= pageEndIndex) - { - if (pageIndex == pageEndIndex) - { - mask &= endMask; - } - - ref ulong pageRef = ref _pageBitmap[pageIndex++]; - - ulong pte; - ulong mappedMask; - - // Map all 2-bit entries that are unmapped. - do - { - pte = Volatile.Read(ref pageRef); - - mappedMask = pte | (pte >> 1); - mappedMask |= (mappedMask & BlockMappedMask) << 1; - mappedMask |= ~mask; // Treat everything outside the range as mapped, thus unchanged. - } - while (Interlocked.CompareExchange(ref pageRef, (pte & mappedMask) | (BlockMappedMask & (~mappedMask)), pte) != pte); - - mask = ulong.MaxValue; - } - } - - /// <summary> - /// Removes the given address mapping from the page table. - /// </summary> - /// <param name="va">Virtual memory address</param> - /// <param name="size">Size to be unmapped</param> - private void RemoveMapping(ulong va, ulong size) - { - int pages = GetPagesCount(va, size, out _); - ulong pageStart = va >> PageBits; - ulong pageEnd = pageStart + (ulong)pages; - - GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); - - startMask = ~startMask; - endMask = ~endMask; - - ulong mask = startMask; - - while (pageIndex <= pageEndIndex) - { - if (pageIndex == pageEndIndex) - { - mask |= endMask; - } - - ref ulong pageRef = ref _pageBitmap[pageIndex++]; - ulong pte; - - do - { - pte = Volatile.Read(ref pageRef); - } - while (Interlocked.CompareExchange(ref pageRef, pte & mask, pte) != pte); - - mask = 0; - } - } - private ulong GetPhysicalAddressChecked(ulong va) { if (!IsMapped(va)) diff --git a/src/Ryujinx.Cpu/IVirtualMemoryManagerTracked.cs b/src/Ryujinx.Cpu/IVirtualMemoryManagerTracked.cs index 199bff24..e8d11ede 100644 --- a/src/Ryujinx.Cpu/IVirtualMemoryManagerTracked.cs +++ b/src/Ryujinx.Cpu/IVirtualMemoryManagerTracked.cs @@ -28,8 +28,9 @@ namespace Ryujinx.Cpu /// <param name="address">CPU virtual address of the region</param> /// <param name="size">Size of the region</param> /// <param name="id">Handle ID</param> + /// <param name="flags">Region flags</param> /// <returns>The memory tracking handle</returns> - RegionHandle BeginTracking(ulong address, ulong size, int id); + RegionHandle BeginTracking(ulong address, ulong size, int id, RegionFlags flags = RegionFlags.None); /// <summary> /// Obtains a memory tracking handle for the given virtual region, with a specified granularity. This should be disposed when finished with. @@ -39,8 +40,9 @@ namespace Ryujinx.Cpu /// <param name="handles">Handles to inherit state from or reuse. When none are present, provide null</param> /// <param name="granularity">Desired granularity of write tracking</param> /// <param name="id">Handle ID</param> + /// <param name="flags">Region flags</param> /// <returns>The memory tracking handle</returns> - MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable<IRegionHandle> handles, ulong granularity, int id); + MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable<IRegionHandle> handles, ulong granularity, int id, RegionFlags flags = RegionFlags.None); /// <summary> /// Obtains a smart memory tracking handle for the given virtual region, with a specified granularity. This should be disposed when finished with. diff --git a/src/Ryujinx.Cpu/Jit/MemoryManager.cs b/src/Ryujinx.Cpu/Jit/MemoryManager.cs index bbfdf536..c87c8b8c 100644 --- a/src/Ryujinx.Cpu/Jit/MemoryManager.cs +++ b/src/Ryujinx.Cpu/Jit/MemoryManager.cs @@ -33,6 +33,8 @@ namespace Ryujinx.Cpu.Jit private readonly MemoryBlock _pageTable; + private readonly ManagedPageFlags _pages; + /// <summary> /// Page table base pointer. /// </summary> @@ -70,6 +72,8 @@ namespace Ryujinx.Cpu.Jit AddressSpaceSize = asSize; _pageTable = new MemoryBlock((asSize / PageSize) * PteSize); + _pages = new ManagedPageFlags(AddressSpaceBits); + Tracking = new MemoryTracking(this, PageSize); } @@ -89,6 +93,7 @@ namespace Ryujinx.Cpu.Jit remainingSize -= PageSize; } + _pages.AddMapping(oVa, size); Tracking.Map(oVa, size); } @@ -111,6 +116,7 @@ namespace Ryujinx.Cpu.Jit UnmapEvent?.Invoke(va, size); Tracking.Unmap(va, size); + _pages.RemoveMapping(va, size); ulong remainingSize = size; while (remainingSize != 0) @@ -149,6 +155,26 @@ namespace Ryujinx.Cpu.Jit } /// <inheritdoc/> + public T ReadGuest<T>(ulong va) where T : unmanaged + { + try + { + SignalMemoryTrackingImpl(va, (ulong)Unsafe.SizeOf<T>(), false, true); + + return Read<T>(va); + } + catch (InvalidMemoryRegionException) + { + if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) + { + throw; + } + + return default; + } + } + + /// <inheritdoc/> public override void Read(ulong va, Span<byte> data) { try @@ -184,6 +210,16 @@ namespace Ryujinx.Cpu.Jit } /// <inheritdoc/> + public void WriteGuest<T>(ulong va, T value) where T : unmanaged + { + Span<byte> data = MemoryMarshal.Cast<T, byte>(MemoryMarshal.CreateSpan(ref value, 1)); + + SignalMemoryTrackingImpl(va, (ulong)data.Length, true, true); + + WriteImpl(va, data); + } + + /// <inheritdoc/> public void WriteUntracked(ulong va, ReadOnlySpan<byte> data) { if (data.Length == 0) @@ -520,50 +556,57 @@ namespace Ryujinx.Cpu.Jit } /// <inheritdoc/> - public void TrackingReprotect(ulong va, ulong size, MemoryPermission protection) + public void TrackingReprotect(ulong va, ulong size, MemoryPermission protection, bool guest) { AssertValidAddressAndSize(va, size); - // Protection is inverted on software pages, since the default value is 0. - protection = (~protection) & MemoryPermission.ReadAndWrite; - - long tag = protection switch + if (guest) { - MemoryPermission.None => 0L, - MemoryPermission.Write => 2L << PointerTagBit, - _ => 3L << PointerTagBit, - }; + // Protection is inverted on software pages, since the default value is 0. + protection = (~protection) & MemoryPermission.ReadAndWrite; - int pages = GetPagesCount(va, (uint)size, out va); - ulong pageStart = va >> PageBits; - long invTagMask = ~(0xffffL << 48); - - for (int page = 0; page < pages; page++) - { - ref long pageRef = ref _pageTable.GetRef<long>(pageStart * PteSize); + long tag = protection switch + { + MemoryPermission.None => 0L, + MemoryPermission.Write => 2L << PointerTagBit, + _ => 3L << PointerTagBit, + }; - long pte; + int pages = GetPagesCount(va, (uint)size, out va); + ulong pageStart = va >> PageBits; + long invTagMask = ~(0xffffL << 48); - do + for (int page = 0; page < pages; page++) { - pte = Volatile.Read(ref pageRef); - } - while (pte != 0 && Interlocked.CompareExchange(ref pageRef, (pte & invTagMask) | tag, pte) != pte); + ref long pageRef = ref _pageTable.GetRef<long>(pageStart * PteSize); + + long pte; - pageStart++; + do + { + pte = Volatile.Read(ref pageRef); + } + while (pte != 0 && Interlocked.CompareExchange(ref pageRef, (pte & invTagMask) | tag, pte) != pte); + + pageStart++; + } + } + else + { + _pages.TrackingReprotect(va, size, protection); } } /// <inheritdoc/> - public RegionHandle BeginTracking(ulong address, ulong size, int id) + public RegionHandle BeginTracking(ulong address, ulong size, int id, RegionFlags flags = RegionFlags.None) { - return Tracking.BeginTracking(address, size, id); + return Tracking.BeginTracking(address, size, id, flags); } /// <inheritdoc/> - public MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable<IRegionHandle> handles, ulong granularity, int id) + public MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable<IRegionHandle> handles, ulong granularity, int id, RegionFlags flags = RegionFlags.None) { - return Tracking.BeginGranularTracking(address, size, handles, granularity, id); + return Tracking.BeginGranularTracking(address, size, handles, granularity, id, flags); } /// <inheritdoc/> @@ -572,8 +615,7 @@ namespace Ryujinx.Cpu.Jit return Tracking.BeginSmartGranularTracking(address, size, granularity, id); } - /// <inheritdoc/> - public void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null) + private void SignalMemoryTrackingImpl(ulong va, ulong size, bool write, bool guest, bool precise = false, int? exemptId = null) { AssertValidAddressAndSize(va, size); @@ -583,31 +625,47 @@ namespace Ryujinx.Cpu.Jit return; } - // We emulate guard pages for software memory access. This makes for an easy transition to - // tracking using host guard pages in future, but also supporting platforms where this is not possible. - - // Write tag includes read protection, since we don't have any read actions that aren't performed before write too. - long tag = (write ? 3L : 1L) << PointerTagBit; + // If the memory tracking is coming from the guest, use the tag bits in the page table entry. + // Otherwise, use the managed page flags. - int pages = GetPagesCount(va, (uint)size, out _); - ulong pageStart = va >> PageBits; - - for (int page = 0; page < pages; page++) + if (guest) { - ref long pageRef = ref _pageTable.GetRef<long>(pageStart * PteSize); + // We emulate guard pages for software memory access. This makes for an easy transition to + // tracking using host guard pages in future, but also supporting platforms where this is not possible. - long pte; + // Write tag includes read protection, since we don't have any read actions that aren't performed before write too. + long tag = (write ? 3L : 1L) << PointerTagBit; - pte = Volatile.Read(ref pageRef); + int pages = GetPagesCount(va, (uint)size, out _); + ulong pageStart = va >> PageBits; - if ((pte & tag) != 0) + for (int page = 0; page < pages; page++) { - Tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId); - break; - } + ref long pageRef = ref _pageTable.GetRef<long>(pageStart * PteSize); + + long pte; + + pte = Volatile.Read(ref pageRef); - pageStart++; + if ((pte & tag) != 0) + { + Tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId, true); + break; + } + + pageStart++; + } } + else + { + _pages.SignalMemoryTracking(Tracking, va, size, write, exemptId); + } + } + + /// <inheritdoc/> + public void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null) + { + SignalMemoryTrackingImpl(va, size, write, false, precise, exemptId); } private ulong PaToPte(ulong pa) diff --git a/src/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs b/src/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs index 0b6ba260..f410d02e 100644 --- a/src/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs +++ b/src/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs @@ -6,7 +6,6 @@ using System; using System.Collections.Generic; using System.Linq; using System.Runtime.CompilerServices; -using System.Threading; namespace Ryujinx.Cpu.Jit { @@ -15,21 +14,6 @@ namespace Ryujinx.Cpu.Jit /// </summary> public sealed class MemoryManagerHostMapped : VirtualMemoryManagerRefCountedBase<ulong, ulong>, IMemoryManager, IVirtualMemoryManagerTracked, IWritableBlock { - public const int PageToPteShift = 5; // 32 pages (2 bits each) in one ulong page table entry. - public const ulong BlockMappedMask = 0x5555555555555555; // First bit of each table entry set. - - private enum HostMappedPtBits : ulong - { - Unmapped = 0, - Mapped, - WriteTracked, - ReadWriteTracked, - - MappedReplicated = 0x5555555555555555, - WriteTrackedReplicated = 0xaaaaaaaaaaaaaaaa, - ReadWriteTrackedReplicated = ulong.MaxValue, - } - private readonly InvalidAccessHandler _invalidAccessHandler; private readonly bool _unsafeMode; @@ -39,7 +23,7 @@ namespace Ryujinx.Cpu.Jit private readonly MemoryEhMeilleure _memoryEh; - private readonly ulong[] _pageBitmap; + private readonly ManagedPageFlags _pages; /// <inheritdoc/> public bool Supports4KBPages => MemoryBlock.GetPageSize() == PageSize; @@ -81,7 +65,7 @@ namespace Ryujinx.Cpu.Jit AddressSpaceBits = asBits; - _pageBitmap = new ulong[1 << (AddressSpaceBits - (PageBits + PageToPteShift))]; + _pages = new ManagedPageFlags(AddressSpaceBits); Tracking = new MemoryTracking(this, (int)MemoryBlock.GetPageSize(), invalidAccessHandler); _memoryEh = new MemoryEhMeilleure(_addressSpace.Base, _addressSpace.Mirror, Tracking); @@ -94,7 +78,7 @@ namespace Ryujinx.Cpu.Jit /// <param name="size">Size of the range in bytes</param> private void AssertMapped(ulong va, ulong size) { - if (!ValidateAddressAndSize(va, size) || !IsRangeMappedImpl(va, size)) + if (!ValidateAddressAndSize(va, size) || !_pages.IsRangeMapped(va, size)) { throw new InvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}"); } @@ -106,7 +90,7 @@ namespace Ryujinx.Cpu.Jit AssertValidAddressAndSize(va, size); _addressSpace.Map(va, pa, size, flags); - AddMapping(va, size); + _pages.AddMapping(va, size); PtMap(va, pa, size); Tracking.Map(va, size); @@ -126,7 +110,7 @@ namespace Ryujinx.Cpu.Jit UnmapEvent?.Invoke(va, size); Tracking.Unmap(va, size); - RemoveMapping(va, size); + _pages.RemoveMapping(va, size); PtUnmap(va, size); _addressSpace.Unmap(va, size); } @@ -337,22 +321,7 @@ namespace Ryujinx.Cpu.Jit [MethodImpl(MethodImplOptions.AggressiveInlining)] public bool IsMapped(ulong va) { - return ValidateAddress(va) && IsMappedImpl(va); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool IsMappedImpl(ulong va) - { - ulong page = va >> PageBits; - - int bit = (int)((page & 31) << 1); - - int pageIndex = (int)(page >> PageToPteShift); - ref ulong pageRef = ref _pageBitmap[pageIndex]; - - ulong pte = Volatile.Read(ref pageRef); - - return ((pte >> bit) & 3) != 0; + return ValidateAddress(va) && _pages.IsMapped(va); } /// <inheritdoc/> @@ -360,58 +329,7 @@ namespace Ryujinx.Cpu.Jit { AssertValidAddressAndSize(va, size); - return IsRangeMappedImpl(va, size); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void GetPageBlockRange(ulong pageStart, ulong pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex) - { - startMask = ulong.MaxValue << ((int)(pageStart & 31) << 1); - endMask = ulong.MaxValue >> (64 - ((int)(pageEnd & 31) << 1)); - - pageIndex = (int)(pageStart >> PageToPteShift); - pageEndIndex = (int)((pageEnd - 1) >> PageToPteShift); - } - - private bool IsRangeMappedImpl(ulong va, ulong size) - { - int pages = GetPagesCount(va, size, out _); - - if (pages == 1) - { - return IsMappedImpl(va); - } - - ulong pageStart = va >> PageBits; - ulong pageEnd = pageStart + (ulong)pages; - - GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); - - // Check if either bit in each 2 bit page entry is set. - // OR the block with itself shifted down by 1, and check the first bit of each entry. - - ulong mask = BlockMappedMask & startMask; - - while (pageIndex <= pageEndIndex) - { - if (pageIndex == pageEndIndex) - { - mask &= endMask; - } - - ref ulong pageRef = ref _pageBitmap[pageIndex++]; - ulong pte = Volatile.Read(ref pageRef); - - pte |= pte >> 1; - if ((pte & mask) != mask) - { - return false; - } - - mask = BlockMappedMask; - } - - return true; + return _pages.IsRangeMapped(va, size); } /// <inheritdoc/> @@ -486,76 +404,7 @@ namespace Ryujinx.Cpu.Jit return; } - // Software table, used for managed memory tracking. - - int pages = GetPagesCount(va, size, out _); - ulong pageStart = va >> PageBits; - - if (pages == 1) - { - ulong tag = (ulong)(write ? HostMappedPtBits.WriteTracked : HostMappedPtBits.ReadWriteTracked); - - int bit = (int)((pageStart & 31) << 1); - - int pageIndex = (int)(pageStart >> PageToPteShift); - ref ulong pageRef = ref _pageBitmap[pageIndex]; - - ulong pte = Volatile.Read(ref pageRef); - ulong state = ((pte >> bit) & 3); - - if (state >= tag) - { - Tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId); - return; - } - else if (state == 0) - { - ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}"); - } - } - else - { - ulong pageEnd = pageStart + (ulong)pages; - - GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); - - ulong mask = startMask; - - ulong anyTrackingTag = (ulong)HostMappedPtBits.WriteTrackedReplicated; - - while (pageIndex <= pageEndIndex) - { - if (pageIndex == pageEndIndex) - { - mask &= endMask; - } - - ref ulong pageRef = ref _pageBitmap[pageIndex++]; - - ulong pte = Volatile.Read(ref pageRef); - ulong mappedMask = mask & BlockMappedMask; - - ulong mappedPte = pte | (pte >> 1); - if ((mappedPte & mappedMask) != mappedMask) - { - ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}"); - } - - pte &= mask; - if ((pte & anyTrackingTag) != 0) // Search for any tracking. - { - // Writes trigger any tracking. - // Only trigger tracking from reads if both bits are set on any page. - if (write || (pte & (pte >> 1) & BlockMappedMask) != 0) - { - Tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId); - break; - } - } - - mask = ulong.MaxValue; - } - } + _pages.SignalMemoryTracking(Tracking, va, size, write, exemptId); } /// <summary> @@ -582,103 +431,28 @@ namespace Ryujinx.Cpu.Jit } /// <inheritdoc/> - public void TrackingReprotect(ulong va, ulong size, MemoryPermission protection) + public void TrackingReprotect(ulong va, ulong size, MemoryPermission protection, bool guest) { - // Protection is inverted on software pages, since the default value is 0. - protection = (~protection) & MemoryPermission.ReadAndWrite; - - int pages = GetPagesCount(va, size, out va); - ulong pageStart = va >> PageBits; - - if (pages == 1) + if (guest) { - ulong protTag = protection switch - { - MemoryPermission.None => (ulong)HostMappedPtBits.Mapped, - MemoryPermission.Write => (ulong)HostMappedPtBits.WriteTracked, - _ => (ulong)HostMappedPtBits.ReadWriteTracked, - }; - - int bit = (int)((pageStart & 31) << 1); - - ulong tagMask = 3UL << bit; - ulong invTagMask = ~tagMask; - - ulong tag = protTag << bit; - - int pageIndex = (int)(pageStart >> PageToPteShift); - ref ulong pageRef = ref _pageBitmap[pageIndex]; - - ulong pte; - - do - { - pte = Volatile.Read(ref pageRef); - } - while ((pte & tagMask) != 0 && Interlocked.CompareExchange(ref pageRef, (pte & invTagMask) | tag, pte) != pte); + _addressSpace.Base.Reprotect(va, size, protection, false); } else { - ulong pageEnd = pageStart + (ulong)pages; - - GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); - - ulong mask = startMask; - - ulong protTag = protection switch - { - MemoryPermission.None => (ulong)HostMappedPtBits.MappedReplicated, - MemoryPermission.Write => (ulong)HostMappedPtBits.WriteTrackedReplicated, - _ => (ulong)HostMappedPtBits.ReadWriteTrackedReplicated, - }; - - while (pageIndex <= pageEndIndex) - { - if (pageIndex == pageEndIndex) - { - mask &= endMask; - } - - ref ulong pageRef = ref _pageBitmap[pageIndex++]; - - ulong pte; - ulong mappedMask; - - // Change the protection of all 2 bit entries that are mapped. - do - { - pte = Volatile.Read(ref pageRef); - - mappedMask = pte | (pte >> 1); - mappedMask |= (mappedMask & BlockMappedMask) << 1; - mappedMask &= mask; // Only update mapped pages within the given range. - } - while (Interlocked.CompareExchange(ref pageRef, (pte & (~mappedMask)) | (protTag & mappedMask), pte) != pte); - - mask = ulong.MaxValue; - } + _pages.TrackingReprotect(va, size, protection); } - - protection = protection switch - { - MemoryPermission.None => MemoryPermission.ReadAndWrite, - MemoryPermission.Write => MemoryPermission.Read, - _ => MemoryPermission.None, - }; - - _addressSpace.Base.Reprotect(va, size, protection, false); } /// <inheritdoc/> - public RegionHandle BeginTracking(ulong address, ulong size, int id) + public RegionHandle BeginTracking(ulong address, ulong size, int id, RegionFlags flags = RegionFlags.None) { - return Tracking.BeginTracking(address, size, id); + return Tracking.BeginTracking(address, size, id, flags); } /// <inheritdoc/> - public MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable<IRegionHandle> handles, ulong granularity, int id) + public MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable<IRegionHandle> handles, ulong granularity, int id, RegionFlags flags = RegionFlags.None) { - return Tracking.BeginGranularTracking(address, size, handles, granularity, id); + return Tracking.BeginGranularTracking(address, size, handles, granularity, id, flags); } /// <inheritdoc/> @@ -688,86 +462,6 @@ namespace Ryujinx.Cpu.Jit } /// <summary> - /// Adds the given address mapping to the page table. - /// </summary> - /// <param name="va">Virtual memory address</param> - /// <param name="size">Size to be mapped</param> - private void AddMapping(ulong va, ulong size) - { - int pages = GetPagesCount(va, size, out _); - ulong pageStart = va >> PageBits; - ulong pageEnd = pageStart + (ulong)pages; - - GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); - - ulong mask = startMask; - - while (pageIndex <= pageEndIndex) - { - if (pageIndex == pageEndIndex) - { - mask &= endMask; - } - - ref ulong pageRef = ref _pageBitmap[pageIndex++]; - - ulong pte; - ulong mappedMask; - - // Map all 2-bit entries that are unmapped. - do - { - pte = Volatile.Read(ref pageRef); - - mappedMask = pte | (pte >> 1); - mappedMask |= (mappedMask & BlockMappedMask) << 1; - mappedMask |= ~mask; // Treat everything outside the range as mapped, thus unchanged. - } - while (Interlocked.CompareExchange(ref pageRef, (pte & mappedMask) | (BlockMappedMask & (~mappedMask)), pte) != pte); - - mask = ulong.MaxValue; - } - } - - /// <summary> - /// Removes the given address mapping from the page table. - /// </summary> - /// <param name="va">Virtual memory address</param> - /// <param name="size">Size to be unmapped</param> - private void RemoveMapping(ulong va, ulong size) - { - int pages = GetPagesCount(va, size, out _); - ulong pageStart = va >> PageBits; - ulong pageEnd = pageStart + (ulong)pages; - - GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); - - startMask = ~startMask; - endMask = ~endMask; - - ulong mask = startMask; - - while (pageIndex <= pageEndIndex) - { - if (pageIndex == pageEndIndex) - { - mask |= endMask; - } - - ref ulong pageRef = ref _pageBitmap[pageIndex++]; - ulong pte; - - do - { - pte = Volatile.Read(ref pageRef); - } - while (Interlocked.CompareExchange(ref pageRef, pte & mask, pte) != pte); - - mask = 0; - } - } - - /// <summary> /// Disposes of resources used by the memory manager. /// </summary> protected override void Destroy() diff --git a/src/Ryujinx.Cpu/ManagedPageFlags.cs b/src/Ryujinx.Cpu/ManagedPageFlags.cs new file mode 100644 index 00000000..a839dae6 --- /dev/null +++ b/src/Ryujinx.Cpu/ManagedPageFlags.cs @@ -0,0 +1,389 @@ +using Ryujinx.Memory; +using Ryujinx.Memory.Tracking; +using System; +using System.Runtime.CompilerServices; +using System.Threading; + +namespace Ryujinx.Cpu +{ + /// <summary> + /// A page bitmap that keeps track of mapped state and tracking protection + /// for managed memory accesses (not using host page protection). + /// </summary> + internal readonly struct ManagedPageFlags + { + public const int PageBits = 12; + public const int PageSize = 1 << PageBits; + public const int PageMask = PageSize - 1; + + private readonly ulong[] _pageBitmap; + + public const int PageToPteShift = 5; // 32 pages (2 bits each) in one ulong page table entry. + public const ulong BlockMappedMask = 0x5555555555555555; // First bit of each table entry set. + + private enum ManagedPtBits : ulong + { + Unmapped = 0, + Mapped, + WriteTracked, + ReadWriteTracked, + + MappedReplicated = 0x5555555555555555, + WriteTrackedReplicated = 0xaaaaaaaaaaaaaaaa, + ReadWriteTrackedReplicated = ulong.MaxValue, + } + + public ManagedPageFlags(int addressSpaceBits) + { + int bits = Math.Max(0, addressSpaceBits - (PageBits + PageToPteShift)); + _pageBitmap = new ulong[1 << bits]; + } + + /// <summary> + /// Computes the number of pages in a virtual address range. + /// </summary> + /// <param name="va">Virtual address of the range</param> + /// <param name="size">Size of the range</param> + /// <param name="startVa">The virtual address of the beginning of the first page</param> + /// <remarks>This function does not differentiate between allocated and unallocated pages.</remarks> + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int GetPagesCount(ulong va, ulong size, out ulong startVa) + { + // WARNING: Always check if ulong does not overflow during the operations. + startVa = va & ~(ulong)PageMask; + ulong vaSpan = (va - startVa + size + PageMask) & ~(ulong)PageMask; + + return (int)(vaSpan / PageSize); + } + + /// <summary> + /// Checks if the page at a given CPU virtual address is mapped. + /// </summary> + /// <param name="va">Virtual address to check</param> + /// <returns>True if the address is mapped, false otherwise</returns> + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly bool IsMapped(ulong va) + { + ulong page = va >> PageBits; + + int bit = (int)((page & 31) << 1); + + int pageIndex = (int)(page >> PageToPteShift); + ref ulong pageRef = ref _pageBitmap[pageIndex]; + + ulong pte = Volatile.Read(ref pageRef); + + return ((pte >> bit) & 3) != 0; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void GetPageBlockRange(ulong pageStart, ulong pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex) + { + startMask = ulong.MaxValue << ((int)(pageStart & 31) << 1); + endMask = ulong.MaxValue >> (64 - ((int)(pageEnd & 31) << 1)); + + pageIndex = (int)(pageStart >> PageToPteShift); + pageEndIndex = (int)((pageEnd - 1) >> PageToPteShift); + } + + /// <summary> + /// Checks if a memory range is mapped. + /// </summary> + /// <param name="va">Virtual address of the range</param> + /// <param name="size">Size of the range in bytes</param> + /// <returns>True if the entire range is mapped, false otherwise</returns> + public readonly bool IsRangeMapped(ulong va, ulong size) + { + int pages = GetPagesCount(va, size, out _); + + if (pages == 1) + { + return IsMapped(va); + } + + ulong pageStart = va >> PageBits; + ulong pageEnd = pageStart + (ulong)pages; + + GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); + + // Check if either bit in each 2 bit page entry is set. + // OR the block with itself shifted down by 1, and check the first bit of each entry. + + ulong mask = BlockMappedMask & startMask; + + while (pageIndex <= pageEndIndex) + { + if (pageIndex == pageEndIndex) + { + mask &= endMask; + } + + ref ulong pageRef = ref _pageBitmap[pageIndex++]; + ulong pte = Volatile.Read(ref pageRef); + + pte |= pte >> 1; + if ((pte & mask) != mask) + { + return false; + } + + mask = BlockMappedMask; + } + + return true; + } + + /// <summary> + /// Reprotect a region of virtual memory for tracking. + /// </summary> + /// <param name="va">Virtual address base</param> + /// <param name="size">Size of the region to protect</param> + /// <param name="protection">Memory protection to set</param> + public readonly void TrackingReprotect(ulong va, ulong size, MemoryPermission protection) + { + // Protection is inverted on software pages, since the default value is 0. + protection = (~protection) & MemoryPermission.ReadAndWrite; + + int pages = GetPagesCount(va, size, out va); + ulong pageStart = va >> PageBits; + + if (pages == 1) + { + ulong protTag = protection switch + { + MemoryPermission.None => (ulong)ManagedPtBits.Mapped, + MemoryPermission.Write => (ulong)ManagedPtBits.WriteTracked, + _ => (ulong)ManagedPtBits.ReadWriteTracked, + }; + + int bit = (int)((pageStart & 31) << 1); + + ulong tagMask = 3UL << bit; + ulong invTagMask = ~tagMask; + + ulong tag = protTag << bit; + + int pageIndex = (int)(pageStart >> PageToPteShift); + ref ulong pageRef = ref _pageBitmap[pageIndex]; + + ulong pte; + + do + { + pte = Volatile.Read(ref pageRef); + } + while ((pte & tagMask) != 0 && Interlocked.CompareExchange(ref pageRef, (pte & invTagMask) | tag, pte) != pte); + } + else + { + ulong pageEnd = pageStart + (ulong)pages; + + GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); + + ulong mask = startMask; + + ulong protTag = protection switch + { + MemoryPermission.None => (ulong)ManagedPtBits.MappedReplicated, + MemoryPermission.Write => (ulong)ManagedPtBits.WriteTrackedReplicated, + _ => (ulong)ManagedPtBits.ReadWriteTrackedReplicated, + }; + + while (pageIndex <= pageEndIndex) + { + if (pageIndex == pageEndIndex) + { + mask &= endMask; + } + + ref ulong pageRef = ref _pageBitmap[pageIndex++]; + + ulong pte; + ulong mappedMask; + + // Change the protection of all 2 bit entries that are mapped. + do + { + pte = Volatile.Read(ref pageRef); + + mappedMask = pte | (pte >> 1); + mappedMask |= (mappedMask & BlockMappedMask) << 1; + mappedMask &= mask; // Only update mapped pages within the given range. + } + while (Interlocked.CompareExchange(ref pageRef, (pte & (~mappedMask)) | (protTag & mappedMask), pte) != pte); + + mask = ulong.MaxValue; + } + } + } + + /// <summary> + /// Alerts the memory tracking that a given region has been read from or written to. + /// This should be called before read/write is performed. + /// </summary> + /// <param name="tracking">Memory tracking structure to call when pages are protected</param> + /// <param name="va">Virtual address of the region</param> + /// <param name="size">Size of the region</param> + /// <param name="write">True if the region was written, false if read</param> + /// <param name="exemptId">Optional ID of the handles that should not be signalled</param> + /// <remarks> + /// This function also validates that the given range is both valid and mapped, and will throw if it is not. + /// </remarks> + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly void SignalMemoryTracking(MemoryTracking tracking, ulong va, ulong size, bool write, int? exemptId = null) + { + // Software table, used for managed memory tracking. + + int pages = GetPagesCount(va, size, out _); + ulong pageStart = va >> PageBits; + + if (pages == 1) + { + ulong tag = (ulong)(write ? ManagedPtBits.WriteTracked : ManagedPtBits.ReadWriteTracked); + + int bit = (int)((pageStart & 31) << 1); + + int pageIndex = (int)(pageStart >> PageToPteShift); + ref ulong pageRef = ref _pageBitmap[pageIndex]; + + ulong pte = Volatile.Read(ref pageRef); + ulong state = ((pte >> bit) & 3); + + if (state >= tag) + { + tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId); + return; + } + else if (state == 0) + { + ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}"); + } + } + else + { + ulong pageEnd = pageStart + (ulong)pages; + + GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); + + ulong mask = startMask; + + ulong anyTrackingTag = (ulong)ManagedPtBits.WriteTrackedReplicated; + + while (pageIndex <= pageEndIndex) + { + if (pageIndex == pageEndIndex) + { + mask &= endMask; + } + + ref ulong pageRef = ref _pageBitmap[pageIndex++]; + + ulong pte = Volatile.Read(ref pageRef); + ulong mappedMask = mask & BlockMappedMask; + + ulong mappedPte = pte | (pte >> 1); + if ((mappedPte & mappedMask) != mappedMask) + { + ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}"); + } + + pte &= mask; + if ((pte & anyTrackingTag) != 0) // Search for any tracking. + { + // Writes trigger any tracking. + // Only trigger tracking from reads if both bits are set on any page. + if (write || (pte & (pte >> 1) & BlockMappedMask) != 0) + { + tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId); + break; + } + } + + mask = ulong.MaxValue; + } + } + } + + /// <summary> + /// Adds the given address mapping to the page table. + /// </summary> + /// <param name="va">Virtual memory address</param> + /// <param name="size">Size to be mapped</param> + public readonly void AddMapping(ulong va, ulong size) + { + int pages = GetPagesCount(va, size, out _); + ulong pageStart = va >> PageBits; + ulong pageEnd = pageStart + (ulong)pages; + + GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); + + ulong mask = startMask; + + while (pageIndex <= pageEndIndex) + { + if (pageIndex == pageEndIndex) + { + mask &= endMask; + } + + ref ulong pageRef = ref _pageBitmap[pageIndex++]; + + ulong pte; + ulong mappedMask; + + // Map all 2-bit entries that are unmapped. + do + { + pte = Volatile.Read(ref pageRef); + + mappedMask = pte | (pte >> 1); + mappedMask |= (mappedMask & BlockMappedMask) << 1; + mappedMask |= ~mask; // Treat everything outside the range as mapped, thus unchanged. + } + while (Interlocked.CompareExchange(ref pageRef, (pte & mappedMask) | (BlockMappedMask & (~mappedMask)), pte) != pte); + + mask = ulong.MaxValue; + } + } + + /// <summary> + /// Removes the given address mapping from the page table. + /// </summary> + /// <param name="va">Virtual memory address</param> + /// <param name="size">Size to be unmapped</param> + public readonly void RemoveMapping(ulong va, ulong size) + { + int pages = GetPagesCount(va, size, out _); + ulong pageStart = va >> PageBits; + ulong pageEnd = pageStart + (ulong)pages; + + GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); + + startMask = ~startMask; + endMask = ~endMask; + + ulong mask = startMask; + + while (pageIndex <= pageEndIndex) + { + if (pageIndex == pageEndIndex) + { + mask |= endMask; + } + + ref ulong pageRef = ref _pageBitmap[pageIndex++]; + ulong pte; + + do + { + pte = Volatile.Read(ref pageRef); + } + while (Interlocked.CompareExchange(ref pageRef, pte & mask, pte) != pte); + + mask = 0; + } + } + + private static void ThrowInvalidMemoryRegionException(string message) => throw new InvalidMemoryRegionException(message); + } +} |