diff options
| author | gdkchan <gab.dark.100@gmail.com> | 2022-05-02 20:30:02 -0300 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2022-05-02 20:30:02 -0300 |
| commit | 95017b8c66f70406e926b278ecdd6d4ec0a93110 (patch) | |
| tree | 3fc8e9a515f9adda96cf0c9d6983a375045ba68c /Ryujinx.Memory/WindowsShared | |
| parent | 4a892fbdc9059504358ddf41c27576032e1ce414 (diff) | |
Support memory aliasing (#2954)
* Back to the origins: Make memory manager take guest PA rather than host address once again
* Direct mapping with alias support on Windows
* Fixes and remove more of the emulated shared memory
* Linux support
* Make shared and transfer memory not depend on SharedMemoryStorage
* More efficient view mapping on Windows (no more restricted to 4KB pages at a time)
* Handle potential access violations caused by partial unmap
* Implement host mapping using shared memory on Linux
* Add new GetPhysicalAddressChecked method, used to ensure the virtual address is mapped before address translation
Also align GetRef behaviour with software memory manager
* We don't need a mirrorable memory block for software memory manager mode
* Disable memory aliasing tests while we don't have shared memory support on Mac
* Shared memory & SIGBUS handler for macOS
* Fix typo + nits + re-enable memory tests
* Set MAP_JIT_DARWIN on x86 Mac too
* Add back the address space mirror
* Only set MAP_JIT_DARWIN if we are mapping as executable
* Disable aliasing tests again (still fails on Mac)
* Fix UnmapView4KB (by not casting size to int)
* Use ref counting on memory blocks to delay closing the shared memory handle until all blocks using it are disposed
* Address PR feedback
* Make RO hold a reference to the guest process memory manager to avoid early disposal
Co-authored-by: nastys <nastys@users.noreply.github.com>
Diffstat (limited to 'Ryujinx.Memory/WindowsShared')
| -rw-r--r-- | Ryujinx.Memory/WindowsShared/EmulatedSharedMemoryWindows.cs | 703 | ||||
| -rw-r--r-- | Ryujinx.Memory/WindowsShared/IntervalTree.cs | 740 | ||||
| -rw-r--r-- | Ryujinx.Memory/WindowsShared/PlaceholderList.cs | 293 | ||||
| -rw-r--r-- | Ryujinx.Memory/WindowsShared/PlaceholderManager.cs | 633 | ||||
| -rw-r--r-- | Ryujinx.Memory/WindowsShared/WindowsApi.cs | 93 | ||||
| -rw-r--r-- | Ryujinx.Memory/WindowsShared/WindowsApiException.cs | 24 |
6 files changed, 1490 insertions, 996 deletions
diff --git a/Ryujinx.Memory/WindowsShared/EmulatedSharedMemoryWindows.cs b/Ryujinx.Memory/WindowsShared/EmulatedSharedMemoryWindows.cs deleted file mode 100644 index 1417f7d5..00000000 --- a/Ryujinx.Memory/WindowsShared/EmulatedSharedMemoryWindows.cs +++ /dev/null @@ -1,703 +0,0 @@ -using Ryujinx.Memory.Range; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Runtime.InteropServices; - -namespace Ryujinx.Memory.WindowsShared -{ - class EmulatedSharedMemoryWindows : IDisposable - { - private static readonly IntPtr InvalidHandleValue = new IntPtr(-1); - private static readonly IntPtr CurrentProcessHandle = new IntPtr(-1); - - public const int MappingBits = 16; // Windows 64kb granularity. - public const ulong MappingGranularity = 1 << MappingBits; - public const ulong MappingMask = MappingGranularity - 1; - - public const ulong BackingSize32GB = 32UL * 1024UL * 1024UL * 1024UL; // Reasonable max size of 32GB. - - private class SharedMemoryMapping : INonOverlappingRange - { - public ulong Address { get; } - - public ulong Size { get; private set; } - - public ulong EndAddress { get; private set; } - - public List<int> Blocks; - - public SharedMemoryMapping(ulong address, ulong size, List<int> blocks = null) - { - Address = address; - Size = size; - EndAddress = address + size; - - Blocks = blocks ?? new List<int>(); - } - - public bool OverlapsWith(ulong address, ulong size) - { - return Address < address + size && address < EndAddress; - } - - public void ExtendTo(ulong endAddress, RangeList<SharedMemoryMapping> list) - { - EndAddress = endAddress; - Size = endAddress - Address; - - list.UpdateEndAddress(this); - } - - public void AddBlocks(IEnumerable<int> blocks) - { - if (Blocks.Count > 0 && blocks.Count() > 0 && Blocks.Last() == blocks.First()) - { - Blocks.AddRange(blocks.Skip(1)); - } - else - { - Blocks.AddRange(blocks); - } - } - - public INonOverlappingRange Split(ulong splitAddress) - { - SharedMemoryMapping newRegion = new SharedMemoryMapping(splitAddress, EndAddress - splitAddress); - - int end = (int)((EndAddress + MappingMask) >> MappingBits); - int start = (int)(Address >> MappingBits); - - Size = splitAddress - Address; - EndAddress = splitAddress; - - int splitEndBlock = (int)((splitAddress + MappingMask) >> MappingBits); - int splitStartBlock = (int)(splitAddress >> MappingBits); - - newRegion.AddBlocks(Blocks.Skip(splitStartBlock - start)); - Blocks.RemoveRange(splitEndBlock - start, end - splitEndBlock); - - return newRegion; - } - } - - [DllImport("kernel32.dll", SetLastError = true)] - private static extern IntPtr CreateFileMapping( - IntPtr hFile, - IntPtr lpFileMappingAttributes, - FileMapProtection flProtect, - uint dwMaximumSizeHigh, - uint dwMaximumSizeLow, - [MarshalAs(UnmanagedType.LPWStr)] string lpName); - - [DllImport("kernel32.dll", SetLastError = true)] - private static extern bool CloseHandle(IntPtr hObject); - - [DllImport("KernelBase.dll", SetLastError = true)] - private static extern IntPtr VirtualAlloc2( - IntPtr process, - IntPtr lpAddress, - IntPtr dwSize, - AllocationType flAllocationType, - MemoryProtection flProtect, - IntPtr extendedParameters, - ulong parameterCount); - - [DllImport("kernel32.dll", SetLastError = true)] - private static extern bool VirtualFree(IntPtr lpAddress, IntPtr dwSize, AllocationType dwFreeType); - - [DllImport("KernelBase.dll", SetLastError = true)] - private static extern IntPtr MapViewOfFile3( - IntPtr hFileMappingObject, - IntPtr process, - IntPtr baseAddress, - ulong offset, - IntPtr dwNumberOfBytesToMap, - ulong allocationType, - MemoryProtection dwDesiredAccess, - IntPtr extendedParameters, - ulong parameterCount); - - [DllImport("KernelBase.dll", SetLastError = true)] - private static extern bool UnmapViewOfFile2(IntPtr process, IntPtr lpBaseAddress, ulong unmapFlags); - - private ulong _size; - - private object _lock = new object(); - - private ulong _backingSize; - private IntPtr _backingMemHandle; - private int _backingEnd; - private int _backingAllocated; - private Queue<int> _backingFreeList; - - private List<ulong> _mappedBases; - private RangeList<SharedMemoryMapping> _mappings; - private SharedMemoryMapping[] _foundMappings = new SharedMemoryMapping[32]; - private PlaceholderList _placeholders; - - public EmulatedSharedMemoryWindows(ulong size) - { - ulong backingSize = BackingSize32GB; - - _size = size; - _backingSize = backingSize; - - _backingMemHandle = CreateFileMapping( - InvalidHandleValue, - IntPtr.Zero, - FileMapProtection.PageReadWrite | FileMapProtection.SectionReserve, - (uint)(backingSize >> 32), - (uint)backingSize, - null); - - if (_backingMemHandle == IntPtr.Zero) - { - throw new OutOfMemoryException(); - } - - _backingFreeList = new Queue<int>(); - _mappings = new RangeList<SharedMemoryMapping>(); - _mappedBases = new List<ulong>(); - _placeholders = new PlaceholderList(size >> MappingBits); - } - - private (ulong granularStart, ulong granularEnd) GetAlignedRange(ulong address, ulong size) - { - return (address & (~MappingMask), (address + size + MappingMask) & (~MappingMask)); - } - - private void Commit(ulong address, ulong size) - { - (ulong granularStart, ulong granularEnd) = GetAlignedRange(address, size); - - ulong endAddress = address + size; - - lock (_lock) - { - // Search a bit before and after the new mapping. - // When adding our new mapping, we may need to join an existing mapping into our new mapping (or in some cases, to the other side!) - ulong searchStart = granularStart == 0 ? 0 : (granularStart - 1); - int mappingCount = _mappings.FindOverlapsNonOverlapping(searchStart, (granularEnd - searchStart) + 1, ref _foundMappings); - - int first = -1; - int last = -1; - SharedMemoryMapping startOverlap = null; - SharedMemoryMapping endOverlap = null; - - int lastIndex = (int)(address >> MappingBits); - int endIndex = (int)((endAddress + MappingMask) >> MappingBits); - int firstBlock = -1; - int endBlock = -1; - - for (int i = 0; i < mappingCount; i++) - { - SharedMemoryMapping mapping = _foundMappings[i]; - - if (mapping.Address < address) - { - if (mapping.EndAddress >= address) - { - startOverlap = mapping; - } - - if ((int)((mapping.EndAddress - 1) >> MappingBits) == lastIndex) - { - lastIndex = (int)((mapping.EndAddress + MappingMask) >> MappingBits); - firstBlock = mapping.Blocks.Last(); - } - } - - if (mapping.EndAddress > endAddress) - { - if (mapping.Address <= endAddress) - { - endOverlap = mapping; - } - - if ((int)((mapping.Address) >> MappingBits) + 1 == endIndex) - { - endIndex = (int)((mapping.Address) >> MappingBits); - endBlock = mapping.Blocks.First(); - } - } - - if (mapping.OverlapsWith(address, size)) - { - if (first == -1) - { - first = i; - } - - last = i; - } - } - - if (startOverlap == endOverlap && startOverlap != null) - { - // Already fully committed. - return; - } - - var blocks = new List<int>(); - int lastBlock = -1; - - if (firstBlock != -1) - { - blocks.Add(firstBlock); - lastBlock = firstBlock; - } - - bool hasMapped = false; - Action map = () => - { - if (!hasMapped) - { - _placeholders.EnsurePlaceholders(address >> MappingBits, (granularEnd - granularStart) >> MappingBits, SplitPlaceholder); - hasMapped = true; - } - - // There's a gap between this index and the last. Allocate blocks to fill it. - blocks.Add(MapBackingBlock(MappingGranularity * (ulong)lastIndex++)); - }; - - if (first != -1) - { - for (int i = first; i <= last; i++) - { - SharedMemoryMapping mapping = _foundMappings[i]; - int mapIndex = (int)(mapping.Address >> MappingBits); - - while (lastIndex < mapIndex) - { - map(); - } - - if (lastBlock == mapping.Blocks[0]) - { - blocks.AddRange(mapping.Blocks.Skip(1)); - } - else - { - blocks.AddRange(mapping.Blocks); - } - - lastIndex = (int)((mapping.EndAddress - 1) >> MappingBits) + 1; - } - } - - while (lastIndex < endIndex) - { - map(); - } - - if (endBlock != -1 && endBlock != lastBlock) - { - blocks.Add(endBlock); - } - - if (startOverlap != null && endOverlap != null) - { - // Both sides should be coalesced. Extend the start overlap to contain the end overlap, and add together their blocks. - - _mappings.Remove(endOverlap); - - startOverlap.ExtendTo(endOverlap.EndAddress, _mappings); - - startOverlap.AddBlocks(blocks); - startOverlap.AddBlocks(endOverlap.Blocks); - } - else if (startOverlap != null) - { - startOverlap.ExtendTo(endAddress, _mappings); - - startOverlap.AddBlocks(blocks); - } - else - { - var mapping = new SharedMemoryMapping(address, size, blocks); - - if (endOverlap != null) - { - mapping.ExtendTo(endOverlap.EndAddress, _mappings); - - mapping.AddBlocks(endOverlap.Blocks); - - _mappings.Remove(endOverlap); - } - - _mappings.Add(mapping); - } - } - } - - private void Decommit(ulong address, ulong size) - { - (ulong granularStart, ulong granularEnd) = GetAlignedRange(address, size); - ulong endAddress = address + size; - - lock (_lock) - { - int mappingCount = _mappings.FindOverlapsNonOverlapping(granularStart, granularEnd - granularStart, ref _foundMappings); - - int first = -1; - int last = -1; - - for (int i = 0; i < mappingCount; i++) - { - SharedMemoryMapping mapping = _foundMappings[i]; - - if (mapping.OverlapsWith(address, size)) - { - if (first == -1) - { - first = i; - } - - last = i; - } - } - - if (first == -1) - { - return; // Could not find any regions to decommit. - } - - int lastReleasedBlock = -1; - - bool releasedFirst = false; - bool releasedLast = false; - - for (int i = last; i >= first; i--) - { - SharedMemoryMapping mapping = _foundMappings[i]; - bool releaseEnd = true; - bool releaseStart = true; - - if (i == last) - { - // If this is the last region, do not release the block if there is a page ahead of us, or the block continues after us. (it is keeping the block alive) - releaseEnd = last == mappingCount - 1; - - // If the end region starts after the decommit end address, split and readd it after modifying its base address. - if (mapping.EndAddress > endAddress) - { - var newMapping = (SharedMemoryMapping)mapping.Split(endAddress); - _mappings.UpdateEndAddress(mapping); - _mappings.Add(newMapping); - - if ((endAddress & MappingMask) != 0) - { - releaseEnd = false; - } - } - - releasedLast = releaseEnd; - } - - if (i == first) - { - // If this is the first region, do not release the block if there is a region behind us. (it is keeping the block alive) - releaseStart = first == 0; - - // If the first region starts before the decommit address, split it by modifying its end address. - if (mapping.Address < address) - { - var oldMapping = mapping; - mapping = (SharedMemoryMapping)mapping.Split(address); - _mappings.UpdateEndAddress(oldMapping); - - if ((address & MappingMask) != 0) - { - releaseStart = false; - } - } - - releasedFirst = releaseStart; - } - - _mappings.Remove(mapping); - - ulong releasePointer = (mapping.EndAddress + MappingMask) & (~MappingMask); - for (int j = mapping.Blocks.Count - 1; j >= 0; j--) - { - int blockId = mapping.Blocks[j]; - - releasePointer -= MappingGranularity; - - if (lastReleasedBlock == blockId) - { - // When committed regions are fragmented, multiple will have the same block id for their start/end granular block. - // Avoid releasing these blocks twice. - continue; - } - - if ((j != 0 || releaseStart) && (j != mapping.Blocks.Count - 1 || releaseEnd)) - { - ReleaseBackingBlock(releasePointer, blockId); - } - - lastReleasedBlock = blockId; - } - } - - ulong placeholderStart = (granularStart >> MappingBits) + (releasedFirst ? 0UL : 1UL); - ulong placeholderEnd = (granularEnd >> MappingBits) - (releasedLast ? 0UL : 1UL); - - if (placeholderEnd > placeholderStart) - { - _placeholders.RemovePlaceholders(placeholderStart, placeholderEnd - placeholderStart, CoalescePlaceholder); - } - } - } - - public bool CommitMap(IntPtr address, IntPtr size) - { - lock (_lock) - { - foreach (ulong mapping in _mappedBases) - { - ulong offset = (ulong)address - mapping; - - if (offset < _size) - { - Commit(offset, (ulong)size); - return true; - } - } - } - - return false; - } - - public bool DecommitMap(IntPtr address, IntPtr size) - { - lock (_lock) - { - foreach (ulong mapping in _mappedBases) - { - ulong offset = (ulong)address - mapping; - - if (offset < _size) - { - Decommit(offset, (ulong)size); - return true; - } - } - } - - return false; - } - - private int MapBackingBlock(ulong offset) - { - bool allocate = false; - int backing; - - if (_backingFreeList.Count > 0) - { - backing = _backingFreeList.Dequeue(); - } - else - { - if (_backingAllocated == _backingEnd) - { - // Allocate the backing. - _backingAllocated++; - allocate = true; - } - - backing = _backingEnd++; - } - - ulong backingOffset = MappingGranularity * (ulong)backing; - - foreach (ulong baseAddress in _mappedBases) - { - CommitToMap(baseAddress, offset, MappingGranularity, backingOffset, allocate); - allocate = false; - } - - return backing; - } - - private void ReleaseBackingBlock(ulong offset, int id) - { - foreach (ulong baseAddress in _mappedBases) - { - DecommitFromMap(baseAddress, offset); - } - - if (_backingEnd - 1 == id) - { - _backingEnd = id; - } - else - { - _backingFreeList.Enqueue(id); - } - } - - public IntPtr Map() - { - IntPtr newMapping = VirtualAlloc2( - CurrentProcessHandle, - IntPtr.Zero, - (IntPtr)_size, - AllocationType.Reserve | AllocationType.ReservePlaceholder, - MemoryProtection.NoAccess, - IntPtr.Zero, - 0); - - if (newMapping == IntPtr.Zero) - { - throw new OutOfMemoryException(); - } - - // Apply all existing mappings to the new mapping - lock (_lock) - { - int lastBlock = -1; - foreach (SharedMemoryMapping mapping in _mappings) - { - ulong blockAddress = mapping.Address & (~MappingMask); - foreach (int block in mapping.Blocks) - { - if (block != lastBlock) - { - ulong backingOffset = MappingGranularity * (ulong)block; - - CommitToMap((ulong)newMapping, blockAddress, MappingGranularity, backingOffset, false); - - lastBlock = block; - } - - blockAddress += MappingGranularity; - } - } - - _mappedBases.Add((ulong)newMapping); - } - - return newMapping; - } - - private void SplitPlaceholder(ulong address, ulong size) - { - ulong byteAddress = address << MappingBits; - IntPtr byteSize = (IntPtr)(size << MappingBits); - - foreach (ulong mapAddress in _mappedBases) - { - bool result = VirtualFree((IntPtr)(mapAddress + byteAddress), byteSize, AllocationType.PreservePlaceholder | AllocationType.Release); - - if (!result) - { - throw new InvalidOperationException("Placeholder could not be split."); - } - } - } - - private void CoalescePlaceholder(ulong address, ulong size) - { - ulong byteAddress = address << MappingBits; - IntPtr byteSize = (IntPtr)(size << MappingBits); - - foreach (ulong mapAddress in _mappedBases) - { - bool result = VirtualFree((IntPtr)(mapAddress + byteAddress), byteSize, AllocationType.CoalescePlaceholders | AllocationType.Release); - - if (!result) - { - throw new InvalidOperationException("Placeholder could not be coalesced."); - } - } - } - - private void CommitToMap(ulong mapAddress, ulong address, ulong size, ulong backingOffset, bool allocate) - { - IntPtr targetAddress = (IntPtr)(mapAddress + address); - - // Assume the placeholder worked (or already exists) - // Map the backing memory into the mapped location. - - IntPtr mapped = MapViewOfFile3( - _backingMemHandle, - CurrentProcessHandle, - targetAddress, - backingOffset, - (IntPtr)MappingGranularity, - 0x4000, // REPLACE_PLACEHOLDER - MemoryProtection.ReadWrite, - IntPtr.Zero, - 0); - - if (mapped == IntPtr.Zero) - { - throw new InvalidOperationException($"Could not map view of backing memory. (va=0x{address:X16} size=0x{size:X16}, error code {Marshal.GetLastWin32Error()})"); - } - - if (allocate) - { - // Commit this part of the shared memory. - VirtualAlloc2(CurrentProcessHandle, targetAddress, (IntPtr)MappingGranularity, AllocationType.Commit, MemoryProtection.ReadWrite, IntPtr.Zero, 0); - } - } - - private void DecommitFromMap(ulong baseAddress, ulong address) - { - UnmapViewOfFile2(CurrentProcessHandle, (IntPtr)(baseAddress + address), 2); - } - - public bool Unmap(ulong baseAddress) - { - lock (_lock) - { - if (_mappedBases.Remove(baseAddress)) - { - int lastBlock = -1; - - foreach (SharedMemoryMapping mapping in _mappings) - { - ulong blockAddress = mapping.Address & (~MappingMask); - foreach (int block in mapping.Blocks) - { - if (block != lastBlock) - { - DecommitFromMap(baseAddress, blockAddress); - - lastBlock = block; - } - - blockAddress += MappingGranularity; - } - } - - if (!VirtualFree((IntPtr)baseAddress, (IntPtr)0, AllocationType.Release)) - { - throw new InvalidOperationException("Couldn't free mapping placeholder."); - } - - return true; - } - - return false; - } - } - - public void Dispose() - { - // Remove all file mappings - lock (_lock) - { - foreach (ulong baseAddress in _mappedBases.ToArray()) - { - Unmap(baseAddress); - } - } - - // Finally, delete the file mapping. - CloseHandle(_backingMemHandle); - } - } -} diff --git a/Ryujinx.Memory/WindowsShared/IntervalTree.cs b/Ryujinx.Memory/WindowsShared/IntervalTree.cs new file mode 100644 index 00000000..eac5c545 --- /dev/null +++ b/Ryujinx.Memory/WindowsShared/IntervalTree.cs @@ -0,0 +1,740 @@ +using System; +using System.Collections.Generic; + +namespace Ryujinx.Memory.WindowsShared +{ + /// <summary> + /// An Augmented Interval Tree based off of the "TreeDictionary"'s Red-Black Tree. Allows fast overlap checking of ranges. + /// </summary> + /// <typeparam name="K">Key</typeparam> + /// <typeparam name="V">Value</typeparam> + class IntervalTree<K, V> where K : IComparable<K> + { + private const int ArrayGrowthSize = 32; + + private const bool Black = true; + private const bool Red = false; + private IntervalTreeNode<K, V> _root = null; + private int _count = 0; + + public int Count => _count; + + public IntervalTree() { } + + #region Public Methods + + /// <summary> + /// Gets the values of the interval whose key is <paramref name="key"/>. + /// </summary> + /// <param name="key">Key of the node value to get</param> + /// <param name="value">Value with the given <paramref name="key"/></param> + /// <returns>True if the key is on the dictionary, false otherwise</returns> + public bool TryGet(K key, out V value) + { + IntervalTreeNode<K, V> node = GetNode(key); + + if (node == null) + { + value = default; + return false; + } + + value = node.Value; + return true; + } + + /// <summary> + /// Returns the start addresses of the intervals whose start and end keys overlap the given range. + /// </summary> + /// <param name="start">Start of the range</param> + /// <param name="end">End of the range</param> + /// <param name="overlaps">Overlaps array to place results in</param> + /// <param name="overlapCount">Index to start writing results into the array. Defaults to 0</param> + /// <returns>Number of intervals found</returns> + public int Get(K start, K end, ref IntervalTreeNode<K, V>[] overlaps, int overlapCount = 0) + { + GetNodes(_root, start, end, ref overlaps, ref overlapCount); + + return overlapCount; + } + + /// <summary> + /// Adds a new interval into the tree whose start is <paramref name="start"/>, end is <paramref name="end"/> and value is <paramref name="value"/>. + /// </summary> + /// <param name="start">Start of the range to add</param> + /// <param name="end">End of the range to insert</param> + /// <param name="value">Value to add</param> + /// <exception cref="ArgumentNullException"><paramref name="value"/> is null</exception> + public void Add(K start, K end, V value) + { + if (value == null) + { + throw new ArgumentNullException(nameof(value)); + } + + BSTInsert(start, end, value, null, out _); + } + + /// <summary> + /// Removes a value from the tree, searching for it with <paramref name="key"/>. + /// </summary> + /// <param name="key">Key of the node to remove</param> + /// <returns>Number of deleted values</returns> + public int Remove(K key) + { + return Remove(GetNode(key)); + } + + /// <summary> + /// Removes a value from the tree, searching for it with <paramref name="key"/>. + /// </summary> + /// <param name="nodeToDelete">Node to be removed</param> + /// <returns>Number of deleted values</returns> + public int Remove(IntervalTreeNode<K, V> nodeToDelete) + { + if (nodeToDelete == null) + { + return 0; + } + + Delete(nodeToDelete); + + _count--; + + return 1; + } + + /// <summary> + /// Adds all the nodes in the dictionary into <paramref name="list"/>. + /// </summary> + /// <returns>A list of all values sorted by Key Order</returns> + public List<V> AsList() + { + List<V> list = new List<V>(); + + AddToList(_root, list); + + return list; + } + + #endregion + + #region Private Methods (BST) + + /// <summary> + /// Adds all values that are children of or contained within <paramref name="node"/> into <paramref name="list"/>, in Key Order. + /// </summary> + /// <param name="node">The node to search for values within</param> + /// <param name="list">The list to add values to</param> + private void AddToList(IntervalTreeNode<K, V> node, List<V> list) + { + if (node == null) + { + return; + } + + AddToList(node.Left, list); + + list.Add(node.Value); + + AddToList(node.Right, list); + } + + /// <summary> + /// Retrieve the node reference whose key is <paramref name="key"/>, or null if no such node exists. + /// </summary> + /// <param name="key">Key of the node to get</param> + /// <exception cref="ArgumentNullException"><paramref name="key"/> is null</exception> + /// <returns>Node reference in the tree</returns> + private IntervalTreeNode<K, V> GetNode(K key) + { + if (key == null) + { + throw new ArgumentNullException(nameof(key)); + } + + IntervalTreeNode<K, V> node = _root; + while (node != null) + { + int cmp = key.CompareTo(node.Start); + if (cmp < 0) + { + node = node.Left; + } + else if (cmp > 0) + { + node = node.Right; + } + else + { + return node; + } + } + return null; + } + + /// <summary> + /// Retrieve all nodes that overlap the given start and end keys. + /// </summary> + /// <param name="start">Start of the range</param> + /// <param name="end">End of the range</param> + /// <param name="overlaps">Overlaps array to place results in</param> + /// <param name="overlapCount">Overlaps count to update</param> + private void GetNodes(IntervalTreeNode<K, V> node, K start, K end, ref IntervalTreeNode<K, V>[] overlaps, ref int overlapCount) + { + if (node == null || start.CompareTo(node.Max) >= 0) + { + return; + } + + GetNodes(node.Left, start, end, ref overlaps, ref overlapCount); + + bool endsOnRight = end.CompareTo(node.Start) > 0; + if (endsOnRight) + { + if (start.CompareTo(node.End) < 0) + { + if (overlaps.Length >= overlapCount) + { + Array.Resize(ref overlaps, overlapCount + ArrayGrowthSize); + } + + overlaps[overlapCount++] = node; + } + + GetNodes(node.Right, start, end, ref overlaps, ref overlapCount); + } + } + + /// <summary> + /// Propagate an increase in max value starting at the given node, heading up the tree. + /// This should only be called if the max increases - not for rebalancing or removals. + /// </summary> + /// <param name="node">The node to start propagating from</param> + private void PropagateIncrease(IntervalTreeNode<K, V> node) + { + K max = node.Max; + IntervalTreeNode<K, V> ptr = node; + + while ((ptr = ptr.Parent) != null) + { + if (max.CompareTo(ptr.Max) > 0) + { + ptr.Max = max; + } + else + { + break; + } + } + } + + /// <summary> + /// Propagate recalculating max value starting at the given node, heading up the tree. + /// This fully recalculates the max value from all children when there is potential for it to decrease. + /// </summary> + /// <param name="node">The node to start propagating from</param> + private void PropagateFull(IntervalTreeNode<K, V> node) + { + IntervalTreeNode<K, V> ptr = node; + + do + { + K max = ptr.End; + + if (ptr.Left != null && ptr.Left.Max.CompareTo(max) > 0) + { + max = ptr.Left.Max; + } + + if (ptr.Right != null && ptr.Right.Max.CompareTo(max) > 0) + { + max = ptr.Right.Max; + } + + ptr.Max = max; + } while ((ptr = ptr.Parent) != null); + } + + /// <summary> + /// Insertion Mechanism for the interval tree. Similar to a BST insert, with the start of the range as the key. + /// Iterates the tree starting from the root and inserts a new node where all children in the left subtree are less than <paramref name="start"/>, and all children in the right subtree are greater than <paramref name="start"/>. + /// Each node can contain multiple values, and has an end address which is the maximum of all those values. + /// Post insertion, the "max" value of the node and all parents are updated. + /// </summary> + /// <param name="start">Start of the range to insert</param> + /// <param name="end">End of the range to insert</param> + /// <param name="value">Value to insert</param> + /// <param name="updateFactoryCallback">Optional factory used to create a new value if <paramref name="start"/> is already on the tree</param> + /// <param name="outNode">Node that was inserted or modified</param> + /// <returns>True if <paramref name="start"/> was not yet on the tree, false otherwise</returns> + private bool BSTInsert(K start, K end, V value, Func<K, V, V> updateFactoryCallback, out IntervalTreeNode<K, V> outNode) + { + IntervalTreeNode<K, V> parent = null; + IntervalTreeNode<K, V> node = _root; + + while (node != null) + { + parent = node; + int cmp = start.CompareTo(node.Start); + if (cmp < 0) + { + node = node.Left; + } + else if (cmp > 0) + { + node = node.Right; + } + else + { + outNode = node; + + if (updateFactoryCallback != null) + { + // Replace + node.Value = updateFactoryCallback(start, node.Value); + + int endCmp = end.CompareTo(node.End); + + if (endCmp > 0) + { + node.End = end; + if (end.CompareTo(node.Max) > 0) + { + node.Max = end; + PropagateIncrease(node); + RestoreBalanceAfterInsertion(node); + } + } + else if (endCmp < 0) + { + node.End = end; + PropagateFull(node); + } + } + + return false; + } + } + IntervalTreeNode<K, V> newNode = new IntervalTreeNode<K, V>(start, end, value, parent); + if (newNode.Parent == null) + { + _root = newNode; + } + else if (start.CompareTo(parent.Start) < 0) + { + parent.Left = newNode; + } + else + { + parent.Right = newNode; + } + + PropagateIncrease(newNode); + _count++; + RestoreBalanceAfterInsertion(newNode); + outNode = newNode; + return true; + } + + /// <summary> + /// Removes the value from the dictionary after searching for it with <paramref name="key">. + /// </summary> + /// <param name="key">Tree node to be removed</param> + private void Delete(IntervalTreeNode<K, V> nodeToDelete) + { + IntervalTreeNode<K, V> replacementNode; + + if (LeftOf(nodeToDelete) == null || RightOf(nodeToDelete) == null) + { + replacementNode = nodeToDelete; + } + else + { + replacementNode = PredecessorOf(nodeToDelete); + } + + IntervalTreeNode<K, V> tmp = LeftOf(replacementNode) ?? RightOf(replacementNode); + + if (tmp != null) + { + tmp.Parent = ParentOf(replacementNode); + } + + if (ParentOf(replacementNode) == null) + { + _root = tmp; + } + else if (replacementNode == LeftOf(ParentOf(replacementNode))) + { + ParentOf(replacementNode).Left = tmp; + } + else + { + ParentOf(replacementNode).Right = tmp; + } + + if (replacementNode != nodeToDelete) + { + nodeToDelete.Start = replacementNode.Start; + nodeToDelete.Value = replacementNode.Value; + nodeToDelete.End = replacementNode.End; + nodeToDelete.Max = replacementNode.Max; + } + + PropagateFull(replacementNode); + + if (tmp != null && ColorOf(replacementNode) == Black) + { + RestoreBalanceAfterRemoval(tmp); + } + } + + /// <summary> + /// Returns the node with the largest key where <paramref name="node"/> is considered the root node. + /// </summary> + /// <param name="node">Root Node</param> + /// <returns>Node with the maximum key in the tree of <paramref name="node"/></returns> + private static IntervalTreeNode<K, V> Maximum(IntervalTreeNode<K, V> node) + { + IntervalTreeNode<K, V> tmp = node; + while (tmp.Right != null) + { + tmp = tmp.Right; + } + + return tmp; + } + + /// <summary> + /// Finds the node whose key is immediately less than <paramref name="node"/>. + /// </summary> + /// <param name="node">Node to find the predecessor of</param> + /// <returns>Predecessor of <paramref name="node"/></returns> + private static IntervalTreeNode<K, V> PredecessorOf(IntervalTreeNode<K, V> node) + { + if (node.Left != null) + { + return Maximum(node.Left); + } + IntervalTreeNode<K, V> parent = node.Parent; + while (parent != null && node == parent.Left) + { + node = parent; + parent = parent.Parent; + } + return parent; + } + + #endregion + + #region Private Methods (RBL) + + private void RestoreBalanceAfterRemoval(IntervalTreeNode<K, V> balanceNode) + { + IntervalTreeNode<K, V> ptr = balanceNode; + + while (ptr != _root && ColorOf(ptr) == Black) + { + if (ptr == LeftOf(ParentOf(ptr))) + { + IntervalTreeNode<K, V> sibling = RightOf(ParentOf(ptr)); + + if (ColorOf(sibling) == Red) + { + SetColor(sibling, Black); + SetColor(ParentOf(ptr), Red); + RotateLeft(ParentOf(ptr)); + sibling = RightOf(ParentOf(ptr)); + } + if (ColorOf(LeftOf(sibling)) == Black && ColorOf(RightOf(sibling)) == Black) + { + SetColor(sibling, Red); + ptr = ParentOf(ptr); + } + else + { + if (ColorOf(RightOf(sibling)) == Black) + { + SetColor(LeftOf(sibling), Black); + SetColor(sibling, Red); + RotateRight(sibling); + sibling = RightOf(ParentOf(ptr)); + } + SetColor(sibling, ColorOf(ParentOf(ptr))); + SetColor(ParentOf(ptr), Black); + SetColor(RightOf(sibling), Black); + RotateLeft(ParentOf(ptr)); + ptr = _root; + } + } + else + { + IntervalTreeNode<K, V> sibling = LeftOf(ParentOf(ptr)); + + if (ColorOf(sibling) == Red) + { + SetColor(sibling, Black); + SetColor(ParentOf(ptr), Red); + RotateRight(ParentOf(ptr)); + sibling = LeftOf(ParentOf(ptr)); + } + if (ColorOf(RightOf(sibling)) == Black && ColorOf(LeftOf(sibling)) == Black) + { + SetColor(sibling, Red); + ptr = ParentOf(ptr); + } + else + { + if (ColorOf(LeftOf(sibling)) == Black) + { + SetColor(RightOf(sibling), Black); + SetColor(sibling, Red); + RotateLeft(sibling); + sibling = LeftOf(ParentOf(ptr)); + } + SetColor(sibling, ColorOf(ParentOf(ptr))); + SetColor(ParentOf(ptr), Black); + SetColor(LeftOf(sibling), Black); + RotateRight(ParentOf(ptr)); + ptr = _root; + } + } + } + SetColor(ptr, Black); + } + + private void RestoreBalanceAfterInsertion(IntervalTreeNode<K, V> balanceNode) + { + SetColor(balanceNode, Red); + while (balanceNode != null && balanceNode != _root && ColorOf(ParentOf(balanceNode)) == Red) + { + if (ParentOf(balanceNode) == LeftOf(ParentOf(ParentOf(balanceNode)))) + { + IntervalTreeNode<K, V> sibling = RightOf(ParentOf(ParentOf(balanceNode))); + + if (ColorOf(sibling) == Red) + { + SetColor(ParentOf(balanceNode), Black); + SetColor(sibling, Black); + SetColor(ParentOf(ParentOf(balanceNode)), Red); + balanceNode = ParentOf(ParentOf(balanceNode)); + } + else + { + if (balanceNode == RightOf(ParentOf(balanceNode))) + { + balanceNode = ParentOf(balanceNode); + RotateLeft(balanceNode); + } + SetColor(ParentOf(balanceNode), Black); + SetColor(ParentOf(ParentOf(balanceNode)), Red); + RotateRight(ParentOf(ParentOf(balanceNode))); + } + } + else + { + IntervalTreeNode<K, V> sibling = LeftOf(ParentOf(ParentOf(balanceNode))); + + if (ColorOf(sibling) == Red) + { + SetColor(ParentOf(balanceNode), Black); + SetColor(sibling, Black); + SetColor(ParentOf(ParentOf(balanceNode)), Red); + balanceNode = ParentOf(ParentOf(balanceNode)); + } + else + { + if (balanceNode == LeftOf(ParentOf(balanceNode))) + { + balanceNode = ParentOf(balanceNode); + RotateRight(balanceNode); + } + SetColor(ParentOf(balanceNode), Black); + SetColor(ParentOf(ParentOf(balanceNode)), Red); + RotateLeft(ParentOf(ParentOf(balanceNode))); + } + } + } + SetColor(_root, Black); + } + + private void RotateLeft(IntervalTreeNode<K, V> node) + { + if (node != null) + { + IntervalTreeNode<K, V> right = RightOf(node); + node.Right = LeftOf(right); + if (node.Right != null) + { + node.Right.Parent = node; + } + IntervalTreeNode<K, V> nodeParent = ParentOf(node); + right.Parent = nodeParent; + if (nodeParent == null) + { + _root = right; + } + else if (node == LeftOf(nodeParent)) + { + nodeParent.Left = right; + } + else + { + nodeParent.Right = right; + } + right.Left = node; + node.Parent = right; + + PropagateFull(node); + } + } + + private void RotateRight(IntervalTreeNode<K, V> node) + { + if (node != null) + { + IntervalTreeNode<K, V> left = LeftOf(node); + node.Left = RightOf(left); + if (node.Left != null) + { + node.Left.Parent = node; + } + IntervalTreeNode<K, V> nodeParent = ParentOf(node); + left.Parent = nodeParent; + if (nodeParent == null) + { + _root = left; + } + else if (node == RightOf(nodeParent)) + { + nodeParent.Right = left; + } + else + { + nodeParent.Left = left; + } + left.Right = node; + node.Parent = left; + + PropagateFull(node); + } + } + + #endregion + + #region Safety-Methods + + // These methods save memory by allowing us to forego sentinel nil nodes, as well as serve as protection against NullReferenceExceptions. + + /// <summary> + /// Returns the color of <paramref name="node"/>, or Black if it is null. + /// </summary> + /// <param name="node">Node</param> + /// <returns>The boolean color of <paramref name="node"/>, or black if null</returns> + private static bool ColorOf(IntervalTreeNode<K, V> node) + { + return node == null || node.Color; + } + + /// <summary> + /// Sets the color of <paramref name="node"/> node to <paramref name="color"/>. + /// <br></br> + /// This method does nothing if <paramref name="node"/> is null. + /// </summary> + /// <param name="node">Node to set the color of</param> + /// <param name="color">Color (Boolean)</param> + private static void SetColor(IntervalTreeNode<K, V> node, bool color) + { + if (node != null) + { + node.Color = color; + } + } + + /// <summary> + /// This method returns the left node of <paramref name="node"/>, or null if <paramref name="node"/> is null. + /// </summary> + /// <param name="node">Node to retrieve the left child from</param> + /// <returns>Left child of <paramref name="node"/></returns> + private static IntervalTreeNode<K, V> LeftOf(IntervalTreeNode<K, V> node) + { + return node?.Left; + } + + /// <summary> + /// This method returns the right node of <paramref name="node"/>, or null if <paramref name="node"/> is null. + /// </summary> + /// <param name="node">Node to retrieve the right child from</param> + /// <returns>Right child of <paramref name="node"/></returns> + private static IntervalTreeNode<K, V> RightOf(IntervalTreeNode<K, V> node) + { + return node?.Right; + } + + /// <summary> + /// Returns the parent node of <paramref name="node"/>, or null if <paramref name="node"/> is null. + /// </summary> + /// <param name="node">Node to retrieve the parent from</param> + /// <returns>Parent of <paramref name="node"/></returns> + private static IntervalTreeNode<K, V> ParentOf(IntervalTreeNode<K, V> node) + { + return node?.Parent; + } + + #endregion + + public bool ContainsKey(K key) + { + return GetNode(key) != null; + } + + public void Clear() + { + _root = null; + _count = 0; + } + } + + /// <summary> + /// Represents a node in the IntervalTree which contains start and end keys of type K, and a value of generic type V. + /// </summary> + /// <typeparam name="K">Key type of the node</typeparam> + /// <typeparam name="V">Value type of the node</typeparam> + class IntervalTreeNode<K, V> + { + public bool Color = true; + public IntervalTreeNode<K, V> Left = null; + public IntervalTreeNode<K, V> Right = null; + public IntervalTreeNode<K, V> Parent = null; + + /// <summary> + /// The start of the range. + /// </summary> + public K Start; + + /// <summary> + /// The end of the range. + /// </summary> + public K End; + + /// <summary> + /// The maximum end value of this node and all its children. + /// </summary> + public K Max; + + /// <summary> + /// Value stored on this node. + /// </summary> + public V Value; + + public IntervalTreeNode(K start, K end, V value, IntervalTreeNode<K, V> parent) + { + Start = start; + End = end; + Max = end; + Value = value; + Parent = parent; + } + } +} diff --git a/Ryujinx.Memory/WindowsShared/PlaceholderList.cs b/Ryujinx.Memory/WindowsShared/PlaceholderList.cs deleted file mode 100644 index 848e410e..00000000 --- a/Ryujinx.Memory/WindowsShared/PlaceholderList.cs +++ /dev/null @@ -1,293 +0,0 @@ -using Ryujinx.Memory.Range; -using System; -using System.Diagnostics; - -namespace Ryujinx.Memory.WindowsShared -{ - /// <summary> - /// A specialized list used for keeping track of Windows 10's memory placeholders. - /// This is used to make splitting a large placeholder into equally small - /// granular chunks much easier, while avoiding slowdown due to a large number of - /// placeholders by coalescing adjacent granular placeholders after they are unused. - /// </summary> - class PlaceholderList - { - private class PlaceholderBlock : IRange - { - public ulong Address { get; } - public ulong Size { get; private set; } - public ulong EndAddress { get; private set; } - public bool IsGranular { get; set; } - - public PlaceholderBlock(ulong id, ulong size, bool isGranular) - { - Address = id; - Size = size; - EndAddress = id + size; - IsGranular = isGranular; - } - - public bool OverlapsWith(ulong address, ulong size) - { - return Address < address + size && address < EndAddress; - } - - public void ExtendTo(ulong end, RangeList<PlaceholderBlock> list) - { - EndAddress = end; - Size = end - Address; - - list.UpdateEndAddress(this); - } - } - - private RangeList<PlaceholderBlock> _placeholders; - private PlaceholderBlock[] _foundBlocks = new PlaceholderBlock[32]; - - /// <summary> - /// Create a new list to manage placeholders. - /// Note that a size is measured in granular placeholders. - /// If the placeholder granularity is 65536 bytes, then a 65536 region will be covered by 1 placeholder granularity. - /// </summary> - /// <param name="size">Size measured in granular placeholders</param> - public PlaceholderList(ulong size) - { - _placeholders = new RangeList<PlaceholderBlock>(); - - _placeholders.Add(new PlaceholderBlock(0, size, false)); - } - - /// <summary> - /// Ensure that the given range of placeholders is granular. - /// </summary> - /// <param name="id">Start of the range, measured in granular placeholders</param> - /// <param name="size">Size of the range, measured in granular placeholders</param> - /// <param name="splitPlaceholderCallback">Callback function to run when splitting placeholders, calls with (start, middle)</param> - public void EnsurePlaceholders(ulong id, ulong size, Action<ulong, ulong> splitPlaceholderCallback) - { - // Search 1 before and after the placeholders, as we may need to expand/join granular regions surrounding the requested area. - - ulong endId = id + size; - ulong searchStartId = id == 0 ? 0 : (id - 1); - int blockCount = _placeholders.FindOverlapsNonOverlapping(searchStartId, (endId - searchStartId) + 1, ref _foundBlocks); - - PlaceholderBlock first = _foundBlocks[0]; - PlaceholderBlock last = _foundBlocks[blockCount - 1]; - bool overlapStart = first.EndAddress >= id && id != 0; - bool overlapEnd = last.Address <= endId; - - for (int i = 0; i < blockCount; i++) - { - // Go through all non-granular blocks in the range and create placeholders. - PlaceholderBlock block = _foundBlocks[i]; - - if (block.Address <= id && block.EndAddress >= endId && block.IsGranular) - { - return; // The region we're searching for is already granular. - } - - if (!block.IsGranular) - { - ulong placeholderStart = Math.Max(block.Address, id); - ulong placeholderEnd = Math.Min(block.EndAddress - 1, endId); - - if (placeholderStart != block.Address && placeholderStart != block.EndAddress) - { - splitPlaceholderCallback(block.Address, placeholderStart - block.Address); - } - - for (ulong j = placeholderStart; j < placeholderEnd; j++) - { - splitPlaceholderCallback(j, 1); - } - } - - if (!((block == first && overlapStart) || (block == last && overlapEnd))) - { - // Remove blocks that will be replaced - _placeholders.Remove(block); - } - } - - if (overlapEnd) - { - if (!(first == last && overlapStart)) - { - _placeholders.Remove(last); - } - - if (last.IsGranular) - { - endId = last.EndAddress; - } - else if (last.EndAddress != endId) - { - _placeholders.Add(new PlaceholderBlock(endId, last.EndAddress - endId, false)); - } - } - - if (overlapStart && first.IsGranular) - { - first.ExtendTo(endId, _placeholders); - } - else - { - if (overlapStart) - { - first.ExtendTo(id, _placeholders); - } - - _placeholders.Add(new PlaceholderBlock(id, endId - id, true)); - } - - ValidateList(); - } - - /// <summary> - /// Coalesces placeholders in a given region, as they are not being used. - /// This assumes that the region only contains placeholders - all views and allocations must have been replaced with placeholders. - /// </summary> - /// <param name="id">Start of the range, measured in granular placeholders</param> - /// <param name="size">Size of the range, measured in granular placeholders</param> - /// <param name="coalescePlaceholderCallback">Callback function to run when coalescing two placeholders, calls with (start, end)</param> - public void RemovePlaceholders(ulong id, ulong size, Action<ulong, ulong> coalescePlaceholderCallback) - { - ulong endId = id + size; - int blockCount = _placeholders.FindOverlapsNonOverlapping(id, size, ref _foundBlocks); - - PlaceholderBlock first = _foundBlocks[0]; - PlaceholderBlock last = _foundBlocks[blockCount - 1]; - - // All granular blocks must have non-granular blocks surrounding them, unless they start at 0. - // We must extend the non-granular blocks into the granular ones. This does mean that we need to search twice. - - if (first.IsGranular || last.IsGranular) - { - ulong surroundStart = Math.Max(0, (first.IsGranular && first.Address != 0) ? first.Address - 1 : id); - blockCount = _placeholders.FindOverlapsNonOverlapping( - surroundStart, - (last.IsGranular ? last.EndAddress + 1 : endId) - surroundStart, - ref _foundBlocks); - - first = _foundBlocks[0]; - last = _foundBlocks[blockCount - 1]; - } - - if (first == last) - { - return; // Already coalesced. - } - - PlaceholderBlock extendBlock = id == 0 ? null : first; - bool newBlock = false; - for (int i = extendBlock == null ? 0 : 1; i < blockCount; i++) - { - // Go through all granular blocks in the range and extend placeholders. - PlaceholderBlock block = _foundBlocks[i]; - - ulong blockEnd = block.EndAddress; - ulong extendFrom; - ulong extent = Math.Min(blockEnd, endId); - - if (block.Address < id && blockEnd > id) - { - block.ExtendTo(id, _placeholders); - extendBlock = null; - } - else - { - _placeholders.Remove(block); - } - - if (extendBlock == null) - { - extendFrom = id; - extendBlock = new PlaceholderBlock(id, extent - id, false); - _placeholders.Add(extendBlock); - - if (blockEnd > extent) - { - _placeholders.Add(new PlaceholderBlock(extent, blockEnd - extent, true)); - - // Skip the next non-granular block, and extend from that into the granular block afterwards. - // (assuming that one is still in the requested range) - - if (i + 1 < blockCount) - { - extendBlock = _foundBlocks[i + 1]; - } - - i++; - } - - newBlock = true; - } - else - { - extendFrom = extendBlock.Address; - extendBlock.ExtendTo(block.IsGranular ? extent : block.EndAddress, _placeholders); - } - - if (block.IsGranular) - { - ulong placeholderStart = Math.Max(block.Address, id); - ulong placeholderEnd = extent; - - if (newBlock) - { - placeholderStart++; - newBlock = false; - } - - for (ulong j = placeholderStart; j < placeholderEnd; j++) - { - coalescePlaceholderCallback(extendFrom, (j + 1) - extendFrom); - } - - if (extent < block.EndAddress) - { - _placeholders.Add(new PlaceholderBlock(placeholderEnd, block.EndAddress - placeholderEnd, true)); - ValidateList(); - return; - } - } - else - { - coalescePlaceholderCallback(extendFrom, block.EndAddress - extendFrom); - } - } - - ValidateList(); - } - - /// <summary> - /// Ensure that the placeholder list is valid. - /// A valid list should not have any gaps between the placeholders, - /// and there may be no placehonders with the same IsGranular value next to each other. - /// </summary> - [Conditional("DEBUG")] - private void ValidateList() - { - bool isGranular = false; - bool first = true; - ulong lastAddress = 0; - - foreach (var placeholder in _placeholders) - { - if (placeholder.Address != lastAddress) - { - throw new InvalidOperationException("Gap in placeholder list."); - } - - if (isGranular == placeholder.IsGranular && !first) - { - throw new InvalidOperationException("Placeholder list not alternating."); - } - - first = false; - isGranular = placeholder.IsGranular; - lastAddress = placeholder.EndAddress; - } - } - } -} diff --git a/Ryujinx.Memory/WindowsShared/PlaceholderManager.cs b/Ryujinx.Memory/WindowsShared/PlaceholderManager.cs new file mode 100644 index 00000000..a16a8c7f --- /dev/null +++ b/Ryujinx.Memory/WindowsShared/PlaceholderManager.cs @@ -0,0 +1,633 @@ +using System; +using System.Diagnostics; +using System.Threading; + +namespace Ryujinx.Memory.WindowsShared +{ + /// <summary> + /// Windows memory placeholder manager. + /// </summary> + class PlaceholderManager + { + private const ulong MinimumPageSize = 0x1000; + + [ThreadStatic] + private static int _threadLocalPartialUnmapsCount; + + private readonly IntervalTree<ulong, ulong> _mappings; + private readonly IntervalTree<ulong, MemoryPermission> _protections; + private readonly ReaderWriterLock _partialUnmapLock; + private int _partialUnmapsCount; + + /// <summary> + /// Creates a new instance of the Windows memory placeholder manager. + /// </summary> + public PlaceholderManager() + { + _mappings = new IntervalTree<ulong, ulong>(); + _protections = new IntervalTree<ulong, MemoryPermission>(); + _partialUnmapLock = new ReaderWriterLock(); + } + + /// <summary> + /// Reserves a range of the address space to be later mapped as shared memory views. + /// </summary> + /// <param name="address">Start address of the region to reserve</param> + /// <param name="size">Size in bytes of the region to reserve</param> + public void ReserveRange(ulong address, ulong size) + { + lock (_mappings) + { + _mappings.Add(address, address + size, ulong.MaxValue); + } + } + + /// <summary> + /// Maps a shared memory view on a previously reserved memory region. + /// </summary> + /// <param name="sharedMemory">Shared memory that will be the backing storage for the view</param> + /// <param name="srcOffset">Offset in the shared memory to map</param> + /// <param name="location">Address to map the view into</param> + /// <param name="size">Size of the view in bytes</param> + public void MapView(IntPtr sharedMemory, ulong srcOffset, IntPtr location, IntPtr size) + { + _partialUnmapLock.AcquireReaderLock(Timeout.Infinite); + + try + { + UnmapViewInternal(sharedMemory, location, size); + MapViewInternal(sharedMemory, srcOffset, location, size); + } + finally + { + _partialUnmapLock.ReleaseReaderLock(); + } + } + + /// <summary> + /// Maps a shared memory view on a previously reserved memory region. + /// </summary> + /// <param name="sharedMemory">Shared memory that will be the backing storage for the view</param> + /// <param name="srcOffset">Offset in the shared memory to map</param> + /// <param name="location">Address to map the view into</param> + /// <param name="size">Size of the view in bytes</param> + /// <exception cref="WindowsApiException">Thrown when the Windows API returns an error mapping the memory</exception> + private void MapViewInternal(IntPtr sharedMemory, ulong srcOffset, IntPtr location, IntPtr size) + { + SplitForMap((ulong)location, (ulong)size, srcOffset); + + var ptr = WindowsApi.MapViewOfFile3( + sharedMemory, + WindowsApi.CurrentProcessHandle, + location, + srcOffset, + size, + 0x4000, + MemoryProtection.ReadWrite, + IntPtr.Zero, + 0); + + if (ptr == IntPtr.Zero) + { + throw new WindowsApiException("MapViewOfFile3"); + } + } + + /// <summary> + /// Splits a larger placeholder, slicing at the start and end address, for a new memory mapping. + /// </summary> + /// <param name="address">Address to split</param> + /// <param name="size">Size of the new region</param> + /// <param name="backingOffset">Offset in the shared memory that will be mapped</param> + private void SplitForMap(ulong address, ulong size, ulong backingOffset) + { + ulong endAddress = address + size; + + var overlaps = Array.Empty<IntervalTreeNode<ulong, ulong>>(); + + lock (_mappings) + { + int count = _mappings.Get(address, endAddress, ref overlaps); + + Debug.Assert(count == 1); + Debug.Assert(!IsMapped(overlaps[0].Value)); + + var overlap = overlaps[0]; + + // Tree operations might modify the node start/end values, so save a copy before we modify the tree. + ulong overlapStart = overlap.Start; + ulong overlapEnd = overlap.End; + ulong overlapValue = overlap.Value; + + _mappings.Remove(overlap); + + bool overlapStartsBefore = overlapStart < address; + bool overlapEndsAfter = overlapEnd > endAddress; + + if (overlapStartsBefore && overlapEndsAfter) + { + CheckFreeResult(WindowsApi.VirtualFree( + (IntPtr)address, + (IntPtr)size, + AllocationType.Release | AllocationType.PreservePlaceholder)); + + _mappings.Add(overlapStart, address, overlapValue); + _mappings.Add(endAddress, overlapEnd, AddBackingOffset(overlapValue, endAddress - overlapStart)); + } + else if (overlapStartsBefore) + { + ulong overlappedSize = overlapEnd - address; + + CheckFreeResult(WindowsApi.VirtualFree( + (IntPtr)address, + (IntPtr)overlappedSize, + AllocationType.Release | AllocationType.PreservePlaceholder)); + + _mappings.Add(overlapStart, address, overlapValue); + } + else if (overlapEndsAfter) + { + ulong overlappedSize = endAddress - overlapStart; + + CheckFreeResult(WindowsApi.VirtualFree( + (IntPtr)overlapStart, + (IntPtr)overlappedSize, + AllocationType.Release | AllocationType.PreservePlaceholder)); + + _mappings.Add(endAddress, overlapEnd, AddBackingOffset(overlapValue, overlappedSize)); + } + + _mappings.Add(address, endAddress, backingOffset); + } + } + + /// <summary> + /// Unmaps a view that has been previously mapped with <see cref="MapView"/>. + /// </summary> + /// <remarks> + /// For "partial unmaps" (when not the entire mapped range is being unmapped), it might be + /// necessary to unmap the whole range and then remap the sub-ranges that should remain mapped. + /// </remarks> + /// <param name="sharedMemory">Shared memory that the view being unmapped belongs to</param> + /// <param name="location">Address to unmap</param> + /// <param name="size">Size of the region to unmap in bytes</param> + public void UnmapView(IntPtr sharedMemory, IntPtr location, IntPtr size) + { + _partialUnmapLock.AcquireReaderLock(Timeout.Infinite); + + try + { + UnmapViewInternal(sharedMemory, location, size); + } + finally + { + _partialUnmapLock.ReleaseReaderLock(); + } + } + + /// <summary> + /// Unmaps a view that has been previously mapped with <see cref="MapView"/>. + /// </summary> + /// <remarks> + /// For "partial unmaps" (when not the entire mapped range is being unmapped), it might be + /// necessary to unmap the whole range and then remap the sub-ranges that should remain mapped. + /// </remarks> + /// <param name="sharedMemory">Shared memory that the view being unmapped belongs to</param> + /// <param name="location">Address to unmap</param> + /// <param name="size">Size of the region to unmap in bytes</param> + /// <exception cref="WindowsApiException">Thrown when the Windows API returns an error unmapping or remapping the memory</exception> + private void UnmapViewInternal(IntPtr sharedMemory, IntPtr location, IntPtr size) + { + ulong startAddress = (ulong)location; + ulong unmapSize = (ulong)size; + ulong endAddress = startAddress + unmapSize; + + var overlaps = Array.Empty<IntervalTreeNode<ulong, ulong>>(); + int count = 0; + + lock (_mappings) + { + count = _mappings.Get(startAddress, endAddress, ref overlaps); + } + + for (int index = 0; index < count; index++) + { + var overlap = overlaps[index]; + + if (IsMapped(overlap.Value)) + { + if (!WindowsApi.UnmapViewOfFile2(WindowsApi.CurrentProcessHandle, (IntPtr)overlap.Start, 2)) + { + throw new WindowsApiException("UnmapViewOfFile2"); + } + + // Tree operations might modify the node start/end values, so save a copy before we modify the tree. + ulong overlapStart = overlap.Start; + ulong overlapEnd = overlap.End; + ulong overlapValue = overlap.Value; + + _mappings.Remove(overlap); + _mappings.Add(overlapStart, overlapEnd, ulong.MaxValue); + + bool overlapStartsBefore = overlapStart < startAddress; + bool overlapEndsAfter = overlapEnd > endAddress; + + if (overlapStartsBefore || overlapEndsAfter) + { + // If the overlap extends beyond the region we are unmapping, + // then we need to re-map the regions that are supposed to remain mapped. + // This is necessary because Windows does not support partial view unmaps. + // That is, you can only fully unmap a view that was previously mapped, you can't just unmap a chunck of it. + + LockCookie lockCookie = _partialUnmapLock.UpgradeToWriterLock(Timeout.Infinite); + + _partialUnmapsCount++; + + if (overlapStartsBefore) + { + ulong remapSize = startAddress - overlapStart; + + MapViewInternal(sharedMemory, overlapValue, (IntPtr)overlapStart, (IntPtr)remapSize); + RestoreRangeProtection(overlapStart, remapSize); + } + + if (overlapEndsAfter) + { + ulong overlappedSize = endAddress - overlapStart; + ulong remapBackingOffset = overlapValue + overlappedSize; + ulong remapAddress = overlapStart + overlappedSize; + ulong remapSize = overlapEnd - endAddress; + + MapViewInternal(sharedMemory, remapBackingOffset, (IntPtr)remapAddress, (IntPtr)remapSize); + RestoreRangeProtection(remapAddress, remapSize); + } + + _partialUnmapLock.DowngradeFromWriterLock(ref lockCookie); + } + } + } + + CoalesceForUnmap(startAddress, unmapSize); + RemoveProtection(startAddress, unmapSize); + } + + /// <summary> + /// Coalesces adjacent placeholders after unmap. + /// </summary> + /// <param name="address">Address of the region that was unmapped</param> + /// <param name="size">Size of the region that was unmapped in bytes</param> + private void CoalesceForUnmap(ulong address, ulong size) + { + ulong endAddress = address + size; + var overlaps = Array.Empty<IntervalTreeNode<ulong, ulong>>(); + int unmappedCount = 0; + + lock (_mappings) + { + int count = _mappings.Get(address - MinimumPageSize, endAddress + MinimumPageSize, ref overlaps); + if (count < 2) + { + // Nothing to coalesce if we only have 1 or no overlaps. + return; + } + + for (int index = 0; index < count; index++) + { + var overlap = overlaps[index]; + + if (!IsMapped(overlap.Value)) + { + if (address > overlap.Start) + { + address = overlap.Start; + } + + if (endAddress < overlap.End) + { + endAddress = overlap.End; + } + + _mappings.Remove(overlap); + + unmappedCount++; + } + } + + _mappings.Add(address, endAddress, ulong.MaxValue); + } + + if (unmappedCount > 1) + { + size = endAddress - address; + + CheckFreeResult(WindowsApi.VirtualFree( + (IntPtr)address, + (IntPtr)size, + AllocationType.Release | AllocationType.CoalescePlaceholders)); + } + } + + /// <summary> + /// Reprotects a region of memory that has been mapped. + /// </summary> + /// <param name="address">Address of the region to reprotect</param> + /// <param name="size">Size of the region to reprotect in bytes</param> + /// <param name="permission">New permissions</param> + /// <returns>True if the reprotection was successful, false otherwise</returns> + public bool ReprotectView(IntPtr address, IntPtr size, MemoryPermission permission) + { + _partialUnmapLock.AcquireReaderLock(Timeout.Infinite); + + try + { + return ReprotectViewInternal(address, size, permission, false); + } + finally + { + _partialUnmapLock.ReleaseReaderLock(); + } + } + + /// <summary> + /// Reprotects a region of memory that has been mapped. + /// </summary> + /// <param name="address">Address of the region to reprotect</param> + /// <param name="size">Size of the region to reprotect in bytes</param> + /// <param name="permission">New permissions</param> + /// <param name="throwOnError">Throw an exception instead of returning an error if the operation fails</param> + /// <returns>True if the reprotection was successful or if <paramref name="throwOnError"/> is true, false otherwise</returns> + /// <exception cref="WindowsApiException">If <paramref name="throwOnError"/> is true, it is thrown when the Windows API returns an error reprotecting the memory</exception> + private bool ReprotectViewInternal(IntPtr address, IntPtr size, MemoryPermission permission, bool throwOnError) + { + ulong reprotectAddress = (ulong)address; + ulong reprotectSize = (ulong)size; + ulong endAddress = reprotectAddress + reprotectSize; + + var overlaps = Array.Empty<IntervalTreeNode<ulong, ulong>>(); + int count = 0; + + lock (_mappings) + { + count = _mappings.Get(reprotectAddress, endAddress, ref overlaps); + } + + bool success = true; + + for (int index = 0; index < count; index++) + { + var overlap = overlaps[index]; + + ulong mappedAddress = overlap.Start; + ulong mappedSize = overlap.End - overlap.Start; + + if (mappedAddress < reprotectAddress) + { + ulong delta = reprotectAddress - mappedAddress; + mappedAddress = reprotectAddress; + mappedSize -= delta; + } + + ulong mappedEndAddress = mappedAddress + mappedSize; + + if (mappedEndAddress > endAddress) + { + ulong delta = mappedEndAddress - endAddress; + mappedSize -= delta; + } + + if (!WindowsApi.VirtualProtect((IntPtr)mappedAddress, (IntPtr)mappedSize, WindowsApi.GetProtection(permission), out _)) + { + if (throwOnError) + { + throw new WindowsApiException("VirtualProtect"); + } + + success = false; + } + + // We only keep track of "non-standard" protections, + // that is, everything that is not just RW (which is the default when views are mapped). + if (permission == MemoryPermission.ReadAndWrite) + { + RemoveProtection(mappedAddress, mappedSize); + } + else + { + AddProtection(mappedAddress, mappedSize, permission); + } + } + + return success; + } + + /// <summary> + /// Checks the result of a VirtualFree operation, throwing if needed. + /// </summary> + /// <param name="success">Operation result</param> + /// <exception cref="WindowsApiException">Thrown if <paramref name="success"/> is false</exception> + private static void CheckFreeResult(bool success) + { + if (!success) + { + throw new WindowsApiException("VirtualFree"); + } + } + + /// <summary> + /// Adds an offset to a backing offset. This will do nothing if the backing offset is the special "unmapped" value. + /// </summary> + /// <param name="backingOffset">Backing offset</param> + /// <param name="offset">Offset to be added</param> + /// <returns>Added offset or just <paramref name="backingOffset"/> if the region is unmapped</returns> + private static ulong AddBackingOffset(ulong backingOffset, ulong offset) + { + if (backingOffset == ulong.MaxValue) + { + return backingOffset; + } + + return backingOffset + offset; + } + + /// <summary> + /// Checks if a region is unmapped. + /// </summary> + /// <param name="backingOffset">Backing offset to check</param> + /// <returns>True if the backing offset is the special "unmapped" value, false otherwise</returns> + private static bool IsMapped(ulong backingOffset) + { + return backingOffset != ulong.MaxValue; + } + + /// <summary> + /// Adds a protection to the list of protections. + /// </summary> + /// <param name="address">Address of the protected region</param> + /// <param name="size">Size of the protected region in bytes</param> + /// <param name="permission">Memory permissions of the region</param> + private void AddProtection(ulong address, ulong size, MemoryPermission permission) + { + ulong endAddress = address + size; + var overlaps = Array.Empty<IntervalTreeNode<ulong, MemoryPermission>>(); + int count = 0; + + lock (_protections) + { + count = _protections.Get(address, endAddress, ref overlaps); + + Debug.Assert(count > 0); + + if (count == 1 && + overlaps[0].Start <= address && + overlaps[0].End >= endAddress && + overlaps[0].Value == permission) + { + return; + } + + ulong startAddress = address; + + for (int index = 0; index < count; index++) + { + var protection = overlaps[index]; + + ulong protAddress = protection.Start; + ulong protEndAddress = protection.End; + MemoryPermission protPermission = protection.Value; + + _protections.Remove(protection); + + if (protection.Value == permission) + { + if (startAddress > protAddress) + { + startAddress = protAddress; + } + + if (endAddress < protEndAddress) + { + endAddress = protEndAddress; + } + } + else + { + if (startAddress > protAddress) + { + _protections.Add(protAddress, startAddress, protPermission); + } + + if (endAddress < protEndAddress) + { + _protections.Add(endAddress, protEndAddress, protPermission); + } + } + } + + _protections.Add(startAddress, endAddress, permission); + } + } + + /// <summary> + /// Removes protection from the list of protections. + /// </summary> + /// <param name="address">Address of the protected region</param> + /// <param name="size">Size of the protected region in bytes</param> + private void RemoveProtection(ulong address, ulong size) + { + ulong endAddress = address + size; + var overlaps = Array.Empty<IntervalTreeNode<ulong, MemoryPermission>>(); + int count = 0; + + lock (_protections) + { + count = _protections.Get(address, endAddress, ref overlaps); + + for (int index = 0; index < count; index++) + { + var protection = overlaps[index]; + + ulong protAddress = protection.Start; + ulong protEndAddress = protection.End; + MemoryPermission protPermission = protection.Value; + + _protections.Remove(protection); + + if (address > protAddress) + { + _protections.Add(protAddress, address, protPermission); + } + + if (endAddress < protEndAddress) + { + _protections.Add(endAddress, protEndAddress, protPermission); + } + } + } + } + + /// <summary> + /// Restores the protection of a given memory region that was remapped, using the protections list. + /// </summary> + /// <param name="address">Address of the remapped region</param> + /// <param name="size">Size of the remapped region in bytes</param> + private void RestoreRangeProtection(ulong address, ulong size) + { + ulong endAddress = address + size; + var overlaps = Array.Empty<IntervalTreeNode<ulong, MemoryPermission>>(); + int count = 0; + + lock (_protections) + { + count = _protections.Get(address, endAddress, ref overlaps); + } + + ulong startAddress = address; + + for (int index = 0; index < count; index++) + { + var protection = overlaps[index]; + + ulong protAddress = protection.Start; + ulong protEndAddress = protection.End; + + if (protAddress < address) + { + protAddress = address; + } + + if (protEndAddress > endAddress) + { + protEndAddress = endAddress; + } + + ReprotectViewInternal((IntPtr)protAddress, (IntPtr)(protEndAddress - protAddress), protection.Value, true); + } + } + + /// <summary> + /// Checks if an access violation handler should retry execution due to a fault caused by partial unmap. + /// </summary> + /// <remarks> + /// Due to Windows limitations, <see cref="UnmapView"/> might need to unmap more memory than requested. + /// The additional memory that was unmapped is later remapped, however this leaves a time gap where the + /// memory might be accessed but is unmapped. Users of the API must compensate for that by catching the + /// access violation and retrying if it happened between the unmap and remap operation. + /// This method can be used to decide if retrying in such cases is necessary or not. + /// </remarks> + /// <returns>True if execution should be retried, false otherwise</returns> + public bool RetryFromAccessViolation() + { + _partialUnmapLock.AcquireReaderLock(Timeout.Infinite); + + bool retry = _threadLocalPartialUnmapsCount != _partialUnmapsCount; + if (retry) + { + _threadLocalPartialUnmapsCount = _partialUnmapsCount; + } + + _partialUnmapLock.ReleaseReaderLock(); + + return retry; + } + } +}
\ No newline at end of file diff --git a/Ryujinx.Memory/WindowsShared/WindowsApi.cs b/Ryujinx.Memory/WindowsShared/WindowsApi.cs new file mode 100644 index 00000000..297bd1ee --- /dev/null +++ b/Ryujinx.Memory/WindowsShared/WindowsApi.cs @@ -0,0 +1,93 @@ +using System; +using System.Runtime.InteropServices; + +namespace Ryujinx.Memory.WindowsShared +{ + static class WindowsApi + { + public static readonly IntPtr InvalidHandleValue = new IntPtr(-1); + public static readonly IntPtr CurrentProcessHandle = new IntPtr(-1); + + [DllImport("kernel32.dll", SetLastError = true)] + public static extern IntPtr VirtualAlloc( + IntPtr lpAddress, + IntPtr dwSize, + AllocationType flAllocationType, + MemoryProtection flProtect); + + [DllImport("KernelBase.dll", SetLastError = true)] + public static extern IntPtr VirtualAlloc2( + IntPtr process, + IntPtr lpAddress, + IntPtr dwSize, + AllocationType flAllocationType, + MemoryProtection flProtect, + IntPtr extendedParameters, + ulong parameterCount); + + [DllImport("kernel32.dll", SetLastError = true)] + public static extern bool VirtualProtect( + IntPtr lpAddress, + IntPtr dwSize, + MemoryProtection flNewProtect, + out MemoryProtection lpflOldProtect); + + [DllImport("kernel32.dll", SetLastError = true)] + public static extern bool VirtualFree(IntPtr lpAddress, IntPtr dwSize, AllocationType dwFreeType); + + [DllImport("kernel32.dll", SetLastError = true)] + public static extern IntPtr CreateFileMapping( + IntPtr hFile, + IntPtr lpFileMappingAttributes, + FileMapProtection flProtect, + uint dwMaximumSizeHigh, + uint dwMaximumSizeLow, + [MarshalAs(UnmanagedType.LPWStr)] string lpName); + + [DllImport("kernel32.dll", SetLastError = true)] + public static extern bool CloseHandle(IntPtr hObject); + + [DllImport("kernel32.dll", SetLastError = true)] + public static extern IntPtr MapViewOfFile( + IntPtr hFileMappingObject, + uint dwDesiredAccess, + uint dwFileOffsetHigh, + uint dwFileOffsetLow, + IntPtr dwNumberOfBytesToMap); + + [DllImport("KernelBase.dll", SetLastError = true)] + public static extern IntPtr MapViewOfFile3( + IntPtr hFileMappingObject, + IntPtr process, + IntPtr baseAddress, + ulong offset, + IntPtr dwNumberOfBytesToMap, + ulong allocationType, + MemoryProtection dwDesiredAccess, + IntPtr extendedParameters, + ulong parameterCount); + + [DllImport("kernel32.dll", SetLastError = true)] + public static extern bool UnmapViewOfFile(IntPtr lpBaseAddress); + + [DllImport("KernelBase.dll", SetLastError = true)] + public static extern bool UnmapViewOfFile2(IntPtr process, IntPtr lpBaseAddress, ulong unmapFlags); + + [DllImport("kernel32.dll")] + public static extern uint GetLastError(); + + public static MemoryProtection GetProtection(MemoryPermission permission) + { + return permission switch + { + MemoryPermission.None => MemoryProtection.NoAccess, + MemoryPermission.Read => MemoryProtection.ReadOnly, + MemoryPermission.ReadAndWrite => MemoryProtection.ReadWrite, + MemoryPermission.ReadAndExecute => MemoryProtection.ExecuteRead, + MemoryPermission.ReadWriteExecute => MemoryProtection.ExecuteReadWrite, + MemoryPermission.Execute => MemoryProtection.Execute, + _ => throw new MemoryProtectionException(permission) + }; + } + } +}
\ No newline at end of file diff --git a/Ryujinx.Memory/WindowsShared/WindowsApiException.cs b/Ryujinx.Memory/WindowsShared/WindowsApiException.cs new file mode 100644 index 00000000..3140d705 --- /dev/null +++ b/Ryujinx.Memory/WindowsShared/WindowsApiException.cs @@ -0,0 +1,24 @@ +using System; + +namespace Ryujinx.Memory.WindowsShared +{ + class WindowsApiException : Exception + { + public WindowsApiException() + { + } + + public WindowsApiException(string functionName) : base(CreateMessage(functionName)) + { + } + + public WindowsApiException(string functionName, Exception inner) : base(CreateMessage(functionName), inner) + { + } + + private static string CreateMessage(string functionName) + { + return $"{functionName} returned error code 0x{WindowsApi.GetLastError():X}."; + } + } +}
\ No newline at end of file |
