diff options
| author | TSR Berry <20988865+TSRBerry@users.noreply.github.com> | 2023-04-08 01:22:00 +0200 |
|---|---|---|
| committer | Mary <thog@protonmail.com> | 2023-04-27 23:51:14 +0200 |
| commit | cee712105850ac3385cd0091a923438167433f9f (patch) | |
| tree | 4a5274b21d8b7f938c0d0ce18736d3f2993b11b1 /Ryujinx.HLE/HOS/Kernel/Memory | |
| parent | cd124bda587ef09668a971fa1cac1c3f0cfc9f21 (diff) | |
Move solution and projects to src
Diffstat (limited to 'Ryujinx.HLE/HOS/Kernel/Memory')
25 files changed, 0 insertions, 5410 deletions
diff --git a/Ryujinx.HLE/HOS/Kernel/Memory/AddressSpaceType.cs b/Ryujinx.HLE/HOS/Kernel/Memory/AddressSpaceType.cs deleted file mode 100644 index 8395c577..00000000 --- a/Ryujinx.HLE/HOS/Kernel/Memory/AddressSpaceType.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace Ryujinx.HLE.HOS.Kernel.Memory -{ - enum AddressSpaceType - { - Addr32Bits = 0, - Addr36Bits = 1, - Addr32BitsNoMap = 2, - Addr39Bits = 3 - } -}
\ No newline at end of file diff --git a/Ryujinx.HLE/HOS/Kernel/Memory/DramMemoryMap.cs b/Ryujinx.HLE/HOS/Kernel/Memory/DramMemoryMap.cs deleted file mode 100644 index 4941d5b7..00000000 --- a/Ryujinx.HLE/HOS/Kernel/Memory/DramMemoryMap.cs +++ /dev/null @@ -1,18 +0,0 @@ -namespace Ryujinx.HLE.HOS.Kernel.Memory -{ - static class DramMemoryMap - { - public const ulong DramBase = 0x80000000; - - public const ulong KernelReserveBase = DramBase + 0x60000; - - public const ulong SlabHeapBase = KernelReserveBase + 0x85000; - public const ulong SlapHeapSize = 0xa21000; - public const ulong SlabHeapEnd = SlabHeapBase + SlapHeapSize; - - public static bool IsHeapPhysicalAddress(ulong address) - { - return address >= SlabHeapEnd; - } - } -}
\ No newline at end of file diff --git a/Ryujinx.HLE/HOS/Kernel/Memory/KCodeMemory.cs b/Ryujinx.HLE/HOS/Kernel/Memory/KCodeMemory.cs deleted file mode 100644 index 11474e49..00000000 --- a/Ryujinx.HLE/HOS/Kernel/Memory/KCodeMemory.cs +++ /dev/null @@ -1,169 +0,0 @@ -using Ryujinx.Common; -using Ryujinx.HLE.HOS.Kernel.Common; -using Ryujinx.HLE.HOS.Kernel.Process; -using Ryujinx.Horizon.Common; -using System; -using System.Diagnostics; - -namespace Ryujinx.HLE.HOS.Kernel.Memory -{ - class KCodeMemory : KAutoObject - { - public KProcess Owner { get; private set; } - private readonly KPageList _pageList; - private readonly object _lock; - private ulong _address; - private bool _isOwnerMapped; - private bool _isMapped; - - public KCodeMemory(KernelContext context) : base(context) - { - _pageList = new KPageList(); - _lock = new object(); - } - - public Result Initialize(ulong address, ulong size) - { - Owner = KernelStatic.GetCurrentProcess(); - - Result result = Owner.MemoryManager.BorrowCodeMemory(_pageList, address, size); - - if (result != Result.Success) - { - return result; - } - - Owner.CpuMemory.Fill(address, size, 0xff); - Owner.IncrementReferenceCount(); - - _address = address; - _isMapped = false; - _isOwnerMapped = false; - - return Result.Success; - } - - public Result Map(ulong address, ulong size, KMemoryPermission perm) - { - if (_pageList.GetPagesCount() != BitUtils.DivRoundUp<ulong>(size, (ulong)KPageTableBase.PageSize)) - { - return KernelResult.InvalidSize; - } - - lock (_lock) - { - if (_isMapped) - { - return KernelResult.InvalidState; - } - - KProcess process = KernelStatic.GetCurrentProcess(); - - Result result = process.MemoryManager.MapPages(address, _pageList, MemoryState.CodeWritable, KMemoryPermission.ReadAndWrite); - - if (result != Result.Success) - { - return result; - } - - _isMapped = true; - } - - return Result.Success; - } - - public Result MapToOwner(ulong address, ulong size, KMemoryPermission permission) - { - if (_pageList.GetPagesCount() != BitUtils.DivRoundUp<ulong>(size, (ulong)KPageTableBase.PageSize)) - { - return KernelResult.InvalidSize; - } - - lock (_lock) - { - if (_isOwnerMapped) - { - return KernelResult.InvalidState; - } - - Debug.Assert(permission == KMemoryPermission.Read || permission == KMemoryPermission.ReadAndExecute); - - Result result = Owner.MemoryManager.MapPages(address, _pageList, MemoryState.CodeReadOnly, permission); - - if (result != Result.Success) - { - return result; - } - - _isOwnerMapped = true; - } - - return Result.Success; - } - - public Result Unmap(ulong address, ulong size) - { - if (_pageList.GetPagesCount() != BitUtils.DivRoundUp<ulong>(size, (ulong)KPageTableBase.PageSize)) - { - return KernelResult.InvalidSize; - } - - lock (_lock) - { - KProcess process = KernelStatic.GetCurrentProcess(); - - Result result = process.MemoryManager.UnmapPages(address, _pageList, MemoryState.CodeWritable); - - if (result != Result.Success) - { - return result; - } - - Debug.Assert(_isMapped); - - _isMapped = false; - } - - return Result.Success; - } - - public Result UnmapFromOwner(ulong address, ulong size) - { - if (_pageList.GetPagesCount() != BitUtils.DivRoundUp<ulong>(size, KPageTableBase.PageSize)) - { - return KernelResult.InvalidSize; - } - - lock (_lock) - { - Result result = Owner.MemoryManager.UnmapPages(address, _pageList, MemoryState.CodeReadOnly); - - if (result != Result.Success) - { - return result; - } - - Debug.Assert(_isOwnerMapped); - - _isOwnerMapped = false; - } - - return Result.Success; - } - - protected override void Destroy() - { - if (!_isMapped && !_isOwnerMapped) - { - ulong size = _pageList.GetPagesCount() * KPageTableBase.PageSize; - - if (Owner.MemoryManager.UnborrowCodeMemory(_address, size, _pageList) != Result.Success) - { - throw new InvalidOperationException("Unexpected failure restoring transfer memory attributes."); - } - } - - Owner.DecrementReferenceCount(); - } - } -}
\ No newline at end of file diff --git a/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryBlock.cs b/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryBlock.cs deleted file mode 100644 index e082105b..00000000 --- a/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryBlock.cs +++ /dev/null @@ -1,156 +0,0 @@ -using Ryujinx.Common.Collections; -using System; - -namespace Ryujinx.HLE.HOS.Kernel.Memory -{ - class KMemoryBlock : IntrusiveRedBlackTreeNode<KMemoryBlock>, IComparable<KMemoryBlock>, IComparable<ulong> - { - public ulong BaseAddress { get; private set; } - public ulong PagesCount { get; private set; } - - public MemoryState State { get; private set; } - public KMemoryPermission Permission { get; private set; } - public MemoryAttribute Attribute { get; private set; } - public KMemoryPermission SourcePermission { get; private set; } - - public int IpcRefCount { get; private set; } - public int DeviceRefCount { get; private set; } - - public KMemoryBlock( - ulong baseAddress, - ulong pagesCount, - MemoryState state, - KMemoryPermission permission, - MemoryAttribute attribute, - int ipcRefCount = 0, - int deviceRefCount = 0) - { - BaseAddress = baseAddress; - PagesCount = pagesCount; - State = state; - Attribute = attribute; - Permission = permission; - IpcRefCount = ipcRefCount; - DeviceRefCount = deviceRefCount; - } - - public void SetState(KMemoryPermission permission, MemoryState state, MemoryAttribute attribute) - { - Permission = permission; - State = state; - Attribute &= MemoryAttribute.IpcAndDeviceMapped; - Attribute |= attribute; - } - - public void SetIpcMappingPermission(KMemoryPermission newPermission) - { - int oldIpcRefCount = IpcRefCount++; - - if ((ushort)IpcRefCount == 0) - { - throw new InvalidOperationException("IPC reference count increment overflowed."); - } - - if (oldIpcRefCount == 0) - { - SourcePermission = Permission; - - Permission &= ~KMemoryPermission.ReadAndWrite; - Permission |= KMemoryPermission.ReadAndWrite & newPermission; - } - - Attribute |= MemoryAttribute.IpcMapped; - } - - public void RestoreIpcMappingPermission() - { - int oldIpcRefCount = IpcRefCount--; - - if (oldIpcRefCount == 0) - { - throw new InvalidOperationException("IPC reference count decrement underflowed."); - } - - if (oldIpcRefCount == 1) - { - Permission = SourcePermission; - - SourcePermission = KMemoryPermission.None; - - Attribute &= ~MemoryAttribute.IpcMapped; - } - } - - public KMemoryBlock SplitRightAtAddress(ulong address) - { - ulong leftAddress = BaseAddress; - - ulong leftPagesCount = (address - leftAddress) / KPageTableBase.PageSize; - - BaseAddress = address; - - PagesCount -= leftPagesCount; - - return new KMemoryBlock( - leftAddress, - leftPagesCount, - State, - Permission, - Attribute, - IpcRefCount, - DeviceRefCount); - } - - public void AddPages(ulong pagesCount) - { - PagesCount += pagesCount; - } - - public KMemoryInfo GetInfo() - { - ulong size = PagesCount * KPageTableBase.PageSize; - - return new KMemoryInfo( - BaseAddress, - size, - State, - Permission, - Attribute, - SourcePermission, - IpcRefCount, - DeviceRefCount); - } - - public int CompareTo(KMemoryBlock other) - { - if (BaseAddress < other.BaseAddress) - { - return -1; - } - else if (BaseAddress <= other.BaseAddress + other.PagesCount * KPageTableBase.PageSize - 1UL) - { - return 0; - } - else - { - return 1; - } - } - - public int CompareTo(ulong address) - { - if (address < BaseAddress) - { - return 1; - } - else if (address <= BaseAddress + PagesCount * KPageTableBase.PageSize - 1UL) - { - return 0; - } - else - { - return -1; - } - } - } -}
\ No newline at end of file diff --git a/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryBlockManager.cs b/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryBlockManager.cs deleted file mode 100644 index e9146aeb..00000000 --- a/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryBlockManager.cs +++ /dev/null @@ -1,288 +0,0 @@ -using Ryujinx.Common.Collections; -using Ryujinx.Horizon.Common; -using System.Diagnostics; - -namespace Ryujinx.HLE.HOS.Kernel.Memory -{ - class KMemoryBlockManager - { - private const int PageSize = KPageTableBase.PageSize; - - private readonly IntrusiveRedBlackTree<KMemoryBlock> _blockTree; - - public int BlocksCount => _blockTree.Count; - - private KMemoryBlockSlabManager _slabManager; - - private ulong _addrSpaceStart; - private ulong _addrSpaceEnd; - - public KMemoryBlockManager() - { - _blockTree = new IntrusiveRedBlackTree<KMemoryBlock>(); - } - - public Result Initialize(ulong addrSpaceStart, ulong addrSpaceEnd, KMemoryBlockSlabManager slabManager) - { - _slabManager = slabManager; - _addrSpaceStart = addrSpaceStart; - _addrSpaceEnd = addrSpaceEnd; - - // First insertion will always need only a single block, because there's nothing to split. - if (!slabManager.CanAllocate(1)) - { - return KernelResult.OutOfResource; - } - - ulong addrSpacePagesCount = (addrSpaceEnd - addrSpaceStart) / PageSize; - - _blockTree.Add(new KMemoryBlock( - addrSpaceStart, - addrSpacePagesCount, - MemoryState.Unmapped, - KMemoryPermission.None, - MemoryAttribute.None)); - - return Result.Success; - } - - public void InsertBlock( - ulong baseAddress, - ulong pagesCount, - MemoryState oldState, - KMemoryPermission oldPermission, - MemoryAttribute oldAttribute, - MemoryState newState, - KMemoryPermission newPermission, - MemoryAttribute newAttribute) - { - // Insert new block on the list only on areas where the state - // of the block matches the state specified on the old* state - // arguments, otherwise leave it as is. - - int oldCount = _blockTree.Count; - - oldAttribute |= MemoryAttribute.IpcAndDeviceMapped; - - ulong endAddr = baseAddress + pagesCount * PageSize; - - KMemoryBlock currBlock = FindBlock(baseAddress); - - while (currBlock != null) - { - ulong currBaseAddr = currBlock.BaseAddress; - ulong currEndAddr = currBlock.PagesCount * PageSize + currBaseAddr; - - if (baseAddress < currEndAddr && currBaseAddr < endAddr) - { - MemoryAttribute currBlockAttr = currBlock.Attribute | MemoryAttribute.IpcAndDeviceMapped; - - if (currBlock.State != oldState || - currBlock.Permission != oldPermission || - currBlockAttr != oldAttribute) - { - currBlock = currBlock.Successor; - - continue; - } - - if (baseAddress > currBaseAddr) - { - KMemoryBlock newBlock = currBlock.SplitRightAtAddress(baseAddress); - _blockTree.Add(newBlock); - } - - if (endAddr < currEndAddr) - { - KMemoryBlock newBlock = currBlock.SplitRightAtAddress(endAddr); - _blockTree.Add(newBlock); - currBlock = newBlock; - } - - currBlock.SetState(newPermission, newState, newAttribute); - - currBlock = MergeEqualStateNeighbors(currBlock); - } - - if (currEndAddr - 1 >= endAddr - 1) - { - break; - } - - currBlock = currBlock.Successor; - } - - _slabManager.Count += _blockTree.Count - oldCount; - - ValidateInternalState(); - } - - public void InsertBlock( - ulong baseAddress, - ulong pagesCount, - MemoryState state, - KMemoryPermission permission = KMemoryPermission.None, - MemoryAttribute attribute = MemoryAttribute.None) - { - // Inserts new block at the list, replacing and splitting - // existing blocks as needed. - - int oldCount = _blockTree.Count; - - ulong endAddr = baseAddress + pagesCount * PageSize; - - KMemoryBlock currBlock = FindBlock(baseAddress); - - while (currBlock != null) - { - ulong currBaseAddr = currBlock.BaseAddress; - ulong currEndAddr = currBlock.PagesCount * PageSize + currBaseAddr; - - if (baseAddress < currEndAddr && currBaseAddr < endAddr) - { - if (baseAddress > currBaseAddr) - { - KMemoryBlock newBlock = currBlock.SplitRightAtAddress(baseAddress); - _blockTree.Add(newBlock); - } - - if (endAddr < currEndAddr) - { - KMemoryBlock newBlock = currBlock.SplitRightAtAddress(endAddr); - _blockTree.Add(newBlock); - currBlock = newBlock; - } - - currBlock.SetState(permission, state, attribute); - - currBlock = MergeEqualStateNeighbors(currBlock); - } - - if (currEndAddr - 1 >= endAddr - 1) - { - break; - } - - currBlock = currBlock.Successor; - } - - _slabManager.Count += _blockTree.Count - oldCount; - - ValidateInternalState(); - } - - public delegate void BlockMutator(KMemoryBlock block, KMemoryPermission newPerm); - - public void InsertBlock( - ulong baseAddress, - ulong pagesCount, - BlockMutator blockMutate, - KMemoryPermission permission = KMemoryPermission.None) - { - // Inserts new block at the list, replacing and splitting - // existing blocks as needed, then calling the callback - // function on the new block. - - int oldCount = _blockTree.Count; - - ulong endAddr = baseAddress + pagesCount * PageSize; - - KMemoryBlock currBlock = FindBlock(baseAddress); - - while (currBlock != null) - { - ulong currBaseAddr = currBlock.BaseAddress; - ulong currEndAddr = currBlock.PagesCount * PageSize + currBaseAddr; - - if (baseAddress < currEndAddr && currBaseAddr < endAddr) - { - if (baseAddress > currBaseAddr) - { - KMemoryBlock newBlock = currBlock.SplitRightAtAddress(baseAddress); - _blockTree.Add(newBlock); - } - - if (endAddr < currEndAddr) - { - KMemoryBlock newBlock = currBlock.SplitRightAtAddress(endAddr); - _blockTree.Add(newBlock); - currBlock = newBlock; - } - - blockMutate(currBlock, permission); - - currBlock = MergeEqualStateNeighbors(currBlock); - } - - if (currEndAddr - 1 >= endAddr - 1) - { - break; - } - - currBlock = currBlock.Successor; - } - - _slabManager.Count += _blockTree.Count - oldCount; - - ValidateInternalState(); - } - - [Conditional("DEBUG")] - private void ValidateInternalState() - { - ulong expectedAddress = 0; - - KMemoryBlock currBlock = FindBlock(_addrSpaceStart); - - while (currBlock != null) - { - Debug.Assert(currBlock.BaseAddress == expectedAddress); - - expectedAddress = currBlock.BaseAddress + currBlock.PagesCount * PageSize; - - currBlock = currBlock.Successor; - } - - Debug.Assert(expectedAddress == _addrSpaceEnd); - } - - private KMemoryBlock MergeEqualStateNeighbors(KMemoryBlock block) - { - KMemoryBlock previousBlock = block.Predecessor; - KMemoryBlock nextBlock = block.Successor; - - if (previousBlock != null && BlockStateEquals(block, previousBlock)) - { - _blockTree.Remove(block); - - previousBlock.AddPages(block.PagesCount); - - block = previousBlock; - } - - if (nextBlock != null && BlockStateEquals(block, nextBlock)) - { - _blockTree.Remove(nextBlock); - - block.AddPages(nextBlock.PagesCount); - } - - return block; - } - - private static bool BlockStateEquals(KMemoryBlock lhs, KMemoryBlock rhs) - { - return lhs.State == rhs.State && - lhs.Permission == rhs.Permission && - lhs.Attribute == rhs.Attribute && - lhs.SourcePermission == rhs.SourcePermission && - lhs.DeviceRefCount == rhs.DeviceRefCount && - lhs.IpcRefCount == rhs.IpcRefCount; - } - - public KMemoryBlock FindBlock(ulong address) - { - return _blockTree.GetNodeByKey(address); - } - } -} diff --git a/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryBlockSlabManager.cs b/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryBlockSlabManager.cs deleted file mode 100644 index 8732b507..00000000 --- a/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryBlockSlabManager.cs +++ /dev/null @@ -1,19 +0,0 @@ -namespace Ryujinx.HLE.HOS.Kernel.Memory -{ - class KMemoryBlockSlabManager - { - private ulong _capacityElements; - - public int Count { get; set; } - - public KMemoryBlockSlabManager(ulong capacityElements) - { - _capacityElements = capacityElements; - } - - public bool CanAllocate(int count) - { - return (ulong)(Count + count) <= _capacityElements; - } - } -}
\ No newline at end of file diff --git a/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryInfo.cs b/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryInfo.cs deleted file mode 100644 index af070ac2..00000000 --- a/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryInfo.cs +++ /dev/null @@ -1,36 +0,0 @@ -namespace Ryujinx.HLE.HOS.Kernel.Memory -{ - class KMemoryInfo - { - public ulong Address { get; } - public ulong Size { get; } - - public MemoryState State { get; } - public KMemoryPermission Permission { get; } - public MemoryAttribute Attribute { get; } - public KMemoryPermission SourcePermission { get; } - - public int IpcRefCount { get; } - public int DeviceRefCount { get; } - - public KMemoryInfo( - ulong address, - ulong size, - MemoryState state, - KMemoryPermission permission, - MemoryAttribute attribute, - KMemoryPermission sourcePermission, - int ipcRefCount, - int deviceRefCount) - { - Address = address; - Size = size; - State = state; - Permission = permission; - Attribute = attribute; - SourcePermission = sourcePermission; - IpcRefCount = ipcRefCount; - DeviceRefCount = deviceRefCount; - } - } -}
\ No newline at end of file diff --git a/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryManager.cs b/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryManager.cs deleted file mode 100644 index 6d0a1658..00000000 --- a/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryManager.cs +++ /dev/null @@ -1,65 +0,0 @@ -using Ryujinx.HLE.HOS.Kernel.Common; -using System; - -namespace Ryujinx.HLE.HOS.Kernel.Memory -{ - class KMemoryManager - { - public KMemoryRegionManager[] MemoryRegions { get; } - - public KMemoryManager(MemorySize size, MemoryArrange arrange) - { - MemoryRegions = KernelInit.GetMemoryRegions(size, arrange); - } - - private KMemoryRegionManager GetMemoryRegion(ulong address) - { - for (int i = 0; i < MemoryRegions.Length; i++) - { - var region = MemoryRegions[i]; - - if (address >= region.Address && address < region.EndAddr) - { - return region; - } - } - - return null; - } - - public void IncrementPagesReferenceCount(ulong address, ulong pagesCount) - { - IncrementOrDecrementPagesReferenceCount(address, pagesCount, true); - } - - public void DecrementPagesReferenceCount(ulong address, ulong pagesCount) - { - IncrementOrDecrementPagesReferenceCount(address, pagesCount, false); - } - - private void IncrementOrDecrementPagesReferenceCount(ulong address, ulong pagesCount, bool increment) - { - while (pagesCount != 0) - { - var region = GetMemoryRegion(address); - - ulong countToProcess = Math.Min(pagesCount, region.GetPageOffsetFromEnd(address)); - - lock (region) - { - if (increment) - { - region.IncrementPagesReferenceCount(address, countToProcess); - } - else - { - region.DecrementPagesReferenceCount(address, countToProcess); - } - } - - pagesCount -= countToProcess; - address += countToProcess * KPageTableBase.PageSize; - } - } - } -} diff --git a/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryRegionManager.cs b/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryRegionManager.cs deleted file mode 100644 index 4596b15d..00000000 --- a/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryRegionManager.cs +++ /dev/null @@ -1,242 +0,0 @@ -using Ryujinx.Horizon.Common; -using System.Diagnostics; - -namespace Ryujinx.HLE.HOS.Kernel.Memory -{ - class KMemoryRegionManager - { - private readonly KPageHeap _pageHeap; - - public ulong Address { get; } - public ulong Size { get; } - public ulong EndAddr => Address + Size; - - private readonly ushort[] _pageReferenceCounts; - - public KMemoryRegionManager(ulong address, ulong size, ulong endAddr) - { - Address = address; - Size = size; - - _pageReferenceCounts = new ushort[size / KPageTableBase.PageSize]; - - _pageHeap = new KPageHeap(address, size); - _pageHeap.Free(address, size / KPageTableBase.PageSize); - _pageHeap.UpdateUsedSize(); - } - - public Result AllocatePages(out KPageList pageList, ulong pagesCount) - { - if (pagesCount == 0) - { - pageList = new KPageList(); - - return Result.Success; - } - - lock (_pageHeap) - { - Result result = AllocatePagesImpl(out pageList, pagesCount, false); - - if (result == Result.Success) - { - foreach (var node in pageList) - { - IncrementPagesReferenceCount(node.Address, node.PagesCount); - } - } - - return result; - } - } - - public ulong AllocatePagesContiguous(KernelContext context, ulong pagesCount, bool backwards) - { - if (pagesCount == 0) - { - return 0; - } - - lock (_pageHeap) - { - ulong address = AllocatePagesContiguousImpl(pagesCount, 1, backwards); - - if (address != 0) - { - IncrementPagesReferenceCount(address, pagesCount); - context.CommitMemory(address - DramMemoryMap.DramBase, pagesCount * KPageTableBase.PageSize); - } - - return address; - } - } - - private Result AllocatePagesImpl(out KPageList pageList, ulong pagesCount, bool random) - { - pageList = new KPageList(); - - int heapIndex = KPageHeap.GetBlockIndex(pagesCount); - - if (heapIndex < 0) - { - return KernelResult.OutOfMemory; - } - - for (int index = heapIndex; index >= 0; index--) - { - ulong pagesPerAlloc = KPageHeap.GetBlockPagesCount(index); - - while (pagesCount >= pagesPerAlloc) - { - ulong allocatedBlock = _pageHeap.AllocateBlock(index, random); - - if (allocatedBlock == 0) - { - break; - } - - Result result = pageList.AddRange(allocatedBlock, pagesPerAlloc); - - if (result != Result.Success) - { - FreePages(pageList); - _pageHeap.Free(allocatedBlock, pagesPerAlloc); - - return result; - } - - pagesCount -= pagesPerAlloc; - } - } - - if (pagesCount != 0) - { - FreePages(pageList); - - return KernelResult.OutOfMemory; - } - - return Result.Success; - } - - private ulong AllocatePagesContiguousImpl(ulong pagesCount, ulong alignPages, bool random) - { - int heapIndex = KPageHeap.GetAlignedBlockIndex(pagesCount, alignPages); - - ulong allocatedBlock = _pageHeap.AllocateBlock(heapIndex, random); - - if (allocatedBlock == 0) - { - return 0; - } - - ulong allocatedPages = KPageHeap.GetBlockPagesCount(heapIndex); - - if (allocatedPages > pagesCount) - { - _pageHeap.Free(allocatedBlock + pagesCount * KPageTableBase.PageSize, allocatedPages - pagesCount); - } - - return allocatedBlock; - } - - public void FreePage(ulong address) - { - lock (_pageHeap) - { - _pageHeap.Free(address, 1); - } - } - - public void FreePages(KPageList pageList) - { - lock (_pageHeap) - { - foreach (KPageNode pageNode in pageList) - { - _pageHeap.Free(pageNode.Address, pageNode.PagesCount); - } - } - } - - public void FreePages(ulong address, ulong pagesCount) - { - lock (_pageHeap) - { - _pageHeap.Free(address, pagesCount); - } - } - - public ulong GetFreePages() - { - lock (_pageHeap) - { - return _pageHeap.GetFreePagesCount(); - } - } - - public void IncrementPagesReferenceCount(ulong address, ulong pagesCount) - { - ulong index = GetPageOffset(address); - ulong endIndex = index + pagesCount; - - while (index < endIndex) - { - ushort referenceCount = ++_pageReferenceCounts[index]; - Debug.Assert(referenceCount >= 1); - - index++; - } - } - - public void DecrementPagesReferenceCount(ulong address, ulong pagesCount) - { - ulong index = GetPageOffset(address); - ulong endIndex = index + pagesCount; - - ulong freeBaseIndex = 0; - ulong freePagesCount = 0; - - while (index < endIndex) - { - Debug.Assert(_pageReferenceCounts[index] > 0); - ushort referenceCount = --_pageReferenceCounts[index]; - - if (referenceCount == 0) - { - if (freePagesCount != 0) - { - freePagesCount++; - } - else - { - freeBaseIndex = index; - freePagesCount = 1; - } - } - else if (freePagesCount != 0) - { - FreePages(Address + freeBaseIndex * KPageTableBase.PageSize, freePagesCount); - freePagesCount = 0; - } - - index++; - } - - if (freePagesCount != 0) - { - FreePages(Address + freeBaseIndex * KPageTableBase.PageSize, freePagesCount); - } - } - - public ulong GetPageOffset(ulong address) - { - return (address - Address) / KPageTableBase.PageSize; - } - - public ulong GetPageOffsetFromEnd(ulong address) - { - return (EndAddr - address) / KPageTableBase.PageSize; - } - } -}
\ No newline at end of file diff --git a/Ryujinx.HLE/HOS/Kernel/Memory/KPageBitmap.cs b/Ryujinx.HLE/HOS/Kernel/Memory/KPageBitmap.cs deleted file mode 100644 index fa090b02..00000000 --- a/Ryujinx.HLE/HOS/Kernel/Memory/KPageBitmap.cs +++ /dev/null @@ -1,298 +0,0 @@ -using Ryujinx.Common; -using System; -using System.Numerics; - -namespace Ryujinx.HLE.HOS.Kernel.Memory -{ - class KPageBitmap - { - private struct RandomNumberGenerator - { - private uint _entropy; - private uint _bitsAvailable; - - private void RefreshEntropy() - { - _entropy = 0; - _bitsAvailable = sizeof(uint) * 8; - } - - private bool GenerateRandomBit() - { - if (_bitsAvailable == 0) - { - RefreshEntropy(); - } - - bool bit = (_entropy & 1) != 0; - - _entropy >>= 1; - _bitsAvailable--; - - return bit; - } - - public int SelectRandomBit(ulong bitmap) - { - int selected = 0; - - int bitsCount = UInt64BitSize / 2; - ulong mask = (1UL << bitsCount) - 1; - - while (bitsCount != 0) - { - ulong low = bitmap & mask; - ulong high = (bitmap >> bitsCount) & mask; - - bool chooseLow; - - if (high == 0) - { - chooseLow = true; - } - else if (low == 0) - { - chooseLow = false; - } - else - { - chooseLow = GenerateRandomBit(); - } - - if (chooseLow) - { - bitmap = low; - } - else - { - bitmap = high; - selected += bitsCount; - } - - bitsCount /= 2; - mask >>= bitsCount; - } - - return selected; - } - } - - private const int UInt64BitSize = sizeof(ulong) * 8; - private const int MaxDepth = 4; - - private readonly RandomNumberGenerator _rng; - private readonly ArraySegment<ulong>[] _bitStorages; - private int _usedDepths; - - public int BitsCount { get; private set; } - - public int HighestDepthIndex => _usedDepths - 1; - - public KPageBitmap() - { - _rng = new RandomNumberGenerator(); - _bitStorages = new ArraySegment<ulong>[MaxDepth]; - } - - public ArraySegment<ulong> Initialize(ArraySegment<ulong> storage, ulong size) - { - _usedDepths = GetRequiredDepth(size); - - for (int depth = HighestDepthIndex; depth >= 0; depth--) - { - _bitStorages[depth] = storage; - size = BitUtils.DivRoundUp<ulong>(size, (ulong)UInt64BitSize); - storage = storage.Slice((int)size); - } - - return storage; - } - - public ulong FindFreeBlock(bool random) - { - ulong offset = 0; - int depth = 0; - - if (random) - { - do - { - ulong v = _bitStorages[depth][(int)offset]; - - if (v == 0) - { - return ulong.MaxValue; - } - - offset = offset * UInt64BitSize + (ulong)_rng.SelectRandomBit(v); - } - while (++depth < _usedDepths); - } - else - { - do - { - ulong v = _bitStorages[depth][(int)offset]; - - if (v == 0) - { - return ulong.MaxValue; - } - - offset = offset * UInt64BitSize + (ulong)BitOperations.TrailingZeroCount(v); - } - while (++depth < _usedDepths); - } - - return offset; - } - - public void SetBit(ulong offset) - { - SetBit(HighestDepthIndex, offset); - BitsCount++; - } - - public void ClearBit(ulong offset) - { - ClearBit(HighestDepthIndex, offset); - BitsCount--; - } - - public bool ClearRange(ulong offset, int count) - { - int depth = HighestDepthIndex; - var bits = _bitStorages[depth]; - - int bitInd = (int)(offset / UInt64BitSize); - - if (count < UInt64BitSize) - { - int shift = (int)(offset % UInt64BitSize); - - ulong mask = ((1UL << count) - 1) << shift; - - ulong v = bits[bitInd]; - - if ((v & mask) != mask) - { - return false; - } - - v &= ~mask; - bits[bitInd] = v; - - if (v == 0) - { - ClearBit(depth - 1, (ulong)bitInd); - } - } - else - { - int remaining = count; - int i = 0; - - do - { - if (bits[bitInd + i++] != ulong.MaxValue) - { - return false; - } - - remaining -= UInt64BitSize; - } - while (remaining > 0); - - remaining = count; - i = 0; - - do - { - bits[bitInd + i] = 0; - ClearBit(depth - 1, (ulong)(bitInd + i)); - i++; - remaining -= UInt64BitSize; - } - while (remaining > 0); - } - - BitsCount -= count; - return true; - } - - private void SetBit(int depth, ulong offset) - { - while (depth >= 0) - { - int ind = (int)(offset / UInt64BitSize); - int which = (int)(offset % UInt64BitSize); - - ulong mask = 1UL << which; - - ulong v = _bitStorages[depth][ind]; - - _bitStorages[depth][ind] = v | mask; - - if (v != 0) - { - break; - } - - offset = (ulong)ind; - depth--; - } - } - - private void ClearBit(int depth, ulong offset) - { - while (depth >= 0) - { - int ind = (int)(offset / UInt64BitSize); - int which = (int)(offset % UInt64BitSize); - - ulong mask = 1UL << which; - - ulong v = _bitStorages[depth][ind]; - - v &= ~mask; - - _bitStorages[depth][ind] = v; - - if (v != 0) - { - break; - } - - offset = (ulong)ind; - depth--; - } - } - - private static int GetRequiredDepth(ulong regionSize) - { - int depth = 0; - - do - { - regionSize /= UInt64BitSize; - depth++; - } - while (regionSize != 0); - - return depth; - } - - public static int CalculateManagementOverheadSize(ulong regionSize) - { - int overheadBits = 0; - - for (int depth = GetRequiredDepth(regionSize) - 1; depth >= 0; depth--) - { - regionSize = BitUtils.DivRoundUp<ulong>(regionSize, UInt64BitSize); - overheadBits += (int)regionSize; - } - - return overheadBits * sizeof(ulong); - } - } -} diff --git a/Ryujinx.HLE/HOS/Kernel/Memory/KPageHeap.cs b/Ryujinx.HLE/HOS/Kernel/Memory/KPageHeap.cs deleted file mode 100644 index c3586ed7..00000000 --- a/Ryujinx.HLE/HOS/Kernel/Memory/KPageHeap.cs +++ /dev/null @@ -1,283 +0,0 @@ -using Ryujinx.Common; -using System; - -namespace Ryujinx.HLE.HOS.Kernel.Memory -{ - class KPageHeap - { - private class Block - { - private KPageBitmap _bitmap = new KPageBitmap(); - private ulong _heapAddress; - private ulong _endOffset; - - public int Shift { get; private set; } - public int NextShift { get; private set; } - public ulong Size => 1UL << Shift; - public int PagesCount => (int)(Size / KPageTableBase.PageSize); - public int FreeBlocksCount => _bitmap.BitsCount; - public int FreePagesCount => FreeBlocksCount * PagesCount; - - public ArraySegment<ulong> Initialize(ulong address, ulong size, int blockShift, int nextBlockShift, ArraySegment<ulong> bitStorage) - { - Shift = blockShift; - NextShift = nextBlockShift; - - ulong endAddress = address + size; - - ulong align = nextBlockShift != 0 - ? 1UL << nextBlockShift - : 1UL << blockShift; - - address = BitUtils.AlignDown(address, align); - endAddress = BitUtils.AlignUp (endAddress, align); - - _heapAddress = address; - _endOffset = (endAddress - address) / (1UL << blockShift); - - return _bitmap.Initialize(bitStorage, _endOffset); - } - - public ulong PushBlock(ulong address) - { - ulong offset = (address - _heapAddress) >> Shift; - - _bitmap.SetBit(offset); - - if (NextShift != 0) - { - int diff = 1 << (NextShift - Shift); - - offset = BitUtils.AlignDown(offset, (ulong)diff); - - if (_bitmap.ClearRange(offset, diff)) - { - return _heapAddress + (offset << Shift); - } - } - - return 0; - } - - public ulong PopBlock(bool random) - { - long sOffset = (long)_bitmap.FindFreeBlock(random); - - if (sOffset < 0L) - { - return 0; - } - - ulong offset = (ulong)sOffset; - - _bitmap.ClearBit(offset); - - return _heapAddress + (offset << Shift); - } - - public static int CalculateManagementOverheadSize(ulong regionSize, int currBlockShift, int nextBlockShift) - { - ulong currBlockSize = 1UL << currBlockShift; - ulong nextBlockSize = 1UL << nextBlockShift; - ulong align = nextBlockShift != 0 ? nextBlockSize : currBlockSize; - return KPageBitmap.CalculateManagementOverheadSize((align * 2 + BitUtils.AlignUp(regionSize, align)) / currBlockSize); - } - } - - private static readonly int[] _memoryBlockPageShifts = new int[] { 12, 16, 21, 22, 25, 29, 30 }; - - private readonly ulong _heapAddress; - private readonly ulong _heapSize; - private ulong _usedSize; - private readonly int _blocksCount; - private readonly Block[] _blocks; - - public KPageHeap(ulong address, ulong size) : this(address, size, _memoryBlockPageShifts) - { - } - - public KPageHeap(ulong address, ulong size, int[] blockShifts) - { - _heapAddress = address; - _heapSize = size; - _blocksCount = blockShifts.Length; - _blocks = new Block[_memoryBlockPageShifts.Length]; - - var currBitmapStorage = new ArraySegment<ulong>(new ulong[CalculateManagementOverheadSize(size, blockShifts)]); - - for (int i = 0; i < blockShifts.Length; i++) - { - int currBlockShift = blockShifts[i]; - int nextBlockShift = i != blockShifts.Length - 1 ? blockShifts[i + 1] : 0; - - _blocks[i] = new Block(); - - currBitmapStorage = _blocks[i].Initialize(address, size, currBlockShift, nextBlockShift, currBitmapStorage); - } - } - - public void UpdateUsedSize() - { - _usedSize = _heapSize - (GetFreePagesCount() * KPageTableBase.PageSize); - } - - public ulong GetFreePagesCount() - { - ulong freeCount = 0; - - for (int i = 0; i < _blocksCount; i++) - { - freeCount += (ulong)_blocks[i].FreePagesCount; - } - - return freeCount; - } - - public ulong AllocateBlock(int index, bool random) - { - ulong neededSize = _blocks[index].Size; - - for (int i = index; i < _blocksCount; i++) - { - ulong address = _blocks[i].PopBlock(random); - - if (address != 0) - { - ulong allocatedSize = _blocks[i].Size; - - if (allocatedSize > neededSize) - { - Free(address + neededSize, (allocatedSize - neededSize) / KPageTableBase.PageSize); - } - - return address; - } - } - - return 0; - } - - private void FreeBlock(ulong block, int index) - { - do - { - block = _blocks[index++].PushBlock(block); - } - while (block != 0); - } - - public void Free(ulong address, ulong pagesCount) - { - if (pagesCount == 0) - { - return; - } - - int bigIndex = _blocksCount - 1; - - ulong start = address; - ulong end = address + pagesCount * KPageTableBase.PageSize; - ulong beforeStart = start; - ulong beforeEnd = start; - ulong afterStart = end; - ulong afterEnd = end; - - while (bigIndex >= 0) - { - ulong blockSize = _blocks[bigIndex].Size; - - ulong bigStart = BitUtils.AlignUp (start, blockSize); - ulong bigEnd = BitUtils.AlignDown(end, blockSize); - - if (bigStart < bigEnd) - { - for (ulong block = bigStart; block < bigEnd; block += blockSize) - { - FreeBlock(block, bigIndex); - } - - beforeEnd = bigStart; - afterStart = bigEnd; - - break; - } - - bigIndex--; - } - - for (int i = bigIndex - 1; i >= 0; i--) - { - ulong blockSize = _blocks[i].Size; - - while (beforeStart + blockSize <= beforeEnd) - { - beforeEnd -= blockSize; - FreeBlock(beforeEnd, i); - } - } - - for (int i = bigIndex - 1; i >= 0; i--) - { - ulong blockSize = _blocks[i].Size; - - while (afterStart + blockSize <= afterEnd) - { - FreeBlock(afterStart, i); - afterStart += blockSize; - } - } - } - - public static int GetAlignedBlockIndex(ulong pagesCount, ulong alignPages) - { - ulong targetPages = Math.Max(pagesCount, alignPages); - - for (int i = 0; i < _memoryBlockPageShifts.Length; i++) - { - if (targetPages <= GetBlockPagesCount(i)) - { - return i; - } - } - - return -1; - } - - public static int GetBlockIndex(ulong pagesCount) - { - for (int i = _memoryBlockPageShifts.Length - 1; i >= 0; i--) - { - if (pagesCount >= GetBlockPagesCount(i)) - { - return i; - } - } - - return -1; - } - - public static ulong GetBlockSize(int index) - { - return 1UL << _memoryBlockPageShifts[index]; - } - - public static ulong GetBlockPagesCount(int index) - { - return GetBlockSize(index) / KPageTableBase.PageSize; - } - - private static int CalculateManagementOverheadSize(ulong regionSize, int[] blockShifts) - { - int overheadSize = 0; - - for (int i = 0; i < blockShifts.Length; i++) - { - int currBlockShift = blockShifts[i]; - int nextBlockShift = i != blockShifts.Length - 1 ? blockShifts[i + 1] : 0; - overheadSize += Block.CalculateManagementOverheadSize(regionSize, currBlockShift, nextBlockShift); - } - - return BitUtils.AlignUp(overheadSize, KPageTableBase.PageSize); - } - } -} diff --git a/Ryujinx.HLE/HOS/Kernel/Memory/KPageList.cs b/Ryujinx.HLE/HOS/Kernel/Memory/KPageList.cs deleted file mode 100644 index 3149faa9..00000000 --- a/Ryujinx.HLE/HOS/Kernel/Memory/KPageList.cs +++ /dev/null @@ -1,97 +0,0 @@ -using Ryujinx.Horizon.Common; -using System.Collections; -using System.Collections.Generic; - -namespace Ryujinx.HLE.HOS.Kernel.Memory -{ - class KPageList : IEnumerable<KPageNode> - { - public LinkedList<KPageNode> Nodes { get; } - - public KPageList() - { - Nodes = new LinkedList<KPageNode>(); - } - - public Result AddRange(ulong address, ulong pagesCount) - { - if (pagesCount != 0) - { - if (Nodes.Last != null) - { - KPageNode lastNode = Nodes.Last.Value; - - if (lastNode.Address + lastNode.PagesCount * KPageTableBase.PageSize == address) - { - address = lastNode.Address; - pagesCount += lastNode.PagesCount; - - Nodes.RemoveLast(); - } - } - - Nodes.AddLast(new KPageNode(address, pagesCount)); - } - - return Result.Success; - } - - public ulong GetPagesCount() - { - ulong sum = 0; - - foreach (KPageNode node in Nodes) - { - sum += node.PagesCount; - } - - return sum; - } - - public bool IsEqual(KPageList other) - { - LinkedListNode<KPageNode> thisNode = Nodes.First; - LinkedListNode<KPageNode> otherNode = other.Nodes.First; - - while (thisNode != null && otherNode != null) - { - if (thisNode.Value.Address != otherNode.Value.Address || - thisNode.Value.PagesCount != otherNode.Value.PagesCount) - { - return false; - } - - thisNode = thisNode.Next; - otherNode = otherNode.Next; - } - - return thisNode == null && otherNode == null; - } - - public void IncrementPagesReferenceCount(KMemoryManager manager) - { - foreach (var node in this) - { - manager.IncrementPagesReferenceCount(node.Address, node.PagesCount); - } - } - - public void DecrementPagesReferenceCount(KMemoryManager manager) - { - foreach (var node in this) - { - manager.DecrementPagesReferenceCount(node.Address, node.PagesCount); - } - } - - public IEnumerator<KPageNode> GetEnumerator() - { - return Nodes.GetEnumerator(); - } - - IEnumerator IEnumerable.GetEnumerator() - { - return GetEnumerator(); - } - } -}
\ No newline at end of file diff --git a/Ryujinx.HLE/HOS/Kernel/Memory/KPageNode.cs b/Ryujinx.HLE/HOS/Kernel/Memory/KPageNode.cs deleted file mode 100644 index ada41687..00000000 --- a/Ryujinx.HLE/HOS/Kernel/Memory/KPageNode.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace Ryujinx.HLE.HOS.Kernel.Memory -{ - struct KPageNode - { - public ulong Address; - public ulong PagesCount; - - public KPageNode(ulong address, ulong pagesCount) - { - Address = address; - PagesCount = pagesCount; - } - } -}
\ No newline at end of file diff --git a/Ryujinx.HLE/HOS/Kernel/Memory/KPageTable.cs b/Ryujinx.HLE/HOS/Kernel/Memory/KPageTable.cs deleted file mode 100644 index 28e9f90a..00000000 --- a/Ryujinx.HLE/HOS/Kernel/Memory/KPageTable.cs +++ /dev/null @@ -1,229 +0,0 @@ -using Ryujinx.Horizon.Common; -using Ryujinx.Memory; -using Ryujinx.Memory.Range; -using System; -using System.Collections.Generic; -using System.Diagnostics; - -namespace Ryujinx.HLE.HOS.Kernel.Memory -{ - class KPageTable : KPageTableBase - { - private readonly IVirtualMemoryManager _cpuMemory; - - protected override bool Supports4KBPages => _cpuMemory.Supports4KBPages; - - public KPageTable(KernelContext context, IVirtualMemoryManager cpuMemory) : base(context) - { - _cpuMemory = cpuMemory; - } - - /// <inheritdoc/> - protected override IEnumerable<HostMemoryRange> GetHostRegions(ulong va, ulong size) - { - return _cpuMemory.GetHostRegions(va, size); - } - - /// <inheritdoc/> - protected override void GetPhysicalRegions(ulong va, ulong size, KPageList pageList) - { - var ranges = _cpuMemory.GetPhysicalRegions(va, size); - foreach (var range in ranges) - { - pageList.AddRange(range.Address + DramMemoryMap.DramBase, range.Size / PageSize); - } - } - - /// <inheritdoc/> - protected override ReadOnlySpan<byte> GetSpan(ulong va, int size) - { - return _cpuMemory.GetSpan(va, size); - } - - /// <inheritdoc/> - protected override Result MapMemory(ulong src, ulong dst, ulong pagesCount, KMemoryPermission oldSrcPermission, KMemoryPermission newDstPermission) - { - KPageList pageList = new KPageList(); - GetPhysicalRegions(src, pagesCount * PageSize, pageList); - - Result result = Reprotect(src, pagesCount, KMemoryPermission.None); - - if (result != Result.Success) - { - return result; - } - - result = MapPages(dst, pageList, newDstPermission, MemoryMapFlags.Private, false, 0); - - if (result != Result.Success) - { - Result reprotectResult = Reprotect(src, pagesCount, oldSrcPermission); - Debug.Assert(reprotectResult == Result.Success); - } - - return result; - } - - /// <inheritdoc/> - protected override Result UnmapMemory(ulong dst, ulong src, ulong pagesCount, KMemoryPermission oldDstPermission, KMemoryPermission newSrcPermission) - { - ulong size = pagesCount * PageSize; - - KPageList srcPageList = new KPageList(); - KPageList dstPageList = new KPageList(); - - GetPhysicalRegions(src, size, srcPageList); - GetPhysicalRegions(dst, size, dstPageList); - - if (!dstPageList.IsEqual(srcPageList)) - { - return KernelResult.InvalidMemRange; - } - - Result result = Unmap(dst, pagesCount); - - if (result != Result.Success) - { - return result; - } - - result = Reprotect(src, pagesCount, newSrcPermission); - - if (result != Result.Success) - { - Result mapResult = MapPages(dst, dstPageList, oldDstPermission, MemoryMapFlags.Private, false, 0); - Debug.Assert(mapResult == Result.Success); - } - - return result; - } - - /// <inheritdoc/> - protected override Result MapPages( - ulong dstVa, - ulong pagesCount, - ulong srcPa, - KMemoryPermission permission, - MemoryMapFlags flags, - bool shouldFillPages, - byte fillValue) - { - ulong size = pagesCount * PageSize; - - Context.CommitMemory(srcPa - DramMemoryMap.DramBase, size); - - _cpuMemory.Map(dstVa, srcPa - DramMemoryMap.DramBase, size, flags); - - if (DramMemoryMap.IsHeapPhysicalAddress(srcPa)) - { - Context.MemoryManager.IncrementPagesReferenceCount(srcPa, pagesCount); - } - - if (shouldFillPages) - { - _cpuMemory.Fill(dstVa, size, fillValue); - } - - return Result.Success; - } - - /// <inheritdoc/> - protected override Result MapPages( - ulong address, - KPageList pageList, - KMemoryPermission permission, - MemoryMapFlags flags, - bool shouldFillPages, - byte fillValue) - { - using var scopedPageList = new KScopedPageList(Context.MemoryManager, pageList); - - ulong currentVa = address; - - foreach (var pageNode in pageList) - { - ulong addr = pageNode.Address - DramMemoryMap.DramBase; - ulong size = pageNode.PagesCount * PageSize; - - Context.CommitMemory(addr, size); - - _cpuMemory.Map(currentVa, addr, size, flags); - - if (shouldFillPages) - { - _cpuMemory.Fill(currentVa, size, fillValue); - } - - currentVa += size; - } - - scopedPageList.SignalSuccess(); - - return Result.Success; - } - - /// <inheritdoc/> - protected override Result MapForeign(IEnumerable<HostMemoryRange> regions, ulong va, ulong size) - { - ulong offset = 0; - - foreach (var region in regions) - { - _cpuMemory.MapForeign(va + offset, region.Address, region.Size); - - offset += region.Size; - } - - return Result.Success; - } - - /// <inheritdoc/> - protected override Result Unmap(ulong address, ulong pagesCount) - { - KPageList pagesToClose = new KPageList(); - - var regions = _cpuMemory.GetPhysicalRegions(address, pagesCount * PageSize); - - foreach (var region in regions) - { - ulong pa = region.Address + DramMemoryMap.DramBase; - if (DramMemoryMap.IsHeapPhysicalAddress(pa)) - { - pagesToClose.AddRange(pa, region.Size / PageSize); - } - } - - _cpuMemory.Unmap(address, pagesCount * PageSize); - - pagesToClose.DecrementPagesReferenceCount(Context.MemoryManager); - - return Result.Success; - } - - /// <inheritdoc/> - protected override Result Reprotect(ulong address, ulong pagesCount, KMemoryPermission permission) - { - // TODO. - return Result.Success; - } - - /// <inheritdoc/> - protected override Result ReprotectWithAttributes(ulong address, ulong pagesCount, KMemoryPermission permission) - { - // TODO. - return Result.Success; - } - - /// <inheritdoc/> - protected override void SignalMemoryTracking(ulong va, ulong size, bool write) - { - _cpuMemory.SignalMemoryTracking(va, size, write); - } - - /// <inheritdoc/> - protected override void Write(ulong va, ReadOnlySpan<byte> data) - { - _cpuMemory.Write(va, data); - } - } -}
\ No newline at end of file diff --git a/Ryujinx.HLE/HOS/Kernel/Memory/KPageTableBase.cs b/Ryujinx.HLE/HOS/Kernel/Memory/KPageTableBase.cs deleted file mode 100644 index 614eb527..00000000 --- a/Ryujinx.HLE/HOS/Kernel/Memory/KPageTableBase.cs +++ /dev/null @@ -1,3043 +0,0 @@ -using Ryujinx.Common; -using Ryujinx.HLE.HOS.Kernel.Common; -using Ryujinx.HLE.HOS.Kernel.Process; -using Ryujinx.Horizon.Common; -using Ryujinx.Memory; -using Ryujinx.Memory.Range; -using System; -using System.Collections.Generic; -using System.Diagnostics; - -namespace Ryujinx.HLE.HOS.Kernel.Memory -{ - abstract class KPageTableBase - { - private static readonly int[] MappingUnitSizes = new int[] - { - 0x1000, - 0x10000, - 0x200000, - 0x400000, - 0x2000000, - 0x40000000 - }; - - public const int PageSize = 0x1000; - - private const int KMemoryBlockSize = 0x40; - - // We need 2 blocks for the case where a big block - // needs to be split in 2, plus one block that will be the new one inserted. - private const int MaxBlocksNeededForInsertion = 2; - - protected readonly KernelContext Context; - protected virtual bool Supports4KBPages => true; - - public ulong AddrSpaceStart { get; private set; } - public ulong AddrSpaceEnd { get; private set; } - - public ulong CodeRegionStart { get; private set; } - public ulong CodeRegionEnd { get; private set; } - - public ulong HeapRegionStart { get; private set; } - public ulong HeapRegionEnd { get; private set; } - - private ulong _currentHeapAddr; - - public ulong AliasRegionStart { get; private set; } - public ulong AliasRegionEnd { get; private set; } - - public ulong StackRegionStart { get; private set; } - public ulong StackRegionEnd { get; private set; } - - public ulong TlsIoRegionStart { get; private set; } - public ulong TlsIoRegionEnd { get; private set; } - - private ulong _heapCapacity; - - public ulong PhysicalMemoryUsage { get; private set; } - - private readonly KMemoryBlockManager _blockManager; - - private MemoryRegion _memRegion; - - private bool _aslrDisabled; - - public int AddrSpaceWidth { get; private set; } - - private bool _isKernel; - - private bool _aslrEnabled; - - private KMemoryBlockSlabManager _slabManager; - - private int _contextId; - - private MersenneTwister _randomNumberGenerator; - - private MemoryFillValue _heapFillValue; - private MemoryFillValue _ipcFillValue; - - public KPageTableBase(KernelContext context) - { - Context = context; - - _blockManager = new KMemoryBlockManager(); - - _isKernel = false; - - _heapFillValue = MemoryFillValue.Zero; - _ipcFillValue = MemoryFillValue.Zero; - } - - private static readonly int[] AddrSpaceSizes = new int[] { 32, 36, 32, 39 }; - - public Result InitializeForProcess( - AddressSpaceType addrSpaceType, - bool aslrEnabled, - bool aslrDisabled, - MemoryRegion memRegion, - ulong address, - ulong size, - KMemoryBlockSlabManager slabManager) - { - if ((uint)addrSpaceType > (uint)AddressSpaceType.Addr39Bits) - { - throw new ArgumentException(nameof(addrSpaceType)); - } - - _contextId = Context.ContextIdManager.GetId(); - - ulong addrSpaceBase = 0; - ulong addrSpaceSize = 1UL << AddrSpaceSizes[(int)addrSpaceType]; - - Result result = CreateUserAddressSpace( - addrSpaceType, - aslrEnabled, - aslrDisabled, - addrSpaceBase, - addrSpaceSize, - memRegion, - address, - size, - slabManager); - - if (result != Result.Success) - { - Context.ContextIdManager.PutId(_contextId); - } - - return result; - } - - private class Region - { - public ulong Start; - public ulong End; - public ulong Size; - public ulong AslrOffset; - } - - private Result CreateUserAddressSpace( - AddressSpaceType addrSpaceType, - bool aslrEnabled, - bool aslrDisabled, - ulong addrSpaceStart, - ulong addrSpaceEnd, - MemoryRegion memRegion, - ulong address, - ulong size, - KMemoryBlockSlabManager slabManager) - { - ulong endAddr = address + size; - - Region aliasRegion = new Region(); - Region heapRegion = new Region(); - Region stackRegion = new Region(); - Region tlsIoRegion = new Region(); - - ulong codeRegionSize; - ulong stackAndTlsIoStart; - ulong stackAndTlsIoEnd; - ulong baseAddress; - - switch (addrSpaceType) - { - case AddressSpaceType.Addr32Bits: - aliasRegion.Size = 0x40000000; - heapRegion.Size = 0x40000000; - stackRegion.Size = 0; - tlsIoRegion.Size = 0; - CodeRegionStart = 0x200000; - codeRegionSize = 0x3fe00000; - stackAndTlsIoStart = 0x200000; - stackAndTlsIoEnd = 0x40000000; - baseAddress = 0x200000; - AddrSpaceWidth = 32; - break; - - case AddressSpaceType.Addr36Bits: - aliasRegion.Size = 0x180000000; - heapRegion.Size = 0x180000000; - stackRegion.Size = 0; - tlsIoRegion.Size = 0; - CodeRegionStart = 0x8000000; - codeRegionSize = 0x78000000; - stackAndTlsIoStart = 0x8000000; - stackAndTlsIoEnd = 0x80000000; - baseAddress = 0x8000000; - AddrSpaceWidth = 36; - break; - - case AddressSpaceType.Addr32BitsNoMap: - aliasRegion.Size = 0; - heapRegion.Size = 0x80000000; - stackRegion.Size = 0; - tlsIoRegion.Size = 0; - CodeRegionStart = 0x200000; - codeRegionSize = 0x3fe00000; - stackAndTlsIoStart = 0x200000; - stackAndTlsIoEnd = 0x40000000; - baseAddress = 0x200000; - AddrSpaceWidth = 32; - break; - - case AddressSpaceType.Addr39Bits: - aliasRegion.Size = 0x1000000000; - heapRegion.Size = 0x180000000; - stackRegion.Size = 0x80000000; - tlsIoRegion.Size = 0x1000000000; - CodeRegionStart = BitUtils.AlignDown<ulong>(address, 0x200000); - codeRegionSize = BitUtils.AlignUp<ulong>(endAddr, 0x200000) - CodeRegionStart; - stackAndTlsIoStart = 0; - stackAndTlsIoEnd = 0; - baseAddress = 0x8000000; - AddrSpaceWidth = 39; - break; - - default: throw new ArgumentException(nameof(addrSpaceType)); - } - - CodeRegionEnd = CodeRegionStart + codeRegionSize; - - ulong mapBaseAddress; - ulong mapAvailableSize; - - if (CodeRegionStart - baseAddress >= addrSpaceEnd - CodeRegionEnd) - { - // Has more space before the start of the code region. - mapBaseAddress = baseAddress; - mapAvailableSize = CodeRegionStart - baseAddress; - } - else - { - // Has more space after the end of the code region. - mapBaseAddress = CodeRegionEnd; - mapAvailableSize = addrSpaceEnd - CodeRegionEnd; - } - - ulong mapTotalSize = aliasRegion.Size + heapRegion.Size + stackRegion.Size + tlsIoRegion.Size; - - ulong aslrMaxOffset = mapAvailableSize - mapTotalSize; - - _aslrEnabled = aslrEnabled; - - AddrSpaceStart = addrSpaceStart; - AddrSpaceEnd = addrSpaceEnd; - - _slabManager = slabManager; - - if (mapAvailableSize < mapTotalSize) - { - return KernelResult.OutOfMemory; - } - - if (aslrEnabled) - { - aliasRegion.AslrOffset = GetRandomValue(0, aslrMaxOffset >> 21) << 21; - heapRegion.AslrOffset = GetRandomValue(0, aslrMaxOffset >> 21) << 21; - stackRegion.AslrOffset = GetRandomValue(0, aslrMaxOffset >> 21) << 21; - tlsIoRegion.AslrOffset = GetRandomValue(0, aslrMaxOffset >> 21) << 21; - } - - // Regions are sorted based on ASLR offset. - // When ASLR is disabled, the order is Map, Heap, NewMap and TlsIo. - aliasRegion.Start = mapBaseAddress + aliasRegion.AslrOffset; - aliasRegion.End = aliasRegion.Start + aliasRegion.Size; - heapRegion.Start = mapBaseAddress + heapRegion.AslrOffset; - heapRegion.End = heapRegion.Start + heapRegion.Size; - stackRegion.Start = mapBaseAddress + stackRegion.AslrOffset; - stackRegion.End = stackRegion.Start + stackRegion.Size; - tlsIoRegion.Start = mapBaseAddress + tlsIoRegion.AslrOffset; - tlsIoRegion.End = tlsIoRegion.Start + tlsIoRegion.Size; - - SortRegion(heapRegion, aliasRegion); - - if (stackRegion.Size != 0) - { - SortRegion(stackRegion, aliasRegion); - SortRegion(stackRegion, heapRegion); - } - else - { - stackRegion.Start = stackAndTlsIoStart; - stackRegion.End = stackAndTlsIoEnd; - } - - if (tlsIoRegion.Size != 0) - { - SortRegion(tlsIoRegion, aliasRegion); - SortRegion(tlsIoRegion, heapRegion); - SortRegion(tlsIoRegion, stackRegion); - } - else - { - tlsIoRegion.Start = stackAndTlsIoStart; - tlsIoRegion.End = stackAndTlsIoEnd; - } - - AliasRegionStart = aliasRegion.Start; - AliasRegionEnd = aliasRegion.End; - HeapRegionStart = heapRegion.Start; - HeapRegionEnd = heapRegion.End; - StackRegionStart = stackRegion.Start; - StackRegionEnd = stackRegion.End; - TlsIoRegionStart = tlsIoRegion.Start; - TlsIoRegionEnd = tlsIoRegion.End; - - // TODO: Check kernel configuration via secure monitor call when implemented to set memory fill values. - - _currentHeapAddr = HeapRegionStart; - _heapCapacity = 0; - PhysicalMemoryUsage = 0; - - _memRegion = memRegion; - _aslrDisabled = aslrDisabled; - - return _blockManager.Initialize(addrSpaceStart, addrSpaceEnd, slabManager); - } - - private ulong GetRandomValue(ulong min, ulong max) - { - return (ulong)GetRandomValue((long)min, (long)max); - } - - private long GetRandomValue(long min, long max) - { - if (_randomNumberGenerator == null) - { - _randomNumberGenerator = new MersenneTwister(0); - } - - return _randomNumberGenerator.GenRandomNumber(min, max); - } - - private static void SortRegion(Region lhs, Region rhs) - { - if (lhs.AslrOffset < rhs.AslrOffset) - { - rhs.Start += lhs.Size; - rhs.End += lhs.Size; - } - else - { - lhs.Start += rhs.Size; - lhs.End += rhs.Size; - } - } - - public Result MapPages(ulong address, KPageList pageList, MemoryState state, KMemoryPermission permission) - { - ulong pagesCount = pageList.GetPagesCount(); - - ulong size = pagesCount * PageSize; - - if (!CanContain(address, size, state)) - { - return KernelResult.InvalidMemState; - } - - lock (_blockManager) - { - if (!IsUnmapped(address, pagesCount * PageSize)) - { - return KernelResult.InvalidMemState; - } - - if (!_slabManager.CanAllocate(MaxBlocksNeededForInsertion)) - { - return KernelResult.OutOfResource; - } - - Result result = MapPages(address, pageList, permission, MemoryMapFlags.None); - - if (result == Result.Success) - { - _blockManager.InsertBlock(address, pagesCount, state, permission); - } - - return result; - } - } - - public Result UnmapPages(ulong address, KPageList pageList, MemoryState stateExpected) - { - ulong pagesCount = pageList.GetPagesCount(); - ulong size = pagesCount * PageSize; - - ulong endAddr = address + size; - - ulong addrSpacePagesCount = (AddrSpaceEnd - AddrSpaceStart) / PageSize; - - if (AddrSpaceStart > address) - { - return KernelResult.InvalidMemState; - } - - if (addrSpacePagesCount < pagesCount) - { - return KernelResult.InvalidMemState; - } - - if (endAddr - 1 > AddrSpaceEnd - 1) - { - return KernelResult.InvalidMemState; - } - - lock (_blockManager) - { - KPageList currentPageList = new KPageList(); - - GetPhysicalRegions(address, size, currentPageList); - - if (!currentPageList.IsEqual(pageList)) - { - return KernelResult.InvalidMemRange; - } - - if (CheckRange( - address, - size, - MemoryState.Mask, - stateExpected, - KMemoryPermission.None, - KMemoryPermission.None, - MemoryAttribute.Mask, - MemoryAttribute.None, - MemoryAttribute.IpcAndDeviceMapped, - out MemoryState state, - out _, - out _)) - { - if (!_slabManager.CanAllocate(MaxBlocksNeededForInsertion)) - { - return KernelResult.OutOfResource; - } - - Result result = Unmap(address, pagesCount); - - if (result == Result.Success) - { - _blockManager.InsertBlock(address, pagesCount, MemoryState.Unmapped); - } - - return result; - } - else - { - return KernelResult.InvalidMemState; - } - } - } - - public Result MapNormalMemory(long address, long size, KMemoryPermission permission) - { - // TODO. - return Result.Success; - } - - public Result MapIoMemory(long address, long size, KMemoryPermission permission) - { - // TODO. - return Result.Success; - } - - public Result MapPages( - ulong pagesCount, - int alignment, - ulong srcPa, - bool paIsValid, - ulong regionStart, - ulong regionPagesCount, - MemoryState state, - KMemoryPermission permission, - out ulong address) - { - address = 0; - - ulong regionSize = regionPagesCount * PageSize; - - if (!CanContain(regionStart, regionSize, state)) - { - return KernelResult.InvalidMemState; - } - - if (regionPagesCount <= pagesCount) - { - return KernelResult.OutOfMemory; - } - - lock (_blockManager) - { - address = AllocateVa(regionStart, regionPagesCount, pagesCount, alignment); - - if (address == 0) - { - return KernelResult.OutOfMemory; - } - - if (!_slabManager.CanAllocate(MaxBlocksNeededForInsertion)) - { - return KernelResult.OutOfResource; - } - - Result result; - - if (paIsValid) - { - result = MapPages(address, pagesCount, srcPa, permission, MemoryMapFlags.Private); - } - else - { - result = AllocateAndMapPages(address, pagesCount, permission); - } - - if (result != Result.Success) - { - return result; - } - - _blockManager.InsertBlock(address, pagesCount, state, permission); - } - - return Result.Success; - } - - public Result MapPages(ulong address, ulong pagesCount, MemoryState state, KMemoryPermission permission) - { - ulong size = pagesCount * PageSize; - - if (!CanContain(address, size, state)) - { - return KernelResult.InvalidMemState; - } - - lock (_blockManager) - { - if (!IsUnmapped(address, size)) - { - return KernelResult.InvalidMemState; - } - - if (!_slabManager.CanAllocate(MaxBlocksNeededForInsertion)) - { - return KernelResult.OutOfResource; - } - - Result result = AllocateAndMapPages(address, pagesCount, permission); - - if (result == Result.Success) - { - _blockManager.InsertBlock(address, pagesCount, state, permission); - } - - return result; - } - } - - private Result AllocateAndMapPages(ulong address, ulong pagesCount, KMemoryPermission permission) - { - KMemoryRegionManager region = GetMemoryRegionManager(); - - Result result = region.AllocatePages(out KPageList pageList, pagesCount); - - if (result != Result.Success) - { - return result; - } - - using var _ = new OnScopeExit(() => pageList.DecrementPagesReferenceCount(Context.MemoryManager)); - - return MapPages(address, pageList, permission, MemoryMapFlags.Private); - } - - public Result MapProcessCodeMemory(ulong dst, ulong src, ulong size) - { - lock (_blockManager) - { - bool success = CheckRange( - src, - size, - MemoryState.Mask, - MemoryState.Heap, - KMemoryPermission.Mask, - KMemoryPermission.ReadAndWrite, - MemoryAttribute.Mask, - MemoryAttribute.None, - MemoryAttribute.IpcAndDeviceMapped, - out MemoryState state, - out KMemoryPermission permission, - out _); - - success &= IsUnmapped(dst, size); - - if (success) - { - if (!_slabManager.CanAllocate(MaxBlocksNeededForInsertion * 2)) - { - return KernelResult.OutOfResource; - } - - ulong pagesCount = size / PageSize; - - Result result = MapMemory(src, dst, pagesCount, permission, KMemoryPermission.None); - - _blockManager.InsertBlock(src, pagesCount, state, KMemoryPermission.None, MemoryAttribute.Borrowed); - _blockManager.InsertBlock(dst, pagesCount, MemoryState.ModCodeStatic); - - return Result.Success; - } - else - { - return KernelResult.InvalidMemState; - } - } - } - - public Result UnmapProcessCodeMemory(ulong dst, ulong src, ulong size) - { - lock (_blockManager) - { - bool success = CheckRange( - src, - size, - MemoryState.Mask, - MemoryState.Heap, - KMemoryPermission.None, - KMemoryPermission.None, - MemoryAttribute.Mask, - MemoryAttribute.Borrowed, - MemoryAttribute.IpcAndDeviceMapped, - out _, - out _, - out _); - - success &= CheckRange( - dst, - PageSize, - MemoryState.UnmapProcessCodeMemoryAllowed, - MemoryState.UnmapProcessCodeMemoryAllowed, - KMemoryPermission.None, - KMemoryPermission.None, - MemoryAttribute.Mask, - MemoryAttribute.None, - MemoryAttribute.IpcAndDeviceMapped, - out MemoryState state, - out _, - out _); - - success &= CheckRange( - dst, - size, - MemoryState.Mask, - state, - KMemoryPermission.None, - KMemoryPermission.None, - MemoryAttribute.Mask, - MemoryAttribute.None); - - if (success) - { - ulong pagesCount = size / PageSize; - - Result result = Unmap(dst, pagesCount); - - if (result != Result.Success) - { - return result; - } - - // TODO: Missing some checks here. - - if (!_slabManager.CanAllocate(MaxBlocksNeededForInsertion * 2)) - { - return KernelResult.OutOfResource; - } - - _blockManager.InsertBlock(dst, pagesCount, MemoryState.Unmapped); - _blockManager.InsertBlock(src, pagesCount, MemoryState.Heap, KMemoryPermission.ReadAndWrite); - - return Result.Success; - } - else - { - return KernelResult.InvalidMemState; - } - } - } - - public Result SetHeapSize(ulong size, out ulong address) - { - address = 0; - - if (size > HeapRegionEnd - HeapRegionStart) - { - return KernelResult.OutOfMemory; - } - - KProcess currentProcess = KernelStatic.GetCurrentProcess(); - - lock (_blockManager) - { - ulong currentHeapSize = GetHeapSize(); - - if (currentHeapSize <= size) - { - // Expand. - ulong sizeDelta = size - currentHeapSize; - - if (currentProcess.ResourceLimit != null && sizeDelta != 0 && - !currentProcess.ResourceLimit.Reserve(LimitableResource.Memory, sizeDelta)) - { - return KernelResult.ResLimitExceeded; - } - - ulong pagesCount = sizeDelta / PageSize; - - KMemoryRegionManager region = GetMemoryRegionManager(); - - Result result = region.AllocatePages(out KPageList pageList, pagesCount); - - using var _ = new OnScopeExit(() => pageList.DecrementPagesReferenceCount(Context.MemoryManager)); - - void CleanUpForError() - { - if (currentProcess.ResourceLimit != null && sizeDelta != 0) - { - currentProcess.ResourceLimit.Release(LimitableResource.Memory, sizeDelta); - } - } - - if (result != Result.Success) - { - CleanUpForError(); - - return result; - } - - if (!_slabManager.CanAllocate(MaxBlocksNeededForInsertion)) - { - CleanUpForError(); - - return KernelResult.OutOfResource; - } - - if (!IsUnmapped(_currentHeapAddr, sizeDelta)) - { - CleanUpForError(); - - return KernelResult.InvalidMemState; - } - - result = MapPages(_currentHeapAddr, pageList, KMemoryPermission.ReadAndWrite, MemoryMapFlags.Private, true, (byte)_heapFillValue); - - if (result != Result.Success) - { - CleanUpForError(); - - return result; - } - - _blockManager.InsertBlock(_currentHeapAddr, pagesCount, MemoryState.Heap, KMemoryPermission.ReadAndWrite); - } - else - { - // Shrink. - ulong freeAddr = HeapRegionStart + size; - ulong sizeDelta = currentHeapSize - size; - - if (!_slabManager.CanAllocate(MaxBlocksNeededForInsertion)) - { - return KernelResult.OutOfResource; - } - - if (!CheckRange( - freeAddr, - sizeDelta, - MemoryState.Mask, - MemoryState.Heap, - KMemoryPermission.Mask, - KMemoryPermission.ReadAndWrite, - MemoryAttribute.Mask, - MemoryAttribute.None, - MemoryAttribute.IpcAndDeviceMapped, - out _, - out _, - out _)) - { - return KernelResult.InvalidMemState; - } - - ulong pagesCount = sizeDelta / PageSize; - - Result result = Unmap(freeAddr, pagesCount); - - if (result != Result.Success) - { - return result; - } - - currentProcess.ResourceLimit?.Release(LimitableResource.Memory, sizeDelta); - - _blockManager.InsertBlock(freeAddr, pagesCount, MemoryState.Unmapped); - } - - _currentHeapAddr = HeapRegionStart + size; - } - - address = HeapRegionStart; - - return Result.Success; - } - - public Result SetMemoryPermission(ulong address, ulong size, KMemoryPermission permission) - { - lock (_blockManager) - { - if (CheckRange( - address, - size, - MemoryState.PermissionChangeAllowed, - MemoryState.PermissionChangeAllowed, - KMemoryPermission.None, - KMemoryPermission.None, - MemoryAttribute.Mask, - MemoryAttribute.None, - MemoryAttribute.IpcAndDeviceMapped, - out MemoryState oldState, - out KMemoryPermission oldPermission, - out _)) - { - if (permission != oldPermission) - { - if (!_slabManager.CanAllocate(MaxBlocksNeededForInsertion)) - { - return KernelResult.OutOfResource; - } - - ulong pagesCount = size / PageSize; - - Result result = Reprotect(address, pagesCount, permission); - - if (result != Result.Success) - { - return result; - } - - _blockManager.InsertBlock(address, pagesCount, oldState, permission); - } - - return Result.Success; - } - else - { - return KernelResult.InvalidMemState; - } - } - } - - public ulong GetTotalHeapSize() - { - lock (_blockManager) - { - return GetHeapSize() + PhysicalMemoryUsage; - } - } - - private ulong GetHeapSize() - { - return _currentHeapAddr - HeapRegionStart; - } - - public Result SetHeapCapacity(ulong capacity) - { - lock (_blockManager) - { - _heapCapacity = capacity; - } - - return Result.Success; - } - - public Result SetMemoryAttribute( - ulong address, - ulong size, - MemoryAttribute attributeMask, - MemoryAttribute attributeValue) - { - lock (_blockManager) - { - if (CheckRange( - address, - size, - MemoryState.AttributeChangeAllowed, - MemoryState.AttributeChangeAllowed, - KMemoryPermission.None, - KMemoryPermission.None, - MemoryAttribute.BorrowedAndIpcMapped, - MemoryAttribute.None, - MemoryAttribute.DeviceMappedAndUncached, - out MemoryState state, - out KMemoryPermission permission, - out MemoryAttribute attribute)) - { - if (!_slabManager.CanAllocate(MaxBlocksNeededForInsertion)) - { - return KernelResult.OutOfResource; - } - - ulong pagesCount = size / PageSize; - - attribute &= ~attributeMask; - attribute |= attributeMask & attributeValue; - - _blockManager.InsertBlock(address, pagesCount, state, permission, attribute); - - return Result.Success; - } - else - { - return KernelResult.InvalidMemState; - } - } - } - - public KMemoryInfo QueryMemory(ulong address) - { - if (address >= AddrSpaceStart && - address < AddrSpaceEnd) - { - lock (_blockManager) - { - return _blockManager.FindBlock(address).GetInfo(); - } - } - else - { - return new KMemoryInfo( - AddrSpaceEnd, - ~AddrSpaceEnd + 1, - MemoryState.Reserved, - KMemoryPermission.None, - MemoryAttribute.None, - KMemoryPermission.None, - 0, - 0); - } - } - - public Result Map(ulong dst, ulong src, ulong size) - { - bool success; - - lock (_blockManager) - { - success = CheckRange( - src, - size, - MemoryState.MapAllowed, - MemoryState.MapAllowed, - KMemoryPermission.Mask, - KMemoryPermission.ReadAndWrite, - MemoryAttribute.Mask, - MemoryAttribute.None, - MemoryAttribute.IpcAndDeviceMapped, - out MemoryState srcState, - out _, - out _); - - success &= IsUnmapped(dst, size); - - if (success) - { - if (!_slabManager.CanAllocate(MaxBlocksNeededForInsertion * 2)) - { - return KernelResult.OutOfResource; - } - - ulong pagesCount = size / PageSize; - - Result result = MapMemory(src, dst, pagesCount, KMemoryPermission.ReadAndWrite, KMemoryPermission.ReadAndWrite); - - if (result != Result.Success) - { - return result; - } - - _blockManager.InsertBlock(src, pagesCount, srcState, KMemoryPermission.None, MemoryAttribute.Borrowed); - _blockManager.InsertBlock(dst, pagesCount, MemoryState.Stack, KMemoryPermission.ReadAndWrite); - - return Result.Success; - } - else - { - return KernelResult.InvalidMemState; - } - } - } - - public Result UnmapForKernel(ulong address, ulong pagesCount, MemoryState stateExpected) - { - ulong size = pagesCount * PageSize; - - lock (_blockManager) - { - if (CheckRange( - address, - size, - MemoryState.Mask, - stateExpected, - KMemoryPermission.None, - KMemoryPermission.None, - MemoryAttribute.Mask, - MemoryAttribute.None, - MemoryAttribute.IpcAndDeviceMapped, - out _, - out _, - out _)) - { - if (!_slabManager.CanAllocate(MaxBlocksNeededForInsertion)) - { - return KernelResult.OutOfResource; - } - - Result result = Unmap(address, pagesCount); - - if (result == Result.Success) - { - _blockManager.InsertBlock(address, pagesCount, MemoryState.Unmapped); - } - - return Result.Success; - } - else - { - return KernelResult.InvalidMemState; - } - } - } - - public Result Unmap(ulong dst, ulong src, ulong size) - { - bool success; - - lock (_blockManager) - { - success = CheckRange( - src, - size, - MemoryState.MapAllowed, - MemoryState.MapAllowed, - KMemoryPermission.Mask, - KMemoryPermission.None, - MemoryAttribute.Mask, - MemoryAttribute.Borrowed, - MemoryAttribute.IpcAndDeviceMapped, - out MemoryState srcState, - out _, - out _); - - success &= CheckRange( - dst, - size, - MemoryState.Mask, - MemoryState.Stack, - KMemoryPermission.None, - KMemoryPermission.None, - MemoryAttribute.Mask, - MemoryAttribute.None, - MemoryAttribute.IpcAndDeviceMapped, - out _, - out KMemoryPermission dstPermission, - out _); - - if (success) - { - if (!_slabManager.CanAllocate(MaxBlocksNeededForInsertion * 2)) - { - return KernelResult.OutOfResource; - } - - ulong pagesCount = size / PageSize; - - Result result = UnmapMemory(dst, src, pagesCount, dstPermission, KMemoryPermission.ReadAndWrite); - - if (result != Result.Success) - { - return result; - } - - _blockManager.InsertBlock(src, pagesCount, srcState, KMemoryPermission.ReadAndWrite); - _blockManager.InsertBlock(dst, pagesCount, MemoryState.Unmapped); - - return Result.Success; - } - else - { - return KernelResult.InvalidMemState; - } - } - } - - public Result UnmapProcessMemory(ulong dst, ulong size, KPageTableBase srcPageTable, ulong src) - { - lock (_blockManager) - { - lock (srcPageTable._blockManager) - { - bool success = CheckRange( - dst, - size, - MemoryState.Mask, - MemoryState.ProcessMemory, - KMemoryPermission.ReadAndWrite, - KMemoryPermission.ReadAndWrite, - MemoryAttribute.Mask, - MemoryAttribute.None, - MemoryAttribute.IpcAndDeviceMapped, - out _, - out _, - out _); - - success &= srcPageTable.CheckRange( - src, - size, - MemoryState.MapProcessAllowed, - MemoryState.MapProcessAllowed, - KMemoryPermission.None, - KMemoryPermission.None, - MemoryAttribute.Mask, - MemoryAttribute.None, - MemoryAttribute.IpcAndDeviceMapped, - out _, - out _, - out _); - - if (!success) - { - return KernelResult.InvalidMemState; - } - - KPageList srcPageList = new KPageList(); - KPageList dstPageList = new KPageList(); - - srcPageTable.GetPhysicalRegions(src, size, srcPageList); - GetPhysicalRegions(dst, size, dstPageList); - - if (!dstPageList.IsEqual(srcPageList)) - { - return KernelResult.InvalidMemRange; - } - } - - if (!_slabManager.CanAllocate(MaxBlocksNeededForInsertion)) - { - return KernelResult.OutOfResource; - } - - ulong pagesCount = size / PageSize; - - Result result = Unmap(dst, pagesCount); - - if (result != Result.Success) - { - return result; - } - - _blockManager.InsertBlock(dst, pagesCount, MemoryState.Unmapped); - - return Result.Success; - } - } - - public Result SetProcessMemoryPermission(ulong address, ulong size, KMemoryPermission permission) - { - lock (_blockManager) - { - if (CheckRange( - address, - size, - MemoryState.ProcessPermissionChangeAllowed, - MemoryState.ProcessPermissionChangeAllowed, - KMemoryPermission.None, - KMemoryPermission.None, - MemoryAttribute.Mask, - MemoryAttribute.None, - MemoryAttribute.IpcAndDeviceMapped, - out MemoryState oldState, - out KMemoryPermission oldPermission, - out _)) - { - MemoryState newState = oldState; - - // If writing into the code region is allowed, then we need - // to change it to mutable. - if ((permission & KMemoryPermission.Write) != 0) - { - if (oldState == MemoryState.CodeStatic) - { - newState = MemoryState.CodeMutable; - } - else if (oldState == MemoryState.ModCodeStatic) - { - newState = MemoryState.ModCodeMutable; - } - else - { - throw new InvalidOperationException($"Memory state \"{oldState}\" not valid for this operation."); - } - } - - if (newState != oldState || permission != oldPermission) - { - if (!_slabManager.CanAllocate(MaxBlocksNeededForInsertion)) - { - return KernelResult.OutOfResource; - } - - ulong pagesCount = size / PageSize; - - Result result; - - if ((oldPermission & KMemoryPermission.Execute) != 0) - { - result = ReprotectWithAttributes(address, pagesCount, permission); - } - else - { - result = Reprotect(address, pagesCount, permission); - } - - if (result != Result.Success) - { - return result; - } - - _blockManager.InsertBlock(address, pagesCount, newState, permission); - } - - return Result.Success; - } - else - { - return KernelResult.InvalidMemState; - } - } - } - - public Result MapPhysicalMemory(ulong address, ulong size) - { - ulong endAddr = address + size; - - lock (_blockManager) - { - ulong mappedSize = 0; - - foreach (KMemoryInfo info in IterateOverRange(address, endAddr)) - { - if (info.State != MemoryState.Unmapped) - { - mappedSize += GetSizeInRange(info, address, endAddr); - } - } - - if (mappedSize == size) - { - return Result.Success; - } - - ulong remainingSize = size - mappedSize; - - ulong remainingPages = remainingSize / PageSize; - - KProcess currentProcess = KernelStatic.GetCurrentProcess(); - - if (currentProcess.ResourceLimit != null && - !currentProcess.ResourceLimit.Reserve(LimitableResource.Memory, remainingSize)) - { - return KernelResult.ResLimitExceeded; - } - - KMemoryRegionManager region = GetMemoryRegionManager(); - - Result result = region.AllocatePages(out KPageList pageList, remainingPages); - - using var _ = new OnScopeExit(() => pageList.DecrementPagesReferenceCount(Context.MemoryManager)); - - void CleanUpForError() - { - currentProcess.ResourceLimit?.Release(LimitableResource.Memory, remainingSize); - } - - if (result != Result.Success) - { - CleanUpForError(); - - return result; - } - - if (!_slabManager.CanAllocate(MaxBlocksNeededForInsertion)) - { - CleanUpForError(); - - return KernelResult.OutOfResource; - } - - LinkedListNode<KPageNode> pageListNode = pageList.Nodes.First; - - KPageNode pageNode = pageListNode.Value; - - ulong srcPa = pageNode.Address; - ulong srcPaPages = pageNode.PagesCount; - - foreach (KMemoryInfo info in IterateOverRange(address, endAddr)) - { - if (info.State != MemoryState.Unmapped) - { - continue; - } - - ulong blockSize = GetSizeInRange(info, address, endAddr); - - ulong dstVaPages = blockSize / PageSize; - - ulong dstVa = GetAddrInRange(info, address); - - while (dstVaPages > 0) - { - if (srcPaPages == 0) - { - pageListNode = pageListNode.Next; - - pageNode = pageListNode.Value; - - srcPa = pageNode.Address; - srcPaPages = pageNode.PagesCount; - } - - ulong currentPagesCount = Math.Min(srcPaPages, dstVaPages); - - MapPages(dstVa, currentPagesCount, srcPa, KMemoryPermission.ReadAndWrite, MemoryMapFlags.Private); - - dstVa += currentPagesCount * PageSize; - srcPa += currentPagesCount * PageSize; - srcPaPages -= currentPagesCount; - dstVaPages -= currentPagesCount; - } - } - - PhysicalMemoryUsage += remainingSize; - - ulong pagesCount = size / PageSize; - - _blockManager.InsertBlock( - address, - pagesCount, - MemoryState.Unmapped, - KMemoryPermission.None, - MemoryAttribute.None, - MemoryState.Heap, - KMemoryPermission.ReadAndWrite, - MemoryAttribute.None); - } - - return Result.Success; - } - - public Result UnmapPhysicalMemory(ulong address, ulong size) - { - ulong endAddr = address + size; - - lock (_blockManager) - { - // Scan, ensure that the region can be unmapped (all blocks are heap or - // already unmapped), fill pages list for freeing memory. - ulong heapMappedSize = 0; - - foreach (KMemoryInfo info in IterateOverRange(address, endAddr)) - { - if (info.State == MemoryState.Heap) - { - if (info.Attribute != MemoryAttribute.None) - { - return KernelResult.InvalidMemState; - } - - ulong blockSize = GetSizeInRange(info, address, endAddr); - - heapMappedSize += blockSize; - } - else if (info.State != MemoryState.Unmapped) - { - return KernelResult.InvalidMemState; - } - } - - if (heapMappedSize == 0) - { - return Result.Success; - } - - if (!_slabManager.CanAllocate(MaxBlocksNeededForInsertion)) - { - return KernelResult.OutOfResource; - } - - // Try to unmap all the heap mapped memory inside range. - Result result = Result.Success; - - foreach (KMemoryInfo info in IterateOverRange(address, endAddr)) - { - if (info.State == MemoryState.Heap) - { - ulong blockSize = GetSizeInRange(info, address, endAddr); - ulong blockAddress = GetAddrInRange(info, address); - - ulong blockPagesCount = blockSize / PageSize; - - result = Unmap(blockAddress, blockPagesCount); - - // The kernel would attempt to remap if this fails, but we don't because: - // - The implementation may not support remapping if memory aliasing is not supported on the platform. - // - Unmap can't ever fail here anyway. - Debug.Assert(result == Result.Success); - } - } - - if (result == Result.Success) - { - PhysicalMemoryUsage -= heapMappedSize; - - KProcess currentProcess = KernelStatic.GetCurrentProcess(); - - currentProcess.ResourceLimit?.Release(LimitableResource.Memory, heapMappedSize); - - ulong pagesCount = size / PageSize; - - _blockManager.InsertBlock(address, pagesCount, MemoryState.Unmapped); - } - - return result; - } - } - - public Result CopyDataToCurrentProcess( - ulong dst, - ulong size, - ulong src, - MemoryState stateMask, - MemoryState stateExpected, - KMemoryPermission permission, - MemoryAttribute attributeMask, - MemoryAttribute attributeExpected) - { - // Client -> server. - return CopyDataFromOrToCurrentProcess( - size, - src, - dst, - stateMask, - stateExpected, - permission, - attributeMask, - attributeExpected, - toServer: true); - } - - public Result CopyDataFromCurrentProcess( - ulong dst, - ulong size, - MemoryState stateMask, - MemoryState stateExpected, - KMemoryPermission permission, - MemoryAttribute attributeMask, - MemoryAttribute attributeExpected, - ulong src) - { - // Server -> client. - return CopyDataFromOrToCurrentProcess( - size, - dst, - src, - stateMask, - stateExpected, - permission, - attributeMask, - attributeExpected, - toServer: false); - } - - private Result CopyDataFromOrToCurrentProcess( - ulong size, - ulong clientAddress, - ulong serverAddress, - MemoryState stateMask, - MemoryState stateExpected, - KMemoryPermission permission, - MemoryAttribute attributeMask, - MemoryAttribute attributeExpected, - bool toServer) - { - if (AddrSpaceStart > clientAddress) - { - return KernelResult.InvalidMemState; - } - - ulong srcEndAddr = clientAddress + size; - - if (srcEndAddr <= clientAddress || srcEndAddr - 1 > AddrSpaceEnd - 1) - { - return KernelResult.InvalidMemState; - } - - lock (_blockManager) - { - if (CheckRange( - clientAddress, - size, - stateMask, - stateExpected, - permission, - permission, - attributeMask | MemoryAttribute.Uncached, - attributeExpected)) - { - KProcess currentProcess = KernelStatic.GetCurrentProcess(); - - while (size > 0) - { - ulong copySize = 0x100000; // Copy chunck size. Any value will do, moderate sizes are recommended. - - if (copySize > size) - { - copySize = size; - } - - if (toServer) - { - currentProcess.CpuMemory.Write(serverAddress, GetSpan(clientAddress, (int)copySize)); - } - else - { - Write(clientAddress, currentProcess.CpuMemory.GetSpan(serverAddress, (int)copySize)); - } - - serverAddress += copySize; - clientAddress += copySize; - size -= copySize; - } - - return Result.Success; - } - else - { - return KernelResult.InvalidMemState; - } - } - } - - public Result MapBufferFromClientProcess( - ulong size, - ulong src, - KPageTableBase srcPageTable, - KMemoryPermission permission, - MemoryState state, - bool send, - out ulong dst) - { - dst = 0; - - lock (srcPageTable._blockManager) - { - lock (_blockManager) - { - Result result = srcPageTable.ReprotectClientProcess( - src, - size, - permission, - state, - out int blocksNeeded); - - if (result != Result.Success) - { - return result; - } - - if (!srcPageTable._slabManager.CanAllocate(blocksNeeded)) - { - return KernelResult.OutOfResource; - } - - ulong srcMapAddress = BitUtils.AlignUp<ulong>(src, PageSize); - ulong srcMapEndAddr = BitUtils.AlignDown<ulong>(src + size, PageSize); - ulong srcMapSize = srcMapEndAddr - srcMapAddress; - - result = MapPagesFromClientProcess(size, src, permission, state, srcPageTable, send, out ulong va); - - if (result != Result.Success) - { - if (srcMapEndAddr > srcMapAddress) - { - srcPageTable.UnmapIpcRestorePermission(src, size, state); - } - - return result; - } - - if (srcMapAddress < srcMapEndAddr) - { - KMemoryPermission permissionMask = permission == KMemoryPermission.ReadAndWrite - ? KMemoryPermission.None - : KMemoryPermission.Read; - - srcPageTable._blockManager.InsertBlock(srcMapAddress, srcMapSize / PageSize, SetIpcMappingPermissions, permissionMask); - } - - dst = va; - } - } - - return Result.Success; - } - - private Result ReprotectClientProcess( - ulong address, - ulong size, - KMemoryPermission permission, - MemoryState state, - out int blocksNeeded) - { - blocksNeeded = 0; - - if (AddrSpaceStart > address) - { - return KernelResult.InvalidMemState; - } - - ulong endAddr = address + size; - - if (endAddr <= address || endAddr - 1 > AddrSpaceEnd - 1) - { - return KernelResult.InvalidMemState; - } - - MemoryState stateMask; - - switch (state) - { - case MemoryState.IpcBuffer0: stateMask = MemoryState.IpcSendAllowedType0; break; - case MemoryState.IpcBuffer1: stateMask = MemoryState.IpcSendAllowedType1; break; - case MemoryState.IpcBuffer3: stateMask = MemoryState.IpcSendAllowedType3; break; - - default: return KernelResult.InvalidCombination; - } - - KMemoryPermission permissionMask = permission == KMemoryPermission.ReadAndWrite - ? KMemoryPermission.None - : KMemoryPermission.Read; - - MemoryAttribute attributeMask = MemoryAttribute.Borrowed | MemoryAttribute.Uncached; - - if (state == MemoryState.IpcBuffer0) - { - attributeMask |= MemoryAttribute.DeviceMapped; - } - - ulong addressRounded = BitUtils.AlignUp<ulong>(address, PageSize); - ulong addressTruncated = BitUtils.AlignDown<ulong>(address, PageSize); - ulong endAddrRounded = BitUtils.AlignUp<ulong>(endAddr, PageSize); - ulong endAddrTruncated = BitUtils.AlignDown<ulong>(endAddr, PageSize); - - if (!_slabManager.CanAllocate(MaxBlocksNeededForInsertion)) - { - return KernelResult.OutOfResource; - } - - ulong visitedSize = 0; - - void CleanUpForError() - { - if (visitedSize == 0) - { - return; - } - - ulong endAddrVisited = address + visitedSize; - - foreach (KMemoryInfo info in IterateOverRange(addressRounded, endAddrVisited)) - { - if ((info.Permission & KMemoryPermission.ReadAndWrite) != permissionMask && info.IpcRefCount == 0) - { - ulong blockAddress = GetAddrInRange(info, addressRounded); - ulong blockSize = GetSizeInRange(info, addressRounded, endAddrVisited); - - ulong blockPagesCount = blockSize / PageSize; - - Result reprotectResult = Reprotect(blockAddress, blockPagesCount, info.Permission); - Debug.Assert(reprotectResult == Result.Success); - } - } - } - - // Signal a read for any resources tracking reads in the region, as the other process is likely to use their data. - SignalMemoryTracking(addressTruncated, endAddrRounded - addressTruncated, false); - - // Reprotect the aligned pages range on the client to make them inaccessible from the client process. - Result result; - - if (addressRounded < endAddrTruncated) - { - foreach (KMemoryInfo info in IterateOverRange(addressRounded, endAddrTruncated)) - { - // Check if the block state matches what we expect. - if ((info.State & stateMask) != stateMask || - (info.Permission & permission) != permission || - (info.Attribute & attributeMask) != MemoryAttribute.None) - { - CleanUpForError(); - - return KernelResult.InvalidMemState; - } - - ulong blockAddress = GetAddrInRange(info, addressRounded); - ulong blockSize = GetSizeInRange(info, addressRounded, endAddrTruncated); - - ulong blockPagesCount = blockSize / PageSize; - - // If the first block starts before the aligned range, it will need to be split. - if (info.Address < addressRounded) - { - blocksNeeded++; - } - - // If the last block ends after the aligned range, it will need to be split. - if (endAddrTruncated - 1 < info.Address + info.Size - 1) - { - blocksNeeded++; - } - - if ((info.Permission & KMemoryPermission.ReadAndWrite) != permissionMask && info.IpcRefCount == 0) - { - result = Reprotect(blockAddress, blockPagesCount, permissionMask); - - if (result != Result.Success) - { - CleanUpForError(); - - return result; - } - } - - visitedSize += blockSize; - } - } - - return Result.Success; - } - - private Result MapPagesFromClientProcess( - ulong size, - ulong address, - KMemoryPermission permission, - MemoryState state, - KPageTableBase srcPageTable, - bool send, - out ulong dst) - { - dst = 0; - - if (!_slabManager.CanAllocate(MaxBlocksNeededForInsertion)) - { - return KernelResult.OutOfResource; - } - - ulong endAddr = address + size; - - ulong addressTruncated = BitUtils.AlignDown<ulong>(address, PageSize); - ulong addressRounded = BitUtils.AlignUp<ulong>(address, PageSize); - ulong endAddrTruncated = BitUtils.AlignDown<ulong>(endAddr, PageSize); - ulong endAddrRounded = BitUtils.AlignUp<ulong>(endAddr, PageSize); - - ulong neededSize = endAddrRounded - addressTruncated; - - ulong neededPagesCount = neededSize / PageSize; - - ulong regionPagesCount = (AliasRegionEnd - AliasRegionStart) / PageSize; - - ulong va = 0; - - for (int unit = MappingUnitSizes.Length - 1; unit >= 0 && va == 0; unit--) - { - int alignment = MappingUnitSizes[unit]; - - va = AllocateVa(AliasRegionStart, regionPagesCount, neededPagesCount, alignment); - } - - if (va == 0) - { - return KernelResult.OutOfVaSpace; - } - - ulong dstFirstPagePa = 0; - ulong dstLastPagePa = 0; - ulong currentVa = va; - - using var _ = new OnScopeExit(() => - { - if (dstFirstPagePa != 0) - { - Context.MemoryManager.DecrementPagesReferenceCount(dstFirstPagePa, 1); - } - - if (dstLastPagePa != 0) - { - Context.MemoryManager.DecrementPagesReferenceCount(dstLastPagePa, 1); - } - }); - - void CleanUpForError() - { - if (currentVa != va) - { - Unmap(va, (currentVa - va) / PageSize); - } - } - - // Is the first page address aligned? - // If not, allocate a new page and copy the unaligned chunck. - if (addressTruncated < addressRounded) - { - dstFirstPagePa = GetMemoryRegionManager().AllocatePagesContiguous(Context, 1, _aslrDisabled); - - if (dstFirstPagePa == 0) - { - CleanUpForError(); - - return KernelResult.OutOfMemory; - } - } - - // Is the last page end address aligned? - // If not, allocate a new page and copy the unaligned chunck. - if (endAddrTruncated < endAddrRounded && (addressTruncated == addressRounded || addressTruncated < endAddrTruncated)) - { - dstLastPagePa = GetMemoryRegionManager().AllocatePagesContiguous(Context, 1, _aslrDisabled); - - if (dstLastPagePa == 0) - { - CleanUpForError(); - - return KernelResult.OutOfMemory; - } - } - - if (dstFirstPagePa != 0) - { - ulong firstPageFillAddress = dstFirstPagePa; - ulong unusedSizeAfter; - - if (send) - { - ulong unusedSizeBefore = address - addressTruncated; - - Context.Memory.Fill(GetDramAddressFromPa(dstFirstPagePa), unusedSizeBefore, (byte)_ipcFillValue); - - ulong copySize = addressRounded <= endAddr ? addressRounded - address : size; - var data = srcPageTable.GetSpan(addressTruncated + unusedSizeBefore, (int)copySize); - - Context.Memory.Write(GetDramAddressFromPa(dstFirstPagePa + unusedSizeBefore), data); - - firstPageFillAddress += unusedSizeBefore + copySize; - - unusedSizeAfter = addressRounded > endAddr ? addressRounded - endAddr : 0; - } - else - { - unusedSizeAfter = PageSize; - } - - if (unusedSizeAfter != 0) - { - Context.Memory.Fill(GetDramAddressFromPa(firstPageFillAddress), unusedSizeAfter, (byte)_ipcFillValue); - } - - Result result = MapPages(currentVa, 1, dstFirstPagePa, permission, MemoryMapFlags.Private); - - if (result != Result.Success) - { - CleanUpForError(); - - return result; - } - - currentVa += PageSize; - } - - if (endAddrTruncated > addressRounded) - { - ulong alignedSize = endAddrTruncated - addressRounded; - - Result result; - - if (srcPageTable.Supports4KBPages) - { - KPageList pageList = new KPageList(); - srcPageTable.GetPhysicalRegions(addressRounded, alignedSize, pageList); - - result = MapPages(currentVa, pageList, permission, MemoryMapFlags.None); - } - else - { - result = MapForeign(srcPageTable.GetHostRegions(addressRounded, alignedSize), currentVa, alignedSize); - } - - if (result != Result.Success) - { - CleanUpForError(); - - return result; - } - - currentVa += alignedSize; - } - - if (dstLastPagePa != 0) - { - ulong lastPageFillAddr = dstLastPagePa; - ulong unusedSizeAfter; - - if (send) - { - ulong copySize = endAddr - endAddrTruncated; - var data = srcPageTable.GetSpan(endAddrTruncated, (int)copySize); - - Context.Memory.Write(GetDramAddressFromPa(dstLastPagePa), data); - - lastPageFillAddr += copySize; - - unusedSizeAfter = PageSize - copySize; - } - else - { - unusedSizeAfter = PageSize; - } - - Context.Memory.Fill(GetDramAddressFromPa(lastPageFillAddr), unusedSizeAfter, (byte)_ipcFillValue); - - Result result = MapPages(currentVa, 1, dstLastPagePa, permission, MemoryMapFlags.Private); - - if (result != Result.Success) - { - CleanUpForError(); - - return result; - } - } - - _blockManager.InsertBlock(va, neededPagesCount, state, permission); - - dst = va + (address - addressTruncated); - - return Result.Success; - } - - public Result UnmapNoAttributeIfStateEquals(ulong address, ulong size, MemoryState state) - { - if (AddrSpaceStart > address) - { - return KernelResult.InvalidMemState; - } - - ulong endAddr = address + size; - - if (endAddr <= address || endAddr - 1 > AddrSpaceEnd - 1) - { - return KernelResult.InvalidMemState; - } - - lock (_blockManager) - { - if (CheckRange( - address, - size, - MemoryState.Mask, - state, - KMemoryPermission.Read, - KMemoryPermission.Read, - MemoryAttribute.Mask, - MemoryAttribute.None, - MemoryAttribute.IpcAndDeviceMapped, - out _, - out _, - out _)) - { - if (!_slabManager.CanAllocate(MaxBlocksNeededForInsertion)) - { - return KernelResult.OutOfResource; - } - - ulong addressTruncated = BitUtils.AlignDown<ulong>(address, PageSize); - ulong addressRounded = BitUtils.AlignUp<ulong>(address, PageSize); - ulong endAddrTruncated = BitUtils.AlignDown<ulong>(endAddr, PageSize); - ulong endAddrRounded = BitUtils.AlignUp<ulong>(endAddr, PageSize); - - ulong pagesCount = (endAddrRounded - addressTruncated) / PageSize; - - Result result = Unmap(addressTruncated, pagesCount); - - if (result == Result.Success) - { - _blockManager.InsertBlock(addressTruncated, pagesCount, MemoryState.Unmapped); - } - - return result; - } - else - { - return KernelResult.InvalidMemState; - } - } - } - - public Result UnmapIpcRestorePermission(ulong address, ulong size, MemoryState state) - { - ulong endAddr = address + size; - - ulong addressRounded = BitUtils.AlignUp<ulong>(address, PageSize); - ulong addressTruncated = BitUtils.AlignDown<ulong>(address, PageSize); - ulong endAddrRounded = BitUtils.AlignUp<ulong>(endAddr, PageSize); - ulong endAddrTruncated = BitUtils.AlignDown<ulong>(endAddr, PageSize); - - ulong pagesCount = addressRounded < endAddrTruncated ? (endAddrTruncated - addressRounded) / PageSize : 0; - - if (pagesCount == 0) - { - return Result.Success; - } - - MemoryState stateMask; - - switch (state) - { - case MemoryState.IpcBuffer0: stateMask = MemoryState.IpcSendAllowedType0; break; - case MemoryState.IpcBuffer1: stateMask = MemoryState.IpcSendAllowedType1; break; - case MemoryState.IpcBuffer3: stateMask = MemoryState.IpcSendAllowedType3; break; - - default: return KernelResult.InvalidCombination; - } - - MemoryAttribute attributeMask = - MemoryAttribute.Borrowed | - MemoryAttribute.IpcMapped | - MemoryAttribute.Uncached; - - if (state == MemoryState.IpcBuffer0) - { - attributeMask |= MemoryAttribute.DeviceMapped; - } - - if (!_slabManager.CanAllocate(MaxBlocksNeededForInsertion)) - { - return KernelResult.OutOfResource; - } - - // Anything on the client side should see this memory as modified. - SignalMemoryTracking(addressTruncated, endAddrRounded - addressTruncated, true); - - lock (_blockManager) - { - foreach (KMemoryInfo info in IterateOverRange(addressRounded, endAddrTruncated)) - { - // Check if the block state matches what we expect. - if ((info.State & stateMask) != stateMask || - (info.Attribute & attributeMask) != MemoryAttribute.IpcMapped) - { - return KernelResult.InvalidMemState; - } - - if (info.Permission != info.SourcePermission && info.IpcRefCount == 1) - { - ulong blockAddress = GetAddrInRange(info, addressRounded); - ulong blockSize = GetSizeInRange(info, addressRounded, endAddrTruncated); - - ulong blockPagesCount = blockSize / PageSize; - - Result result = Reprotect(blockAddress, blockPagesCount, info.SourcePermission); - - if (result != Result.Success) - { - return result; - } - } - } - - _blockManager.InsertBlock(addressRounded, pagesCount, RestoreIpcMappingPermissions); - - return Result.Success; - } - } - - private static void SetIpcMappingPermissions(KMemoryBlock block, KMemoryPermission permission) - { - block.SetIpcMappingPermission(permission); - } - - private static void RestoreIpcMappingPermissions(KMemoryBlock block, KMemoryPermission permission) - { - block.RestoreIpcMappingPermission(); - } - - public Result GetPagesIfStateEquals( - ulong address, - ulong size, - MemoryState stateMask, - MemoryState stateExpected, - KMemoryPermission permissionMask, - KMemoryPermission permissionExpected, - MemoryAttribute attributeMask, - MemoryAttribute attributeExpected, - KPageList pageList) - { - if (!InsideAddrSpace(address, size)) - { - return KernelResult.InvalidMemState; - } - - lock (_blockManager) - { - if (CheckRange( - address, - size, - stateMask | MemoryState.IsPoolAllocated, - stateExpected | MemoryState.IsPoolAllocated, - permissionMask, - permissionExpected, - attributeMask, - attributeExpected, - MemoryAttribute.IpcAndDeviceMapped, - out _, - out _, - out _)) - { - GetPhysicalRegions(address, size, pageList); - - return Result.Success; - } - else - { - return KernelResult.InvalidMemState; - } - } - } - - public Result BorrowIpcBuffer(ulong address, ulong size) - { - return SetAttributesAndChangePermission( - address, - size, - MemoryState.IpcBufferAllowed, - MemoryState.IpcBufferAllowed, - KMemoryPermission.Mask, - KMemoryPermission.ReadAndWrite, - MemoryAttribute.Mask, - MemoryAttribute.None, - KMemoryPermission.None, - MemoryAttribute.Borrowed); - } - - public Result BorrowTransferMemory(KPageList pageList, ulong address, ulong size, KMemoryPermission permission) - { - return SetAttributesAndChangePermission( - address, - size, - MemoryState.TransferMemoryAllowed, - MemoryState.TransferMemoryAllowed, - KMemoryPermission.Mask, - KMemoryPermission.ReadAndWrite, - MemoryAttribute.Mask, - MemoryAttribute.None, - permission, - MemoryAttribute.Borrowed, - pageList); - } - - public Result BorrowCodeMemory(KPageList pageList, ulong address, ulong size) - { - return SetAttributesAndChangePermission( - address, - size, - MemoryState.CodeMemoryAllowed, - MemoryState.CodeMemoryAllowed, - KMemoryPermission.Mask, - KMemoryPermission.ReadAndWrite, - MemoryAttribute.Mask, - MemoryAttribute.None, - KMemoryPermission.None, - MemoryAttribute.Borrowed, - pageList); - } - - private Result SetAttributesAndChangePermission( - ulong address, - ulong size, - MemoryState stateMask, - MemoryState stateExpected, - KMemoryPermission permissionMask, - KMemoryPermission permissionExpected, - MemoryAttribute attributeMask, - MemoryAttribute attributeExpected, - KMemoryPermission newPermission, - MemoryAttribute attributeSetMask, - KPageList pageList = null) - { - if (address + size <= address || !InsideAddrSpace(address, size)) - { - return KernelResult.InvalidMemState; - } - - lock (_blockManager) - { - if (CheckRange( - address, - size, - stateMask | MemoryState.IsPoolAllocated, - stateExpected | MemoryState.IsPoolAllocated, - permissionMask, - permissionExpected, - attributeMask, - attributeExpected, - MemoryAttribute.IpcAndDeviceMapped, - out MemoryState oldState, - out KMemoryPermission oldPermission, - out MemoryAttribute oldAttribute)) - { - ulong pagesCount = size / PageSize; - - if (pageList != null) - { - GetPhysicalRegions(address, pagesCount * PageSize, pageList); - } - - if (!_slabManager.CanAllocate(MaxBlocksNeededForInsertion)) - { - return KernelResult.OutOfResource; - } - - if (newPermission == KMemoryPermission.None) - { - newPermission = oldPermission; - } - - if (newPermission != oldPermission) - { - Result result = Reprotect(address, pagesCount, newPermission); - - if (result != Result.Success) - { - return result; - } - } - - MemoryAttribute newAttribute = oldAttribute | attributeSetMask; - - _blockManager.InsertBlock(address, pagesCount, oldState, newPermission, newAttribute); - - return Result.Success; - } - else - { - return KernelResult.InvalidMemState; - } - } - } - - public Result UnborrowIpcBuffer(ulong address, ulong size) - { - return ClearAttributesAndChangePermission( - address, - size, - MemoryState.IpcBufferAllowed, - MemoryState.IpcBufferAllowed, - KMemoryPermission.None, - KMemoryPermission.None, - MemoryAttribute.Mask, - MemoryAttribute.Borrowed, - KMemoryPermission.ReadAndWrite, - MemoryAttribute.Borrowed); - } - - public Result UnborrowTransferMemory(ulong address, ulong size, KPageList pageList) - { - return ClearAttributesAndChangePermission( - address, - size, - MemoryState.TransferMemoryAllowed, - MemoryState.TransferMemoryAllowed, - KMemoryPermission.None, - KMemoryPermission.None, - MemoryAttribute.Mask, - MemoryAttribute.Borrowed, - KMemoryPermission.ReadAndWrite, - MemoryAttribute.Borrowed, - pageList); - } - - public Result UnborrowCodeMemory(ulong address, ulong size, KPageList pageList) - { - return ClearAttributesAndChangePermission( - address, - size, - MemoryState.CodeMemoryAllowed, - MemoryState.CodeMemoryAllowed, - KMemoryPermission.None, - KMemoryPermission.None, - MemoryAttribute.Mask, - MemoryAttribute.Borrowed, - KMemoryPermission.ReadAndWrite, - MemoryAttribute.Borrowed, - pageList); - } - - private Result ClearAttributesAndChangePermission( - ulong address, - ulong size, - MemoryState stateMask, - MemoryState stateExpected, - KMemoryPermission permissionMask, - KMemoryPermission permissionExpected, - MemoryAttribute attributeMask, - MemoryAttribute attributeExpected, - KMemoryPermission newPermission, - MemoryAttribute attributeClearMask, - KPageList pageList = null) - { - if (address + size <= address || !InsideAddrSpace(address, size)) - { - return KernelResult.InvalidMemState; - } - - lock (_blockManager) - { - if (CheckRange( - address, - size, - stateMask | MemoryState.IsPoolAllocated, - stateExpected | MemoryState.IsPoolAllocated, - permissionMask, - permissionExpected, - attributeMask, - attributeExpected, - MemoryAttribute.IpcAndDeviceMapped, - out MemoryState oldState, - out KMemoryPermission oldPermission, - out MemoryAttribute oldAttribute)) - { - ulong pagesCount = size / PageSize; - - if (pageList != null) - { - KPageList currentPageList = new KPageList(); - - GetPhysicalRegions(address, pagesCount * PageSize, currentPageList); - - if (!currentPageList.IsEqual(pageList)) - { - return KernelResult.InvalidMemRange; - } - } - - if (!_slabManager.CanAllocate(MaxBlocksNeededForInsertion)) - { - return KernelResult.OutOfResource; - } - - if (newPermission == KMemoryPermission.None) - { - newPermission = oldPermission; - } - - if (newPermission != oldPermission) - { - Result result = Reprotect(address, pagesCount, newPermission); - - if (result != Result.Success) - { - return result; - } - } - - MemoryAttribute newAttribute = oldAttribute & ~attributeClearMask; - - _blockManager.InsertBlock(address, pagesCount, oldState, newPermission, newAttribute); - - return Result.Success; - } - else - { - return KernelResult.InvalidMemState; - } - } - } - - private static ulong GetAddrInRange(KMemoryInfo info, ulong start) - { - if (info.Address < start) - { - return start; - } - - return info.Address; - } - - private static ulong GetSizeInRange(KMemoryInfo info, ulong start, ulong end) - { - ulong endAddr = info.Size + info.Address; - ulong size = info.Size; - - if (info.Address < start) - { - size -= start - info.Address; - } - - if (endAddr > end) - { - size -= endAddr - end; - } - - return size; - } - - private bool IsUnmapped(ulong address, ulong size) - { - return CheckRange( - address, - size, - MemoryState.Mask, - MemoryState.Unmapped, - KMemoryPermission.Mask, - KMemoryPermission.None, - MemoryAttribute.Mask, - MemoryAttribute.None, - MemoryAttribute.IpcAndDeviceMapped, - out _, - out _, - out _); - } - - private bool CheckRange( - ulong address, - ulong size, - MemoryState stateMask, - MemoryState stateExpected, - KMemoryPermission permissionMask, - KMemoryPermission permissionExpected, - MemoryAttribute attributeMask, - MemoryAttribute attributeExpected, - MemoryAttribute attributeIgnoreMask, - out MemoryState outState, - out KMemoryPermission outPermission, - out MemoryAttribute outAttribute) - { - ulong endAddr = address + size; - - KMemoryBlock currBlock = _blockManager.FindBlock(address); - - KMemoryInfo info = currBlock.GetInfo(); - - MemoryState firstState = info.State; - KMemoryPermission firstPermission = info.Permission; - MemoryAttribute firstAttribute = info.Attribute; - - do - { - info = currBlock.GetInfo(); - - // Check if the block state matches what we expect. - if (firstState != info.State || - firstPermission != info.Permission || - (info.Attribute & attributeMask) != attributeExpected || - (firstAttribute | attributeIgnoreMask) != (info.Attribute | attributeIgnoreMask) || - (firstState & stateMask) != stateExpected || - (firstPermission & permissionMask) != permissionExpected) - { - outState = MemoryState.Unmapped; - outPermission = KMemoryPermission.None; - outAttribute = MemoryAttribute.None; - - return false; - } - } - while (info.Address + info.Size - 1 < endAddr - 1 && (currBlock = currBlock.Successor) != null); - - outState = firstState; - outPermission = firstPermission; - outAttribute = firstAttribute & ~attributeIgnoreMask; - - return true; - } - - private bool CheckRange( - ulong address, - ulong size, - MemoryState stateMask, - MemoryState stateExpected, - KMemoryPermission permissionMask, - KMemoryPermission permissionExpected, - MemoryAttribute attributeMask, - MemoryAttribute attributeExpected) - { - foreach (KMemoryInfo info in IterateOverRange(address, address + size)) - { - // Check if the block state matches what we expect. - if ((info.State & stateMask) != stateExpected || - (info.Permission & permissionMask) != permissionExpected || - (info.Attribute & attributeMask) != attributeExpected) - { - return false; - } - } - - return true; - } - - private IEnumerable<KMemoryInfo> IterateOverRange(ulong start, ulong end) - { - KMemoryBlock currBlock = _blockManager.FindBlock(start); - - KMemoryInfo info; - - do - { - info = currBlock.GetInfo(); - - yield return info; - } - while (info.Address + info.Size - 1 < end - 1 && (currBlock = currBlock.Successor) != null); - } - - private ulong AllocateVa(ulong regionStart, ulong regionPagesCount, ulong neededPagesCount, int alignment) - { - ulong address = 0; - - ulong regionEndAddr = regionStart + regionPagesCount * PageSize; - - ulong reservedPagesCount = _isKernel ? 1UL : 4UL; - - if (_aslrEnabled) - { - ulong totalNeededSize = (reservedPagesCount + neededPagesCount) * PageSize; - - ulong remainingPages = regionPagesCount - neededPagesCount; - - ulong aslrMaxOffset = ((remainingPages + reservedPagesCount) * PageSize) / (ulong)alignment; - - for (int attempt = 0; attempt < 8; attempt++) - { - ulong aslrAddress = BitUtils.AlignDown(regionStart + GetRandomValue(0, aslrMaxOffset) * (ulong)alignment, (ulong)alignment); - ulong aslrEndAddr = aslrAddress + totalNeededSize; - - KMemoryInfo info = _blockManager.FindBlock(aslrAddress).GetInfo(); - - if (info.State != MemoryState.Unmapped) - { - continue; - } - - ulong currBaseAddr = info.Address + reservedPagesCount * PageSize; - ulong currEndAddr = info.Address + info.Size; - - if (aslrAddress >= regionStart && - aslrAddress >= currBaseAddr && - aslrEndAddr - 1 <= regionEndAddr - 1 && - aslrEndAddr - 1 <= currEndAddr - 1) - { - address = aslrAddress; - break; - } - } - - if (address == 0) - { - ulong aslrPage = GetRandomValue(0, aslrMaxOffset); - - address = FindFirstFit( - regionStart + aslrPage * PageSize, - regionPagesCount - aslrPage, - neededPagesCount, - alignment, - 0, - reservedPagesCount); - } - } - - if (address == 0) - { - address = FindFirstFit( - regionStart, - regionPagesCount, - neededPagesCount, - alignment, - 0, - reservedPagesCount); - } - - return address; - } - - private ulong FindFirstFit( - ulong regionStart, - ulong regionPagesCount, - ulong neededPagesCount, - int alignment, - ulong reservedStart, - ulong reservedPagesCount) - { - ulong reservedSize = reservedPagesCount * PageSize; - - ulong totalNeededSize = reservedSize + neededPagesCount * PageSize; - - ulong regionEndAddr = (regionStart + regionPagesCount * PageSize) - 1; - - KMemoryBlock currBlock = _blockManager.FindBlock(regionStart); - - KMemoryInfo info = currBlock.GetInfo(); - - while (regionEndAddr >= info.Address) - { - if (info.State == MemoryState.Unmapped) - { - ulong currBaseAddr = info.Address <= regionStart ? regionStart : info.Address; - ulong currEndAddr = info.Address + info.Size - 1; - - currBaseAddr += reservedSize; - - ulong address = BitUtils.AlignDown<ulong>(currBaseAddr, (ulong)alignment) + reservedStart; - - if (currBaseAddr > address) - { - address += (ulong)alignment; - } - - ulong allocationEndAddr = address + totalNeededSize - 1; - - if (info.Address <= address && - address < allocationEndAddr && - allocationEndAddr <= regionEndAddr && - allocationEndAddr <= currEndAddr) - { - return address; - } - } - - currBlock = currBlock.Successor; - - if (currBlock == null) - { - break; - } - - info = currBlock.GetInfo(); - } - - return 0; - } - - public bool CanContain(ulong address, ulong size, MemoryState state) - { - ulong endAddr = address + size; - - ulong regionBaseAddr = GetBaseAddress(state); - ulong regionEndAddr = regionBaseAddr + GetSize(state); - - bool InsideRegion() - { - return regionBaseAddr <= address && - endAddr > address && - endAddr - 1 <= regionEndAddr - 1; - } - - bool OutsideHeapRegion() - { - return endAddr <= HeapRegionStart || address >= HeapRegionEnd; - } - - bool OutsideAliasRegion() - { - return endAddr <= AliasRegionStart || address >= AliasRegionEnd; - } - - switch (state) - { - case MemoryState.Io: - case MemoryState.Normal: - case MemoryState.CodeStatic: - case MemoryState.CodeMutable: - case MemoryState.SharedMemory: - case MemoryState.ModCodeStatic: - case MemoryState.ModCodeMutable: - case MemoryState.Stack: - case MemoryState.ThreadLocal: - case MemoryState.TransferMemoryIsolated: - case MemoryState.TransferMemory: - case MemoryState.ProcessMemory: - case MemoryState.CodeReadOnly: - case MemoryState.CodeWritable: - return InsideRegion() && OutsideHeapRegion() && OutsideAliasRegion(); - - case MemoryState.Heap: - return InsideRegion() && OutsideAliasRegion(); - - case MemoryState.IpcBuffer0: - case MemoryState.IpcBuffer1: - case MemoryState.IpcBuffer3: - return InsideRegion() && OutsideHeapRegion(); - - case MemoryState.KernelStack: - return InsideRegion(); - } - - throw new ArgumentException($"Invalid state value \"{state}\"."); - } - - private ulong GetBaseAddress(MemoryState state) - { - switch (state) - { - case MemoryState.Io: - case MemoryState.Normal: - case MemoryState.ThreadLocal: - return TlsIoRegionStart; - - case MemoryState.CodeStatic: - case MemoryState.CodeMutable: - case MemoryState.SharedMemory: - case MemoryState.ModCodeStatic: - case MemoryState.ModCodeMutable: - case MemoryState.TransferMemoryIsolated: - case MemoryState.TransferMemory: - case MemoryState.ProcessMemory: - case MemoryState.CodeReadOnly: - case MemoryState.CodeWritable: - return GetAddrSpaceBaseAddr(); - - case MemoryState.Heap: - return HeapRegionStart; - - case MemoryState.IpcBuffer0: - case MemoryState.IpcBuffer1: - case MemoryState.IpcBuffer3: - return AliasRegionStart; - - case MemoryState.Stack: - return StackRegionStart; - - case MemoryState.KernelStack: - return AddrSpaceStart; - } - - throw new ArgumentException($"Invalid state value \"{state}\"."); - } - - private ulong GetSize(MemoryState state) - { - switch (state) - { - case MemoryState.Io: - case MemoryState.Normal: - case MemoryState.ThreadLocal: - return TlsIoRegionEnd - TlsIoRegionStart; - - case MemoryState.CodeStatic: - case MemoryState.CodeMutable: - case MemoryState.SharedMemory: - case MemoryState.ModCodeStatic: - case MemoryState.ModCodeMutable: - case MemoryState.TransferMemoryIsolated: - case MemoryState.TransferMemory: - case MemoryState.ProcessMemory: - case MemoryState.CodeReadOnly: - case MemoryState.CodeWritable: - return GetAddrSpaceSize(); - - case MemoryState.Heap: - return HeapRegionEnd - HeapRegionStart; - - case MemoryState.IpcBuffer0: - case MemoryState.IpcBuffer1: - case MemoryState.IpcBuffer3: - return AliasRegionEnd - AliasRegionStart; - - case MemoryState.Stack: - return StackRegionEnd - StackRegionStart; - - case MemoryState.KernelStack: - return AddrSpaceEnd - AddrSpaceStart; - } - - throw new ArgumentException($"Invalid state value \"{state}\"."); - } - - public ulong GetAddrSpaceBaseAddr() - { - if (AddrSpaceWidth == 36 || AddrSpaceWidth == 39) - { - return 0x8000000; - } - else if (AddrSpaceWidth == 32) - { - return 0x200000; - } - else - { - throw new InvalidOperationException("Invalid address space width!"); - } - } - - public ulong GetAddrSpaceSize() - { - if (AddrSpaceWidth == 36) - { - return 0xff8000000; - } - else if (AddrSpaceWidth == 39) - { - return 0x7ff8000000; - } - else if (AddrSpaceWidth == 32) - { - return 0xffe00000; - } - else - { - throw new InvalidOperationException("Invalid address space width!"); - } - } - - private static ulong GetDramAddressFromPa(ulong pa) - { - return pa - DramMemoryMap.DramBase; - } - - protected KMemoryRegionManager GetMemoryRegionManager() - { - return Context.MemoryManager.MemoryRegions[(int)_memRegion]; - } - - public ulong GetMmUsedPages() - { - lock (_blockManager) - { - return BitUtils.DivRoundUp<ulong>(GetMmUsedSize(), PageSize); - } - } - - private ulong GetMmUsedSize() - { - return (ulong)(_blockManager.BlocksCount * KMemoryBlockSize); - } - - public bool IsInvalidRegion(ulong address, ulong size) - { - return address + size - 1 > GetAddrSpaceBaseAddr() + GetAddrSpaceSize() - 1; - } - - public bool InsideAddrSpace(ulong address, ulong size) - { - return AddrSpaceStart <= address && address + size - 1 <= AddrSpaceEnd - 1; - } - - public bool InsideAliasRegion(ulong address, ulong size) - { - return address + size > AliasRegionStart && AliasRegionEnd > address; - } - - public bool InsideHeapRegion(ulong address, ulong size) - { - return address + size > HeapRegionStart && HeapRegionEnd > address; - } - - public bool InsideStackRegion(ulong address, ulong size) - { - return address + size > StackRegionStart && StackRegionEnd > address; - } - - public bool OutsideAliasRegion(ulong address, ulong size) - { - return AliasRegionStart > address || address + size - 1 > AliasRegionEnd - 1; - } - - public bool OutsideAddrSpace(ulong address, ulong size) - { - return AddrSpaceStart > address || address + size - 1 > AddrSpaceEnd - 1; - } - - public bool OutsideStackRegion(ulong address, ulong size) - { - return StackRegionStart > address || address + size - 1 > StackRegionEnd - 1; - } - - /// <summary> - /// Gets the host regions that make up the given virtual address region. - /// If any part of the virtual region is unmapped, null is returned. - /// </summary> - /// <param name="va">Virtual address of the range</param> - /// <param name="size">Size of the range</param> - /// <returns>The host regions</returns> - /// <exception cref="Ryujinx.Memory.InvalidMemoryRegionException">Throw for unhandled invalid or unmapped memory accesses</exception> - protected abstract IEnumerable<HostMemoryRange> GetHostRegions(ulong va, ulong size); - - /// <summary> - /// Gets the physical regions that make up the given virtual address region. - /// If any part of the virtual region is unmapped, null is returned. - /// </summary> - /// <param name="va">Virtual address of the range</param> - /// <param name="size">Size of the range</param> - /// <param name="pageList">Page list where the ranges will be added</param> - protected abstract void GetPhysicalRegions(ulong va, ulong size, KPageList pageList); - - /// <summary> - /// Gets a read-only span of data from CPU mapped memory. - /// </summary> - /// <remarks> - /// This may perform a allocation if the data is not contiguous in memory. - /// For this reason, the span is read-only, you can't modify the data. - /// </remarks> - /// <param name="va">Virtual address of the data</param> - /// <param name="size">Size of the data</param> - /// <param name="tracked">True if read tracking is triggered on the span</param> - /// <returns>A read-only span of the data</returns> - /// <exception cref="Ryujinx.Memory.InvalidMemoryRegionException">Throw for unhandled invalid or unmapped memory accesses</exception> - protected abstract ReadOnlySpan<byte> GetSpan(ulong va, int size); - - /// <summary> - /// Maps a new memory region with the contents of a existing memory region. - /// </summary> - /// <param name="src">Source memory region where the data will be taken from</param> - /// <param name="dst">Destination memory region to map</param> - /// <param name="pagesCount">Number of pages to map</param> - /// <param name="oldSrcPermission">Current protection of the source memory region</param> - /// <param name="newDstPermission">Desired protection for the destination memory region</param> - /// <returns>Result of the mapping operation</returns> - protected abstract Result MapMemory(ulong src, ulong dst, ulong pagesCount, KMemoryPermission oldSrcPermission, KMemoryPermission newDstPermission); - - /// <summary> - /// Unmaps a region of memory that was previously mapped with <see cref="MapMemory"/>. - /// </summary> - /// <param name="dst">Destination memory region to be unmapped</param> - /// <param name="src">Source memory region that was originally remapped</param> - /// <param name="pagesCount">Number of pages to unmap</param> - /// <param name="oldDstPermission">Current protection of the destination memory region</param> - /// <param name="newSrcPermission">Desired protection of the source memory region</param> - /// <returns>Result of the unmapping operation</returns> - protected abstract Result UnmapMemory(ulong dst, ulong src, ulong pagesCount, KMemoryPermission oldDstPermission, KMemoryPermission newSrcPermission); - - /// <summary> - /// Maps a region of memory into the specified physical memory region. - /// </summary> - /// <param name="dstVa">Destination virtual address that should be mapped</param> - /// <param name="pagesCount">Number of pages to map</param> - /// <param name="srcPa">Physical address where the pages should be mapped. May be ignored if aliasing is not supported</param> - /// <param name="permission">Permission of the region to be mapped</param> - /// <param name="flags">Flags controlling the memory map operation</param> - /// <param name="shouldFillPages">Indicate if the pages should be filled with the <paramref name="fillValue"/> value</param> - /// <param name="fillValue">The value used to fill pages when <paramref name="shouldFillPages"/> is set to true</param> - /// <returns>Result of the mapping operation</returns> - protected abstract Result MapPages( - ulong dstVa, - ulong pagesCount, - ulong srcPa, - KMemoryPermission permission, - MemoryMapFlags flags, - bool shouldFillPages = false, - byte fillValue = 0); - - /// <summary> - /// Maps a region of memory into the specified physical memory region. - /// </summary> - /// <param name="address">Destination virtual address that should be mapped</param> - /// <param name="pageList">List of physical memory pages where the pages should be mapped. May be ignored if aliasing is not supported</param> - /// <param name="permission">Permission of the region to be mapped</param> - /// <param name="flags">Flags controlling the memory map operation</param> - /// <param name="shouldFillPages">Indicate if the pages should be filled with the <paramref name="fillValue"/> value</param> - /// <param name="fillValue">The value used to fill pages when <paramref name="shouldFillPages"/> is set to true</param> - /// <returns>Result of the mapping operation</returns> - protected abstract Result MapPages( - ulong address, - KPageList pageList, - KMemoryPermission permission, - MemoryMapFlags flags, - bool shouldFillPages = false, - byte fillValue = 0); - - /// <summary> - /// Maps pages into an arbitrary host memory location. - /// </summary> - /// <param name="regions">Host regions to be mapped into the specified virtual memory region</param> - /// <param name="va">Destination virtual address of the range on this page table</param> - /// <param name="size">Size of the range</param> - /// <returns>Result of the mapping operation</returns> - protected abstract Result MapForeign(IEnumerable<HostMemoryRange> regions, ulong va, ulong size); - - /// <summary> - /// Unmaps a region of memory that was previously mapped with one of the page mapping methods. - /// </summary> - /// <param name="address">Virtual address of the region to unmap</param> - /// <param name="pagesCount">Number of pages to unmap</param> - /// <returns>Result of the unmapping operation</returns> - protected abstract Result Unmap(ulong address, ulong pagesCount); - - /// <summary> - /// Changes the permissions of a given virtual memory region. - /// </summary> - /// <param name="address">Virtual address of the region to have the permission changes</param> - /// <param name="pagesCount">Number of pages to have their permissions changed</param> - /// <param name="permission">New permission</param> - /// <returns>Result of the permission change operation</returns> - protected abstract Result Reprotect(ulong address, ulong pagesCount, KMemoryPermission permission); - - /// <summary> - /// Changes the permissions of a given virtual memory region. - /// </summary> - /// <param name="address">Virtual address of the region to have the permission changes</param> - /// <param name="pagesCount">Number of pages to have their permissions changed</param> - /// <param name="permission">New permission</param> - /// <returns>Result of the permission change operation</returns> - protected abstract Result ReprotectWithAttributes(ulong address, ulong pagesCount, KMemoryPermission permission); - - /// <summary> - /// Alerts the memory tracking that a given region has been read from or written to. - /// This should be called before read/write is performed. - /// </summary> - /// <param name="va">Virtual address of the region</param> - /// <param name="size">Size of the region</param> - protected abstract void SignalMemoryTracking(ulong va, ulong size, bool write); - - /// <summary> - /// Writes data to CPU mapped memory, with write tracking. - /// </summary> - /// <param name="va">Virtual address to write the data into</param> - /// <param name="data">Data to be written</param> - /// <exception cref="Ryujinx.Memory.InvalidMemoryRegionException">Throw for unhandled invalid or unmapped memory accesses</exception> - protected abstract void Write(ulong va, ReadOnlySpan<byte> data); - } -}
\ No newline at end of file diff --git a/Ryujinx.HLE/HOS/Kernel/Memory/KScopedPageList.cs b/Ryujinx.HLE/HOS/Kernel/Memory/KScopedPageList.cs deleted file mode 100644 index a0c19f9c..00000000 --- a/Ryujinx.HLE/HOS/Kernel/Memory/KScopedPageList.cs +++ /dev/null @@ -1,27 +0,0 @@ -using System; - -namespace Ryujinx.HLE.HOS.Kernel.Memory -{ - struct KScopedPageList : IDisposable - { - private readonly KMemoryManager _manager; - private KPageList _pageList; - - public KScopedPageList(KMemoryManager manager, KPageList pageList) - { - _manager = manager; - _pageList = pageList; - pageList.IncrementPagesReferenceCount(manager); - } - - public void SignalSuccess() - { - _pageList = null; - } - - public void Dispose() - { - _pageList?.DecrementPagesReferenceCount(_manager); - } - } -} diff --git a/Ryujinx.HLE/HOS/Kernel/Memory/KSharedMemory.cs b/Ryujinx.HLE/HOS/Kernel/Memory/KSharedMemory.cs deleted file mode 100644 index 5ec3cd72..00000000 --- a/Ryujinx.HLE/HOS/Kernel/Memory/KSharedMemory.cs +++ /dev/null @@ -1,75 +0,0 @@ -using Ryujinx.Common; -using Ryujinx.HLE.HOS.Kernel.Common; -using Ryujinx.HLE.HOS.Kernel.Process; -using Ryujinx.Horizon.Common; -using Ryujinx.Memory; - -namespace Ryujinx.HLE.HOS.Kernel.Memory -{ - class KSharedMemory : KAutoObject - { - private readonly KPageList _pageList; - - private readonly ulong _ownerPid; - - private readonly KMemoryPermission _ownerPermission; - private readonly KMemoryPermission _userPermission; - - public KSharedMemory( - KernelContext context, - SharedMemoryStorage storage, - ulong ownerPid, - KMemoryPermission ownerPermission, - KMemoryPermission userPermission) : base(context) - { - _pageList = storage.GetPageList(); - _ownerPid = ownerPid; - _ownerPermission = ownerPermission; - _userPermission = userPermission; - } - - public Result MapIntoProcess( - KPageTableBase memoryManager, - ulong address, - ulong size, - KProcess process, - KMemoryPermission permission) - { - if (_pageList.GetPagesCount() != BitUtils.DivRoundUp<ulong>(size, KPageTableBase.PageSize)) - { - return KernelResult.InvalidSize; - } - - KMemoryPermission expectedPermission = process.Pid == _ownerPid - ? _ownerPermission - : _userPermission; - - if (permission != expectedPermission) - { - return KernelResult.InvalidPermission; - } - - // On platforms with page size > 4 KB, this can fail due to the address not being page aligned, - // we can return an error to force the application to retry with a different address. - - try - { - return memoryManager.MapPages(address, _pageList, MemoryState.SharedMemory, permission); - } - catch (InvalidMemoryRegionException) - { - return KernelResult.InvalidMemState; - } - } - - public Result UnmapFromProcess(KPageTableBase memoryManager, ulong address, ulong size, KProcess process) - { - if (_pageList.GetPagesCount() != BitUtils.DivRoundUp<ulong>(size, KPageTableBase.PageSize)) - { - return KernelResult.InvalidSize; - } - - return memoryManager.UnmapPages(address, _pageList, MemoryState.SharedMemory); - } - } -}
\ No newline at end of file diff --git a/Ryujinx.HLE/HOS/Kernel/Memory/KSlabHeap.cs b/Ryujinx.HLE/HOS/Kernel/Memory/KSlabHeap.cs deleted file mode 100644 index 9051e84c..00000000 --- a/Ryujinx.HLE/HOS/Kernel/Memory/KSlabHeap.cs +++ /dev/null @@ -1,50 +0,0 @@ -using System.Collections.Generic; - -namespace Ryujinx.HLE.HOS.Kernel.Memory -{ - class KSlabHeap - { - private LinkedList<ulong> _items; - - public KSlabHeap(ulong pa, ulong itemSize, ulong size) - { - _items = new LinkedList<ulong>(); - - int itemsCount = (int)(size / itemSize); - - for (int index = 0; index < itemsCount; index++) - { - _items.AddLast(pa); - - pa += itemSize; - } - } - - public bool TryGetItem(out ulong pa) - { - lock (_items) - { - if (_items.First != null) - { - pa = _items.First.Value; - - _items.RemoveFirst(); - - return true; - } - } - - pa = 0; - - return false; - } - - public void Free(ulong pa) - { - lock (_items) - { - _items.AddFirst(pa); - } - } - } -}
\ No newline at end of file diff --git a/Ryujinx.HLE/HOS/Kernel/Memory/KTransferMemory.cs b/Ryujinx.HLE/HOS/Kernel/Memory/KTransferMemory.cs deleted file mode 100644 index b2449598..00000000 --- a/Ryujinx.HLE/HOS/Kernel/Memory/KTransferMemory.cs +++ /dev/null @@ -1,130 +0,0 @@ -using Ryujinx.Common; -using Ryujinx.HLE.HOS.Kernel.Common; -using Ryujinx.HLE.HOS.Kernel.Process; -using Ryujinx.Horizon.Common; -using System; - -namespace Ryujinx.HLE.HOS.Kernel.Memory -{ - class KTransferMemory : KAutoObject - { - private KProcess _creator; - - // TODO: Remove when we no longer need to read it from the owner directly. - public KProcess Creator => _creator; - - private readonly KPageList _pageList; - - public ulong Address { get; private set; } - public ulong Size { get; private set; } - - public KMemoryPermission Permission { get; private set; } - - private bool _hasBeenInitialized; - private bool _isMapped; - - public KTransferMemory(KernelContext context) : base(context) - { - _pageList = new KPageList(); - } - - public KTransferMemory(KernelContext context, SharedMemoryStorage storage) : base(context) - { - _pageList = storage.GetPageList(); - Permission = KMemoryPermission.ReadAndWrite; - - _hasBeenInitialized = true; - _isMapped = false; - } - - public Result Initialize(ulong address, ulong size, KMemoryPermission permission) - { - KProcess creator = KernelStatic.GetCurrentProcess(); - - _creator = creator; - - Result result = creator.MemoryManager.BorrowTransferMemory(_pageList, address, size, permission); - - if (result != Result.Success) - { - return result; - } - - creator.IncrementReferenceCount(); - - Permission = permission; - Address = address; - Size = size; - _hasBeenInitialized = true; - _isMapped = false; - - return result; - } - - public Result MapIntoProcess( - KPageTableBase memoryManager, - ulong address, - ulong size, - KProcess process, - KMemoryPermission permission) - { - if (_pageList.GetPagesCount() != BitUtils.DivRoundUp<ulong>(size, KPageTableBase.PageSize)) - { - return KernelResult.InvalidSize; - } - - if (permission != Permission || _isMapped) - { - return KernelResult.InvalidState; - } - - MemoryState state = Permission == KMemoryPermission.None ? MemoryState.TransferMemoryIsolated : MemoryState.TransferMemory; - - Result result = memoryManager.MapPages(address, _pageList, state, KMemoryPermission.ReadAndWrite); - - if (result == Result.Success) - { - _isMapped = true; - } - - return result; - } - - public Result UnmapFromProcess( - KPageTableBase memoryManager, - ulong address, - ulong size, - KProcess process) - { - if (_pageList.GetPagesCount() != BitUtils.DivRoundUp<ulong>(size, (ulong)KPageTableBase.PageSize)) - { - return KernelResult.InvalidSize; - } - - MemoryState state = Permission == KMemoryPermission.None ? MemoryState.TransferMemoryIsolated : MemoryState.TransferMemory; - - Result result = memoryManager.UnmapPages(address, _pageList, state); - - if (result == Result.Success) - { - _isMapped = false; - } - - return result; - } - - protected override void Destroy() - { - if (_hasBeenInitialized) - { - if (!_isMapped && _creator.MemoryManager.UnborrowTransferMemory(Address, Size, _pageList) != Result.Success) - { - throw new InvalidOperationException("Unexpected failure restoring transfer memory attributes."); - } - - _creator.ResourceLimit?.Release(LimitableResource.TransferMemory, 1); - _creator.DecrementReferenceCount(); - } - } - } -}
\ No newline at end of file diff --git a/Ryujinx.HLE/HOS/Kernel/Memory/MemoryAttribute.cs b/Ryujinx.HLE/HOS/Kernel/Memory/MemoryAttribute.cs deleted file mode 100644 index 42407ffe..00000000 --- a/Ryujinx.HLE/HOS/Kernel/Memory/MemoryAttribute.cs +++ /dev/null @@ -1,22 +0,0 @@ -using System; - -namespace Ryujinx.HLE.HOS.Kernel.Memory -{ - [Flags] - enum MemoryAttribute : byte - { - None = 0, - Mask = 0xff, - - Borrowed = 1 << 0, - IpcMapped = 1 << 1, - DeviceMapped = 1 << 2, - Uncached = 1 << 3, - - IpcAndDeviceMapped = IpcMapped | DeviceMapped, - - BorrowedAndIpcMapped = Borrowed | IpcMapped, - - DeviceMappedAndUncached = DeviceMapped | Uncached - } -}
\ No newline at end of file diff --git a/Ryujinx.HLE/HOS/Kernel/Memory/MemoryFillValue.cs b/Ryujinx.HLE/HOS/Kernel/Memory/MemoryFillValue.cs deleted file mode 100644 index cdc892fc..00000000 --- a/Ryujinx.HLE/HOS/Kernel/Memory/MemoryFillValue.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace Ryujinx.HLE.HOS.Kernel.Memory -{ - enum MemoryFillValue : byte - { - Zero = 0, - Stack = 0x58, - Ipc = 0x59, - Heap = 0x5A, - } -} diff --git a/Ryujinx.HLE/HOS/Kernel/Memory/MemoryPermission.cs b/Ryujinx.HLE/HOS/Kernel/Memory/MemoryPermission.cs deleted file mode 100644 index 563b817d..00000000 --- a/Ryujinx.HLE/HOS/Kernel/Memory/MemoryPermission.cs +++ /dev/null @@ -1,20 +0,0 @@ -using System; - -namespace Ryujinx.HLE.HOS.Kernel.Memory -{ - [Flags] - enum KMemoryPermission : uint - { - None = 0, - UserMask = Read | Write | Execute, - Mask = uint.MaxValue, - - Read = 1 << 0, - Write = 1 << 1, - Execute = 1 << 2, - DontCare = 1 << 28, - - ReadAndWrite = Read | Write, - ReadAndExecute = Read | Execute - } -}
\ No newline at end of file diff --git a/Ryujinx.HLE/HOS/Kernel/Memory/MemoryRegion.cs b/Ryujinx.HLE/HOS/Kernel/Memory/MemoryRegion.cs deleted file mode 100644 index ad719bde..00000000 --- a/Ryujinx.HLE/HOS/Kernel/Memory/MemoryRegion.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace Ryujinx.HLE.HOS.Kernel.Memory -{ - enum MemoryRegion - { - Application = 0, - Applet = 1, - Service = 2, - NvServices = 3 - } -}
\ No newline at end of file diff --git a/Ryujinx.HLE/HOS/Kernel/Memory/MemoryState.cs b/Ryujinx.HLE/HOS/Kernel/Memory/MemoryState.cs deleted file mode 100644 index d3b61780..00000000 --- a/Ryujinx.HLE/HOS/Kernel/Memory/MemoryState.cs +++ /dev/null @@ -1,50 +0,0 @@ -using System; - -namespace Ryujinx.HLE.HOS.Kernel.Memory -{ - [Flags] - enum MemoryState : uint - { - Unmapped = 0x00000000, - Io = 0x00002001, - Normal = 0x00042002, - CodeStatic = 0x00DC7E03, - CodeMutable = 0x03FEBD04, - Heap = 0x037EBD05, - SharedMemory = 0x00402006, - ModCodeStatic = 0x00DD7E08, - ModCodeMutable = 0x03FFBD09, - IpcBuffer0 = 0x005C3C0A, - Stack = 0x005C3C0B, - ThreadLocal = 0x0040200C, - TransferMemoryIsolated = 0x015C3C0D, - TransferMemory = 0x005C380E, - ProcessMemory = 0x0040380F, - Reserved = 0x00000010, - IpcBuffer1 = 0x005C3811, - IpcBuffer3 = 0x004C2812, - KernelStack = 0x00002013, - CodeReadOnly = 0x00402214, - CodeWritable = 0x00402015, - UserMask = 0xff, - Mask = 0xffffffff, - - PermissionChangeAllowed = 1 << 8, - ForceReadWritableByDebugSyscalls = 1 << 9, - IpcSendAllowedType0 = 1 << 10, - IpcSendAllowedType3 = 1 << 11, - IpcSendAllowedType1 = 1 << 12, - ProcessPermissionChangeAllowed = 1 << 14, - MapAllowed = 1 << 15, - UnmapProcessCodeMemoryAllowed = 1 << 16, - TransferMemoryAllowed = 1 << 17, - QueryPhysicalAddressAllowed = 1 << 18, - MapDeviceAllowed = 1 << 19, - MapDeviceAlignedAllowed = 1 << 20, - IpcBufferAllowed = 1 << 21, - IsPoolAllocated = 1 << 22, - MapProcessAllowed = 1 << 23, - AttributeChangeAllowed = 1 << 24, - CodeMemoryAllowed = 1 << 25 - } -}
\ No newline at end of file diff --git a/Ryujinx.HLE/HOS/Kernel/Memory/SharedMemoryStorage.cs b/Ryujinx.HLE/HOS/Kernel/Memory/SharedMemoryStorage.cs deleted file mode 100644 index c68b7369..00000000 --- a/Ryujinx.HLE/HOS/Kernel/Memory/SharedMemoryStorage.cs +++ /dev/null @@ -1,49 +0,0 @@ -using System; - -namespace Ryujinx.HLE.HOS.Kernel.Memory -{ - class SharedMemoryStorage - { - private readonly KernelContext _context; - private readonly KPageList _pageList; - private readonly ulong _size; - - public SharedMemoryStorage(KernelContext context, KPageList pageList) - { - _context = context; - _pageList = pageList; - _size = pageList.GetPagesCount() * KPageTableBase.PageSize; - - foreach (KPageNode pageNode in pageList) - { - ulong address = pageNode.Address - DramMemoryMap.DramBase; - ulong size = pageNode.PagesCount * KPageTableBase.PageSize; - context.CommitMemory(address, size); - } - } - - public void ZeroFill() - { - for (ulong offset = 0; offset < _size; offset += sizeof(ulong)) - { - GetRef<ulong>(offset) = 0; - } - } - - public ref T GetRef<T>(ulong offset) where T : unmanaged - { - if (_pageList.Nodes.Count == 1) - { - ulong address = _pageList.Nodes.First.Value.Address - DramMemoryMap.DramBase; - return ref _context.Memory.GetRef<T>(address + offset); - } - - throw new NotImplementedException("Non-contiguous shared memory is not yet supported."); - } - - public KPageList GetPageList() - { - return _pageList; - } - } -} |
