diff options
| author | gdkchan <gab.dark.100@gmail.com> | 2019-01-18 20:26:39 -0200 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2019-01-18 20:26:39 -0200 |
| commit | 22bacc618815170c0d186a82e1ea4558e36b7063 (patch) | |
| tree | 79b97959481fea1ac301da6d4e9dea9b991ece6f /Ryujinx.HLE/HOS/Kernel/Memory | |
| parent | 3731d0ce8412c3c48286c242842bcb4940b4ca6d (diff) | |
Improve kernel IPC implementation (#550)
* Implement some IPC related kernel SVCs properly
* Fix BLZ decompression when the segment also has a uncompressed chunck
* Set default cpu core on process start from ProgramLoader, remove debug message
* Load process capabilities properly on KIPs
* Fix a copy/paste error in UnmapPhysicalMemory64
* Implement smarter switching between old and new IPC system to support the old HLE services implementation without the manual switch
* Implement RegisterService on sm and AcceptSession (partial)
* Misc fixes and improvements on new IPC methods
* Move IPC related SVCs into a separate file, and logging on RegisterService (sm)
* Some small fixes related to receive list buffers and error cases
* Load NSOs using the correct pool partition
* Fix corner case on GetMaskFromMinMax where range is 64, doesn't happen in pratice however
* Fix send static buffer copy
* Session release, implement closing requests on client disconnect
* Implement ConnectToPort SVC
* KLightSession init
Diffstat (limited to 'Ryujinx.HLE/HOS/Kernel/Memory')
| -rw-r--r-- | Ryujinx.HLE/HOS/Kernel/Memory/KMemoryBlock.cs | 106 | ||||
| -rw-r--r-- | Ryujinx.HLE/HOS/Kernel/Memory/KMemoryInfo.cs | 31 | ||||
| -rw-r--r-- | Ryujinx.HLE/HOS/Kernel/Memory/KMemoryManager.cs | 1270 | ||||
| -rw-r--r-- | Ryujinx.HLE/HOS/Kernel/Memory/KMemoryRegionManager.cs | 285 | ||||
| -rw-r--r-- | Ryujinx.HLE/HOS/Kernel/Memory/KSharedMemory.cs | 5 | ||||
| -rw-r--r-- | Ryujinx.HLE/HOS/Kernel/Memory/KTransferMemory.cs | 6 |
6 files changed, 1287 insertions, 416 deletions
diff --git a/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryBlock.cs b/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryBlock.cs index 89a19498..b7c2b309 100644 --- a/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryBlock.cs +++ b/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryBlock.cs @@ -1,29 +1,108 @@ +using System; + namespace Ryujinx.HLE.HOS.Kernel.Memory { class KMemoryBlock { - public ulong BaseAddress { get; set; } - public ulong PagesCount { get; set; } + public ulong BaseAddress { get; private set; } + public ulong PagesCount { get; private set; } - public MemoryState State { get; set; } - public MemoryPermission Permission { get; set; } - public MemoryAttribute Attribute { get; set; } + public MemoryState State { get; private set; } + public MemoryPermission Permission { get; private set; } + public MemoryAttribute Attribute { get; private set; } + public MemoryPermission SourcePermission { get; private set; } - public int IpcRefCount { get; set; } - public int DeviceRefCount { get; set; } + public int IpcRefCount { get; private set; } + public int DeviceRefCount { get; private set; } public KMemoryBlock( ulong baseAddress, ulong pagesCount, MemoryState state, MemoryPermission permission, - MemoryAttribute attribute) + MemoryAttribute attribute, + int ipcRefCount = 0, + int deviceRefCount = 0) + { + BaseAddress = baseAddress; + PagesCount = pagesCount; + State = state; + Attribute = attribute; + Permission = permission; + IpcRefCount = ipcRefCount; + DeviceRefCount = deviceRefCount; + } + + public void SetState(MemoryPermission permission, MemoryState state, MemoryAttribute attribute) + { + Permission = permission; + State = state; + Attribute &= MemoryAttribute.IpcAndDeviceMapped; + Attribute |= attribute; + } + + public void SetIpcMappingPermission(MemoryPermission permission) + { + int oldIpcRefCount = IpcRefCount++; + + if ((ushort)IpcRefCount == 0) + { + throw new InvalidOperationException("IPC reference count increment overflowed."); + } + + if (oldIpcRefCount == 0) + { + SourcePermission = permission; + + Permission &= ~MemoryPermission.ReadAndWrite; + Permission |= MemoryPermission.ReadAndWrite & permission; + } + + Attribute |= MemoryAttribute.IpcMapped; + } + + public void RestoreIpcMappingPermission() + { + int oldIpcRefCount = IpcRefCount--; + + if (oldIpcRefCount == 0) + { + throw new InvalidOperationException("IPC reference count decrement underflowed."); + } + + if (oldIpcRefCount == 1) + { + Permission = SourcePermission; + + SourcePermission = MemoryPermission.None; + + Attribute &= ~MemoryAttribute.IpcMapped; + } + } + + public KMemoryBlock SplitRightAtAddress(ulong address) + { + ulong leftAddress = BaseAddress; + + ulong leftPagesCount = (address - leftAddress) / KMemoryManager.PageSize; + + BaseAddress = address; + + PagesCount -= leftPagesCount; + + return new KMemoryBlock( + leftAddress, + leftPagesCount, + State, + Permission, + Attribute, + IpcRefCount, + DeviceRefCount); + } + + public void AddPages(ulong pagesCount) { - BaseAddress = baseAddress; - PagesCount = pagesCount; - State = state; - Attribute = attribute; - Permission = permission; + PagesCount += pagesCount; } public KMemoryInfo GetInfo() @@ -36,6 +115,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory State, Permission, Attribute, + SourcePermission, IpcRefCount, DeviceRefCount); } diff --git a/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryInfo.cs b/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryInfo.cs index 226ce77c..21e9e494 100644 --- a/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryInfo.cs +++ b/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryInfo.cs @@ -2,15 +2,16 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory { class KMemoryInfo { - public ulong Address { get; private set; } - public ulong Size { get; private set; } + public ulong Address { get; } + public ulong Size { get; } - public MemoryState State { get; private set; } - public MemoryPermission Permission { get; private set; } - public MemoryAttribute Attribute { get; private set; } + public MemoryState State { get; } + public MemoryPermission Permission { get; } + public MemoryAttribute Attribute { get; } + public MemoryPermission SourcePermission { get; } - public int IpcRefCount { get; private set; } - public int DeviceRefCount { get; private set; } + public int IpcRefCount { get; } + public int DeviceRefCount { get; } public KMemoryInfo( ulong address, @@ -18,16 +19,18 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory MemoryState state, MemoryPermission permission, MemoryAttribute attribute, + MemoryPermission sourcePermission, int ipcRefCount, int deviceRefCount) { - Address = address; - Size = size; - State = state; - Attribute = attribute; - Permission = permission; - IpcRefCount = ipcRefCount; - DeviceRefCount = deviceRefCount; + Address = address; + Size = size; + State = state; + Permission = permission; + Attribute = attribute; + SourcePermission = sourcePermission; + IpcRefCount = ipcRefCount; + DeviceRefCount = deviceRefCount; } } }
\ No newline at end of file diff --git a/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryManager.cs b/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryManager.cs index fb5dec04..7a40139c 100644 --- a/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryManager.cs +++ b/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryManager.cs @@ -9,6 +9,16 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory { class KMemoryManager { + private static readonly int[] MappingUnitSizes = new int[] + { + 0x1000, + 0x10000, + 0x200000, + 0x400000, + 0x2000000, + 0x40000000 + }; + public const int PageSize = 0x1000; private const int KMemoryBlockSize = 0x40; @@ -335,7 +345,12 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory ulong addrSpacePagesCount = (addrSpaceEnd - addrSpaceStart) / PageSize; - InsertBlock(addrSpaceStart, addrSpacePagesCount, MemoryState.Unmapped); + _blocks.AddFirst(new KMemoryBlock( + addrSpaceStart, + addrSpacePagesCount, + MemoryState.Unmapped, + MemoryPermission.None, + MemoryAttribute.None)); return KernelResult.Success; } @@ -488,67 +503,9 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory return KernelResult.OutOfMemory; } - ulong reservedPagesCount = _isKernel ? 1UL : 4UL; - lock (_blocks) { - if (_aslrEnabled) - { - ulong totalNeededSize = (reservedPagesCount + neededPagesCount) * PageSize; - - ulong remainingPages = regionPagesCount - neededPagesCount; - - ulong aslrMaxOffset = ((remainingPages + reservedPagesCount) * PageSize) / (ulong)alignment; - - for (int attempt = 0; attempt < 8; attempt++) - { - address = BitUtils.AlignDown(regionStart + GetRandomValue(0, aslrMaxOffset) * (ulong)alignment, alignment); - - ulong endAddr = address + totalNeededSize; - - KMemoryInfo info = FindBlock(address).GetInfo(); - - if (info.State != MemoryState.Unmapped) - { - continue; - } - - ulong currBaseAddr = info.Address + reservedPagesCount * PageSize; - ulong currEndAddr = info.Address + info.Size; - - if (address >= regionStart && - address >= currBaseAddr && - endAddr - 1 <= regionEndAddr - 1 && - endAddr - 1 <= currEndAddr - 1) - { - break; - } - } - - if (address == 0) - { - ulong aslrPage = GetRandomValue(0, aslrMaxOffset); - - address = FindFirstFit( - regionStart + aslrPage * PageSize, - regionPagesCount - aslrPage, - neededPagesCount, - alignment, - 0, - reservedPagesCount); - } - } - - if (address == 0) - { - address = FindFirstFit( - regionStart, - regionPagesCount, - neededPagesCount, - alignment, - 0, - reservedPagesCount); - } + address = AllocateVa(regionStart, regionPagesCount, neededPagesCount, alignment); if (address == 0) { @@ -977,6 +934,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory MemoryState.Reserved, MemoryPermission.None, MemoryAttribute.None, + MemoryPermission.None, 0, 0); } @@ -1325,22 +1283,13 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory { ulong mappedSize = 0; - KMemoryInfo info; - - LinkedListNode<KMemoryBlock> node = FindBlockNode(address); - - do + foreach (KMemoryInfo info in IterateOverRange(address, endAddr)) { - info = node.Value.GetInfo(); - if (info.State != MemoryState.Unmapped) { mappedSize += GetSizeInRange(info, address, endAddr); } - - node = node.Next; } - while (info.Address + info.Size < endAddr && node != null); if (mappedSize == size) { @@ -1419,16 +1368,8 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory KPageList pageList = new KPageList(); - KMemoryInfo info; - - LinkedListNode<KMemoryBlock> baseNode = FindBlockNode(address); - - LinkedListNode<KMemoryBlock> node = baseNode; - - do + foreach (KMemoryInfo info in IterateOverRange(address, endAddr)) { - info = node.Value.GetInfo(); - if (info.State == MemoryState.Heap) { if (info.Attribute != MemoryAttribute.None) @@ -1447,10 +1388,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory { return KernelResult.InvalidMemState; } - - node = node.Next; } - while (info.Address + info.Size < endAddr && node != null); if (heapMappedSize == 0) { @@ -1465,12 +1403,8 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory //Try to unmap all the heap mapped memory inside range. KernelResult result = KernelResult.Success; - node = baseNode; - - do + foreach (KMemoryInfo info in IterateOverRange(address, endAddr)) { - info = node.Value.GetInfo(); - if (info.State == MemoryState.Heap) { ulong blockSize = GetSizeInRange(info, address, endAddr); @@ -1488,10 +1422,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory break; } } - - node = node.Next; } - while (info.Address + info.Size < endAddr && node != null); if (result == KernelResult.Success) { @@ -1514,10 +1445,6 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory private void MapPhysicalMemory(KPageList pageList, ulong address, ulong endAddr) { - KMemoryInfo info; - - LinkedListNode<KMemoryBlock> node = FindBlockNode(address); - LinkedListNode<KPageNode> pageListNode = pageList.Nodes.First; KPageNode pageNode = pageListNode.Value; @@ -1525,10 +1452,8 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory ulong srcPa = pageNode.Address; ulong srcPaPages = pageNode.PagesCount; - do + foreach (KMemoryInfo info in IterateOverRange(address, endAddr)) { - info = node.Value.GetInfo(); - if (info.State == MemoryState.Unmapped) { ulong blockSize = GetSizeInRange(info, address, endAddr); @@ -1570,38 +1495,750 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory dstVaPages -= pagesCount; } } + } + } - node = node.Next; + public KernelResult CopyDataToCurrentProcess( + ulong dst, + ulong size, + ulong src, + MemoryState stateMask, + MemoryState stateExpected, + MemoryPermission permission, + MemoryAttribute attributeMask, + MemoryAttribute attributeExpected) + { + //Client -> server. + return CopyDataFromOrToCurrentProcess( + size, + src, + dst, + stateMask, + stateExpected, + permission, + attributeMask, + attributeExpected, + toServer: true); + } + + public KernelResult CopyDataFromCurrentProcess( + ulong dst, + ulong size, + MemoryState stateMask, + MemoryState stateExpected, + MemoryPermission permission, + MemoryAttribute attributeMask, + MemoryAttribute attributeExpected, + ulong src) + { + //Server -> client. + return CopyDataFromOrToCurrentProcess( + size, + dst, + src, + stateMask, + stateExpected, + permission, + attributeMask, + attributeExpected, + toServer: false); + } + + private KernelResult CopyDataFromOrToCurrentProcess( + ulong size, + ulong clientAddress, + ulong serverAddress, + MemoryState stateMask, + MemoryState stateExpected, + MemoryPermission permission, + MemoryAttribute attributeMask, + MemoryAttribute attributeExpected, + bool toServer) + { + if (AddrSpaceStart > clientAddress) + { + return KernelResult.InvalidMemState; + } + + ulong srcEndAddr = clientAddress + size; + + if (srcEndAddr <= clientAddress || srcEndAddr - 1 > AddrSpaceEnd - 1) + { + return KernelResult.InvalidMemState; + } + + lock (_blocks) + { + if (CheckRange( + clientAddress, + size, + stateMask, + stateExpected, + permission, + permission, + attributeMask | MemoryAttribute.Uncached, + attributeExpected)) + { + KProcess currentProcess = _system.Scheduler.GetCurrentProcess(); + + serverAddress = currentProcess.MemoryManager.GetDramAddressFromVa(serverAddress); + + if (toServer) + { + _system.Device.Memory.Copy(serverAddress, GetDramAddressFromVa(clientAddress), size); + } + else + { + _system.Device.Memory.Copy(GetDramAddressFromVa(clientAddress), serverAddress, size); + } + + return KernelResult.Success; + } + else + { + return KernelResult.InvalidMemState; + } } - while (info.Address + info.Size < endAddr && node != null); } - private static ulong GetSizeInRange(KMemoryInfo info, ulong start, ulong end) + public KernelResult MapBufferFromClientProcess( + ulong size, + ulong src, + KMemoryManager sourceMemMgr, + MemoryPermission permission, + MemoryState state, + bool copyData, + out ulong dst) { - ulong endAddr = info.Size + info.Address; - ulong size = info.Size; + dst = 0; - if (info.Address < start) + KernelResult result = sourceMemMgr.GetPagesForMappingIntoAnotherProcess( + src, + size, + permission, + state, + copyData, + _aslrDisabled, + _memRegion, + out KPageList pageList); + + if (result != KernelResult.Success) { - size -= start - info.Address; + return result; } - if (endAddr > end) + result = MapPagesFromAnotherProcess(size, src, permission, state, pageList, out ulong va); + + if (result != KernelResult.Success) { - size -= endAddr - end; + sourceMemMgr.UnmapIpcRestorePermission(src, size, state); + } + else + { + dst = va; } - return size; + return result; } - private static ulong GetAddrInRange(KMemoryInfo info, ulong start) + private KernelResult GetPagesForMappingIntoAnotherProcess( + ulong address, + ulong size, + MemoryPermission permission, + MemoryState state, + bool copyData, + bool aslrDisabled, + MemoryRegion region, + out KPageList pageList) { - if (info.Address < start) + pageList = null; + + if (AddrSpaceStart > address) { - return start; + return KernelResult.InvalidMemState; } - return info.Address; + ulong endAddr = address + size; + + if (endAddr <= address || endAddr - 1 > AddrSpaceEnd - 1) + { + return KernelResult.InvalidMemState; + } + + MemoryState stateMask; + + switch (state) + { + case MemoryState.IpcBuffer0: stateMask = MemoryState.IpcSendAllowedType0; break; + case MemoryState.IpcBuffer1: stateMask = MemoryState.IpcSendAllowedType1; break; + case MemoryState.IpcBuffer3: stateMask = MemoryState.IpcSendAllowedType3; break; + + default: return KernelResult.InvalidCombination; + } + + MemoryPermission permissionMask = permission == MemoryPermission.ReadAndWrite + ? MemoryPermission.None + : MemoryPermission.Read; + + MemoryAttribute attributeMask = MemoryAttribute.Borrowed | MemoryAttribute.Uncached; + + if (state == MemoryState.IpcBuffer0) + { + attributeMask |= MemoryAttribute.DeviceMapped; + } + + ulong addressRounded = BitUtils.AlignUp (address, PageSize); + ulong endAddrRounded = BitUtils.AlignUp (endAddr, PageSize); + ulong endAddrTruncated = BitUtils.AlignDown(endAddr, PageSize); + + if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) + { + return KernelResult.OutOfResource; + } + + ulong visitedSize = 0; + + void CleanUpForError() + { + ulong endAddrVisited = address + visitedSize; + + foreach (KMemoryInfo info in IterateOverRange(address, endAddrVisited)) + { + if ((info.Permission & MemoryPermission.ReadAndWrite) != permissionMask && info.IpcRefCount == 0) + { + ulong blockAddress = GetAddrInRange(info, addressRounded); + ulong blockSize = GetSizeInRange(info, addressRounded, endAddrVisited); + + ulong blockPagesCount = blockSize / PageSize; + + if (DoMmuOperation( + blockAddress, + blockPagesCount, + 0, + false, + info.Permission, + MemoryOperation.ChangePermRw) != KernelResult.Success) + { + throw new InvalidOperationException("Unexpected failure trying to restore permission."); + } + } + } + } + + lock (_blocks) + { + KernelResult result; + + foreach (KMemoryInfo info in IterateOverRange(address, endAddrRounded)) + { + //Check if the block state matches what we expect. + if ((info.State & stateMask) != stateMask || + (info.Permission & permission) != permission || + (info.Attribute & attributeMask) != MemoryAttribute.None) + { + CleanUpForError(); + + return KernelResult.InvalidMemState; + } + + ulong blockAddress = GetAddrInRange(info, addressRounded); + ulong blockSize = GetSizeInRange(info, addressRounded, endAddrTruncated); + + ulong blockPagesCount = blockSize / PageSize; + + if ((info.Permission & MemoryPermission.ReadAndWrite) != permissionMask && info.IpcRefCount == 0) + { + result = DoMmuOperation( + blockAddress, + blockPagesCount, + 0, + false, + permissionMask, + MemoryOperation.ChangePermRw); + + if (result != KernelResult.Success) + { + CleanUpForError(); + + return result; + } + } + + visitedSize += blockSize; + } + + result = GetPagesForIpcTransfer(address, size, copyData, aslrDisabled, region, out pageList); + + if (result != KernelResult.Success) + { + CleanUpForError(); + + return result; + } + + if (visitedSize != 0) + { + InsertBlock(address, visitedSize / PageSize, SetIpcMappingPermissions, permissionMask); + } + } + + return KernelResult.Success; + } + + private KernelResult GetPagesForIpcTransfer( + ulong address, + ulong size, + bool copyData, + bool aslrDisabled, + MemoryRegion region, + out KPageList pageList) + { + pageList = null; + + ulong addressTruncated = BitUtils.AlignDown(address, PageSize); + ulong addressRounded = BitUtils.AlignUp (address, PageSize); + + ulong endAddr = address + size; + + ulong dstFirstPagePa = AllocateSinglePage(region, aslrDisabled); + + if (dstFirstPagePa == 0) + { + return KernelResult.OutOfMemory; + } + + ulong dstLastPagePa = 0; + + void CleanUpForError() + { + FreeSinglePage(region, dstFirstPagePa); + + if (dstLastPagePa != 0) + { + FreeSinglePage(region, dstLastPagePa); + } + } + + ulong firstPageFillAddress = dstFirstPagePa; + + if (!ConvertVaToPa(addressTruncated, out ulong srcFirstPagePa)) + { + CleanUpForError(); + + return KernelResult.InvalidMemState; + } + + ulong unusedSizeAfter; + + //When the start address is unaligned, we can't safely map the + //first page as it would expose other undesirable information on the + //target process. So, instead we allocate new pages, copy the data + //inside the range, and then clear the remaining space. + //The same also holds for the last page, if the end address + //(address + size) is also not aligned. + if (copyData) + { + ulong unusedSizeBefore = address - addressTruncated; + + _system.Device.Memory.Set(dstFirstPagePa, 0, unusedSizeBefore); + + ulong copySize = addressRounded <= endAddr ? addressRounded - address : size; + + _system.Device.Memory.Copy( + GetDramAddressFromPa(dstFirstPagePa + unusedSizeBefore), + GetDramAddressFromPa(srcFirstPagePa + unusedSizeBefore), copySize); + + firstPageFillAddress += unusedSizeBefore + copySize; + + unusedSizeAfter = addressRounded > endAddr ? addressRounded - endAddr : 0; + } + else + { + unusedSizeAfter = PageSize; + } + + if (unusedSizeAfter != 0) + { + _system.Device.Memory.Set(firstPageFillAddress, 0, unusedSizeAfter); + } + + KPageList pages = new KPageList(); + + if (pages.AddRange(dstFirstPagePa, 1) != KernelResult.Success) + { + CleanUpForError(); + + return KernelResult.OutOfResource; + } + + ulong endAddrTruncated = BitUtils.AlignDown(endAddr, PageSize); + ulong endAddrRounded = BitUtils.AlignUp (endAddr, PageSize); + + if (endAddrTruncated > addressRounded) + { + ulong alignedPagesCount = (endAddrTruncated - addressRounded) / PageSize; + + AddVaRangeToPageList(pages, addressRounded, alignedPagesCount); + } + + if (endAddrTruncated != endAddrRounded) + { + //End is also not aligned... + dstLastPagePa = AllocateSinglePage(region, aslrDisabled); + + if (dstLastPagePa == 0) + { + CleanUpForError(); + + return KernelResult.OutOfMemory; + } + + ulong lastPageFillAddr = dstLastPagePa; + + if (!ConvertVaToPa(endAddrTruncated, out ulong srcLastPagePa)) + { + CleanUpForError(); + + return KernelResult.InvalidMemState; + } + + if (copyData) + { + ulong copySize = endAddr - endAddrTruncated; + + _system.Device.Memory.Copy( + GetDramAddressFromPa(dstLastPagePa), + GetDramAddressFromPa(srcLastPagePa), copySize); + + lastPageFillAddr += copySize; + + unusedSizeAfter = PageSize - copySize; + } + else + { + unusedSizeAfter = PageSize; + } + + _system.Device.Memory.Set(lastPageFillAddr, 0, unusedSizeAfter); + + if (pages.AddRange(dstFirstPagePa, 1) != KernelResult.Success) + { + CleanUpForError(); + + return KernelResult.OutOfResource; + } + } + + pageList = pages; + + return KernelResult.Success; + } + + private ulong AllocateSinglePage(MemoryRegion region, bool aslrDisabled) + { + KMemoryRegionManager regionMgr = _system.MemoryRegions[(int)region]; + + return regionMgr.AllocatePagesContiguous(1, aslrDisabled); + } + + private void FreeSinglePage(MemoryRegion region, ulong address) + { + KMemoryRegionManager regionMgr = _system.MemoryRegions[(int)region]; + + regionMgr.FreePage(address); + } + + private KernelResult MapPagesFromAnotherProcess( + ulong size, + ulong address, + MemoryPermission permission, + MemoryState state, + KPageList pageList, + out ulong mappedVa) + { + mappedVa = 0; + + lock (_blocks) + { + if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) + { + return KernelResult.OutOfResource; + } + + ulong endAddr = address + size; + + ulong addressTruncated = BitUtils.AlignDown(address, PageSize); + ulong endAddrRounded = BitUtils.AlignUp (endAddr, PageSize); + + ulong neededSize = endAddrRounded - addressTruncated; + + ulong neededPagesCount = neededSize / PageSize; + + ulong regionPagesCount = (AliasRegionEnd - AliasRegionStart) / PageSize; + + ulong va = 0; + + for (int unit = MappingUnitSizes.Length - 1; unit >= 0 && va == 0; unit--) + { + int alignemnt = MappingUnitSizes[unit]; + + va = AllocateVa(AliasRegionStart, regionPagesCount, neededPagesCount, alignemnt); + } + + if (va == 0) + { + return KernelResult.OutOfVaSpace; + } + + if (pageList.Nodes.Count != 0) + { + KernelResult result = MapPages(va, pageList, permission); + + if (result != KernelResult.Success) + { + return result; + } + } + + InsertBlock(va, neededPagesCount, state, permission); + + mappedVa = va; + } + + return KernelResult.Success; + } + + public KernelResult UnmapNoAttributeIfStateEquals(ulong address, ulong size, MemoryState state) + { + if (AddrSpaceStart > address) + { + return KernelResult.InvalidMemState; + } + + ulong endAddr = address + size; + + if (endAddr <= address || endAddr - 1 > AddrSpaceEnd - 1) + { + return KernelResult.InvalidMemState; + } + + lock (_blocks) + { + if (CheckRange( + address, + size, + MemoryState.Mask, + state, + MemoryPermission.Read, + MemoryPermission.Read, + MemoryAttribute.Mask, + MemoryAttribute.None, + MemoryAttribute.IpcAndDeviceMapped, + out _, + out _, + out _)) + { + if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) + { + return KernelResult.OutOfResource; + } + + ulong addressTruncated = BitUtils.AlignDown(address, PageSize); + ulong endAddrRounded = BitUtils.AlignUp (endAddr, PageSize); + + ulong pagesCount = (endAddrRounded - addressTruncated) / PageSize; + + KernelResult result = DoMmuOperation( + addressTruncated, + pagesCount, + 0, + false, + MemoryPermission.None, + MemoryOperation.Unmap); + + if (result == KernelResult.Success) + { + InsertBlock(addressTruncated, pagesCount, MemoryState.Unmapped); + } + + return result; + } + else + { + return KernelResult.InvalidMemState; + } + } + } + + public KernelResult UnmapIpcRestorePermission(ulong address, ulong size, MemoryState state) + { + ulong endAddr = address + size; + + ulong addressRounded = BitUtils.AlignUp (address, PageSize); + ulong endAddrTruncated = BitUtils.AlignDown(endAddr, PageSize); + + ulong pagesCount = (endAddrTruncated - addressRounded) / PageSize; + + MemoryState stateMask; + + switch (state) + { + case MemoryState.IpcBuffer0: stateMask = MemoryState.IpcSendAllowedType0; break; + case MemoryState.IpcBuffer1: stateMask = MemoryState.IpcSendAllowedType1; break; + case MemoryState.IpcBuffer3: stateMask = MemoryState.IpcSendAllowedType3; break; + + default: return KernelResult.InvalidCombination; + } + + MemoryAttribute attributeMask = + MemoryAttribute.Borrowed | + MemoryAttribute.IpcMapped | + MemoryAttribute.Uncached; + + if (state == MemoryState.IpcBuffer0) + { + attributeMask |= MemoryAttribute.DeviceMapped; + } + + if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) + { + return KernelResult.OutOfResource; + } + + lock (_blocks) + { + foreach (KMemoryInfo info in IterateOverRange(address, endAddrTruncated)) + { + //Check if the block state matches what we expect. + if ((info.State & stateMask) != stateMask || + (info.Attribute & attributeMask) != MemoryAttribute.IpcMapped) + { + return KernelResult.InvalidMemState; + } + + if (info.Permission != info.SourcePermission && info.IpcRefCount == 1) + { + ulong blockAddress = GetAddrInRange(info, addressRounded); + ulong blockSize = GetSizeInRange(info, addressRounded, endAddrTruncated); + + ulong blockPagesCount = blockSize / PageSize; + + KernelResult result = DoMmuOperation( + blockAddress, + blockPagesCount, + 0, + false, + info.SourcePermission, + MemoryOperation.ChangePermRw); + + if (result != KernelResult.Success) + { + return result; + } + } + } + } + + InsertBlock(address, pagesCount, RestoreIpcMappingPermissions); + + return KernelResult.Success; + } + + public KernelResult UnborrowIpcBuffer(ulong address, ulong size) + { + return ClearAttributesAndChangePermission( + address, + size, + MemoryState.IpcBufferAllowed, + MemoryState.IpcBufferAllowed, + MemoryPermission.None, + MemoryPermission.None, + MemoryAttribute.Mask, + MemoryAttribute.Borrowed, + MemoryPermission.ReadAndWrite, + MemoryAttribute.Borrowed); + } + + private KernelResult ClearAttributesAndChangePermission( + ulong address, + ulong size, + MemoryState stateMask, + MemoryState stateExpected, + MemoryPermission permissionMask, + MemoryPermission permissionExpected, + MemoryAttribute attributeMask, + MemoryAttribute attributeExpected, + MemoryPermission newPermission, + MemoryAttribute attributeClearMask, + KPageList pageList = null) + { + lock (_blocks) + { + if (CheckRange( + address, + size, + stateMask | MemoryState.IsPoolAllocated, + stateExpected | MemoryState.IsPoolAllocated, + permissionMask, + permissionExpected, + attributeMask, + attributeExpected, + MemoryAttribute.IpcAndDeviceMapped, + out MemoryState oldState, + out MemoryPermission oldPermission, + out MemoryAttribute oldAttribute)) + { + ulong pagesCount = size / PageSize; + + if (pageList != null) + { + KPageList currPageList = new KPageList(); + + AddVaRangeToPageList(currPageList, address, pagesCount); + + if (!currPageList.IsEqual(pageList)) + { + return KernelResult.InvalidMemRange; + } + } + + if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) + { + return KernelResult.OutOfResource; + } + + if (newPermission == MemoryPermission.None) + { + newPermission = oldPermission; + } + + if (newPermission != oldPermission) + { + KernelResult result = DoMmuOperation( + address, + pagesCount, + 0, + false, + newPermission, + MemoryOperation.ChangePermRw); + + if (result != KernelResult.Success) + { + return result; + } + } + + MemoryAttribute newAttribute = oldAttribute & ~attributeClearMask; + + InsertBlock(address, pagesCount, oldState, newPermission, newAttribute); + + return KernelResult.Success; + } + else + { + return KernelResult.InvalidMemState; + } + } } private void AddVaRangeToPageList(KPageList pageList, ulong start, ulong pagesCount) @@ -1610,9 +2247,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory while (address < start + pagesCount * PageSize) { - KernelResult result = ConvertVaToPa(address, out ulong pa); - - if (result != KernelResult.Success) + if (!ConvertVaToPa(address, out ulong pa)) { throw new InvalidOperationException("Unexpected failure translating virtual address."); } @@ -1623,6 +2258,34 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory } } + private static ulong GetAddrInRange(KMemoryInfo info, ulong start) + { + if (info.Address < start) + { + return start; + } + + return info.Address; + } + + private static ulong GetSizeInRange(KMemoryInfo info, ulong start, ulong end) + { + ulong endAddr = info.Size + info.Address; + ulong size = info.Size; + + if (info.Address < start) + { + size -= start - info.Address; + } + + if (endAddr > end) + { + size -= endAddr - end; + } + + return size; + } + private bool IsUnmapped(ulong address, ulong size) { return CheckRange( @@ -1654,7 +2317,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory out MemoryPermission outPermission, out MemoryAttribute outAttribute) { - ulong endAddr = address + size - 1; + ulong endAddr = address + size; LinkedListNode<KMemoryBlock> node = FindBlockNode(address); @@ -1676,28 +2339,20 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory (firstState & stateMask) != stateExpected || (firstPermission & permissionMask) != permissionExpected) { - break; - } - - //Check if this is the last block on the range, if so return success. - if (endAddr <= info.Address + info.Size - 1) - { - outState = firstState; - outPermission = firstPermission; - outAttribute = firstAttribute & ~attributeIgnoreMask; + outState = MemoryState.Unmapped; + outPermission = MemoryPermission.None; + outAttribute = MemoryAttribute.None; - return true; + return false; } - - node = node.Next; } - while (node != null); + while (info.Address + info.Size - 1 < endAddr - 1 && (node = node.Next) != null); - outState = MemoryState.Unmapped; - outPermission = MemoryPermission.None; - outAttribute = MemoryAttribute.None; + outState = firstState; + outPermission = firstPermission; + outAttribute = firstAttribute & ~attributeIgnoreMask; - return false; + return true; } private bool CheckRange( @@ -1710,33 +2365,33 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory MemoryAttribute attributeMask, MemoryAttribute attributeExpected) { - ulong endAddr = address + size - 1; - - LinkedListNode<KMemoryBlock> node = FindBlockNode(address); - - do + foreach (KMemoryInfo info in IterateOverRange(address, address + size)) { - KMemoryInfo info = node.Value.GetInfo(); - //Check if the block state matches what we expect. if ((info.State & stateMask) != stateExpected || (info.Permission & permissionMask) != permissionExpected || (info.Attribute & attributeMask) != attributeExpected) { - break; + return false; } + } - //Check if this is the last block on the range, if so return success. - if (endAddr <= info.Address + info.Size - 1) - { - return true; - } + return true; + } - node = node.Next; - } - while (node != null); + private IEnumerable<KMemoryInfo> IterateOverRange(ulong start, ulong end) + { + LinkedListNode<KMemoryBlock> node = FindBlockNode(start); + + KMemoryInfo info; + + do + { + info = node.Value.GetInfo(); - return false; + yield return info; + } + while (info.Address + info.Size - 1 < end - 1 && (node = node.Next) != null); } private void InsertBlock( @@ -1750,23 +2405,22 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory MemoryAttribute newAttribute) { //Insert new block on the list only on areas where the state - //of the block matches the state specified on the Old* state + //of the block matches the state specified on the old* state //arguments, otherwise leave it as is. int oldCount = _blocks.Count; oldAttribute |= MemoryAttribute.IpcAndDeviceMapped; - ulong endAddr = pagesCount * PageSize + baseAddress; + ulong endAddr = baseAddress + pagesCount * PageSize; LinkedListNode<KMemoryBlock> node = _blocks.First; while (node != null) { - LinkedListNode<KMemoryBlock> newNode = node; - LinkedListNode<KMemoryBlock> nextNode = node.Next; - KMemoryBlock currBlock = node.Value; + LinkedListNode<KMemoryBlock> nextNode = node.Next; + ulong currBaseAddr = currBlock.BaseAddress; ulong currEndAddr = currBlock.PagesCount * PageSize + currBaseAddr; @@ -1783,67 +2437,28 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory continue; } - if (currBaseAddr >= baseAddress && currEndAddr <= endAddr) + LinkedListNode<KMemoryBlock> newNode = node; + + if (baseAddress > currBaseAddr) { - currBlock.State = newState; - currBlock.Permission = newPermission; - currBlock.Attribute &= ~MemoryAttribute.IpcAndDeviceMapped; - currBlock.Attribute |= newAttribute; + _blocks.AddBefore(node, currBlock.SplitRightAtAddress(baseAddress)); } - else if (currBaseAddr >= baseAddress) - { - currBlock.BaseAddress = endAddr; - currBlock.PagesCount = (currEndAddr - endAddr) / PageSize; - - ulong newPagesCount = (endAddr - currBaseAddr) / PageSize; - - newNode = _blocks.AddBefore(node, new KMemoryBlock( - currBaseAddr, - newPagesCount, - newState, - newPermission, - newAttribute)); - } - else if (currEndAddr <= endAddr) + if (endAddr < currEndAddr) { - currBlock.PagesCount = (baseAddress - currBaseAddr) / PageSize; - - ulong newPagesCount = (currEndAddr - baseAddress) / PageSize; - - newNode = _blocks.AddAfter(node, new KMemoryBlock( - baseAddress, - newPagesCount, - newState, - newPermission, - newAttribute)); + newNode = _blocks.AddBefore(node, currBlock.SplitRightAtAddress(endAddr)); } - else - { - currBlock.PagesCount = (baseAddress - currBaseAddr) / PageSize; - - ulong nextPagesCount = (currEndAddr - endAddr) / PageSize; - - newNode = _blocks.AddAfter(node, new KMemoryBlock( - baseAddress, - pagesCount, - newState, - newPermission, - newAttribute)); - - _blocks.AddAfter(newNode, new KMemoryBlock( - endAddr, - nextPagesCount, - currBlock.State, - currBlock.Permission, - currBlock.Attribute)); - nextNode = null; - } + newNode.Value.SetState(newPermission, newState, newAttribute); MergeEqualStateNeighbours(newNode); } + if (currEndAddr - 1 >= endAddr - 1) + { + break; + } + node = nextNode; } @@ -1859,13 +2474,9 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory { //Inserts new block at the list, replacing and spliting //existing blocks as needed. - KMemoryBlock block = new KMemoryBlock(baseAddress, pagesCount, state, permission, attribute); - int oldCount = _blocks.Count; - ulong endAddr = pagesCount * PageSize + baseAddress; - - LinkedListNode<KMemoryBlock> newNode = null; + ulong endAddr = baseAddress + pagesCount * PageSize; LinkedListNode<KMemoryBlock> node = _blocks.First; @@ -1880,69 +2491,99 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory if (baseAddress < currEndAddr && currBaseAddr < endAddr) { - if (baseAddress >= currBaseAddr && endAddr <= currEndAddr) + LinkedListNode<KMemoryBlock> newNode = node; + + if (baseAddress > currBaseAddr) { - block.Attribute |= currBlock.Attribute & MemoryAttribute.IpcAndDeviceMapped; + _blocks.AddBefore(node, currBlock.SplitRightAtAddress(baseAddress)); } - if (baseAddress > currBaseAddr && endAddr < currEndAddr) + if (endAddr < currEndAddr) { - currBlock.PagesCount = (baseAddress - currBaseAddr) / PageSize; + newNode = _blocks.AddBefore(node, currBlock.SplitRightAtAddress(endAddr)); + } - ulong nextPagesCount = (currEndAddr - endAddr) / PageSize; + newNode.Value.SetState(permission, state, attribute); - newNode = _blocks.AddAfter(node, block); + MergeEqualStateNeighbours(newNode); + } - _blocks.AddAfter(newNode, new KMemoryBlock( - endAddr, - nextPagesCount, - currBlock.State, - currBlock.Permission, - currBlock.Attribute)); + if (currEndAddr - 1 >= endAddr - 1) + { + break; + } - break; - } - else if (baseAddress <= currBaseAddr && endAddr < currEndAddr) - { - currBlock.BaseAddress = endAddr; + node = nextNode; + } - currBlock.PagesCount = (currEndAddr - endAddr) / PageSize; + _blockAllocator.Count += _blocks.Count - oldCount; + } - if (newNode == null) - { - newNode = _blocks.AddBefore(node, block); - } - } - else if (baseAddress > currBaseAddr && endAddr >= currEndAddr) - { - currBlock.PagesCount = (baseAddress - currBaseAddr) / PageSize; + private static void SetIpcMappingPermissions(KMemoryBlock block, MemoryPermission permission) + { + block.SetIpcMappingPermission(permission); + } - if (newNode == null) - { - newNode = _blocks.AddAfter(node, block); - } - } - else + private static void RestoreIpcMappingPermissions(KMemoryBlock block, MemoryPermission permission) + { + block.RestoreIpcMappingPermission(); + } + + private delegate void BlockMutator(KMemoryBlock block, MemoryPermission newPerm); + + private void InsertBlock( + ulong baseAddress, + ulong pagesCount, + BlockMutator blockMutate, + MemoryPermission permission = MemoryPermission.None) + { + //Inserts new block at the list, replacing and spliting + //existing blocks as needed, then calling the callback + //function on the new block. + int oldCount = _blocks.Count; + + ulong endAddr = baseAddress + pagesCount * PageSize; + + LinkedListNode<KMemoryBlock> node = _blocks.First; + + while (node != null) + { + KMemoryBlock currBlock = node.Value; + + LinkedListNode<KMemoryBlock> nextNode = node.Next; + + ulong currBaseAddr = currBlock.BaseAddress; + ulong currEndAddr = currBlock.PagesCount * PageSize + currBaseAddr; + + if (baseAddress < currEndAddr && currBaseAddr < endAddr) + { + LinkedListNode<KMemoryBlock> newNode = node; + + if (baseAddress > currBaseAddr) { - if (newNode == null) - { - newNode = _blocks.AddBefore(node, block); - } + _blocks.AddBefore(node, currBlock.SplitRightAtAddress(baseAddress)); + } - _blocks.Remove(node); + if (endAddr < currEndAddr) + { + newNode = _blocks.AddBefore(node, currBlock.SplitRightAtAddress(endAddr)); } + + KMemoryBlock newBlock = newNode.Value; + + blockMutate(newBlock, permission); + + MergeEqualStateNeighbours(newNode); } - node = nextNode; - } + if (currEndAddr - 1 >= endAddr - 1) + { + break; + } - if (newNode == null) - { - newNode = _blocks.AddFirst(block); + node = nextNode; } - MergeEqualStateNeighbours(newNode); - _blockAllocator.Count += _blocks.Count - oldCount; } @@ -1950,42 +2591,117 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory { KMemoryBlock block = node.Value; - ulong endAddr = block.PagesCount * PageSize + block.BaseAddress; - if (node.Previous != null) { - KMemoryBlock previous = node.Previous.Value; + KMemoryBlock previousBlock = node.Previous.Value; - if (BlockStateEquals(block, previous)) + if (BlockStateEquals(block, previousBlock)) { - _blocks.Remove(node.Previous); + LinkedListNode<KMemoryBlock> previousNode = node.Previous; - block.BaseAddress = previous.BaseAddress; + _blocks.Remove(node); + + previousBlock.AddPages(block.PagesCount); + + node = previousNode; + block = previousBlock; } } if (node.Next != null) { - KMemoryBlock next = node.Next.Value; + KMemoryBlock nextBlock = node.Next.Value; - if (BlockStateEquals(block, next)) + if (BlockStateEquals(block, nextBlock)) { _blocks.Remove(node.Next); - endAddr = next.BaseAddress + next.PagesCount * PageSize; + block.AddPages(nextBlock.PagesCount); } } - - block.PagesCount = (endAddr - block.BaseAddress) / PageSize; } private static bool BlockStateEquals(KMemoryBlock lhs, KMemoryBlock rhs) { - return lhs.State == rhs.State && - lhs.Permission == rhs.Permission && - lhs.Attribute == rhs.Attribute && - lhs.DeviceRefCount == rhs.DeviceRefCount && - lhs.IpcRefCount == rhs.IpcRefCount; + return lhs.State == rhs.State && + lhs.Permission == rhs.Permission && + lhs.Attribute == rhs.Attribute && + lhs.SourcePermission == rhs.SourcePermission && + lhs.DeviceRefCount == rhs.DeviceRefCount && + lhs.IpcRefCount == rhs.IpcRefCount; + } + + private ulong AllocateVa( + ulong regionStart, + ulong regionPagesCount, + ulong neededPagesCount, + int alignment) + { + ulong address = 0; + + ulong regionEndAddr = regionStart + regionPagesCount * PageSize; + + ulong reservedPagesCount = _isKernel ? 1UL : 4UL; + + if (_aslrEnabled) + { + ulong totalNeededSize = (reservedPagesCount + neededPagesCount) * PageSize; + + ulong remainingPages = regionPagesCount - neededPagesCount; + + ulong aslrMaxOffset = ((remainingPages + reservedPagesCount) * PageSize) / (ulong)alignment; + + for (int attempt = 0; attempt < 8; attempt++) + { + address = BitUtils.AlignDown(regionStart + GetRandomValue(0, aslrMaxOffset) * (ulong)alignment, alignment); + + ulong endAddr = address + totalNeededSize; + + KMemoryInfo info = FindBlock(address).GetInfo(); + + if (info.State != MemoryState.Unmapped) + { + continue; + } + + ulong currBaseAddr = info.Address + reservedPagesCount * PageSize; + ulong currEndAddr = info.Address + info.Size; + + if (address >= regionStart && + address >= currBaseAddr && + endAddr - 1 <= regionEndAddr - 1 && + endAddr - 1 <= currEndAddr - 1) + { + break; + } + } + + if (address == 0) + { + ulong aslrPage = GetRandomValue(0, aslrMaxOffset); + + address = FindFirstFit( + regionStart + aslrPage * PageSize, + regionPagesCount - aslrPage, + neededPagesCount, + alignment, + 0, + reservedPagesCount); + } + } + + if (address == 0) + { + address = FindFirstFit( + regionStart, + regionPagesCount, + neededPagesCount, + alignment, + 0, + reservedPagesCount); + } + + return address; } private ulong FindFirstFit( @@ -2397,11 +3113,21 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory return KernelResult.Success; } - public KernelResult ConvertVaToPa(ulong va, out ulong pa) + public ulong GetDramAddressFromVa(ulong va) + { + return (ulong)_cpuMemory.GetPhysicalAddress((long)va); + } + + public bool ConvertVaToPa(ulong va, out ulong pa) { pa = DramMemoryMap.DramBase + (ulong)_cpuMemory.GetPhysicalAddress((long)va); - return KernelResult.Success; + return true; + } + + public static ulong GetDramAddressFromPa(ulong pa) + { + return pa - DramMemoryMap.DramBase; } public long GetMmUsedPages() diff --git a/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryRegionManager.cs b/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryRegionManager.cs index 777e9aa9..92cef559 100644 --- a/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryRegionManager.cs +++ b/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryRegionManager.cs @@ -94,6 +94,14 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory } } + public ulong AllocatePagesContiguous(ulong pagesCount, bool backwards) + { + lock (_blocks) + { + return AllocatePagesContiguousImpl(pagesCount, backwards); + } + } + private KernelResult AllocatePagesImpl(ulong pagesCount, bool backwards, out KPageList pageList) { pageList = new KPageList(); @@ -122,165 +130,216 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory //If so, try allocating as much requested pages as possible. while (blockPagesCount <= pagesCount) { - ulong address = 0; + ulong address = AllocatePagesForOrder(blockIndex, backwards, bestFitBlockSize); - for (int currBlockIndex = blockIndex; - currBlockIndex < _blockOrdersCount && address == 0; - currBlockIndex++) + //The address being zero means that no free space was found on that order, + //just give up and try with the next one. + if (address == 0) { - block = _blocks[currBlockIndex]; + break; + } - int index = 0; + //Add new allocated page(s) to the pages list. + //If an error occurs, then free all allocated pages and fail. + KernelResult result = pageList.AddRange(address, blockPagesCount); - bool zeroMask = false; + if (result != KernelResult.Success) + { + FreePages(address, blockPagesCount); - for (int level = 0; level < block.MaxLevel; level++) + foreach (KPageNode pageNode in pageList) { - long mask = block.Masks[level][index]; - - if (mask == 0) - { - zeroMask = true; - - break; - } - - if (backwards) - { - index = (index * 64 + 63) - BitUtils.CountLeadingZeros64(mask); - } - else - { - index = index * 64 + BitUtils.CountLeadingZeros64(BitUtils.ReverseBits64(mask)); - } + FreePages(pageNode.Address, pageNode.PagesCount); } - if (block.SizeInBlocksTruncated <= (ulong)index || zeroMask) - { - continue; - } + return result; + } - block.FreeCount--; + pagesCount -= blockPagesCount; + } + } - int tempIdx = index; + //Success case, all requested pages were allocated successfully. + if (pagesCount == 0) + { + return KernelResult.Success; + } - for (int level = block.MaxLevel - 1; level >= 0; level--, tempIdx /= 64) - { - block.Masks[level][tempIdx / 64] &= ~(1L << (tempIdx & 63)); + //Error case, free allocated pages and return out of memory. + foreach (KPageNode pageNode in pageList) + { + FreePages(pageNode.Address, pageNode.PagesCount); + } - if (block.Masks[level][tempIdx / 64] != 0) - { - break; - } - } + pageList = null; - address = block.StartAligned + ((ulong)index << block.Order); - } + return KernelResult.OutOfMemory; + } - for (int currBlockIndex = blockIndex; - currBlockIndex < _blockOrdersCount && address == 0; - currBlockIndex++) - { - block = _blocks[currBlockIndex]; + private ulong AllocatePagesContiguousImpl(ulong pagesCount, bool backwards) + { + if (pagesCount == 0 || _blocks.Length < 1) + { + return 0; + } - int index = 0; + int blockIndex = 0; - bool zeroMask = false; + while ((1UL << _blocks[blockIndex].Order) / KMemoryManager.PageSize < pagesCount) + { + if (++blockIndex >= _blocks.Length) + { + return 0; + } + } - for (int level = 0; level < block.MaxLevel; level++) - { - long mask = block.Masks[level][index]; - - if (mask == 0) - { - zeroMask = true; - - break; - } - - if (backwards) - { - index = index * 64 + BitUtils.CountLeadingZeros64(BitUtils.ReverseBits64(mask)); - } - else - { - index = (index * 64 + 63) - BitUtils.CountLeadingZeros64(mask); - } - } + ulong tightestFitBlockSize = 1UL << _blocks[blockIndex].Order; - if (block.SizeInBlocksTruncated <= (ulong)index || zeroMask) - { - continue; - } + ulong address = AllocatePagesForOrder(blockIndex, backwards, tightestFitBlockSize); - block.FreeCount--; + ulong requiredSize = pagesCount * KMemoryManager.PageSize; - int tempIdx = index; + if (address != 0 && tightestFitBlockSize > requiredSize) + { + FreePages(address + requiredSize, (tightestFitBlockSize - requiredSize) / KMemoryManager.PageSize); + } - for (int level = block.MaxLevel - 1; level >= 0; level--, tempIdx /= 64) - { - block.Masks[level][tempIdx / 64] &= ~(1L << (tempIdx & 63)); + return address; + } - if (block.Masks[level][tempIdx / 64] != 0) - { - break; - } - } + private ulong AllocatePagesForOrder(int blockIndex, bool backwards, ulong bestFitBlockSize) + { + ulong address = 0; - address = block.StartAligned + ((ulong)index << block.Order); - } + KMemoryRegionBlock block = null; - //The address being zero means that no free space was found on that order, - //just give up and try with the next one. - if (address == 0) + for (int currBlockIndex = blockIndex; + currBlockIndex < _blockOrdersCount && address == 0; + currBlockIndex++) + { + block = _blocks[currBlockIndex]; + + int index = 0; + + bool zeroMask = false; + + for (int level = 0; level < block.MaxLevel; level++) + { + long mask = block.Masks[level][index]; + + if (mask == 0) { + zeroMask = true; + break; } - //If we are using a larger order than best fit, then we should - //split it into smaller blocks. - ulong firstFreeBlockSize = 1UL << block.Order; + if (backwards) + { + index = (index * 64 + 63) - BitUtils.CountLeadingZeros64(mask); + } + else + { + index = index * 64 + BitUtils.CountLeadingZeros64(BitUtils.ReverseBits64(mask)); + } + } - if (firstFreeBlockSize > bestFitBlockSize) + if (block.SizeInBlocksTruncated <= (ulong)index || zeroMask) + { + continue; + } + + block.FreeCount--; + + int tempIdx = index; + + for (int level = block.MaxLevel - 1; level >= 0; level--, tempIdx /= 64) + { + block.Masks[level][tempIdx / 64] &= ~(1L << (tempIdx & 63)); + + if (block.Masks[level][tempIdx / 64] != 0) { - FreePages(address + bestFitBlockSize, (firstFreeBlockSize - bestFitBlockSize) / KMemoryManager.PageSize); + break; } + } - //Add new allocated page(s) to the pages list. - //If an error occurs, then free all allocated pages and fail. - KernelResult result = pageList.AddRange(address, blockPagesCount); + address = block.StartAligned + ((ulong)index << block.Order); + } - if (result != KernelResult.Success) + for (int currBlockIndex = blockIndex; + currBlockIndex < _blockOrdersCount && address == 0; + currBlockIndex++) + { + block = _blocks[currBlockIndex]; + + int index = 0; + + bool zeroMask = false; + + for (int level = 0; level < block.MaxLevel; level++) + { + long mask = block.Masks[level][index]; + + if (mask == 0) { - FreePages(address, blockPagesCount); + zeroMask = true; - foreach (KPageNode pageNode in pageList) - { - FreePages(pageNode.Address, pageNode.PagesCount); - } + break; + } - return result; + if (backwards) + { + index = index * 64 + BitUtils.CountLeadingZeros64(BitUtils.ReverseBits64(mask)); } + else + { + index = (index * 64 + 63) - BitUtils.CountLeadingZeros64(mask); + } + } - pagesCount -= blockPagesCount; + if (block.SizeInBlocksTruncated <= (ulong)index || zeroMask) + { + continue; } - } - //Success case, all requested pages were allocated successfully. - if (pagesCount == 0) - { - return KernelResult.Success; + block.FreeCount--; + + int tempIdx = index; + + for (int level = block.MaxLevel - 1; level >= 0; level--, tempIdx /= 64) + { + block.Masks[level][tempIdx / 64] &= ~(1L << (tempIdx & 63)); + + if (block.Masks[level][tempIdx / 64] != 0) + { + break; + } + } + + address = block.StartAligned + ((ulong)index << block.Order); } - //Error case, free allocated pages and return out of memory. - foreach (KPageNode pageNode in pageList) + if (address != 0) { - FreePages(pageNode.Address, pageNode.PagesCount); + //If we are using a larger order than best fit, then we should + //split it into smaller blocks. + ulong firstFreeBlockSize = 1UL << block.Order; + + if (firstFreeBlockSize > bestFitBlockSize) + { + FreePages(address + bestFitBlockSize, (firstFreeBlockSize - bestFitBlockSize) / KMemoryManager.PageSize); + } } - pageList = null; + return address; + } - return KernelResult.OutOfMemory; + public void FreePage(ulong address) + { + lock (_blocks) + { + FreePages(address, 1); + } } public void FreePages(KPageList pageList) diff --git a/Ryujinx.HLE/HOS/Kernel/Memory/KSharedMemory.cs b/Ryujinx.HLE/HOS/Kernel/Memory/KSharedMemory.cs index f2a05bda..6b92ed30 100644 --- a/Ryujinx.HLE/HOS/Kernel/Memory/KSharedMemory.cs +++ b/Ryujinx.HLE/HOS/Kernel/Memory/KSharedMemory.cs @@ -4,7 +4,7 @@ using Ryujinx.HLE.HOS.Kernel.Process; namespace Ryujinx.HLE.HOS.Kernel.Memory { - class KSharedMemory + class KSharedMemory : KAutoObject { private KPageList _pageList; @@ -14,10 +14,11 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory private MemoryPermission _userPermission; public KSharedMemory( + Horizon system, KPageList pageList, long ownerPid, MemoryPermission ownerPermission, - MemoryPermission userPermission) + MemoryPermission userPermission) : base(system) { _pageList = pageList; _ownerPid = ownerPid; diff --git a/Ryujinx.HLE/HOS/Kernel/Memory/KTransferMemory.cs b/Ryujinx.HLE/HOS/Kernel/Memory/KTransferMemory.cs index 02367e89..a0929eca 100644 --- a/Ryujinx.HLE/HOS/Kernel/Memory/KTransferMemory.cs +++ b/Ryujinx.HLE/HOS/Kernel/Memory/KTransferMemory.cs @@ -1,11 +1,13 @@ +using Ryujinx.HLE.HOS.Kernel.Common; + namespace Ryujinx.HLE.HOS.Kernel.Memory { - class KTransferMemory + class KTransferMemory : KAutoObject { public ulong Address { get; private set; } public ulong Size { get; private set; } - public KTransferMemory(ulong address, ulong size) + public KTransferMemory(Horizon system, ulong address, ulong size) : base(system) { Address = address; Size = size; |
