diff options
Diffstat (limited to 'Ryujinx.HLE/HOS/Kernel/Memory/KMemoryManager.cs')
| -rw-r--r-- | Ryujinx.HLE/HOS/Kernel/Memory/KMemoryManager.cs | 3322 |
1 files changed, 26 insertions, 3296 deletions
diff --git a/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryManager.cs b/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryManager.cs index 5b6df53b..6d0a1658 100644 --- a/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryManager.cs +++ b/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryManager.cs @@ -1,3335 +1,65 @@ -using Ryujinx.Common; -using Ryujinx.HLE.HOS.Kernel.Common; -using Ryujinx.HLE.HOS.Kernel.Process; -using Ryujinx.Memory; +using Ryujinx.HLE.HOS.Kernel.Common; using System; -using System.Collections.Generic; -using System.Diagnostics; namespace Ryujinx.HLE.HOS.Kernel.Memory { class KMemoryManager { - private static readonly int[] MappingUnitSizes = new int[] - { - 0x1000, - 0x10000, - 0x200000, - 0x400000, - 0x2000000, - 0x40000000 - }; - - public const int PageSize = 0x1000; - - private const int KMemoryBlockSize = 0x40; - - // We need 2 blocks for the case where a big block - // needs to be split in 2, plus one block that will be the new one inserted. - private const int MaxBlocksNeededForInsertion = 2; - - private readonly LinkedList<KMemoryBlock> _blocks; - - private readonly IVirtualMemoryManager _cpuMemory; - - private readonly KernelContext _context; - - public ulong AddrSpaceStart { get; private set; } - public ulong AddrSpaceEnd { get; private set; } - - public ulong CodeRegionStart { get; private set; } - public ulong CodeRegionEnd { get; private set; } - - public ulong HeapRegionStart { get; private set; } - public ulong HeapRegionEnd { get; private set; } - - private ulong _currentHeapAddr; - - public ulong AliasRegionStart { get; private set; } - public ulong AliasRegionEnd { get; private set; } - - public ulong StackRegionStart { get; private set; } - public ulong StackRegionEnd { get; private set; } - - public ulong TlsIoRegionStart { get; private set; } - public ulong TlsIoRegionEnd { get; private set; } - - private ulong _heapCapacity; - - public ulong PhysicalMemoryUsage { get; private set; } - - private MemoryRegion _memRegion; - - private bool _aslrDisabled; - - public int AddrSpaceWidth { get; private set; } - - private bool _isKernel; - - private bool _aslrEnabled; - - private KMemoryBlockAllocator _blockAllocator; - - private int _contextId; - - private MersenneTwister _randomNumberGenerator; - - public KMemoryManager(KernelContext context, IVirtualMemoryManager cpuMemory) - { - _context = context; - _cpuMemory = cpuMemory; - - _blocks = new LinkedList<KMemoryBlock>(); - - _isKernel = false; - } - - private static readonly int[] AddrSpaceSizes = new int[] { 32, 36, 32, 39 }; - - public KernelResult InitializeForProcess( - AddressSpaceType addrSpaceType, - bool aslrEnabled, - bool aslrDisabled, - MemoryRegion memRegion, - ulong address, - ulong size, - KMemoryBlockAllocator blockAllocator) - { - if ((uint)addrSpaceType > (uint)AddressSpaceType.Addr39Bits) - { - throw new ArgumentException(nameof(addrSpaceType)); - } - - _contextId = _context.ContextIdManager.GetId(); - - ulong addrSpaceBase = 0; - ulong addrSpaceSize = 1UL << AddrSpaceSizes[(int)addrSpaceType]; - - KernelResult result = CreateUserAddressSpace( - addrSpaceType, - aslrEnabled, - aslrDisabled, - addrSpaceBase, - addrSpaceSize, - memRegion, - address, - size, - blockAllocator); - - if (result != KernelResult.Success) - { - _context.ContextIdManager.PutId(_contextId); - } - - return result; - } - - private class Region - { - public ulong Start; - public ulong End; - public ulong Size; - public ulong AslrOffset; - } - - private KernelResult CreateUserAddressSpace( - AddressSpaceType addrSpaceType, - bool aslrEnabled, - bool aslrDisabled, - ulong addrSpaceStart, - ulong addrSpaceEnd, - MemoryRegion memRegion, - ulong address, - ulong size, - KMemoryBlockAllocator blockAllocator) - { - ulong endAddr = address + size; - - Region aliasRegion = new Region(); - Region heapRegion = new Region(); - Region stackRegion = new Region(); - Region tlsIoRegion = new Region(); - - ulong codeRegionSize; - ulong stackAndTlsIoStart; - ulong stackAndTlsIoEnd; - ulong baseAddress; - - switch (addrSpaceType) - { - case AddressSpaceType.Addr32Bits: - aliasRegion.Size = 0x40000000; - heapRegion.Size = 0x40000000; - stackRegion.Size = 0; - tlsIoRegion.Size = 0; - CodeRegionStart = 0x200000; - codeRegionSize = 0x3fe00000; - stackAndTlsIoStart = 0x200000; - stackAndTlsIoEnd = 0x40000000; - baseAddress = 0x200000; - AddrSpaceWidth = 32; - break; - - case AddressSpaceType.Addr36Bits: - aliasRegion.Size = 0x180000000; - heapRegion.Size = 0x180000000; - stackRegion.Size = 0; - tlsIoRegion.Size = 0; - CodeRegionStart = 0x8000000; - codeRegionSize = 0x78000000; - stackAndTlsIoStart = 0x8000000; - stackAndTlsIoEnd = 0x80000000; - baseAddress = 0x8000000; - AddrSpaceWidth = 36; - break; - - case AddressSpaceType.Addr32BitsNoMap: - aliasRegion.Size = 0; - heapRegion.Size = 0x80000000; - stackRegion.Size = 0; - tlsIoRegion.Size = 0; - CodeRegionStart = 0x200000; - codeRegionSize = 0x3fe00000; - stackAndTlsIoStart = 0x200000; - stackAndTlsIoEnd = 0x40000000; - baseAddress = 0x200000; - AddrSpaceWidth = 32; - break; - - case AddressSpaceType.Addr39Bits: - aliasRegion.Size = 0x1000000000; - heapRegion.Size = 0x180000000; - stackRegion.Size = 0x80000000; - tlsIoRegion.Size = 0x1000000000; - CodeRegionStart = BitUtils.AlignDown(address, 0x200000); - codeRegionSize = BitUtils.AlignUp (endAddr, 0x200000) - CodeRegionStart; - stackAndTlsIoStart = 0; - stackAndTlsIoEnd = 0; - baseAddress = 0x8000000; - AddrSpaceWidth = 39; - break; - - default: throw new ArgumentException(nameof(addrSpaceType)); - } - - CodeRegionEnd = CodeRegionStart + codeRegionSize; - - ulong mapBaseAddress; - ulong mapAvailableSize; - - if (CodeRegionStart - baseAddress >= addrSpaceEnd - CodeRegionEnd) - { - // Has more space before the start of the code region. - mapBaseAddress = baseAddress; - mapAvailableSize = CodeRegionStart - baseAddress; - } - else - { - // Has more space after the end of the code region. - mapBaseAddress = CodeRegionEnd; - mapAvailableSize = addrSpaceEnd - CodeRegionEnd; - } - - ulong mapTotalSize = aliasRegion.Size + heapRegion.Size + stackRegion.Size + tlsIoRegion.Size; - - ulong aslrMaxOffset = mapAvailableSize - mapTotalSize; - - _aslrEnabled = aslrEnabled; - - AddrSpaceStart = addrSpaceStart; - AddrSpaceEnd = addrSpaceEnd; - - _blockAllocator = blockAllocator; - - if (mapAvailableSize < mapTotalSize) - { - return KernelResult.OutOfMemory; - } - - if (aslrEnabled) - { - aliasRegion.AslrOffset = GetRandomValue(0, aslrMaxOffset >> 21) << 21; - heapRegion.AslrOffset = GetRandomValue(0, aslrMaxOffset >> 21) << 21; - stackRegion.AslrOffset = GetRandomValue(0, aslrMaxOffset >> 21) << 21; - tlsIoRegion.AslrOffset = GetRandomValue(0, aslrMaxOffset >> 21) << 21; - } - - // Regions are sorted based on ASLR offset. - // When ASLR is disabled, the order is Map, Heap, NewMap and TlsIo. - aliasRegion.Start = mapBaseAddress + aliasRegion.AslrOffset; - aliasRegion.End = aliasRegion.Start + aliasRegion.Size; - heapRegion.Start = mapBaseAddress + heapRegion.AslrOffset; - heapRegion.End = heapRegion.Start + heapRegion.Size; - stackRegion.Start = mapBaseAddress + stackRegion.AslrOffset; - stackRegion.End = stackRegion.Start + stackRegion.Size; - tlsIoRegion.Start = mapBaseAddress + tlsIoRegion.AslrOffset; - tlsIoRegion.End = tlsIoRegion.Start + tlsIoRegion.Size; - - SortRegion(heapRegion, aliasRegion); - - if (stackRegion.Size != 0) - { - SortRegion(stackRegion, aliasRegion); - SortRegion(stackRegion, heapRegion); - } - else - { - stackRegion.Start = stackAndTlsIoStart; - stackRegion.End = stackAndTlsIoEnd; - } - - if (tlsIoRegion.Size != 0) - { - SortRegion(tlsIoRegion, aliasRegion); - SortRegion(tlsIoRegion, heapRegion); - SortRegion(tlsIoRegion, stackRegion); - } - else - { - tlsIoRegion.Start = stackAndTlsIoStart; - tlsIoRegion.End = stackAndTlsIoEnd; - } - - AliasRegionStart = aliasRegion.Start; - AliasRegionEnd = aliasRegion.End; - HeapRegionStart = heapRegion.Start; - HeapRegionEnd = heapRegion.End; - StackRegionStart = stackRegion.Start; - StackRegionEnd = stackRegion.End; - TlsIoRegionStart = tlsIoRegion.Start; - TlsIoRegionEnd = tlsIoRegion.End; - - _currentHeapAddr = HeapRegionStart; - _heapCapacity = 0; - PhysicalMemoryUsage = 0; - - _memRegion = memRegion; - _aslrDisabled = aslrDisabled; - - return InitializeBlocks(addrSpaceStart, addrSpaceEnd); - } - - private ulong GetRandomValue(ulong min, ulong max) - { - return (ulong)GetRandomValue((long)min, (long)max); - } - - private long GetRandomValue(long min, long max) - { - if (_randomNumberGenerator == null) - { - _randomNumberGenerator = new MersenneTwister(0); - } - - return _randomNumberGenerator.GenRandomNumber(min, max); - } - - private static void SortRegion(Region lhs, Region rhs) - { - if (lhs.AslrOffset < rhs.AslrOffset) - { - rhs.Start += lhs.Size; - rhs.End += lhs.Size; - } - else - { - lhs.Start += rhs.Size; - lhs.End += rhs.Size; - } - } - - private KernelResult InitializeBlocks(ulong addrSpaceStart, ulong addrSpaceEnd) - { - // First insertion will always need only a single block, - // because there's nothing else to split. - if (!_blockAllocator.CanAllocate(1)) - { - return KernelResult.OutOfResource; - } - - ulong addrSpacePagesCount = (addrSpaceEnd - addrSpaceStart) / PageSize; - - _blocks.AddFirst(new KMemoryBlock( - addrSpaceStart, - addrSpacePagesCount, - MemoryState.Unmapped, - KMemoryPermission.None, - MemoryAttribute.None)); - - return KernelResult.Success; - } - - public KernelResult MapPages( - ulong address, - KPageList pageList, - MemoryState state, - KMemoryPermission permission) - { - ulong pagesCount = pageList.GetPagesCount(); - - ulong size = pagesCount * PageSize; - - if (!CanContain(address, size, state)) - { - return KernelResult.InvalidMemState; - } - - lock (_blocks) - { - if (!IsUnmapped(address, pagesCount * PageSize)) - { - return KernelResult.InvalidMemState; - } - - if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) - { - return KernelResult.OutOfResource; - } - - KernelResult result = MapPages(address, pageList, permission); - - if (result == KernelResult.Success) - { - InsertBlock(address, pagesCount, state, permission); - } - - return result; - } - } - - public KernelResult UnmapPages(ulong address, KPageList pageList, MemoryState stateExpected) - { - ulong pagesCount = pageList.GetPagesCount(); - - ulong size = pagesCount * PageSize; - - ulong endAddr = address + size; - - ulong addrSpacePagesCount = (AddrSpaceEnd - AddrSpaceStart) / PageSize; - - if (AddrSpaceStart > address) - { - return KernelResult.InvalidMemState; - } - - if (addrSpacePagesCount < pagesCount) - { - return KernelResult.InvalidMemState; - } - - if (endAddr - 1 > AddrSpaceEnd - 1) - { - return KernelResult.InvalidMemState; - } - - lock (_blocks) - { - KPageList currentPageList = new KPageList(); - - AddVaRangeToPageList(currentPageList, address, pagesCount); - - if (!currentPageList.IsEqual(pageList)) - { - return KernelResult.InvalidMemRange; - } - - if (CheckRange( - address, - size, - MemoryState.Mask, - stateExpected, - KMemoryPermission.None, - KMemoryPermission.None, - MemoryAttribute.Mask, - MemoryAttribute.None, - MemoryAttribute.IpcAndDeviceMapped, - out MemoryState state, - out _, - out _)) - { - if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) - { - return KernelResult.OutOfResource; - } - - KernelResult result = MmuUnmap(address, pagesCount); - - if (result == KernelResult.Success) - { - InsertBlock(address, pagesCount, MemoryState.Unmapped); - } - - return result; - } - else - { - return KernelResult.InvalidMemState; - } - } - } - - public KernelResult MapNormalMemory(long address, long size, KMemoryPermission permission) - { - // TODO. - return KernelResult.Success; - } - - public KernelResult MapIoMemory(long address, long size, KMemoryPermission permission) - { - // TODO. - return KernelResult.Success; - } - - public KernelResult AllocateOrMapPa( - ulong neededPagesCount, - int alignment, - ulong srcPa, - bool map, - ulong regionStart, - ulong regionPagesCount, - MemoryState state, - KMemoryPermission permission, - out ulong address) - { - address = 0; - - ulong regionSize = regionPagesCount * PageSize; - - ulong regionEndAddr = regionStart + regionSize; - - if (!CanContain(regionStart, regionSize, state)) - { - return KernelResult.InvalidMemState; - } - - if (regionPagesCount <= neededPagesCount) - { - return KernelResult.OutOfMemory; - } - - lock (_blocks) - { - address = AllocateVa(regionStart, regionPagesCount, neededPagesCount, alignment); - - if (address == 0) - { - return KernelResult.OutOfMemory; - } - - if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) - { - return KernelResult.OutOfResource; - } - - MemoryOperation operation = map - ? MemoryOperation.MapPa - : MemoryOperation.Allocate; - - KernelResult result = DoMmuOperation( - address, - neededPagesCount, - srcPa, - map, - permission, - operation); - - if (result != KernelResult.Success) - { - return result; - } - - InsertBlock(address, neededPagesCount, state, permission); - } - - return KernelResult.Success; - } - - public KernelResult MapNewProcessCode( - ulong address, - ulong pagesCount, - MemoryState state, - KMemoryPermission permission) - { - ulong size = pagesCount * PageSize; - - if (!CanContain(address, size, state)) - { - return KernelResult.InvalidMemState; - } - - lock (_blocks) - { - if (!IsUnmapped(address, size)) - { - return KernelResult.InvalidMemState; - } - - if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) - { - return KernelResult.OutOfResource; - } - - KernelResult result = DoMmuOperation( - address, - pagesCount, - 0, - false, - permission, - MemoryOperation.Allocate); - - if (result == KernelResult.Success) - { - InsertBlock(address, pagesCount, state, permission); - } - - return result; - } - } - - public KernelResult MapProcessCodeMemory(ulong dst, ulong src, ulong size) - { - ulong pagesCount = size / PageSize; - - lock (_blocks) - { - bool success = CheckRange( - src, - size, - MemoryState.Mask, - MemoryState.Heap, - KMemoryPermission.Mask, - KMemoryPermission.ReadAndWrite, - MemoryAttribute.Mask, - MemoryAttribute.None, - MemoryAttribute.IpcAndDeviceMapped, - out MemoryState state, - out KMemoryPermission permission, - out _); - - success &= IsUnmapped(dst, size); - - if (success) - { - if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion * 2)) - { - return KernelResult.OutOfResource; - } - - KPageList pageList = new KPageList(); - - AddVaRangeToPageList(pageList, src, pagesCount); - - KernelResult result = MmuChangePermission(src, pagesCount, KMemoryPermission.None); - - if (result != KernelResult.Success) - { - return result; - } - - result = MapPages(dst, pageList, KMemoryPermission.None); - - if (result != KernelResult.Success) - { - MmuChangePermission(src, pagesCount, permission); - - return result; - } - - InsertBlock(src, pagesCount, state, KMemoryPermission.None, MemoryAttribute.Borrowed); - InsertBlock(dst, pagesCount, MemoryState.ModCodeStatic); - - return KernelResult.Success; - } - else - { - return KernelResult.InvalidMemState; - } - } - } - - public KernelResult UnmapProcessCodeMemory(ulong dst, ulong src, ulong size) - { - ulong pagesCount = size / PageSize; - - lock (_blocks) - { - bool success = CheckRange( - src, - size, - MemoryState.Mask, - MemoryState.Heap, - KMemoryPermission.None, - KMemoryPermission.None, - MemoryAttribute.Mask, - MemoryAttribute.Borrowed, - MemoryAttribute.IpcAndDeviceMapped, - out _, - out _, - out _); - - success &= CheckRange( - dst, - PageSize, - MemoryState.UnmapProcessCodeMemoryAllowed, - MemoryState.UnmapProcessCodeMemoryAllowed, - KMemoryPermission.None, - KMemoryPermission.None, - MemoryAttribute.Mask, - MemoryAttribute.None, - MemoryAttribute.IpcAndDeviceMapped, - out MemoryState state, - out _, - out _); - - success &= CheckRange( - dst, - size, - MemoryState.Mask, - state, - KMemoryPermission.None, - KMemoryPermission.None, - MemoryAttribute.Mask, - MemoryAttribute.None); - - if (success) - { - KernelResult result = MmuUnmap(dst, pagesCount); - - if (result != KernelResult.Success) - { - return result; - } - - // TODO: Missing some checks here. - - if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion * 2)) - { - return KernelResult.OutOfResource; - } - - InsertBlock(dst, pagesCount, MemoryState.Unmapped); - InsertBlock(src, pagesCount, MemoryState.Heap, KMemoryPermission.ReadAndWrite); - - return KernelResult.Success; - } - else - { - return KernelResult.InvalidMemState; - } - } - } - - public KernelResult SetHeapSize(ulong size, out ulong address) - { - address = 0; - - if (size > HeapRegionEnd - HeapRegionStart) - { - return KernelResult.OutOfMemory; - } - - KProcess currentProcess = KernelStatic.GetCurrentProcess(); - - lock (_blocks) - { - ulong currentHeapSize = GetHeapSize(); - - if (currentHeapSize <= size) - { - // Expand. - ulong sizeDelta = size - currentHeapSize; - - if (currentProcess.ResourceLimit != null && sizeDelta != 0 && - !currentProcess.ResourceLimit.Reserve(LimitableResource.Memory, sizeDelta)) - { - return KernelResult.ResLimitExceeded; - } - - ulong pagesCount = sizeDelta / PageSize; - - KMemoryRegionManager region = GetMemoryRegionManager(); - - KernelResult result = region.AllocatePages(pagesCount, _aslrDisabled, out KPageList pageList); - - void CleanUpForError() - { - if (pageList != null) - { - region.FreePages(pageList); - } - - if (currentProcess.ResourceLimit != null && sizeDelta != 0) - { - currentProcess.ResourceLimit.Release(LimitableResource.Memory, sizeDelta); - } - } - - if (result != KernelResult.Success) - { - CleanUpForError(); - - return result; - } - - if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) - { - CleanUpForError(); - - return KernelResult.OutOfResource; - } - - if (!IsUnmapped(_currentHeapAddr, sizeDelta)) - { - CleanUpForError(); - - return KernelResult.InvalidMemState; - } - - result = DoMmuOperation( - _currentHeapAddr, - pagesCount, - pageList, - KMemoryPermission.ReadAndWrite, - MemoryOperation.MapVa); - - if (result != KernelResult.Success) - { - CleanUpForError(); - - return result; - } - - InsertBlock(_currentHeapAddr, pagesCount, MemoryState.Heap, KMemoryPermission.ReadAndWrite); - } - else - { - // Shrink. - ulong freeAddr = HeapRegionStart + size; - ulong sizeDelta = currentHeapSize - size; - - if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) - { - return KernelResult.OutOfResource; - } - - if (!CheckRange( - freeAddr, - sizeDelta, - MemoryState.Mask, - MemoryState.Heap, - KMemoryPermission.Mask, - KMemoryPermission.ReadAndWrite, - MemoryAttribute.Mask, - MemoryAttribute.None, - MemoryAttribute.IpcAndDeviceMapped, - out _, - out _, - out _)) - { - return KernelResult.InvalidMemState; - } - - ulong pagesCount = sizeDelta / PageSize; - - KernelResult result = MmuUnmap(freeAddr, pagesCount); - - if (result != KernelResult.Success) - { - return result; - } - - currentProcess.ResourceLimit?.Release(LimitableResource.Memory, sizeDelta); - - InsertBlock(freeAddr, pagesCount, MemoryState.Unmapped); - } - - _currentHeapAddr = HeapRegionStart + size; - } - - address = HeapRegionStart; - - return KernelResult.Success; - } - - public ulong GetTotalHeapSize() - { - lock (_blocks) - { - return GetHeapSize() + PhysicalMemoryUsage; - } - } - - private ulong GetHeapSize() - { - return _currentHeapAddr - HeapRegionStart; - } - - public KernelResult SetHeapCapacity(ulong capacity) - { - lock (_blocks) - { - _heapCapacity = capacity; - } - - return KernelResult.Success; - } - - public KernelResult SetMemoryAttribute( - ulong address, - ulong size, - MemoryAttribute attributeMask, - MemoryAttribute attributeValue) - { - lock (_blocks) - { - if (CheckRange( - address, - size, - MemoryState.AttributeChangeAllowed, - MemoryState.AttributeChangeAllowed, - KMemoryPermission.None, - KMemoryPermission.None, - MemoryAttribute.BorrowedAndIpcMapped, - MemoryAttribute.None, - MemoryAttribute.DeviceMappedAndUncached, - out MemoryState state, - out KMemoryPermission permission, - out MemoryAttribute attribute)) - { - if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) - { - return KernelResult.OutOfResource; - } - - ulong pagesCount = size / PageSize; - - attribute &= ~attributeMask; - attribute |= attributeMask & attributeValue; - - InsertBlock(address, pagesCount, state, permission, attribute); - - return KernelResult.Success; - } - else - { - return KernelResult.InvalidMemState; - } - } - } - - public KMemoryInfo QueryMemory(ulong address) - { - if (address >= AddrSpaceStart && - address < AddrSpaceEnd) - { - lock (_blocks) - { - return FindBlock(address).GetInfo(); - } - } - else - { - return new KMemoryInfo( - AddrSpaceEnd, - ~AddrSpaceEnd + 1, - MemoryState.Reserved, - KMemoryPermission.None, - MemoryAttribute.None, - KMemoryPermission.None, - 0, - 0); - } - } - - public KernelResult Map(ulong dst, ulong src, ulong size) - { - bool success; - - lock (_blocks) - { - success = CheckRange( - src, - size, - MemoryState.MapAllowed, - MemoryState.MapAllowed, - KMemoryPermission.Mask, - KMemoryPermission.ReadAndWrite, - MemoryAttribute.Mask, - MemoryAttribute.None, - MemoryAttribute.IpcAndDeviceMapped, - out MemoryState srcState, - out _, - out _); - - success &= IsUnmapped(dst, size); - - if (success) - { - if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion * 2)) - { - return KernelResult.OutOfResource; - } - - ulong pagesCount = size / PageSize; - - KPageList pageList = new KPageList(); - - AddVaRangeToPageList(pageList, src, pagesCount); - - KernelResult result = MmuChangePermission(src, pagesCount, KMemoryPermission.None); - - if (result != KernelResult.Success) - { - return result; - } - - result = MapPages(dst, pageList, KMemoryPermission.ReadAndWrite); - - if (result != KernelResult.Success) - { - if (MmuChangePermission(src, pagesCount, KMemoryPermission.ReadAndWrite) != KernelResult.Success) - { - throw new InvalidOperationException("Unexpected failure reverting memory permission."); - } - - return result; - } - - InsertBlock(src, pagesCount, srcState, KMemoryPermission.None, MemoryAttribute.Borrowed); - InsertBlock(dst, pagesCount, MemoryState.Stack, KMemoryPermission.ReadAndWrite); - - return KernelResult.Success; - } - else - { - return KernelResult.InvalidMemState; - } - } - } - - public KernelResult UnmapForKernel(ulong address, ulong pagesCount, MemoryState stateExpected) - { - ulong size = pagesCount * PageSize; - - lock (_blocks) - { - if (CheckRange( - address, - size, - MemoryState.Mask, - stateExpected, - KMemoryPermission.None, - KMemoryPermission.None, - MemoryAttribute.Mask, - MemoryAttribute.None, - MemoryAttribute.IpcAndDeviceMapped, - out _, - out _, - out _)) - { - if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) - { - return KernelResult.OutOfResource; - } - - KernelResult result = MmuUnmap(address, pagesCount); - - if (result == KernelResult.Success) - { - InsertBlock(address, pagesCount, MemoryState.Unmapped); - } - - return KernelResult.Success; - } - else - { - return KernelResult.InvalidMemState; - } - } - } - - public KernelResult Unmap(ulong dst, ulong src, ulong size) - { - bool success; - - lock (_blocks) - { - success = CheckRange( - src, - size, - MemoryState.MapAllowed, - MemoryState.MapAllowed, - KMemoryPermission.Mask, - KMemoryPermission.None, - MemoryAttribute.Mask, - MemoryAttribute.Borrowed, - MemoryAttribute.IpcAndDeviceMapped, - out MemoryState srcState, - out _, - out _); - - success &= CheckRange( - dst, - size, - MemoryState.Mask, - MemoryState.Stack, - KMemoryPermission.None, - KMemoryPermission.None, - MemoryAttribute.Mask, - MemoryAttribute.None, - MemoryAttribute.IpcAndDeviceMapped, - out _, - out KMemoryPermission dstPermission, - out _); - - if (success) - { - if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion * 2)) - { - return KernelResult.OutOfResource; - } - - ulong pagesCount = size / PageSize; - - KPageList srcPageList = new KPageList(); - KPageList dstPageList = new KPageList(); - - AddVaRangeToPageList(srcPageList, src, pagesCount); - AddVaRangeToPageList(dstPageList, dst, pagesCount); - - if (!dstPageList.IsEqual(srcPageList)) - { - return KernelResult.InvalidMemRange; - } - - KernelResult result = MmuUnmap(dst, pagesCount); - - if (result != KernelResult.Success) - { - return result; - } - - result = MmuChangePermission(src, pagesCount, KMemoryPermission.ReadAndWrite); - - if (result != KernelResult.Success) - { - MapPages(dst, dstPageList, dstPermission); - - return result; - } - - InsertBlock(src, pagesCount, srcState, KMemoryPermission.ReadAndWrite); - InsertBlock(dst, pagesCount, MemoryState.Unmapped); - - return KernelResult.Success; - } - else - { - return KernelResult.InvalidMemState; - } - } - } - - public KernelResult SetProcessMemoryPermission(ulong address, ulong size, KMemoryPermission permission) - { - lock (_blocks) - { - if (CheckRange( - address, - size, - MemoryState.ProcessPermissionChangeAllowed, - MemoryState.ProcessPermissionChangeAllowed, - KMemoryPermission.None, - KMemoryPermission.None, - MemoryAttribute.Mask, - MemoryAttribute.None, - MemoryAttribute.IpcAndDeviceMapped, - out MemoryState oldState, - out KMemoryPermission oldPermission, - out _)) - { - MemoryState newState = oldState; - - // If writing into the code region is allowed, then we need - // to change it to mutable. - if ((permission & KMemoryPermission.Write) != 0) - { - if (oldState == MemoryState.CodeStatic) - { - newState = MemoryState.CodeMutable; - } - else if (oldState == MemoryState.ModCodeStatic) - { - newState = MemoryState.ModCodeMutable; - } - else - { - throw new InvalidOperationException($"Memory state \"{oldState}\" not valid for this operation."); - } - } - - if (newState != oldState || permission != oldPermission) - { - if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) - { - return KernelResult.OutOfResource; - } - - ulong pagesCount = size / PageSize; - - MemoryOperation operation = (permission & KMemoryPermission.Execute) != 0 - ? MemoryOperation.ChangePermsAndAttributes - : MemoryOperation.ChangePermRw; - - KernelResult result = DoMmuOperation(address, pagesCount, 0, false, permission, operation); - - if (result != KernelResult.Success) - { - return result; - } - - InsertBlock(address, pagesCount, newState, permission); - } - - return KernelResult.Success; - } - else - { - return KernelResult.InvalidMemState; - } - } - } - - public KernelResult MapPhysicalMemory(ulong address, ulong size) - { - ulong endAddr = address + size; - - lock (_blocks) - { - ulong mappedSize = 0; - - foreach (KMemoryInfo info in IterateOverRange(address, endAddr)) - { - if (info.State != MemoryState.Unmapped) - { - mappedSize += GetSizeInRange(info, address, endAddr); - } - } - - if (mappedSize == size) - { - return KernelResult.Success; - } - - ulong remainingSize = size - mappedSize; - - ulong remainingPages = remainingSize / PageSize; - - KProcess currentProcess = KernelStatic.GetCurrentProcess(); - - if (currentProcess.ResourceLimit != null && - !currentProcess.ResourceLimit.Reserve(LimitableResource.Memory, remainingSize)) - { - return KernelResult.ResLimitExceeded; - } - - KMemoryRegionManager region = GetMemoryRegionManager(); - - KernelResult result = region.AllocatePages(remainingPages, _aslrDisabled, out KPageList pageList); - - void CleanUpForError() - { - if (pageList != null) - { - region.FreePages(pageList); - } - - currentProcess.ResourceLimit?.Release(LimitableResource.Memory, remainingSize); - } - - if (result != KernelResult.Success) - { - CleanUpForError(); - - return result; - } - - if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) - { - CleanUpForError(); - - return KernelResult.OutOfResource; - } - - MapPhysicalMemory(pageList, address, endAddr); - - PhysicalMemoryUsage += remainingSize; - - ulong pagesCount = size / PageSize; - - InsertBlock( - address, - pagesCount, - MemoryState.Unmapped, - KMemoryPermission.None, - MemoryAttribute.None, - MemoryState.Heap, - KMemoryPermission.ReadAndWrite, - MemoryAttribute.None); - } - - return KernelResult.Success; - } - - public KernelResult UnmapPhysicalMemory(ulong address, ulong size) - { - ulong endAddr = address + size; - - lock (_blocks) - { - // Scan, ensure that the region can be unmapped (all blocks are heap or - // already unmapped), fill pages list for freeing memory. - ulong heapMappedSize = 0; - - KPageList pageList = new KPageList(); - - foreach (KMemoryInfo info in IterateOverRange(address, endAddr)) - { - if (info.State == MemoryState.Heap) - { - if (info.Attribute != MemoryAttribute.None) - { - return KernelResult.InvalidMemState; - } - - ulong blockSize = GetSizeInRange(info, address, endAddr); - ulong blockAddress = GetAddrInRange(info, address); - - AddVaRangeToPageList(pageList, blockAddress, blockSize / PageSize); - - heapMappedSize += blockSize; - } - else if (info.State != MemoryState.Unmapped) - { - return KernelResult.InvalidMemState; - } - } - - if (heapMappedSize == 0) - { - return KernelResult.Success; - } - - if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) - { - return KernelResult.OutOfResource; - } - - // Try to unmap all the heap mapped memory inside range. - KernelResult result = KernelResult.Success; - - foreach (KMemoryInfo info in IterateOverRange(address, endAddr)) - { - if (info.State == MemoryState.Heap) - { - ulong blockSize = GetSizeInRange(info, address, endAddr); - ulong blockAddress = GetAddrInRange(info, address); - - ulong blockPagesCount = blockSize / PageSize; - - result = MmuUnmap(blockAddress, blockPagesCount); - - if (result != KernelResult.Success) - { - // If we failed to unmap, we need to remap everything back again. - MapPhysicalMemory(pageList, address, blockAddress + blockSize); - - break; - } - } - } - - if (result == KernelResult.Success) - { - GetMemoryRegionManager().FreePages(pageList); - - PhysicalMemoryUsage -= heapMappedSize; - - KProcess currentProcess = KernelStatic.GetCurrentProcess(); - - currentProcess.ResourceLimit?.Release(LimitableResource.Memory, heapMappedSize); - - ulong pagesCount = size / PageSize; - - InsertBlock(address, pagesCount, MemoryState.Unmapped); - } - - return result; - } - } - - private void MapPhysicalMemory(KPageList pageList, ulong address, ulong endAddr) - { - LinkedListNode<KPageNode> pageListNode = pageList.Nodes.First; - - KPageNode pageNode = pageListNode.Value; - - ulong srcPa = pageNode.Address; - ulong srcPaPages = pageNode.PagesCount; - - foreach (KMemoryInfo info in IterateOverRange(address, endAddr)) - { - if (info.State == MemoryState.Unmapped) - { - ulong blockSize = GetSizeInRange(info, address, endAddr); - - ulong dstVaPages = blockSize / PageSize; - - ulong dstVa = GetAddrInRange(info, address); - - while (dstVaPages > 0) - { - if (srcPaPages == 0) - { - pageListNode = pageListNode.Next; - - pageNode = pageListNode.Value; - - srcPa = pageNode.Address; - srcPaPages = pageNode.PagesCount; - } - - ulong pagesCount = srcPaPages; - - if (pagesCount > dstVaPages) - { - pagesCount = dstVaPages; - } - - DoMmuOperation( - dstVa, - pagesCount, - srcPa, - true, - KMemoryPermission.ReadAndWrite, - MemoryOperation.MapPa); - - dstVa += pagesCount * PageSize; - srcPa += pagesCount * PageSize; - srcPaPages -= pagesCount; - dstVaPages -= pagesCount; - } - } - } - } - - public KernelResult CopyDataToCurrentProcess( - ulong dst, - ulong size, - ulong src, - MemoryState stateMask, - MemoryState stateExpected, - KMemoryPermission permission, - MemoryAttribute attributeMask, - MemoryAttribute attributeExpected) - { - // Client -> server. - return CopyDataFromOrToCurrentProcess( - size, - src, - dst, - stateMask, - stateExpected, - permission, - attributeMask, - attributeExpected, - toServer: true); - } - - public KernelResult CopyDataFromCurrentProcess( - ulong dst, - ulong size, - MemoryState stateMask, - MemoryState stateExpected, - KMemoryPermission permission, - MemoryAttribute attributeMask, - MemoryAttribute attributeExpected, - ulong src) - { - // Server -> client. - return CopyDataFromOrToCurrentProcess( - size, - dst, - src, - stateMask, - stateExpected, - permission, - attributeMask, - attributeExpected, - toServer: false); - } - - private KernelResult CopyDataFromOrToCurrentProcess( - ulong size, - ulong clientAddress, - ulong serverAddress, - MemoryState stateMask, - MemoryState stateExpected, - KMemoryPermission permission, - MemoryAttribute attributeMask, - MemoryAttribute attributeExpected, - bool toServer) - { - if (AddrSpaceStart > clientAddress) - { - return KernelResult.InvalidMemState; - } - - ulong srcEndAddr = clientAddress + size; - - if (srcEndAddr <= clientAddress || srcEndAddr - 1 > AddrSpaceEnd - 1) - { - return KernelResult.InvalidMemState; - } - - lock (_blocks) - { - if (CheckRange( - clientAddress, - size, - stateMask, - stateExpected, - permission, - permission, - attributeMask | MemoryAttribute.Uncached, - attributeExpected)) - { - KProcess currentProcess = KernelStatic.GetCurrentProcess(); - - while (size > 0) - { - ulong copySize = Math.Min(PageSize - (serverAddress & (PageSize - 1)), PageSize - (clientAddress & (PageSize - 1))); - - if (copySize > size) - { - copySize = size; - } - - ulong serverDramAddr = currentProcess.MemoryManager.GetDramAddressFromVa(serverAddress); - ulong clientDramAddr = GetDramAddressFromVa(clientAddress); - - if (serverDramAddr != clientDramAddr) - { - if (toServer) - { - _context.Memory.Copy(serverDramAddr, clientDramAddr, copySize); - } - else - { - _context.Memory.Copy(clientDramAddr, serverDramAddr, copySize); - } - } - - serverAddress += copySize; - clientAddress += copySize; - size -= copySize; - } - - return KernelResult.Success; - } - else - { - return KernelResult.InvalidMemState; - } - } - } - - public KernelResult MapBufferFromClientProcess( - ulong size, - ulong src, - KMemoryManager sourceMemMgr, - KMemoryPermission permission, - MemoryState state, - bool copyData, - out ulong dst) - { - dst = 0; - - KernelResult result = sourceMemMgr.GetPagesForMappingIntoAnotherProcess( - src, - size, - permission, - state, - copyData, - _aslrDisabled, - _memRegion, - out KPageList pageList); - - if (result != KernelResult.Success) - { - return result; - } - - result = MapPagesFromAnotherProcess(size, src, permission, state, pageList, out ulong va); - - if (result != KernelResult.Success) - { - sourceMemMgr.UnmapIpcRestorePermission(src, size, state); - } - else - { - dst = va; - } - - return result; - } - - private KernelResult GetPagesForMappingIntoAnotherProcess( - ulong address, - ulong size, - KMemoryPermission permission, - MemoryState state, - bool copyData, - bool aslrDisabled, - MemoryRegion region, - out KPageList pageList) - { - pageList = null; - - if (AddrSpaceStart > address) - { - return KernelResult.InvalidMemState; - } - - ulong endAddr = address + size; - - if (endAddr <= address || endAddr - 1 > AddrSpaceEnd - 1) - { - return KernelResult.InvalidMemState; - } - - MemoryState stateMask; - - switch (state) - { - case MemoryState.IpcBuffer0: stateMask = MemoryState.IpcSendAllowedType0; break; - case MemoryState.IpcBuffer1: stateMask = MemoryState.IpcSendAllowedType1; break; - case MemoryState.IpcBuffer3: stateMask = MemoryState.IpcSendAllowedType3; break; - - default: return KernelResult.InvalidCombination; - } - - KMemoryPermission permissionMask = permission == KMemoryPermission.ReadAndWrite - ? KMemoryPermission.None - : KMemoryPermission.Read; - - MemoryAttribute attributeMask = MemoryAttribute.Borrowed | MemoryAttribute.Uncached; - - if (state == MemoryState.IpcBuffer0) - { - attributeMask |= MemoryAttribute.DeviceMapped; - } - - ulong addressRounded = BitUtils.AlignUp (address, PageSize); - ulong addressTruncated = BitUtils.AlignDown(address, PageSize); - ulong endAddrRounded = BitUtils.AlignUp (endAddr, PageSize); - ulong endAddrTruncated = BitUtils.AlignDown(endAddr, PageSize); - - if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) - { - return KernelResult.OutOfResource; - } - - ulong visitedSize = 0; - - void CleanUpForError() - { - if (visitedSize == 0) - { - return; - } - - ulong endAddrVisited = address + visitedSize; - - foreach (KMemoryInfo info in IterateOverRange(addressRounded, endAddrVisited)) - { - if ((info.Permission & KMemoryPermission.ReadAndWrite) != permissionMask && info.IpcRefCount == 0) - { - ulong blockAddress = GetAddrInRange(info, addressRounded); - ulong blockSize = GetSizeInRange(info, addressRounded, endAddrVisited); - - ulong blockPagesCount = blockSize / PageSize; - - if (DoMmuOperation( - blockAddress, - blockPagesCount, - 0, - false, - info.Permission, - MemoryOperation.ChangePermRw) != KernelResult.Success) - { - throw new InvalidOperationException("Unexpected failure trying to restore permission."); - } - } - } - } - - // Signal a read for any resources tracking reads in the region, as the other process is likely to use their data. - _cpuMemory.SignalMemoryTracking(addressTruncated, endAddrRounded - addressTruncated, false); - - lock (_blocks) - { - KernelResult result; - - if (addressRounded < endAddrTruncated) - { - foreach (KMemoryInfo info in IterateOverRange(addressRounded, endAddrTruncated)) - { - // Check if the block state matches what we expect. - if ((info.State & stateMask) != stateMask || - (info.Permission & permission) != permission || - (info.Attribute & attributeMask) != MemoryAttribute.None) - { - CleanUpForError(); - - return KernelResult.InvalidMemState; - } - - ulong blockAddress = GetAddrInRange(info, addressRounded); - ulong blockSize = GetSizeInRange(info, addressRounded, endAddrTruncated); - - ulong blockPagesCount = blockSize / PageSize; - - if ((info.Permission & KMemoryPermission.ReadAndWrite) != permissionMask && info.IpcRefCount == 0) - { - result = DoMmuOperation( - blockAddress, - blockPagesCount, - 0, - false, - permissionMask, - MemoryOperation.ChangePermRw); - - if (result != KernelResult.Success) - { - CleanUpForError(); - - return result; - } - } - - visitedSize += blockSize; - } - } - - result = GetPagesForIpcTransfer(address, size, copyData, aslrDisabled, region, out pageList); - - if (result != KernelResult.Success) - { - CleanUpForError(); - - return result; - } - - if (visitedSize != 0) - { - InsertBlock(addressRounded, visitedSize / PageSize, SetIpcMappingPermissions, permissionMask); - } - } - - return KernelResult.Success; - } - - private KernelResult GetPagesForIpcTransfer( - ulong address, - ulong size, - bool copyData, - bool aslrDisabled, - MemoryRegion region, - out KPageList pageList) - { - // When the start address is unaligned, we can't safely map the - // first page as it would expose other undesirable information on the - // target process. So, instead we allocate new pages, copy the data - // inside the range, and then clear the remaining space. - // The same also holds for the last page, if the end address - // (address + size) is also not aligned. - - pageList = null; - - KPageList pages = new KPageList(); - - ulong addressTruncated = BitUtils.AlignDown(address, PageSize); - ulong addressRounded = BitUtils.AlignUp (address, PageSize); - - ulong endAddr = address + size; - - ulong dstFirstPagePa = 0; - ulong dstLastPagePa = 0; - - void CleanUpForError() - { - if (dstFirstPagePa != 0) - { - FreeSinglePage(region, dstFirstPagePa); - } - - if (dstLastPagePa != 0) - { - FreeSinglePage(region, dstLastPagePa); - } - } - - // Is the first page address aligned? - // If not, allocate a new page and copy the unaligned chunck. - if (addressTruncated < addressRounded) - { - dstFirstPagePa = AllocateSinglePage(region, aslrDisabled); - - if (dstFirstPagePa == 0) - { - return KernelResult.OutOfMemory; - } - - ulong firstPageFillAddress = dstFirstPagePa; - - if (!TryConvertVaToPa(addressTruncated, out ulong srcFirstPagePa)) - { - CleanUpForError(); - - return KernelResult.InvalidMemState; - } - - ulong unusedSizeAfter; - - if (copyData) - { - ulong unusedSizeBefore = address - addressTruncated; - - _context.Memory.ZeroFill(GetDramAddressFromPa(dstFirstPagePa), unusedSizeBefore); - - ulong copySize = addressRounded <= endAddr ? addressRounded - address : size; - - _context.Memory.Copy( - GetDramAddressFromPa(dstFirstPagePa + unusedSizeBefore), - GetDramAddressFromPa(srcFirstPagePa + unusedSizeBefore), copySize); - - firstPageFillAddress += unusedSizeBefore + copySize; - - unusedSizeAfter = addressRounded > endAddr ? addressRounded - endAddr : 0; - } - else - { - unusedSizeAfter = PageSize; - } - - if (unusedSizeAfter != 0) - { - _context.Memory.ZeroFill(GetDramAddressFromPa(firstPageFillAddress), unusedSizeAfter); - } - - if (pages.AddRange(dstFirstPagePa, 1) != KernelResult.Success) - { - CleanUpForError(); - - return KernelResult.OutOfResource; - } - } - - ulong endAddrTruncated = BitUtils.AlignDown(endAddr, PageSize); - ulong endAddrRounded = BitUtils.AlignUp (endAddr, PageSize); - - if (endAddrTruncated > addressRounded) - { - ulong alignedPagesCount = (endAddrTruncated - addressRounded) / PageSize; - - AddVaRangeToPageList(pages, addressRounded, alignedPagesCount); - } - - // Is the last page end address aligned? - // If not, allocate a new page and copy the unaligned chunck. - if (endAddrTruncated < endAddrRounded && (addressTruncated == addressRounded || addressTruncated < endAddrTruncated)) - { - dstLastPagePa = AllocateSinglePage(region, aslrDisabled); - - if (dstLastPagePa == 0) - { - CleanUpForError(); - - return KernelResult.OutOfMemory; - } - - ulong lastPageFillAddr = dstLastPagePa; - - if (!TryConvertVaToPa(endAddrTruncated, out ulong srcLastPagePa)) - { - CleanUpForError(); - - return KernelResult.InvalidMemState; - } - - ulong unusedSizeAfter; - - if (copyData) - { - ulong copySize = endAddr - endAddrTruncated; - - _context.Memory.Copy( - GetDramAddressFromPa(dstLastPagePa), - GetDramAddressFromPa(srcLastPagePa), copySize); - - lastPageFillAddr += copySize; - - unusedSizeAfter = PageSize - copySize; - } - else - { - unusedSizeAfter = PageSize; - } - - _context.Memory.ZeroFill(GetDramAddressFromPa(lastPageFillAddr), unusedSizeAfter); - - if (pages.AddRange(dstLastPagePa, 1) != KernelResult.Success) - { - CleanUpForError(); - - return KernelResult.OutOfResource; - } - } - - pageList = pages; - - return KernelResult.Success; - } - - private ulong AllocateSinglePage(MemoryRegion region, bool aslrDisabled) - { - KMemoryRegionManager regionMgr = _context.MemoryRegions[(int)region]; - - return regionMgr.AllocatePagesContiguous(1, aslrDisabled); - } - - private void FreeSinglePage(MemoryRegion region, ulong address) - { - KMemoryRegionManager regionMgr = _context.MemoryRegions[(int)region]; - - regionMgr.FreePage(address); - } - - private KernelResult MapPagesFromAnotherProcess( - ulong size, - ulong address, - KMemoryPermission permission, - MemoryState state, - KPageList pageList, - out ulong dst) - { - dst = 0; - - lock (_blocks) - { - if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) - { - return KernelResult.OutOfResource; - } - - ulong endAddr = address + size; - - ulong addressTruncated = BitUtils.AlignDown(address, PageSize); - ulong endAddrRounded = BitUtils.AlignUp (endAddr, PageSize); - - ulong neededSize = endAddrRounded - addressTruncated; - - ulong neededPagesCount = neededSize / PageSize; - - ulong regionPagesCount = (AliasRegionEnd - AliasRegionStart) / PageSize; - - ulong va = 0; - - for (int unit = MappingUnitSizes.Length - 1; unit >= 0 && va == 0; unit--) - { - int alignment = MappingUnitSizes[unit]; - - va = AllocateVa(AliasRegionStart, regionPagesCount, neededPagesCount, alignment); - } - - if (va == 0) - { - return KernelResult.OutOfVaSpace; - } - - if (pageList.Nodes.Count != 0) - { - KernelResult result = MapPages(va, pageList, permission); - - if (result != KernelResult.Success) - { - return result; - } - } - - InsertBlock(va, neededPagesCount, state, permission); - - dst = va + (address - addressTruncated); - } - - return KernelResult.Success; - } - - public KernelResult UnmapNoAttributeIfStateEquals(ulong address, ulong size, MemoryState state) - { - if (AddrSpaceStart > address) - { - return KernelResult.InvalidMemState; - } - - ulong endAddr = address + size; - - if (endAddr <= address || endAddr - 1 > AddrSpaceEnd - 1) - { - return KernelResult.InvalidMemState; - } - - lock (_blocks) - { - if (CheckRange( - address, - size, - MemoryState.Mask, - state, - KMemoryPermission.Read, - KMemoryPermission.Read, - MemoryAttribute.Mask, - MemoryAttribute.None, - MemoryAttribute.IpcAndDeviceMapped, - out _, - out _, - out _)) - { - if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) - { - return KernelResult.OutOfResource; - } - - ulong addressTruncated = BitUtils.AlignDown(address, PageSize); - ulong addressRounded = BitUtils.AlignUp (address, PageSize); - ulong endAddrTruncated = BitUtils.AlignDown(endAddr, PageSize); - ulong endAddrRounded = BitUtils.AlignUp (endAddr, PageSize); - - ulong pagesCount = (endAddrRounded - addressTruncated) / PageSize; - - // Free pages we had to create on-demand, if any of the buffer was not page aligned. - // Real kernel has page ref counting, so this is done as part of the unmap operation. - if (addressTruncated != addressRounded) - { - FreeSinglePage(_memRegion, ConvertVaToPa(addressTruncated)); - } - - if (endAddrTruncated < endAddrRounded && (addressTruncated == addressRounded || addressTruncated < endAddrTruncated)) - { - FreeSinglePage(_memRegion, ConvertVaToPa(endAddrTruncated)); - } - - KernelResult result = DoMmuOperation( - addressTruncated, - pagesCount, - 0, - false, - KMemoryPermission.None, - MemoryOperation.Unmap); - - if (result == KernelResult.Success) - { - InsertBlock(addressTruncated, pagesCount, MemoryState.Unmapped); - } - - return result; - } - else - { - return KernelResult.InvalidMemState; - } - } - } - - public KernelResult UnmapIpcRestorePermission(ulong address, ulong size, MemoryState state) - { - ulong endAddr = address + size; - - ulong addressRounded = BitUtils.AlignUp (address, PageSize); - ulong addressTruncated = BitUtils.AlignDown(address, PageSize); - ulong endAddrRounded = BitUtils.AlignUp (endAddr, PageSize); - ulong endAddrTruncated = BitUtils.AlignDown(endAddr, PageSize); - - ulong pagesCount = addressRounded < endAddrTruncated ? (endAddrTruncated - addressRounded) / PageSize : 0; - - if (pagesCount == 0) - { - return KernelResult.Success; - } - - MemoryState stateMask; - - switch (state) - { - case MemoryState.IpcBuffer0: stateMask = MemoryState.IpcSendAllowedType0; break; - case MemoryState.IpcBuffer1: stateMask = MemoryState.IpcSendAllowedType1; break; - case MemoryState.IpcBuffer3: stateMask = MemoryState.IpcSendAllowedType3; break; - - default: return KernelResult.InvalidCombination; - } - - MemoryAttribute attributeMask = - MemoryAttribute.Borrowed | - MemoryAttribute.IpcMapped | - MemoryAttribute.Uncached; - - if (state == MemoryState.IpcBuffer0) - { - attributeMask |= MemoryAttribute.DeviceMapped; - } - - if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) - { - return KernelResult.OutOfResource; - } - - // Anything on the client side should see this memory as modified. - _cpuMemory.SignalMemoryTracking(addressTruncated, endAddrRounded - addressTruncated, true); - - lock (_blocks) - { - foreach (KMemoryInfo info in IterateOverRange(addressRounded, endAddrTruncated)) - { - // Check if the block state matches what we expect. - if ((info.State & stateMask) != stateMask || - (info.Attribute & attributeMask) != MemoryAttribute.IpcMapped) - { - return KernelResult.InvalidMemState; - } - - if (info.Permission != info.SourcePermission && info.IpcRefCount == 1) - { - ulong blockAddress = GetAddrInRange(info, addressRounded); - ulong blockSize = GetSizeInRange(info, addressRounded, endAddrTruncated); - - ulong blockPagesCount = blockSize / PageSize; - - KernelResult result = DoMmuOperation( - blockAddress, - blockPagesCount, - 0, - false, - info.SourcePermission, - MemoryOperation.ChangePermRw); - - if (result != KernelResult.Success) - { - return result; - } - } - } - - InsertBlock(addressRounded, pagesCount, RestoreIpcMappingPermissions); - - return KernelResult.Success; - } - } - - public KernelResult BorrowIpcBuffer(ulong address, ulong size) - { - return SetAttributesAndChangePermission( - address, - size, - MemoryState.IpcBufferAllowed, - MemoryState.IpcBufferAllowed, - KMemoryPermission.Mask, - KMemoryPermission.ReadAndWrite, - MemoryAttribute.Mask, - MemoryAttribute.None, - KMemoryPermission.None, - MemoryAttribute.Borrowed); - } - - public KernelResult BorrowTransferMemory(KPageList pageList, ulong address, ulong size, KMemoryPermission permission) - { - return SetAttributesAndChangePermission( - address, - size, - MemoryState.TransferMemoryAllowed, - MemoryState.TransferMemoryAllowed, - KMemoryPermission.Mask, - KMemoryPermission.ReadAndWrite, - MemoryAttribute.Mask, - MemoryAttribute.None, - permission, - MemoryAttribute.Borrowed, - pageList); - } - - private KernelResult SetAttributesAndChangePermission( - ulong address, - ulong size, - MemoryState stateMask, - MemoryState stateExpected, - KMemoryPermission permissionMask, - KMemoryPermission permissionExpected, - MemoryAttribute attributeMask, - MemoryAttribute attributeExpected, - KMemoryPermission newPermission, - MemoryAttribute attributeSetMask, - KPageList pageList = null) - { - if (address + size <= address || !InsideAddrSpace(address, size)) - { - return KernelResult.InvalidMemState; - } - - lock (_blocks) - { - if (CheckRange( - address, - size, - stateMask | MemoryState.IsPoolAllocated, - stateExpected | MemoryState.IsPoolAllocated, - permissionMask, - permissionExpected, - attributeMask, - attributeExpected, - MemoryAttribute.IpcAndDeviceMapped, - out MemoryState oldState, - out KMemoryPermission oldPermission, - out MemoryAttribute oldAttribute)) - { - ulong pagesCount = size / PageSize; - - if (pageList != null) - { - AddVaRangeToPageList(pageList, address, pagesCount); - } - - if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) - { - return KernelResult.OutOfResource; - } - - if (newPermission == KMemoryPermission.None) - { - newPermission = oldPermission; - } - - if (newPermission != oldPermission) - { - KernelResult result = DoMmuOperation( - address, - pagesCount, - 0, - false, - newPermission, - MemoryOperation.ChangePermRw); - - if (result != KernelResult.Success) - { - return result; - } - } - - MemoryAttribute newAttribute = oldAttribute | attributeSetMask; - - InsertBlock(address, pagesCount, oldState, newPermission, newAttribute); - - return KernelResult.Success; - } - else - { - return KernelResult.InvalidMemState; - } - } - } - - public KernelResult UnborrowIpcBuffer(ulong address, ulong size) - { - return ClearAttributesAndChangePermission( - address, - size, - MemoryState.IpcBufferAllowed, - MemoryState.IpcBufferAllowed, - KMemoryPermission.None, - KMemoryPermission.None, - MemoryAttribute.Mask, - MemoryAttribute.Borrowed, - KMemoryPermission.ReadAndWrite, - MemoryAttribute.Borrowed); - } - - public KernelResult UnborrowTransferMemory(ulong address, ulong size, KPageList pageList) - { - return ClearAttributesAndChangePermission( - address, - size, - MemoryState.TransferMemoryAllowed, - MemoryState.TransferMemoryAllowed, - KMemoryPermission.None, - KMemoryPermission.None, - MemoryAttribute.Mask, - MemoryAttribute.Borrowed, - KMemoryPermission.ReadAndWrite, - MemoryAttribute.Borrowed, - pageList); - } - - private KernelResult ClearAttributesAndChangePermission( - ulong address, - ulong size, - MemoryState stateMask, - MemoryState stateExpected, - KMemoryPermission permissionMask, - KMemoryPermission permissionExpected, - MemoryAttribute attributeMask, - MemoryAttribute attributeExpected, - KMemoryPermission newPermission, - MemoryAttribute attributeClearMask, - KPageList pageList = null) - { - if (address + size <= address || !InsideAddrSpace(address, size)) - { - return KernelResult.InvalidMemState; - } - - lock (_blocks) - { - if (CheckRange( - address, - size, - stateMask | MemoryState.IsPoolAllocated, - stateExpected | MemoryState.IsPoolAllocated, - permissionMask, - permissionExpected, - attributeMask, - attributeExpected, - MemoryAttribute.IpcAndDeviceMapped, - out MemoryState oldState, - out KMemoryPermission oldPermission, - out MemoryAttribute oldAttribute)) - { - ulong pagesCount = size / PageSize; - - if (pageList != null) - { - KPageList currPageList = new KPageList(); - - AddVaRangeToPageList(currPageList, address, pagesCount); - - if (!currPageList.IsEqual(pageList)) - { - return KernelResult.InvalidMemRange; - } - } - - if (!_blockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) - { - return KernelResult.OutOfResource; - } - - if (newPermission == KMemoryPermission.None) - { - newPermission = oldPermission; - } - - if (newPermission != oldPermission) - { - KernelResult result = DoMmuOperation( - address, - pagesCount, - 0, - false, - newPermission, - MemoryOperation.ChangePermRw); - - if (result != KernelResult.Success) - { - return result; - } - } - - MemoryAttribute newAttribute = oldAttribute & ~attributeClearMask; - - InsertBlock(address, pagesCount, oldState, newPermission, newAttribute); - - return KernelResult.Success; - } - else - { - return KernelResult.InvalidMemState; - } - } - } - - private void AddVaRangeToPageList(KPageList pageList, ulong start, ulong pagesCount) - { - ulong address = start; - - while (address < start + pagesCount * PageSize) - { - if (!TryConvertVaToPa(address, out ulong pa)) - { - throw new InvalidOperationException("Unexpected failure translating virtual address."); - } - - pageList.AddRange(pa, 1); - - address += PageSize; - } - } - - private static ulong GetAddrInRange(KMemoryInfo info, ulong start) - { - if (info.Address < start) - { - return start; - } - - return info.Address; - } - - private static ulong GetSizeInRange(KMemoryInfo info, ulong start, ulong end) - { - ulong endAddr = info.Size + info.Address; - ulong size = info.Size; - - if (info.Address < start) - { - size -= start - info.Address; - } - - if (endAddr > end) - { - size -= endAddr - end; - } - - return size; - } - - private bool IsUnmapped(ulong address, ulong size) - { - return CheckRange( - address, - size, - MemoryState.Mask, - MemoryState.Unmapped, - KMemoryPermission.Mask, - KMemoryPermission.None, - MemoryAttribute.Mask, - MemoryAttribute.None, - MemoryAttribute.IpcAndDeviceMapped, - out _, - out _, - out _); - } - - private bool CheckRange( - ulong address, - ulong size, - MemoryState stateMask, - MemoryState stateExpected, - KMemoryPermission permissionMask, - KMemoryPermission permissionExpected, - MemoryAttribute attributeMask, - MemoryAttribute attributeExpected, - MemoryAttribute attributeIgnoreMask, - out MemoryState outState, - out KMemoryPermission outPermission, - out MemoryAttribute outAttribute) - { - ulong endAddr = address + size; - - LinkedListNode<KMemoryBlock> node = FindBlockNode(address); - - KMemoryInfo info = node.Value.GetInfo(); - - MemoryState firstState = info.State; - KMemoryPermission firstPermission = info.Permission; - MemoryAttribute firstAttribute = info.Attribute; - - do - { - info = node.Value.GetInfo(); - - // Check if the block state matches what we expect. - if ( firstState != info.State || - firstPermission != info.Permission || - (info.Attribute & attributeMask) != attributeExpected || - (firstAttribute | attributeIgnoreMask) != (info.Attribute | attributeIgnoreMask) || - (firstState & stateMask) != stateExpected || - (firstPermission & permissionMask) != permissionExpected) - { - outState = MemoryState.Unmapped; - outPermission = KMemoryPermission.None; - outAttribute = MemoryAttribute.None; - - return false; - } - } - while (info.Address + info.Size - 1 < endAddr - 1 && (node = node.Next) != null); - - outState = firstState; - outPermission = firstPermission; - outAttribute = firstAttribute & ~attributeIgnoreMask; - - return true; - } - - private bool CheckRange( - ulong address, - ulong size, - MemoryState stateMask, - MemoryState stateExpected, - KMemoryPermission permissionMask, - KMemoryPermission permissionExpected, - MemoryAttribute attributeMask, - MemoryAttribute attributeExpected) - { - foreach (KMemoryInfo info in IterateOverRange(address, address + size)) - { - // Check if the block state matches what we expect. - if ((info.State & stateMask) != stateExpected || - (info.Permission & permissionMask) != permissionExpected || - (info.Attribute & attributeMask) != attributeExpected) - { - return false; - } - } - - return true; - } - - private IEnumerable<KMemoryInfo> IterateOverRange(ulong start, ulong end) - { - LinkedListNode<KMemoryBlock> node = FindBlockNode(start); - - KMemoryInfo info; - - do - { - info = node.Value.GetInfo(); - - yield return info; - } - while (info.Address + info.Size - 1 < end - 1 && (node = node.Next) != null); - } - - private void InsertBlock( - ulong baseAddress, - ulong pagesCount, - MemoryState oldState, - KMemoryPermission oldPermission, - MemoryAttribute oldAttribute, - MemoryState newState, - KMemoryPermission newPermission, - MemoryAttribute newAttribute) - { - // Insert new block on the list only on areas where the state - // of the block matches the state specified on the old* state - // arguments, otherwise leave it as is. - int oldCount = _blocks.Count; - - oldAttribute |= MemoryAttribute.IpcAndDeviceMapped; - - ulong endAddr = baseAddress + pagesCount * PageSize; - - LinkedListNode<KMemoryBlock> node = _blocks.First; - - while (node != null) - { - LinkedListNode<KMemoryBlock> newNode = node; - - KMemoryBlock currBlock = node.Value; - - ulong currBaseAddr = currBlock.BaseAddress; - ulong currEndAddr = currBlock.PagesCount * PageSize + currBaseAddr; - - if (baseAddress < currEndAddr && currBaseAddr < endAddr) - { - MemoryAttribute currBlockAttr = currBlock.Attribute | MemoryAttribute.IpcAndDeviceMapped; + public KMemoryRegionManager[] MemoryRegions { get; } - if (currBlock.State != oldState || - currBlock.Permission != oldPermission || - currBlockAttr != oldAttribute) - { - node = node.Next; - - continue; - } - - if (baseAddress > currBaseAddr) - { - _blocks.AddBefore(node, currBlock.SplitRightAtAddress(baseAddress)); - } - - if (endAddr < currEndAddr) - { - newNode = _blocks.AddBefore(node, currBlock.SplitRightAtAddress(endAddr)); - } - - newNode.Value.SetState(newPermission, newState, newAttribute); - - newNode = MergeEqualStateNeighbors(newNode); - } - - if (currEndAddr - 1 >= endAddr - 1) - { - break; - } - - node = newNode.Next; - } - - _blockAllocator.Count += _blocks.Count - oldCount; - - ValidateInternalState(); - } - - private void InsertBlock( - ulong baseAddress, - ulong pagesCount, - MemoryState state, - KMemoryPermission permission = KMemoryPermission.None, - MemoryAttribute attribute = MemoryAttribute.None) - { - // Inserts new block at the list, replacing and splitting - // existing blocks as needed. - int oldCount = _blocks.Count; - - ulong endAddr = baseAddress + pagesCount * PageSize; - - LinkedListNode<KMemoryBlock> node = _blocks.First; - - while (node != null) - { - LinkedListNode<KMemoryBlock> newNode = node; - - KMemoryBlock currBlock = node.Value; - - ulong currBaseAddr = currBlock.BaseAddress; - ulong currEndAddr = currBlock.PagesCount * PageSize + currBaseAddr; - - if (baseAddress < currEndAddr && currBaseAddr < endAddr) - { - if (baseAddress > currBaseAddr) - { - _blocks.AddBefore(node, currBlock.SplitRightAtAddress(baseAddress)); - } - - if (endAddr < currEndAddr) - { - newNode = _blocks.AddBefore(node, currBlock.SplitRightAtAddress(endAddr)); - } - - newNode.Value.SetState(permission, state, attribute); - - newNode = MergeEqualStateNeighbors(newNode); - } - - if (currEndAddr - 1 >= endAddr - 1) - { - break; - } - - node = newNode.Next; - } - - _blockAllocator.Count += _blocks.Count - oldCount; - - ValidateInternalState(); - } - - private static void SetIpcMappingPermissions(KMemoryBlock block, KMemoryPermission permission) + public KMemoryManager(MemorySize size, MemoryArrange arrange) { - block.SetIpcMappingPermission(permission); + MemoryRegions = KernelInit.GetMemoryRegions(size, arrange); } - private static void RestoreIpcMappingPermissions(KMemoryBlock block, KMemoryPermission permission) + private KMemoryRegionManager GetMemoryRegion(ulong address) { - block.RestoreIpcMappingPermission(); - } - - private delegate void BlockMutator(KMemoryBlock block, KMemoryPermission newPerm); - - private void InsertBlock( - ulong baseAddress, - ulong pagesCount, - BlockMutator blockMutate, - KMemoryPermission permission = KMemoryPermission.None) - { - // Inserts new block at the list, replacing and splitting - // existing blocks as needed, then calling the callback - // function on the new block. - int oldCount = _blocks.Count; - - ulong endAddr = baseAddress + pagesCount * PageSize; - - LinkedListNode<KMemoryBlock> node = _blocks.First; - - while (node != null) + for (int i = 0; i < MemoryRegions.Length; i++) { - LinkedListNode<KMemoryBlock> newNode = node; - - KMemoryBlock currBlock = node.Value; - - ulong currBaseAddr = currBlock.BaseAddress; - ulong currEndAddr = currBlock.PagesCount * PageSize + currBaseAddr; - - if (baseAddress < currEndAddr && currBaseAddr < endAddr) - { - if (baseAddress > currBaseAddr) - { - _blocks.AddBefore(node, currBlock.SplitRightAtAddress(baseAddress)); - } - - if (endAddr < currEndAddr) - { - newNode = _blocks.AddBefore(node, currBlock.SplitRightAtAddress(endAddr)); - } + var region = MemoryRegions[i]; - KMemoryBlock newBlock = newNode.Value; - - blockMutate(newBlock, permission); - - newNode = MergeEqualStateNeighbors(newNode); - } - - if (currEndAddr - 1 >= endAddr - 1) + if (address >= region.Address && address < region.EndAddr) { - break; + return region; } - - node = newNode.Next; } - _blockAllocator.Count += _blocks.Count - oldCount; - - ValidateInternalState(); + return null; } - [Conditional("DEBUG")] - private void ValidateInternalState() + public void IncrementPagesReferenceCount(ulong address, ulong pagesCount) { - ulong expectedAddress = 0; - - LinkedListNode<KMemoryBlock> node = _blocks.First; - - while (node != null) - { - LinkedListNode<KMemoryBlock> newNode = node; - - KMemoryBlock currBlock = node.Value; - - Debug.Assert(currBlock.BaseAddress == expectedAddress); - - expectedAddress = currBlock.BaseAddress + currBlock.PagesCount * PageSize; - - node = newNode.Next; - } - - Debug.Assert(expectedAddress == AddrSpaceEnd); + IncrementOrDecrementPagesReferenceCount(address, pagesCount, true); } - private LinkedListNode<KMemoryBlock> MergeEqualStateNeighbors(LinkedListNode<KMemoryBlock> node) + public void DecrementPagesReferenceCount(ulong address, ulong pagesCount) { - KMemoryBlock block = node.Value; - - if (node.Previous != null) - { - KMemoryBlock previousBlock = node.Previous.Value; - - if (BlockStateEquals(block, previousBlock)) - { - LinkedListNode<KMemoryBlock> previousNode = node.Previous; - - _blocks.Remove(node); - - previousBlock.AddPages(block.PagesCount); - - node = previousNode; - block = previousBlock; - } - } - - if (node.Next != null) - { - KMemoryBlock nextBlock = node.Next.Value; - - if (BlockStateEquals(block, nextBlock)) - { - _blocks.Remove(node.Next); - - block.AddPages(nextBlock.PagesCount); - } - } - - return node; + IncrementOrDecrementPagesReferenceCount(address, pagesCount, false); } - private static bool BlockStateEquals(KMemoryBlock lhs, KMemoryBlock rhs) + private void IncrementOrDecrementPagesReferenceCount(ulong address, ulong pagesCount, bool increment) { - return lhs.State == rhs.State && - lhs.Permission == rhs.Permission && - lhs.Attribute == rhs.Attribute && - lhs.SourcePermission == rhs.SourcePermission && - lhs.DeviceRefCount == rhs.DeviceRefCount && - lhs.IpcRefCount == rhs.IpcRefCount; - } - - private ulong AllocateVa( - ulong regionStart, - ulong regionPagesCount, - ulong neededPagesCount, - int alignment) - { - ulong address = 0; - - ulong regionEndAddr = regionStart + regionPagesCount * PageSize; - - ulong reservedPagesCount = _isKernel ? 1UL : 4UL; - - if (_aslrEnabled) + while (pagesCount != 0) { - ulong totalNeededSize = (reservedPagesCount + neededPagesCount) * PageSize; - - ulong remainingPages = regionPagesCount - neededPagesCount; + var region = GetMemoryRegion(address); - ulong aslrMaxOffset = ((remainingPages + reservedPagesCount) * PageSize) / (ulong)alignment; + ulong countToProcess = Math.Min(pagesCount, region.GetPageOffsetFromEnd(address)); - for (int attempt = 0; attempt < 8; attempt++) + lock (region) { - address = BitUtils.AlignDown(regionStart + GetRandomValue(0, aslrMaxOffset) * (ulong)alignment, alignment); - - ulong endAddr = address + totalNeededSize; - - KMemoryInfo info = FindBlock(address).GetInfo(); - - if (info.State != MemoryState.Unmapped) + if (increment) { - continue; + region.IncrementPagesReferenceCount(address, countToProcess); } - - ulong currBaseAddr = info.Address + reservedPagesCount * PageSize; - ulong currEndAddr = info.Address + info.Size; - - if (address >= regionStart && - address >= currBaseAddr && - endAddr - 1 <= regionEndAddr - 1 && - endAddr - 1 <= currEndAddr - 1) + else { - break; + region.DecrementPagesReferenceCount(address, countToProcess); } } - if (address == 0) - { - ulong aslrPage = GetRandomValue(0, aslrMaxOffset); - - address = FindFirstFit( - regionStart + aslrPage * PageSize, - regionPagesCount - aslrPage, - neededPagesCount, - alignment, - 0, - reservedPagesCount); - } + pagesCount -= countToProcess; + address += countToProcess * KPageTableBase.PageSize; } - - if (address == 0) - { - address = FindFirstFit( - regionStart, - regionPagesCount, - neededPagesCount, - alignment, - 0, - reservedPagesCount); - } - - return address; - } - - private ulong FindFirstFit( - ulong regionStart, - ulong regionPagesCount, - ulong neededPagesCount, - int alignment, - ulong reservedStart, - ulong reservedPagesCount) - { - ulong reservedSize = reservedPagesCount * PageSize; - - ulong totalNeededSize = reservedSize + neededPagesCount * PageSize; - - ulong regionEndAddr = regionStart + regionPagesCount * PageSize; - - LinkedListNode<KMemoryBlock> node = FindBlockNode(regionStart); - - KMemoryInfo info = node.Value.GetInfo(); - - while (regionEndAddr >= info.Address) - { - if (info.State == MemoryState.Unmapped) - { - ulong currBaseAddr = info.Address + reservedSize; - ulong currEndAddr = info.Address + info.Size - 1; - - ulong address = BitUtils.AlignDown(currBaseAddr, alignment) + reservedStart; - - if (currBaseAddr > address) - { - address += (ulong)alignment; - } - - ulong allocationEndAddr = address + totalNeededSize - 1; - - if (allocationEndAddr <= regionEndAddr && - allocationEndAddr <= currEndAddr && - address < allocationEndAddr) - { - return address; - } - } - - node = node.Next; - - if (node == null) - { - break; - } - - info = node.Value.GetInfo(); - } - - return 0; - } - - private KMemoryBlock FindBlock(ulong address) - { - return FindBlockNode(address)?.Value; - } - - private LinkedListNode<KMemoryBlock> FindBlockNode(ulong address) - { - lock (_blocks) - { - LinkedListNode<KMemoryBlock> node = _blocks.First; - - while (node != null) - { - KMemoryBlock block = node.Value; - - ulong currEndAddr = block.PagesCount * PageSize + block.BaseAddress; - - if (block.BaseAddress <= address && currEndAddr - 1 >= address) - { - return node; - } - - node = node.Next; - } - } - - return null; - } - - public bool CanContain(ulong address, ulong size, MemoryState state) - { - ulong endAddr = address + size; - - ulong regionBaseAddr = GetBaseAddress(state); - ulong regionEndAddr = regionBaseAddr + GetSize(state); - - bool InsideRegion() - { - return regionBaseAddr <= address && - endAddr > address && - endAddr - 1 <= regionEndAddr - 1; - } - - bool OutsideHeapRegion() => endAddr <= HeapRegionStart || address >= HeapRegionEnd; - bool OutsideAliasRegion() => endAddr <= AliasRegionStart || address >= AliasRegionEnd; - - switch (state) - { - case MemoryState.Io: - case MemoryState.Normal: - case MemoryState.CodeStatic: - case MemoryState.CodeMutable: - case MemoryState.SharedMemory: - case MemoryState.ModCodeStatic: - case MemoryState.ModCodeMutable: - case MemoryState.Stack: - case MemoryState.ThreadLocal: - case MemoryState.TransferMemoryIsolated: - case MemoryState.TransferMemory: - case MemoryState.ProcessMemory: - case MemoryState.CodeReadOnly: - case MemoryState.CodeWritable: - return InsideRegion() && OutsideHeapRegion() && OutsideAliasRegion(); - - case MemoryState.Heap: - return InsideRegion() && OutsideAliasRegion(); - - case MemoryState.IpcBuffer0: - case MemoryState.IpcBuffer1: - case MemoryState.IpcBuffer3: - return InsideRegion() && OutsideHeapRegion(); - - case MemoryState.KernelStack: - return InsideRegion(); - } - - throw new ArgumentException($"Invalid state value \"{state}\"."); - } - - private ulong GetBaseAddress(MemoryState state) - { - switch (state) - { - case MemoryState.Io: - case MemoryState.Normal: - case MemoryState.ThreadLocal: - return TlsIoRegionStart; - - case MemoryState.CodeStatic: - case MemoryState.CodeMutable: - case MemoryState.SharedMemory: - case MemoryState.ModCodeStatic: - case MemoryState.ModCodeMutable: - case MemoryState.TransferMemoryIsolated: - case MemoryState.TransferMemory: - case MemoryState.ProcessMemory: - case MemoryState.CodeReadOnly: - case MemoryState.CodeWritable: - return GetAddrSpaceBaseAddr(); - - case MemoryState.Heap: - return HeapRegionStart; - - case MemoryState.IpcBuffer0: - case MemoryState.IpcBuffer1: - case MemoryState.IpcBuffer3: - return AliasRegionStart; - - case MemoryState.Stack: - return StackRegionStart; - - case MemoryState.KernelStack: - return AddrSpaceStart; - } - - throw new ArgumentException($"Invalid state value \"{state}\"."); - } - - private ulong GetSize(MemoryState state) - { - switch (state) - { - case MemoryState.Io: - case MemoryState.Normal: - case MemoryState.ThreadLocal: - return TlsIoRegionEnd - TlsIoRegionStart; - - case MemoryState.CodeStatic: - case MemoryState.CodeMutable: - case MemoryState.SharedMemory: - case MemoryState.ModCodeStatic: - case MemoryState.ModCodeMutable: - case MemoryState.TransferMemoryIsolated: - case MemoryState.TransferMemory: - case MemoryState.ProcessMemory: - case MemoryState.CodeReadOnly: - case MemoryState.CodeWritable: - return GetAddrSpaceSize(); - - case MemoryState.Heap: - return HeapRegionEnd - HeapRegionStart; - - case MemoryState.IpcBuffer0: - case MemoryState.IpcBuffer1: - case MemoryState.IpcBuffer3: - return AliasRegionEnd - AliasRegionStart; - - case MemoryState.Stack: - return StackRegionEnd - StackRegionStart; - - case MemoryState.KernelStack: - return AddrSpaceEnd - AddrSpaceStart; - } - - throw new ArgumentException($"Invalid state value \"{state}\"."); - } - - public ulong GetAddrSpaceBaseAddr() - { - if (AddrSpaceWidth == 36 || AddrSpaceWidth == 39) - { - return 0x8000000; - } - else if (AddrSpaceWidth == 32) - { - return 0x200000; - } - else - { - throw new InvalidOperationException("Invalid address space width!"); - } - } - - public ulong GetAddrSpaceSize() - { - if (AddrSpaceWidth == 36) - { - return 0xff8000000; - } - else if (AddrSpaceWidth == 39) - { - return 0x7ff8000000; - } - else if (AddrSpaceWidth == 32) - { - return 0xffe00000; - } - else - { - throw new InvalidOperationException("Invalid address space width!"); - } - } - - private KernelResult MapPages(ulong address, KPageList pageList, KMemoryPermission permission) - { - ulong currAddr = address; - - KernelResult result = KernelResult.Success; - - foreach (KPageNode pageNode in pageList) - { - result = DoMmuOperation( - currAddr, - pageNode.PagesCount, - pageNode.Address, - true, - permission, - MemoryOperation.MapPa); - - if (result != KernelResult.Success) - { - KMemoryInfo info = FindBlock(currAddr).GetInfo(); - - ulong pagesCount = (address - currAddr) / PageSize; - - result = MmuUnmap(address, pagesCount); - - break; - } - - currAddr += pageNode.PagesCount * PageSize; - } - - return result; - } - - private KernelResult MmuUnmap(ulong address, ulong pagesCount) - { - return DoMmuOperation( - address, - pagesCount, - 0, - false, - KMemoryPermission.None, - MemoryOperation.Unmap); - } - - private KernelResult MmuChangePermission(ulong address, ulong pagesCount, KMemoryPermission permission) - { - return DoMmuOperation( - address, - pagesCount, - 0, - false, - permission, - MemoryOperation.ChangePermRw); - } - - private KernelResult DoMmuOperation( - ulong dstVa, - ulong pagesCount, - ulong srcPa, - bool map, - KMemoryPermission permission, - MemoryOperation operation) - { - if (map != (operation == MemoryOperation.MapPa)) - { - throw new ArgumentException(nameof(map) + " value is invalid for this operation."); - } - - KernelResult result; - - switch (operation) - { - case MemoryOperation.MapPa: - { - ulong size = pagesCount * PageSize; - - _cpuMemory.Map(dstVa, srcPa - DramMemoryMap.DramBase, size); - - result = KernelResult.Success; - - break; - } - - case MemoryOperation.Allocate: - { - KMemoryRegionManager region = GetMemoryRegionManager(); - - result = region.AllocatePages(pagesCount, _aslrDisabled, out KPageList pageList); - - if (result == KernelResult.Success) - { - result = MmuMapPages(dstVa, pageList); - } - - break; - } - - case MemoryOperation.Unmap: - { - ulong size = pagesCount * PageSize; - - _cpuMemory.Unmap(dstVa, size); - - result = KernelResult.Success; - - break; - } - - case MemoryOperation.ChangePermRw: result = KernelResult.Success; break; - case MemoryOperation.ChangePermsAndAttributes: result = KernelResult.Success; break; - - default: throw new ArgumentException($"Invalid operation \"{operation}\"."); - } - - return result; - } - - private KernelResult DoMmuOperation( - ulong address, - ulong pagesCount, - KPageList pageList, - KMemoryPermission permission, - MemoryOperation operation) - { - if (operation != MemoryOperation.MapVa) - { - throw new ArgumentException($"Invalid memory operation \"{operation}\" specified."); - } - - return MmuMapPages(address, pageList); - } - - private KMemoryRegionManager GetMemoryRegionManager() - { - return _context.MemoryRegions[(int)_memRegion]; - } - - private KernelResult MmuMapPages(ulong address, KPageList pageList) - { - foreach (KPageNode pageNode in pageList) - { - ulong size = pageNode.PagesCount * PageSize; - - _cpuMemory.Map(address, pageNode.Address - DramMemoryMap.DramBase, size); - - address += size; - } - - return KernelResult.Success; - } - - public ulong GetDramAddressFromVa(ulong va) - { - return _cpuMemory.GetPhysicalAddress(va); - } - - public ulong ConvertVaToPa(ulong va) - { - if (!TryConvertVaToPa(va, out ulong pa)) - { - throw new ArgumentException($"Invalid virtual address 0x{va:X} specified."); - } - - return pa; - } - - public bool TryConvertVaToPa(ulong va, out ulong pa) - { - pa = DramMemoryMap.DramBase + _cpuMemory.GetPhysicalAddress(va); - - return true; - } - - public static ulong GetDramAddressFromPa(ulong pa) - { - return pa - DramMemoryMap.DramBase; - } - - public long GetMmUsedPages() - { - lock (_blocks) - { - return BitUtils.DivRoundUp(GetMmUsedSize(), PageSize); - } - } - - private long GetMmUsedSize() - { - return _blocks.Count * KMemoryBlockSize; - } - - public bool IsInvalidRegion(ulong address, ulong size) - { - return address + size - 1 > GetAddrSpaceBaseAddr() + GetAddrSpaceSize() - 1; - } - - public bool InsideAddrSpace(ulong address, ulong size) - { - return AddrSpaceStart <= address && address + size - 1 <= AddrSpaceEnd - 1; - } - - public bool InsideAliasRegion(ulong address, ulong size) - { - return address + size > AliasRegionStart && AliasRegionEnd > address; - } - - public bool InsideHeapRegion(ulong address, ulong size) - { - return address + size > HeapRegionStart && HeapRegionEnd > address; - } - - public bool InsideStackRegion(ulong address, ulong size) - { - return address + size > StackRegionStart && StackRegionEnd > address; - } - - public bool OutsideAliasRegion(ulong address, ulong size) - { - return AliasRegionStart > address || address + size - 1 > AliasRegionEnd - 1; - } - - public bool OutsideAddrSpace(ulong address, ulong size) - { - return AddrSpaceStart > address || address + size - 1 > AddrSpaceEnd - 1; - } - - public bool OutsideStackRegion(ulong address, ulong size) - { - return StackRegionStart > address || address + size - 1 > StackRegionEnd - 1; } } -}
\ No newline at end of file +} |