diff options
| author | gdkchan <gab.dark.100@gmail.com> | 2018-11-28 20:18:09 -0200 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2018-11-28 20:18:09 -0200 |
| commit | 00579927e43bf55ee06ae02933c1e755fb4120eb (patch) | |
| tree | 0fd06db7b28e0accf87b465ec6f4dc74691febab /Ryujinx.HLE/HOS/Kernel/KMemoryManager.cs | |
| parent | e7fe7d724778535f8eff390abef54274a343c0b7 (diff) | |
Better process implementation (#491)
* Initial implementation of KProcess
* Some improvements to the memory manager, implement back guest stack trace printing
* Better GetInfo implementation, improve checking in some places with information from process capabilities
* Allow the cpu to read/write from the correct memory locations for accesses crossing a page boundary
* Change long -> ulong for address/size on memory related methods to avoid unnecessary casts
* Attempt at implementing ldr:ro with new KProcess
* Allow BSS with size 0 on ldr:ro
* Add checking for memory block slab heap usage, return errors if full, exit gracefully
* Use KMemoryBlockSize const from KMemoryManager
* Allow all methods to read from non-contiguous locations
* Fix for TransactParcelAuto
* Address PR feedback, additionally fix some small issues related to the KIP loader and implement SVCs GetProcessId, GetProcessList, GetSystemInfo, CreatePort and ManageNamedPort
* Fix wrong check for source pages count from page list on MapPhysicalMemory
* Fix some issues with UnloadNro on ldr:ro
Diffstat (limited to 'Ryujinx.HLE/HOS/Kernel/KMemoryManager.cs')
| -rw-r--r-- | Ryujinx.HLE/HOS/Kernel/KMemoryManager.cs | 2155 |
1 files changed, 1711 insertions, 444 deletions
diff --git a/Ryujinx.HLE/HOS/Kernel/KMemoryManager.cs b/Ryujinx.HLE/HOS/Kernel/KMemoryManager.cs index 4cfb2aa2..0aa21e3f 100644 --- a/Ryujinx.HLE/HOS/Kernel/KMemoryManager.cs +++ b/Ryujinx.HLE/HOS/Kernel/KMemoryManager.cs @@ -1,331 +1,858 @@ using ChocolArm64.Memory; -using Ryujinx.HLE.Memory; +using Ryujinx.Common; using System; using System.Collections.Generic; -using static Ryujinx.HLE.HOS.ErrorCode; - namespace Ryujinx.HLE.HOS.Kernel { class KMemoryManager { public const int PageSize = 0x1000; + private const int KMemoryBlockSize = 0x40; + + //We need 2 blocks for the case where a big block + //needs to be split in 2, plus one block that will be the new one inserted. + private const int MaxBlocksNeededForInsertion = 2; + private LinkedList<KMemoryBlock> Blocks; private MemoryManager CpuMemory; - private ArenaAllocator Allocator; + private Horizon System; + + public ulong AddrSpaceStart { get; private set; } + public ulong AddrSpaceEnd { get; private set; } + + public ulong CodeRegionStart { get; private set; } + public ulong CodeRegionEnd { get; private set; } + + public ulong HeapRegionStart { get; private set; } + public ulong HeapRegionEnd { get; private set; } + + private ulong CurrentHeapAddr; - public long AddrSpaceStart { get; private set; } - public long AddrSpaceEnd { get; private set; } + public ulong AliasRegionStart { get; private set; } + public ulong AliasRegionEnd { get; private set; } - public long CodeRegionStart { get; private set; } - public long CodeRegionEnd { get; private set; } + public ulong StackRegionStart { get; private set; } + public ulong StackRegionEnd { get; private set; } - public long MapRegionStart { get; private set; } - public long MapRegionEnd { get; private set; } + public ulong TlsIoRegionStart { get; private set; } + public ulong TlsIoRegionEnd { get; private set; } - public long HeapRegionStart { get; private set; } - public long HeapRegionEnd { get; private set; } + private ulong HeapCapacity; - public long NewMapRegionStart { get; private set; } - public long NewMapRegionEnd { get; private set; } + public ulong PhysicalMemoryUsage { get; private set; } - public long TlsIoRegionStart { get; private set; } - public long TlsIoRegionEnd { get; private set; } + private MemoryRegion MemRegion; - public long PersonalMmHeapUsage { get; private set; } + private bool AslrDisabled; - private long CurrentHeapAddr; + public int AddrSpaceWidth { get; private set; } - public KMemoryManager(Process Process) + private bool IsKernel; + private bool AslrEnabled; + + private KMemoryBlockAllocator BlockAllocator; + + private int ContextId; + + private MersenneTwister RandomNumberGenerator; + + public KMemoryManager(Horizon System, MemoryManager CpuMemory) { - CpuMemory = Process.Memory; - Allocator = Process.Device.Memory.Allocator; + this.System = System; + this.CpuMemory = CpuMemory; - long CodeRegionSize; - long MapRegionSize; - long HeapRegionSize; - long NewMapRegionSize; - long TlsIoRegionSize; - int AddrSpaceWidth; + Blocks = new LinkedList<KMemoryBlock>(); + } - AddressSpaceType AddrType = AddressSpaceType.Addr39Bits; + private static readonly int[] AddrSpaceSizes = new int[] { 32, 36, 32, 39 }; - if (Process.MetaData != null) + public KernelResult InitializeForProcess( + AddressSpaceType AddrSpaceType, + bool AslrEnabled, + bool AslrDisabled, + MemoryRegion MemRegion, + ulong Address, + ulong Size, + KMemoryBlockAllocator BlockAllocator) + { + if ((uint)AddrSpaceType > (uint)AddressSpaceType.Addr39Bits) { - AddrType = (AddressSpaceType)Process.MetaData.AddressSpaceWidth; + throw new ArgumentException(nameof(AddrSpaceType)); } - switch (AddrType) + ContextId = System.ContextIdManager.GetId(); + + ulong AddrSpaceBase = 0; + ulong AddrSpaceSize = 1UL << AddrSpaceSizes[(int)AddrSpaceType]; + + KernelResult Result = CreateUserAddressSpace( + AddrSpaceType, + AslrEnabled, + AslrDisabled, + AddrSpaceBase, + AddrSpaceSize, + MemRegion, + Address, + Size, + BlockAllocator); + + if (Result != KernelResult.Success) + { + System.ContextIdManager.PutId(ContextId); + } + + return Result; + } + + private class Region + { + public ulong Start; + public ulong End; + public ulong Size; + public ulong AslrOffset; + } + + private KernelResult CreateUserAddressSpace( + AddressSpaceType AddrSpaceType, + bool AslrEnabled, + bool AslrDisabled, + ulong AddrSpaceStart, + ulong AddrSpaceEnd, + MemoryRegion MemRegion, + ulong Address, + ulong Size, + KMemoryBlockAllocator BlockAllocator) + { + ulong EndAddr = Address + Size; + + Region AliasRegion = new Region(); + Region HeapRegion = new Region(); + Region StackRegion = new Region(); + Region TlsIoRegion = new Region(); + + ulong CodeRegionSize; + ulong StackAndTlsIoStart; + ulong StackAndTlsIoEnd; + ulong BaseAddress; + + switch (AddrSpaceType) { case AddressSpaceType.Addr32Bits: - CodeRegionStart = 0x200000; - CodeRegionSize = 0x3fe00000; - MapRegionSize = 0x40000000; - HeapRegionSize = 0x40000000; - NewMapRegionSize = 0; - TlsIoRegionSize = 0; - AddrSpaceWidth = 32; + AliasRegion.Size = 0x40000000; + HeapRegion.Size = 0x40000000; + StackRegion.Size = 0; + TlsIoRegion.Size = 0; + CodeRegionStart = 0x200000; + CodeRegionSize = 0x3fe00000; + StackAndTlsIoStart = 0x200000; + StackAndTlsIoEnd = 0x40000000; + BaseAddress = 0x200000; + AddrSpaceWidth = 32; break; case AddressSpaceType.Addr36Bits: - CodeRegionStart = 0x8000000; - CodeRegionSize = 0x78000000; - MapRegionSize = 0x180000000; - HeapRegionSize = 0x180000000; - NewMapRegionSize = 0; - TlsIoRegionSize = 0; - AddrSpaceWidth = 36; + AliasRegion.Size = 0x180000000; + HeapRegion.Size = 0x180000000; + StackRegion.Size = 0; + TlsIoRegion.Size = 0; + CodeRegionStart = 0x8000000; + CodeRegionSize = 0x78000000; + StackAndTlsIoStart = 0x8000000; + StackAndTlsIoEnd = 0x80000000; + BaseAddress = 0x8000000; + AddrSpaceWidth = 36; break; - case AddressSpaceType.Addr36BitsNoMap: - CodeRegionStart = 0x200000; - CodeRegionSize = 0x3fe00000; - MapRegionSize = 0; - HeapRegionSize = 0x80000000; - NewMapRegionSize = 0; - TlsIoRegionSize = 0; - AddrSpaceWidth = 36; + case AddressSpaceType.Addr32BitsNoMap: + AliasRegion.Size = 0; + HeapRegion.Size = 0x80000000; + StackRegion.Size = 0; + TlsIoRegion.Size = 0; + CodeRegionStart = 0x200000; + CodeRegionSize = 0x3fe00000; + StackAndTlsIoStart = 0x200000; + StackAndTlsIoEnd = 0x40000000; + BaseAddress = 0x200000; + AddrSpaceWidth = 32; break; case AddressSpaceType.Addr39Bits: - CodeRegionStart = 0x8000000; - CodeRegionSize = 0x80000000; - MapRegionSize = 0x1000000000; - HeapRegionSize = 0x180000000; - NewMapRegionSize = 0x80000000; - TlsIoRegionSize = 0x1000000000; - AddrSpaceWidth = 39; + AliasRegion.Size = 0x1000000000; + HeapRegion.Size = 0x180000000; + StackRegion.Size = 0x80000000; + TlsIoRegion.Size = 0x1000000000; + CodeRegionStart = BitUtils.AlignDown(Address, 0x200000); + CodeRegionSize = BitUtils.AlignUp (EndAddr, 0x200000) - CodeRegionStart; + StackAndTlsIoStart = 0; + StackAndTlsIoEnd = 0; + BaseAddress = 0x8000000; + AddrSpaceWidth = 39; break; - default: throw new InvalidOperationException(); + default: throw new ArgumentException(nameof(AddrSpaceType)); + } + + CodeRegionEnd = CodeRegionStart + CodeRegionSize; + + ulong MapBaseAddress; + ulong MapAvailableSize; + + if (CodeRegionStart - BaseAddress >= AddrSpaceEnd - CodeRegionEnd) + { + //Has more space before the start of the code region. + MapBaseAddress = BaseAddress; + MapAvailableSize = CodeRegionStart - BaseAddress; + } + else + { + //Has more space after the end of the code region. + MapBaseAddress = CodeRegionEnd; + MapAvailableSize = AddrSpaceEnd - CodeRegionEnd; } - AddrSpaceStart = 0; - AddrSpaceEnd = 1L << AddrSpaceWidth; + ulong MapTotalSize = AliasRegion.Size + HeapRegion.Size + StackRegion.Size + TlsIoRegion.Size; + + ulong AslrMaxOffset = MapAvailableSize - MapTotalSize; - CodeRegionEnd = CodeRegionStart + CodeRegionSize; - MapRegionStart = CodeRegionEnd; - MapRegionEnd = CodeRegionEnd + MapRegionSize; - HeapRegionStart = MapRegionEnd; - HeapRegionEnd = MapRegionEnd + HeapRegionSize; - NewMapRegionStart = HeapRegionEnd; - NewMapRegionEnd = HeapRegionEnd + NewMapRegionSize; - TlsIoRegionStart = NewMapRegionEnd; - TlsIoRegionEnd = NewMapRegionEnd + TlsIoRegionSize; + this.AslrEnabled = AslrEnabled; - CurrentHeapAddr = HeapRegionStart; + this.AddrSpaceStart = AddrSpaceStart; + this.AddrSpaceEnd = AddrSpaceEnd; - if (NewMapRegionSize == 0) + this.BlockAllocator = BlockAllocator; + + if (MapAvailableSize < MapTotalSize) { - NewMapRegionStart = AddrSpaceStart; - NewMapRegionEnd = AddrSpaceEnd; + return KernelResult.OutOfMemory; } - Blocks = new LinkedList<KMemoryBlock>(); + if (AslrEnabled) + { + AliasRegion.AslrOffset = GetRandomValue(0, AslrMaxOffset >> 21) << 21; + HeapRegion.AslrOffset = GetRandomValue(0, AslrMaxOffset >> 21) << 21; + StackRegion.AslrOffset = GetRandomValue(0, AslrMaxOffset >> 21) << 21; + TlsIoRegion.AslrOffset = GetRandomValue(0, AslrMaxOffset >> 21) << 21; + } + + //Regions are sorted based on ASLR offset. + //When ASLR is disabled, the order is Map, Heap, NewMap and TlsIo. + AliasRegion.Start = MapBaseAddress + AliasRegion.AslrOffset; + AliasRegion.End = AliasRegion.Start + AliasRegion.Size; + HeapRegion.Start = MapBaseAddress + HeapRegion.AslrOffset; + HeapRegion.End = HeapRegion.Start + HeapRegion.Size; + StackRegion.Start = MapBaseAddress + StackRegion.AslrOffset; + StackRegion.End = StackRegion.Start + StackRegion.Size; + TlsIoRegion.Start = MapBaseAddress + TlsIoRegion.AslrOffset; + TlsIoRegion.End = TlsIoRegion.Start + TlsIoRegion.Size; - long AddrSpacePagesCount = (AddrSpaceEnd - AddrSpaceStart) / PageSize; + SortRegion(HeapRegion, AliasRegion); + + if (StackRegion.Size != 0) + { + SortRegion(StackRegion, AliasRegion); + SortRegion(StackRegion, HeapRegion); + } + else + { + StackRegion.Start = StackAndTlsIoStart; + StackRegion.End = StackAndTlsIoEnd; + } + + if (TlsIoRegion.Size != 0) + { + SortRegion(TlsIoRegion, AliasRegion); + SortRegion(TlsIoRegion, HeapRegion); + SortRegion(TlsIoRegion, StackRegion); + } + else + { + TlsIoRegion.Start = StackAndTlsIoStart; + TlsIoRegion.End = StackAndTlsIoEnd; + } + + AliasRegionStart = AliasRegion.Start; + AliasRegionEnd = AliasRegion.End; + HeapRegionStart = HeapRegion.Start; + HeapRegionEnd = HeapRegion.End; + StackRegionStart = StackRegion.Start; + StackRegionEnd = StackRegion.End; + TlsIoRegionStart = TlsIoRegion.Start; + TlsIoRegionEnd = TlsIoRegion.End; + + CurrentHeapAddr = HeapRegionStart; + HeapCapacity = 0; + PhysicalMemoryUsage = 0; + + this.MemRegion = MemRegion; + this.AslrDisabled = AslrDisabled; + + return InitializeBlocks(AddrSpaceStart, AddrSpaceEnd); + } + + private ulong GetRandomValue(ulong Min, ulong Max) + { + return (ulong)GetRandomValue((long)Min, (long)Max); + } + + private long GetRandomValue(long Min, long Max) + { + if (RandomNumberGenerator == null) + { + RandomNumberGenerator = new MersenneTwister(0); + } + + return RandomNumberGenerator.GenRandomNumber(Min, Max); + } + + private static void SortRegion(Region Lhs, Region Rhs) + { + if (Lhs.AslrOffset < Rhs.AslrOffset) + { + Rhs.Start += Lhs.Size; + Rhs.End += Lhs.Size; + } + else + { + Lhs.Start += Rhs.Size; + Lhs.End += Rhs.Size; + } + } + + private KernelResult InitializeBlocks(ulong AddrSpaceStart, ulong AddrSpaceEnd) + { + //First insertion will always need only a single block, + //because there's nothing else to split. + if (!BlockAllocator.CanAllocate(1)) + { + return KernelResult.OutOfResource; + } + + ulong AddrSpacePagesCount = (AddrSpaceEnd - AddrSpaceStart) / PageSize; InsertBlock(AddrSpaceStart, AddrSpacePagesCount, MemoryState.Unmapped); + + return KernelResult.Success; } - public void HleMapProcessCode(long Position, long Size) + public KernelResult MapPages( + ulong Address, + KPageList PageList, + MemoryState State, + MemoryPermission Permission) { - long PagesCount = Size / PageSize; + ulong PagesCount = PageList.GetPagesCount(); + + ulong Size = PagesCount * PageSize; - if (!Allocator.TryAllocate(Size, out long PA)) + if (!ValidateRegionForState(Address, Size, State)) { - throw new InvalidOperationException(); + return KernelResult.InvalidMemState; } lock (Blocks) { - InsertBlock(Position, PagesCount, MemoryState.CodeStatic, MemoryPermission.ReadAndExecute); + if (!IsUnmapped(Address, PagesCount * PageSize)) + { + return KernelResult.InvalidMemState; + } + + if (!BlockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) + { + return KernelResult.OutOfResource; + } + + KernelResult Result = MapPages(Address, PageList, Permission); + + if (Result == KernelResult.Success) + { + InsertBlock(Address, PagesCount, State, Permission); + } - CpuMemory.Map(Position, PA, Size); + return Result; } } - public long MapProcessCodeMemory(long Dst, long Src, long Size) + public KernelResult UnmapPages(ulong Address, KPageList PageList, MemoryState StateExpected) { + ulong PagesCount = PageList.GetPagesCount(); + + ulong Size = PagesCount * PageSize; + + ulong EndAddr = Address + Size; + + ulong AddrSpacePagesCount = (AddrSpaceEnd - AddrSpaceStart) / PageSize; + + if (AddrSpaceStart > Address) + { + return KernelResult.InvalidMemState; + } + + if (AddrSpacePagesCount < PagesCount) + { + return KernelResult.InvalidMemState; + } + + if (EndAddr - 1 > AddrSpaceEnd - 1) + { + return KernelResult.InvalidMemState; + } + lock (Blocks) { - long PagesCount = Size / PageSize; + KPageList CurrentPageList = new KPageList(); - bool Success = IsUnmapped(Dst, Size); + AddVaRangeToPageList(CurrentPageList, Address, PagesCount); - Success &= CheckRange( - Src, - Size, - MemoryState.Mask, - MemoryState.Heap, - MemoryPermission.Mask, - MemoryPermission.ReadAndWrite, - MemoryAttribute.Mask, - MemoryAttribute.None, - MemoryAttribute.IpcAndDeviceMapped, - out _, - out _, - out _); + if (!CurrentPageList.IsEqual(PageList)) + { + return KernelResult.InvalidMemRange; + } - if (Success) + if (CheckRange( + Address, + Size, + MemoryState.Mask, + StateExpected, + MemoryPermission.None, + MemoryPermission.None, + MemoryAttribute.Mask, + MemoryAttribute.None, + MemoryAttribute.IpcAndDeviceMapped, + out MemoryState State, + out _, + out _)) { - long PA = CpuMemory.GetPhysicalAddress(Src); + if (!BlockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) + { + return KernelResult.OutOfResource; + } - InsertBlock(Dst, PagesCount, MemoryState.CodeStatic, MemoryPermission.ReadAndExecute); - InsertBlock(Src, PagesCount, MemoryState.Heap, MemoryPermission.None); + KernelResult Result = MmuUnmap(Address, PagesCount); - CpuMemory.Map(Dst, PA, Size); + if (Result == KernelResult.Success) + { + InsertBlock(Address, PagesCount, MemoryState.Unmapped); + } - return 0; + return Result; + } + else + { + return KernelResult.InvalidMemState; } } + } + + public KernelResult MapNormalMemory(long Address, long Size, MemoryPermission Permission) + { + //TODO. + return KernelResult.Success; + } - return MakeError(ErrorModule.Kernel, KernelErr.NoAccessPerm); + public KernelResult MapIoMemory(long Address, long Size, MemoryPermission Permission) + { + //TODO. + return KernelResult.Success; } - public long UnmapProcessCodeMemory(long Dst, long Src, long Size) + public KernelResult AllocateOrMapPa( + ulong NeededPagesCount, + int Alignment, + ulong SrcPa, + bool Map, + ulong RegionStart, + ulong RegionPagesCount, + MemoryState State, + MemoryPermission Permission, + out ulong Address) { + Address = 0; + + ulong RegionSize = RegionPagesCount * PageSize; + + ulong RegionEndAddr = RegionStart + RegionSize; + + if (!ValidateRegionForState(RegionStart, RegionSize, State)) + { + return KernelResult.InvalidMemState; + } + + if (RegionPagesCount <= NeededPagesCount) + { + return KernelResult.OutOfMemory; + } + + ulong ReservedPagesCount = IsKernel ? 1UL : 4UL; + lock (Blocks) { - long PagesCount = Size / PageSize; + if (AslrEnabled) + { + ulong TotalNeededSize = (ReservedPagesCount + NeededPagesCount) * PageSize; - bool Success = CheckRange( - Dst, - Size, - MemoryState.Mask, - MemoryState.CodeStatic, - MemoryPermission.None, - MemoryPermission.None, - MemoryAttribute.Mask, - MemoryAttribute.None, - MemoryAttribute.IpcAndDeviceMapped, - out _, - out _, - out _); + ulong RemainingPages = RegionPagesCount - NeededPagesCount; - Success &= CheckRange( - Src, - Size, - MemoryState.Mask, - MemoryState.Heap, - MemoryPermission.Mask, - MemoryPermission.None, - MemoryAttribute.Mask, - MemoryAttribute.None, - MemoryAttribute.IpcAndDeviceMapped, - out _, - out _, - out _); + ulong AslrMaxOffset = ((RemainingPages + ReservedPagesCount) * PageSize) / (ulong)Alignment; - if (Success) + for (int Attempt = 0; Attempt < 8; Attempt++) + { + Address = BitUtils.AlignDown(RegionStart + GetRandomValue(0, AslrMaxOffset) * (ulong)Alignment, Alignment); + + ulong EndAddr = Address + TotalNeededSize; + + KMemoryInfo Info = FindBlock(Address).GetInfo(); + + if (Info.State != MemoryState.Unmapped) + { + continue; + } + + ulong CurrBaseAddr = Info.Address + ReservedPagesCount * PageSize; + ulong CurrEndAddr = Info.Address + Info.Size; + + if (Address >= RegionStart && + Address >= CurrBaseAddr && + EndAddr - 1 <= RegionEndAddr - 1 && + EndAddr - 1 <= CurrEndAddr - 1) + { + break; + } + } + + if (Address == 0) + { + ulong AslrPage = GetRandomValue(0, AslrMaxOffset); + + Address = FindFirstFit( + RegionStart + AslrPage * PageSize, + RegionPagesCount - AslrPage, + NeededPagesCount, + Alignment, + 0, + ReservedPagesCount); + } + } + + if (Address == 0) { - InsertBlock(Dst, PagesCount, MemoryState.Unmapped); - InsertBlock(Src, PagesCount, MemoryState.Heap, MemoryPermission.ReadAndWrite); + Address = FindFirstFit( + RegionStart, + RegionPagesCount, + NeededPagesCount, + Alignment, + 0, + ReservedPagesCount); + } - CpuMemory.Unmap(Dst, Size); + if (Address == 0) + { + return KernelResult.OutOfMemory; + } + + if (!BlockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) + { + return KernelResult.OutOfResource; + } + + MemoryOperation Operation = Map + ? MemoryOperation.MapPa + : MemoryOperation.Allocate; - return 0; + KernelResult Result = DoMmuOperation( + Address, + NeededPagesCount, + SrcPa, + Map, + Permission, + Operation); + + if (Result != KernelResult.Success) + { + return Result; } + + InsertBlock(Address, NeededPagesCount, State, Permission); } - return MakeError(ErrorModule.Kernel, KernelErr.NoAccessPerm); + return KernelResult.Success; } - public void HleMapCustom(long Position, long Size, MemoryState State, MemoryPermission Permission) + public KernelResult MapNewProcessCode( + ulong Address, + ulong PagesCount, + MemoryState State, + MemoryPermission Permission) { - long PagesCount = Size / PageSize; + ulong Size = PagesCount * PageSize; - if (!Allocator.TryAllocate(Size, out long PA)) + if (!ValidateRegionForState(Address, Size, State)) { - throw new InvalidOperationException(); + return KernelResult.InvalidMemState; } lock (Blocks) { - InsertBlock(Position, PagesCount, State, Permission); + if (!IsUnmapped(Address, Size)) + { + return KernelResult.InvalidMemState; + } - CpuMemory.Map(Position, PA, Size); + if (!BlockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) + { + return KernelResult.OutOfResource; + } + + KernelResult Result = DoMmuOperation( + Address, + PagesCount, + 0, + false, + Permission, + MemoryOperation.Allocate); + + if (Result == KernelResult.Success) + { + InsertBlock(Address, PagesCount, State, Permission); + } + + return Result; } } - public long HleMapTlsPage() + public KernelResult MapProcessCodeMemory(ulong Dst, ulong Src, ulong Size) { - bool HasTlsIoRegion = TlsIoRegionStart != TlsIoRegionEnd; - - long Position = HasTlsIoRegion ? TlsIoRegionStart : CodeRegionStart; + ulong PagesCount = Size / PageSize; lock (Blocks) { - while (Position < (HasTlsIoRegion ? TlsIoRegionEnd : CodeRegionEnd)) + bool Success = CheckRange( + Src, + Size, + MemoryState.Mask, + MemoryState.Heap, + MemoryPermission.Mask, + MemoryPermission.ReadAndWrite, + MemoryAttribute.Mask, + MemoryAttribute.None, + MemoryAttribute.IpcAndDeviceMapped, + out MemoryState State, + out MemoryPermission Permission, + out _); + + Success &= IsUnmapped(Dst, Size); + + if (Success) { - if (FindBlock(Position).State == MemoryState.Unmapped) + if (!BlockAllocator.CanAllocate(MaxBlocksNeededForInsertion * 2)) { - InsertBlock(Position, 1, MemoryState.ThreadLocal, MemoryPermission.ReadAndWrite); + return KernelResult.OutOfResource; + } - if (!Allocator.TryAllocate(PageSize, out long PA)) - { - throw new InvalidOperationException(); - } + KPageList PageList = new KPageList(); + + AddVaRangeToPageList(PageList, Src, PagesCount); - CpuMemory.Map(Position, PA, PageSize); + KernelResult Result = MmuChangePermission(Src, PagesCount, MemoryPermission.None); + + if (Result != KernelResult.Success) + { + return Result; + } + + Result = MapPages(Dst, PageList, MemoryPermission.None); + + if (Result != KernelResult.Success) + { + MmuChangePermission(Src, PagesCount, Permission); - return Position; + return Result; } - Position += PageSize; + InsertBlock(Src, PagesCount, State, MemoryPermission.None, MemoryAttribute.Borrowed); + InsertBlock(Dst, PagesCount, MemoryState.ModCodeStatic); + + return KernelResult.Success; + } + else + { + return KernelResult.InvalidMemState; } + } + } + + public KernelResult UnmapProcessCodeMemory(ulong Dst, ulong Src, ulong Size) + { + ulong PagesCount = Size / PageSize; + + lock (Blocks) + { + bool Success = CheckRange( + Src, + Size, + MemoryState.Mask, + MemoryState.Heap, + MemoryPermission.None, + MemoryPermission.None, + MemoryAttribute.Mask, + MemoryAttribute.Borrowed, + MemoryAttribute.IpcAndDeviceMapped, + out _, + out _, + out _); - throw new InvalidOperationException(); + Success &= CheckRange( + Dst, + PageSize, + MemoryState.UnmapProcessCodeMemoryAllowed, + MemoryState.UnmapProcessCodeMemoryAllowed, + MemoryPermission.None, + MemoryPermission.None, + MemoryAttribute.Mask, + MemoryAttribute.None, + MemoryAttribute.IpcAndDeviceMapped, + out MemoryState State, + out _, + out _); + + Success &= CheckRange( + Dst, + Size, + MemoryState.Mask, + State, + MemoryPermission.None, + MemoryPermission.None, + MemoryAttribute.Mask, + MemoryAttribute.None); + + if (Success) + { + KernelResult Result = MmuUnmap(Dst, PagesCount); + + if (Result != KernelResult.Success) + { + return Result; + } + + //TODO: Missing some checks here. + + if (!BlockAllocator.CanAllocate(MaxBlocksNeededForInsertion * 2)) + { + return KernelResult.OutOfResource; + } + + InsertBlock(Dst, PagesCount, MemoryState.Unmapped); + InsertBlock(Src, PagesCount, MemoryState.Heap, MemoryPermission.ReadAndWrite); + + return KernelResult.Success; + } + else + { + return KernelResult.InvalidMemState; + } } } - public long TrySetHeapSize(long Size, out long Position) + public KernelResult SetHeapSize(ulong Size, out ulong Address) { - Position = 0; + Address = 0; - if ((ulong)Size > (ulong)(HeapRegionEnd - HeapRegionStart)) + if (Size > HeapRegionEnd - HeapRegionStart) { - return MakeError(ErrorModule.Kernel, KernelErr.OutOfMemory); + return KernelResult.OutOfMemory; } - bool Success = false; + KProcess CurrentProcess = System.Scheduler.GetCurrentProcess(); - long CurrentHeapSize = GetHeapSize(); + ulong CurrentHeapSize = GetHeapSize(); - if ((ulong)CurrentHeapSize <= (ulong)Size) + if (CurrentHeapSize <= Size) { //Expand. - long DiffSize = Size - CurrentHeapSize; + ulong DiffSize = Size - CurrentHeapSize; lock (Blocks) { - if (Success = IsUnmapped(CurrentHeapAddr, DiffSize)) + if (CurrentProcess.ResourceLimit != null && DiffSize != 0 && + !CurrentProcess.ResourceLimit.Reserve(LimitableResource.Memory, DiffSize)) + { + return KernelResult.ResLimitExceeded; + } + + ulong PagesCount = DiffSize / PageSize; + + KMemoryRegionManager Region = GetMemoryRegionManager(); + + KernelResult Result = Region.AllocatePages(PagesCount, AslrDisabled, out KPageList PageList); + + void CleanUpForError() { - if (!Allocator.TryAllocate(DiffSize, out long PA)) + if (PageList != null) + { + Region.FreePages(PageList); + } + + if (CurrentProcess.ResourceLimit != null && DiffSize != 0) { - return MakeError(ErrorModule.Kernel, KernelErr.OutOfMemory); + CurrentProcess.ResourceLimit.Release(LimitableResource.Memory, DiffSize); } + } + + if (Result != KernelResult.Success) + { + CleanUpForError(); + + return Result; + } - long PagesCount = DiffSize / PageSize; + if (!BlockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) + { + CleanUpForError(); - InsertBlock(CurrentHeapAddr, PagesCount, MemoryState.Heap, MemoryPermission.ReadAndWrite); + return KernelResult.OutOfResource; + } - CpuMemory.Map(CurrentHeapAddr, PA, DiffSize); + if (!IsUnmapped(CurrentHeapAddr, DiffSize)) + { + CleanUpForError(); + + return KernelResult.InvalidMemState; } + + Result = DoMmuOperation( + CurrentHeapAddr, + PagesCount, + PageList, + MemoryPermission.ReadAndWrite, + MemoryOperation.MapVa); + + if (Result != KernelResult.Success) + { + CleanUpForError(); + + return Result; + } + + InsertBlock(CurrentHeapAddr, PagesCount, MemoryState.Heap, MemoryPermission.ReadAndWrite); } } else { //Shrink. - long FreeAddr = HeapRegionStart + Size; - long DiffSize = CurrentHeapSize - Size; + ulong FreeAddr = HeapRegionStart + Size; + ulong DiffSize = CurrentHeapSize - Size; lock (Blocks) { - Success = CheckRange( + if (!BlockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) + { + return KernelResult.OutOfResource; + } + + if (!CheckRange( FreeAddr, DiffSize, MemoryState.Mask, @@ -337,48 +864,66 @@ namespace Ryujinx.HLE.HOS.Kernel MemoryAttribute.IpcAndDeviceMapped, out _, out _, - out _); - - if (Success) + out _)) { - long PagesCount = DiffSize / PageSize; + return KernelResult.InvalidMemState; + } - InsertBlock(FreeAddr, PagesCount, MemoryState.Unmapped); + ulong PagesCount = DiffSize / PageSize; - FreePages(FreeAddr, PagesCount); + KernelResult Result = MmuUnmap(FreeAddr, PagesCount); - CpuMemory.Unmap(FreeAddr, DiffSize); + if (Result != KernelResult.Success) + { + return Result; } + + CurrentProcess.ResourceLimit?.Release(LimitableResource.Memory, BitUtils.AlignDown(DiffSize, PageSize)); + + InsertBlock(FreeAddr, PagesCount, MemoryState.Unmapped); } } CurrentHeapAddr = HeapRegionStart + Size; - if (Success) - { - Position = HeapRegionStart; + Address = HeapRegionStart; - return 0; - } + return KernelResult.Success; + } - return MakeError(ErrorModule.Kernel, KernelErr.NoAccessPerm); + public ulong GetTotalHeapSize() + { + lock (Blocks) + { + return GetHeapSize() + PhysicalMemoryUsage; + } } - public long GetHeapSize() + private ulong GetHeapSize() { return CurrentHeapAddr - HeapRegionStart; } - public long SetMemoryAttribute( - long Position, - long Size, + public KernelResult SetHeapCapacity(ulong Capacity) + { + lock (Blocks) + { + HeapCapacity = Capacity; + } + + return KernelResult.Success; + } + + public KernelResult SetMemoryAttribute( + ulong Address, + ulong Size, MemoryAttribute AttributeMask, MemoryAttribute AttributeValue) { lock (Blocks) { if (CheckRange( - Position, + Address, Size, MemoryState.AttributeChangeAllowed, MemoryState.AttributeChangeAllowed, @@ -391,35 +936,42 @@ namespace Ryujinx.HLE.HOS.Kernel out MemoryPermission Permission, out MemoryAttribute Attribute)) { - long PagesCount = Size / PageSize; + if (!BlockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) + { + return KernelResult.OutOfResource; + } + + ulong PagesCount = Size / PageSize; Attribute &= ~AttributeMask; Attribute |= AttributeMask & AttributeValue; - InsertBlock(Position, PagesCount, State, Permission, Attribute); + InsertBlock(Address, PagesCount, State, Permission, Attribute); - return 0; + return KernelResult.Success; + } + else + { + return KernelResult.InvalidMemState; } } - - return MakeError(ErrorModule.Kernel, KernelErr.NoAccessPerm); } - public KMemoryInfo QueryMemory(long Position) + public KMemoryInfo QueryMemory(ulong Address) { - if ((ulong)Position >= (ulong)AddrSpaceStart && - (ulong)Position < (ulong)AddrSpaceEnd) + if (Address >= AddrSpaceStart && + Address < AddrSpaceEnd) { lock (Blocks) { - return FindBlock(Position).GetInfo(); + return FindBlock(Address).GetInfo(); } } else { return new KMemoryInfo( AddrSpaceEnd, - -AddrSpaceEnd, + ~AddrSpaceEnd + 1, MemoryState.Reserved, MemoryPermission.None, MemoryAttribute.None, @@ -428,7 +980,7 @@ namespace Ryujinx.HLE.HOS.Kernel } } - public long Map(long Src, long Dst, long Size) + public KernelResult Map(ulong Dst, ulong Src, ulong Size) { bool Success; @@ -452,22 +1004,90 @@ namespace Ryujinx.HLE.HOS.Kernel if (Success) { - long PagesCount = Size / PageSize; + if (!BlockAllocator.CanAllocate(MaxBlocksNeededForInsertion * 2)) + { + return KernelResult.OutOfResource; + } - InsertBlock(Src, PagesCount, SrcState, MemoryPermission.None, MemoryAttribute.Borrowed); + ulong PagesCount = Size / PageSize; - InsertBlock(Dst, PagesCount, MemoryState.MappedMemory, MemoryPermission.ReadAndWrite); + KPageList PageList = new KPageList(); - long PA = CpuMemory.GetPhysicalAddress(Src); + AddVaRangeToPageList(PageList, Src, PagesCount); - CpuMemory.Map(Dst, PA, Size); + KernelResult Result = MmuChangePermission(Src, PagesCount, MemoryPermission.None); + + if (Result != KernelResult.Success) + { + return Result; + } + + Result = MapPages(Dst, PageList, MemoryPermission.ReadAndWrite); + + if (Result != KernelResult.Success) + { + if (MmuChangePermission(Src, PagesCount, MemoryPermission.ReadAndWrite) != KernelResult.Success) + { + throw new InvalidOperationException("Unexpected failure reverting memory permission."); + } + + return Result; + } + + InsertBlock(Src, PagesCount, SrcState, MemoryPermission.None, MemoryAttribute.Borrowed); + InsertBlock(Dst, PagesCount, MemoryState.Stack, MemoryPermission.ReadAndWrite); + + return KernelResult.Success; + } + else + { + return KernelResult.InvalidMemState; } } + } + + public KernelResult UnmapForKernel(ulong Address, ulong PagesCount, MemoryState StateExpected) + { + ulong Size = PagesCount * PageSize; + + lock (Blocks) + { + if (CheckRange( + Address, + Size, + MemoryState.Mask, + StateExpected, + MemoryPermission.None, + MemoryPermission.None, + MemoryAttribute.Mask, + MemoryAttribute.None, + MemoryAttribute.IpcAndDeviceMapped, + out _, + out _, + out _)) + { + if (!BlockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) + { + return KernelResult.OutOfResource; + } + + KernelResult Result = MmuUnmap(Address, PagesCount); - return Success ? 0 : MakeError(ErrorModule.Kernel, KernelErr.NoAccessPerm); + if (Result == KernelResult.Success) + { + InsertBlock(Address, PagesCount, MemoryState.Unmapped); + } + + return KernelResult.Success; + } + else + { + return KernelResult.InvalidMemState; + } + } } - public long Unmap(long Src, long Dst, long Size) + public KernelResult Unmap(ulong Dst, ulong Src, ulong Size) { bool Success; @@ -491,87 +1111,70 @@ namespace Ryujinx.HLE.HOS.Kernel Dst, Size, MemoryState.Mask, - MemoryState.MappedMemory, + MemoryState.Stack, MemoryPermission.None, MemoryPermission.None, MemoryAttribute.Mask, MemoryAttribute.None, MemoryAttribute.IpcAndDeviceMapped, out _, - out _, + out MemoryPermission DstPermission, out _); if (Success) { - long PagesCount = Size / PageSize; - - InsertBlock(Src, PagesCount, SrcState, MemoryPermission.ReadAndWrite); + if (!BlockAllocator.CanAllocate(MaxBlocksNeededForInsertion * 2)) + { + return KernelResult.OutOfResource; + } - InsertBlock(Dst, PagesCount, MemoryState.Unmapped); + ulong PagesCount = Size / PageSize; - CpuMemory.Unmap(Dst, Size); - } - } + KPageList SrcPageList = new KPageList(); + KPageList DstPageList = new KPageList(); - return Success ? 0 : MakeError(ErrorModule.Kernel, KernelErr.NoAccessPerm); - } + AddVaRangeToPageList(SrcPageList, Src, PagesCount); + AddVaRangeToPageList(DstPageList, Dst, PagesCount); - public long MapSharedMemory(KSharedMemory SharedMemory, MemoryPermission Permission, long Position) - { - lock (Blocks) - { - if (IsUnmapped(Position, SharedMemory.Size)) - { - long PagesCount = SharedMemory.Size / PageSize; - - InsertBlock(Position, PagesCount, MemoryState.SharedMemory, Permission); + if (!DstPageList.IsEqual(SrcPageList)) + { + return KernelResult.InvalidMemRange; + } - CpuMemory.Map(Position, SharedMemory.PA, SharedMemory.Size); + KernelResult Result = MmuUnmap(Dst, PagesCount); - return 0; - } - } + if (Result != KernelResult.Success) + { + return Result; + } - return MakeError(ErrorModule.Kernel, KernelErr.NoAccessPerm); - } + Result = MmuChangePermission(Src, PagesCount, MemoryPermission.ReadAndWrite); - public long UnmapSharedMemory(long Position, long Size) - { - lock (Blocks) - { - if (CheckRange( - Position, - Size, - MemoryState.Mask, - MemoryState.SharedMemory, - MemoryPermission.None, - MemoryPermission.None, - MemoryAttribute.Mask, - MemoryAttribute.None, - MemoryAttribute.IpcAndDeviceMapped, - out MemoryState State, - out _, - out _)) - { - long PagesCount = Size / PageSize; + if (Result != KernelResult.Success) + { + MapPages(Dst, DstPageList, DstPermission); - InsertBlock(Position, PagesCount, MemoryState.Unmapped); + return Result; + } - CpuMemory.Unmap(Position, Size); + InsertBlock(Src, PagesCount, SrcState, MemoryPermission.ReadAndWrite); + InsertBlock(Dst, PagesCount, MemoryState.Unmapped); - return 0; + return KernelResult.Success; + } + else + { + return KernelResult.InvalidMemState; } } - - return MakeError(ErrorModule.Kernel, KernelErr.NoAccessPerm); } - public long ReserveTransferMemory(long Position, long Size, MemoryPermission Permission) + public KernelResult ReserveTransferMemory(ulong Address, ulong Size, MemoryPermission Permission) { lock (Blocks) { if (CheckRange( - Position, + Address, Size, MemoryState.TransferMemoryAllowed | MemoryState.IsPoolAllocated, MemoryState.TransferMemoryAllowed | MemoryState.IsPoolAllocated, @@ -584,25 +1187,34 @@ namespace Ryujinx.HLE.HOS.Kernel out _, out MemoryAttribute Attribute)) { - long PagesCount = Size / PageSize; + //TODO: Missing checks. + + if (!BlockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) + { + return KernelResult.OutOfResource; + } + + ulong PagesCount = Size / PageSize; Attribute |= MemoryAttribute.Borrowed; - InsertBlock(Position, PagesCount, State, Permission, Attribute); + InsertBlock(Address, PagesCount, State, Permission, Attribute); - return 0; + return KernelResult.Success; + } + else + { + return KernelResult.InvalidMemState; } } - - return MakeError(ErrorModule.Kernel, KernelErr.NoAccessPerm); } - public long ResetTransferMemory(long Position, long Size) + public KernelResult ResetTransferMemory(ulong Address, ulong Size) { lock (Blocks) { if (CheckRange( - Position, + Address, Size, MemoryState.TransferMemoryAllowed | MemoryState.IsPoolAllocated, MemoryState.TransferMemoryAllowed | MemoryState.IsPoolAllocated, @@ -615,23 +1227,30 @@ namespace Ryujinx.HLE.HOS.Kernel out _, out _)) { - long PagesCount = Size / PageSize; + if (!BlockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) + { + return KernelResult.OutOfResource; + } + + ulong PagesCount = Size / PageSize; - InsertBlock(Position, PagesCount, State, MemoryPermission.ReadAndWrite); + InsertBlock(Address, PagesCount, State, MemoryPermission.ReadAndWrite); - return 0; + return KernelResult.Success; + } + else + { + return KernelResult.InvalidMemState; } } - - return MakeError(ErrorModule.Kernel, KernelErr.NoAccessPerm); } - public long SetProcessMemoryPermission(long Position, long Size, MemoryPermission Permission) + public KernelResult SetProcessMemoryPermission(ulong Address, ulong Size, MemoryPermission Permission) { lock (Blocks) { if (CheckRange( - Position, + Address, Size, MemoryState.ProcessPermissionChangeAllowed, MemoryState.ProcessPermissionChangeAllowed, @@ -640,47 +1259,73 @@ namespace Ryujinx.HLE.HOS.Kernel MemoryAttribute.Mask, MemoryAttribute.None, MemoryAttribute.IpcAndDeviceMapped, - out MemoryState State, - out _, + out MemoryState OldState, + out MemoryPermission OldPermission, out _)) { - if (State == MemoryState.CodeStatic) - { - State = MemoryState.CodeMutable; - } - else if (State == MemoryState.ModCodeStatic) + MemoryState NewState = OldState; + + //If writing into the code region is allowed, then we need + //to change it to mutable. + if ((Permission & MemoryPermission.Write) != 0) { - State = MemoryState.ModCodeMutable; + if (OldState == MemoryState.CodeStatic) + { + NewState = MemoryState.CodeMutable; + } + else if (OldState == MemoryState.ModCodeStatic) + { + NewState = MemoryState.ModCodeMutable; + } + else + { + throw new InvalidOperationException($"Memory state \"{OldState}\" not valid for this operation."); + } } - else + + if (NewState != OldState || Permission != OldPermission) { - throw new InvalidOperationException(); - } + if (!BlockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) + { + return KernelResult.OutOfResource; + } - long PagesCount = Size / PageSize; + ulong PagesCount = Size / PageSize; - InsertBlock(Position, PagesCount, State, Permission); + MemoryOperation Operation = (Permission & MemoryPermission.Execute) != 0 + ? MemoryOperation.ChangePermsAndAttributes + : MemoryOperation.ChangePermRw; - return 0; + KernelResult Result = DoMmuOperation(Address, PagesCount, 0, false, Permission, Operation); + + if (Result != KernelResult.Success) + { + return Result; + } + + InsertBlock(Address, PagesCount, NewState, Permission); + } + + return KernelResult.Success; + } + else + { + return KernelResult.InvalidMemState; } } - - return MakeError(ErrorModule.Kernel, KernelErr.NoAccessPerm); } - public long MapPhysicalMemory(long Position, long Size) + public KernelResult MapPhysicalMemory(ulong Address, ulong Size) { - long End = Position + Size; + ulong EndAddr = Address + Size; lock (Blocks) { - long MappedSize = 0; + ulong MappedSize = 0; KMemoryInfo Info; - LinkedListNode<KMemoryBlock> BaseNode = FindBlockNode(Position); - - LinkedListNode<KMemoryBlock> Node = BaseNode; + LinkedListNode<KMemoryBlock> Node = FindBlockNode(Address); do { @@ -688,57 +1333,66 @@ namespace Ryujinx.HLE.HOS.Kernel if (Info.State != MemoryState.Unmapped) { - MappedSize += GetSizeInRange(Info, Position, End); + MappedSize += GetSizeInRange(Info, Address, EndAddr); } Node = Node.Next; } - while ((ulong)(Info.Position + Info.Size) < (ulong)End && Node != null); + while (Info.Address + Info.Size < EndAddr && Node != null); if (MappedSize == Size) { - return 0; + return KernelResult.Success; } - long RemainingSize = Size - MappedSize; + ulong RemainingSize = Size - MappedSize; + + ulong RemainingPages = RemainingSize / PageSize; - if (!Allocator.TryAllocate(RemainingSize, out long PA)) + KProcess CurrentProcess = System.Scheduler.GetCurrentProcess(); + + if (CurrentProcess.ResourceLimit != null && + !CurrentProcess.ResourceLimit.Reserve(LimitableResource.Memory, RemainingSize)) { - return MakeError(ErrorModule.Kernel, KernelErr.OutOfMemory); + return KernelResult.ResLimitExceeded; } - Node = BaseNode; + KMemoryRegionManager Region = GetMemoryRegionManager(); - do - { - Info = Node.Value.GetInfo(); + KernelResult Result = Region.AllocatePages(RemainingPages, AslrDisabled, out KPageList PageList); - if (Info.State == MemoryState.Unmapped) + void CleanUpForError() + { + if (PageList != null) { - long CurrSize = GetSizeInRange(Info, Position, End); + Region.FreePages(PageList); + } - long MapPosition = Info.Position; + CurrentProcess.ResourceLimit?.Release(LimitableResource.Memory, RemainingSize); + } - if ((ulong)MapPosition < (ulong)Position) - { - MapPosition = Position; - } + if (Result != KernelResult.Success) + { + CleanUpForError(); - CpuMemory.Map(MapPosition, PA, CurrSize); + return Result; + } - PA += CurrSize; - } + if (!BlockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) + { + CleanUpForError(); - Node = Node.Next; + return KernelResult.OutOfResource; } - while ((ulong)(Info.Position + Info.Size) < (ulong)End && Node != null); - PersonalMmHeapUsage += RemainingSize; + MapPhysicalMemory(PageList, Address, EndAddr); - long PagesCount = Size / PageSize; + PhysicalMemoryUsage += RemainingSize; + + ulong PagesCount = Size / PageSize; InsertBlock( - Position, + Address, PagesCount, MemoryState.Unmapped, MemoryPermission.None, @@ -748,22 +1402,26 @@ namespace Ryujinx.HLE.HOS.Kernel MemoryAttribute.None); } - return 0; + return KernelResult.Success; } - public long UnmapPhysicalMemory(long Position, long Size) + public KernelResult UnmapPhysicalMemory(ulong Address, ulong Size) { - long End = Position + Size; + ulong EndAddr = Address + Size; lock (Blocks) { - long HeapMappedSize = 0; + //Scan, ensure that the region can be unmapped (all blocks are heap or + //already unmapped), fill pages list for freeing memory. + ulong HeapMappedSize = 0; - long CurrPosition = Position; + KPageList PageList = new KPageList(); KMemoryInfo Info; - LinkedListNode<KMemoryBlock> Node = FindBlockNode(CurrPosition); + LinkedListNode<KMemoryBlock> BaseNode = FindBlockNode(Address); + + LinkedListNode<KMemoryBlock> Node = BaseNode; do { @@ -773,90 +1431,200 @@ namespace Ryujinx.HLE.HOS.Kernel { if (Info.Attribute != MemoryAttribute.None) { - return MakeError(ErrorModule.Kernel, KernelErr.NoAccessPerm); + return KernelResult.InvalidMemState; } - HeapMappedSize += GetSizeInRange(Info, Position, End); + ulong BlockSize = GetSizeInRange(Info, Address, EndAddr); + ulong BlockAddress = GetAddrInRange(Info, Address); + + AddVaRangeToPageList(PageList, BlockAddress, BlockSize / PageSize); + + HeapMappedSize += BlockSize; } else if (Info.State != MemoryState.Unmapped) { - return MakeError(ErrorModule.Kernel, KernelErr.NoAccessPerm); + return KernelResult.InvalidMemState; } Node = Node.Next; } - while ((ulong)(Info.Position + Info.Size) < (ulong)End && Node != null); + while (Info.Address + Info.Size < EndAddr && Node != null); if (HeapMappedSize == 0) { - return 0; + return KernelResult.Success; + } + + if (!BlockAllocator.CanAllocate(MaxBlocksNeededForInsertion)) + { + return KernelResult.OutOfResource; } - PersonalMmHeapUsage -= HeapMappedSize; + //Try to unmap all the heap mapped memory inside range. + KernelResult Result = KernelResult.Success; - long PagesCount = Size / PageSize; + Node = BaseNode; - InsertBlock(Position, PagesCount, MemoryState.Unmapped); + do + { + Info = Node.Value.GetInfo(); + + if (Info.State == MemoryState.Heap) + { + ulong BlockSize = GetSizeInRange(Info, Address, EndAddr); + ulong BlockAddress = GetAddrInRange(Info, Address); - FreePages(Position, PagesCount); + ulong BlockPagesCount = BlockSize / PageSize; - CpuMemory.Unmap(Position, Size); + Result = MmuUnmap(BlockAddress, BlockPagesCount); - return 0; + if (Result != KernelResult.Success) + { + //If we failed to unmap, we need to remap everything back again. + MapPhysicalMemory(PageList, Address, BlockAddress + BlockSize); + + break; + } + } + + Node = Node.Next; + } + while (Info.Address + Info.Size < EndAddr && Node != null); + + if (Result == KernelResult.Success) + { + GetMemoryRegionManager().FreePages(PageList); + + PhysicalMemoryUsage -= HeapMappedSize; + + KProcess CurrentProcess = System.Scheduler.GetCurrentProcess(); + + CurrentProcess.ResourceLimit?.Release(LimitableResource.Memory, HeapMappedSize); + + ulong PagesCount = Size / PageSize; + + InsertBlock(Address, PagesCount, MemoryState.Unmapped); + } + + return Result; } } - private long GetSizeInRange(KMemoryInfo Info, long Start, long End) + private void MapPhysicalMemory(KPageList PageList, ulong Address, ulong EndAddr) { - long CurrEnd = Info.Size + Info.Position; - long CurrSize = Info.Size; + KMemoryInfo Info; - if ((ulong)Info.Position < (ulong)Start) - { - CurrSize -= Start - Info.Position; - } + LinkedListNode<KMemoryBlock> Node = FindBlockNode(Address); - if ((ulong)CurrEnd > (ulong)End) + LinkedListNode<KPageNode> PageListNode = PageList.Nodes.First; + + KPageNode PageNode = PageListNode.Value; + + ulong SrcPa = PageNode.Address; + ulong SrcPaPages = PageNode.PagesCount; + + do { - CurrSize -= CurrEnd - End; - } + Info = Node.Value.GetInfo(); + + if (Info.State == MemoryState.Unmapped) + { + ulong BlockSize = GetSizeInRange(Info, Address, EndAddr); + + ulong DstVaPages = BlockSize / PageSize; + + ulong DstVa = GetAddrInRange(Info, Address); + + while (DstVaPages > 0) + { + if (SrcPaPages == 0) + { + PageListNode = PageListNode.Next; + + PageNode = PageListNode.Value; + + SrcPa = PageNode.Address; + SrcPaPages = PageNode.PagesCount; + } + + ulong PagesCount = SrcPaPages; + + if (PagesCount > DstVaPages) + { + PagesCount = DstVaPages; + } + + DoMmuOperation( + DstVa, + PagesCount, + SrcPa, + true, + MemoryPermission.ReadAndWrite, + MemoryOperation.MapPa); + + DstVa += PagesCount * PageSize; + SrcPa += PagesCount * PageSize; + SrcPaPages -= PagesCount; + DstVaPages -= PagesCount; + } + } - return CurrSize; + Node = Node.Next; + } + while (Info.Address + Info.Size < EndAddr && Node != null); } - private void FreePages(long Position, long PagesCount) + private static ulong GetSizeInRange(KMemoryInfo Info, ulong Start, ulong End) { - for (long Page = 0; Page < PagesCount; Page++) + ulong EndAddr = Info.Size + Info.Address; + ulong Size = Info.Size; + + if (Info.Address < Start) { - long VA = Position + Page * PageSize; + Size -= Start - Info.Address; + } - if (!CpuMemory.IsMapped(VA)) - { - continue; - } + if (EndAddr > End) + { + Size -= EndAddr - End; + } - long PA = CpuMemory.GetPhysicalAddress(VA); + return Size; + } - Allocator.Free(PA, PageSize); + private static ulong GetAddrInRange(KMemoryInfo Info, ulong Start) + { + if (Info.Address < Start) + { + return Start; } + + return Info.Address; } - public bool HleIsUnmapped(long Position, long Size) + private void AddVaRangeToPageList(KPageList PageList, ulong Start, ulong PagesCount) { - bool Result = false; + ulong Address = Start; - lock (Blocks) + while (Address < Start + PagesCount * PageSize) { - Result = IsUnmapped(Position, Size); - } + KernelResult Result = ConvertVaToPa(Address, out ulong Pa); - return Result; + if (Result != KernelResult.Success) + { + throw new InvalidOperationException("Unexpected failure translating virtual address."); + } + + PageList.AddRange(Pa, 1); + + Address += PageSize; + } } - private bool IsUnmapped(long Position, long Size) + private bool IsUnmapped(ulong Address, ulong Size) { return CheckRange( - Position, + Address, Size, MemoryState.Mask, MemoryState.Unmapped, @@ -871,8 +1639,8 @@ namespace Ryujinx.HLE.HOS.Kernel } private bool CheckRange( - long Position, - long Size, + ulong Address, + ulong Size, MemoryState StateMask, MemoryState StateExpected, MemoryPermission PermissionMask, @@ -884,24 +1652,44 @@ namespace Ryujinx.HLE.HOS.Kernel out MemoryPermission OutPermission, out MemoryAttribute OutAttribute) { - KMemoryInfo BlkInfo = FindBlock(Position).GetInfo(); + ulong EndAddr = Address + Size - 1; + + LinkedListNode<KMemoryBlock> Node = FindBlockNode(Address); + + KMemoryInfo Info = Node.Value.GetInfo(); - ulong Start = (ulong)Position; - ulong End = (ulong)Size + Start; + MemoryState FirstState = Info.State; + MemoryPermission FirstPermission = Info.Permission; + MemoryAttribute FirstAttribute = Info.Attribute; - if (End <= (ulong)(BlkInfo.Position + BlkInfo.Size)) + do { - if ((BlkInfo.Attribute & AttributeMask) == AttributeExpected && - (BlkInfo.State & StateMask) == StateExpected && - (BlkInfo.Permission & PermissionMask) == PermissionExpected) + Info = Node.Value.GetInfo(); + + //Check if the block state matches what we expect. + if ( FirstState != Info.State || + FirstPermission != Info.Permission || + (Info.Attribute & AttributeMask) != AttributeExpected || + (FirstAttribute | AttributeIgnoreMask) != (Info.Attribute | AttributeIgnoreMask) || + (FirstState & StateMask) != StateExpected || + (FirstPermission & PermissionMask) != PermissionExpected) + { + break; + } + + //Check if this is the last block on the range, if so return success. + if (EndAddr <= Info.Address + Info.Size - 1) { - OutState = BlkInfo.State; - OutPermission = BlkInfo.Permission; - OutAttribute = BlkInfo.Attribute & ~AttributeIgnoreMask; + OutState = FirstState; + OutPermission = FirstPermission; + OutAttribute = FirstAttribute & ~AttributeIgnoreMask; return true; } + + Node = Node.Next; } + while (Node != null); OutState = MemoryState.Unmapped; OutPermission = MemoryPermission.None; @@ -910,9 +1698,48 @@ namespace Ryujinx.HLE.HOS.Kernel return false; } + private bool CheckRange( + ulong Address, + ulong Size, + MemoryState StateMask, + MemoryState StateExpected, + MemoryPermission PermissionMask, + MemoryPermission PermissionExpected, + MemoryAttribute AttributeMask, + MemoryAttribute AttributeExpected) + { + ulong EndAddr = Address + Size - 1; + + LinkedListNode<KMemoryBlock> Node = FindBlockNode(Address); + + do + { + KMemoryInfo Info = Node.Value.GetInfo(); + + //Check if the block state matches what we expect. + if ((Info.State & StateMask) != StateExpected || + (Info.Permission & PermissionMask) != PermissionExpected || + (Info.Attribute & AttributeMask) != AttributeExpected) + { + break; + } + + //Check if this is the last block on the range, if so return success. + if (EndAddr <= Info.Address + Info.Size - 1) + { + return true; + } + + Node = Node.Next; + } + while (Node != null); + + return false; + } + private void InsertBlock( - long BasePosition, - long PagesCount, + ulong BaseAddress, + ulong PagesCount, MemoryState OldState, MemoryPermission OldPermission, MemoryAttribute OldAttribute, @@ -923,10 +1750,11 @@ namespace Ryujinx.HLE.HOS.Kernel //Insert new block on the list only on areas where the state //of the block matches the state specified on the Old* state //arguments, otherwise leave it as is. + int OldCount = Blocks.Count; + OldAttribute |= MemoryAttribute.IpcAndDeviceMapped; - ulong Start = (ulong)BasePosition; - ulong End = (ulong)PagesCount * PageSize + Start; + ulong EndAddr = PagesCount * PageSize + BaseAddress; LinkedListNode<KMemoryBlock> Node = Blocks.First; @@ -937,10 +1765,10 @@ namespace Ryujinx.HLE.HOS.Kernel KMemoryBlock CurrBlock = Node.Value; - ulong CurrStart = (ulong)CurrBlock.BasePosition; - ulong CurrEnd = (ulong)CurrBlock.PagesCount * PageSize + CurrStart; + ulong CurrBaseAddr = CurrBlock.BaseAddress; + ulong CurrEndAddr = CurrBlock.PagesCount * PageSize + CurrBaseAddr; - if (Start < CurrEnd && CurrStart < End) + if (BaseAddress < CurrEndAddr && CurrBaseAddr < EndAddr) { MemoryAttribute CurrBlockAttr = CurrBlock.Attribute | MemoryAttribute.IpcAndDeviceMapped; @@ -953,36 +1781,36 @@ namespace Ryujinx.HLE.HOS.Kernel continue; } - if (CurrStart >= Start && CurrEnd <= End) + if (CurrBaseAddr >= BaseAddress && CurrEndAddr <= EndAddr) { CurrBlock.State = NewState; CurrBlock.Permission = NewPermission; CurrBlock.Attribute &= ~MemoryAttribute.IpcAndDeviceMapped; CurrBlock.Attribute |= NewAttribute; } - else if (CurrStart >= Start) + else if (CurrBaseAddr >= BaseAddress) { - CurrBlock.BasePosition = (long)End; + CurrBlock.BaseAddress = EndAddr; - CurrBlock.PagesCount = (long)((CurrEnd - End) / PageSize); + CurrBlock.PagesCount = (CurrEndAddr - EndAddr) / PageSize; - long NewPagesCount = (long)((End - CurrStart) / PageSize); + ulong NewPagesCount = (EndAddr - CurrBaseAddr) / PageSize; NewNode = Blocks.AddBefore(Node, new KMemoryBlock( - (long)CurrStart, + CurrBaseAddr, NewPagesCount, NewState, NewPermission, NewAttribute)); } - else if (CurrEnd <= End) + else if (CurrEndAddr <= EndAddr) { - CurrBlock.PagesCount = (long)((Start - CurrStart) / PageSize); + CurrBlock.PagesCount = (BaseAddress - CurrBaseAddr) / PageSize; - long NewPagesCount = (long)((CurrEnd - Start) / PageSize); + ulong NewPagesCount = (CurrEndAddr - BaseAddress) / PageSize; NewNode = Blocks.AddAfter(Node, new KMemoryBlock( - BasePosition, + BaseAddress, NewPagesCount, NewState, NewPermission, @@ -990,19 +1818,19 @@ namespace Ryujinx.HLE.HOS.Kernel } else { - CurrBlock.PagesCount = (long)((Start - CurrStart) / PageSize); + CurrBlock.PagesCount = (BaseAddress - CurrBaseAddr) / PageSize; - long NextPagesCount = (long)((CurrEnd - End) / PageSize); + ulong NextPagesCount = (CurrEndAddr - EndAddr) / PageSize; NewNode = Blocks.AddAfter(Node, new KMemoryBlock( - BasePosition, + BaseAddress, PagesCount, NewState, NewPermission, NewAttribute)); Blocks.AddAfter(NewNode, new KMemoryBlock( - (long)End, + EndAddr, NextPagesCount, CurrBlock.State, CurrBlock.Permission, @@ -1016,21 +1844,24 @@ namespace Ryujinx.HLE.HOS.Kernel Node = NextNode; } + + BlockAllocator.Count += Blocks.Count - OldCount; } private void InsertBlock( - long BasePosition, - long PagesCount, + ulong BaseAddress, + ulong PagesCount, MemoryState State, MemoryPermission Permission = MemoryPermission.None, MemoryAttribute Attribute = MemoryAttribute.None) { //Inserts new block at the list, replacing and spliting //existing blocks as needed. - KMemoryBlock Block = new KMemoryBlock(BasePosition, PagesCount, State, Permission, Attribute); + KMemoryBlock Block = new KMemoryBlock(BaseAddress, PagesCount, State, Permission, Attribute); + + int OldCount = Blocks.Count; - ulong Start = (ulong)BasePosition; - ulong End = (ulong)PagesCount * PageSize + Start; + ulong EndAddr = PagesCount * PageSize + BaseAddress; LinkedListNode<KMemoryBlock> NewNode = null; @@ -1042,26 +1873,26 @@ namespace Ryujinx.HLE.HOS.Kernel LinkedListNode<KMemoryBlock> NextNode = Node.Next; - ulong CurrStart = (ulong)CurrBlock.BasePosition; - ulong CurrEnd = (ulong)CurrBlock.PagesCount * PageSize + CurrStart; + ulong CurrBaseAddr = CurrBlock.BaseAddress; + ulong CurrEndAddr = CurrBlock.PagesCount * PageSize + CurrBaseAddr; - if (Start < CurrEnd && CurrStart < End) + if (BaseAddress < CurrEndAddr && CurrBaseAddr < EndAddr) { - if (Start >= CurrStart && End <= CurrEnd) + if (BaseAddress >= CurrBaseAddr && EndAddr <= CurrEndAddr) { Block.Attribute |= CurrBlock.Attribute & MemoryAttribute.IpcAndDeviceMapped; } - if (Start > CurrStart && End < CurrEnd) + if (BaseAddress > CurrBaseAddr && EndAddr < CurrEndAddr) { - CurrBlock.PagesCount = (long)((Start - CurrStart) / PageSize); + CurrBlock.PagesCount = (BaseAddress - CurrBaseAddr) / PageSize; - long NextPagesCount = (long)((CurrEnd - End) / PageSize); + ulong NextPagesCount = (CurrEndAddr - EndAddr) / PageSize; NewNode = Blocks.AddAfter(Node, Block); Blocks.AddAfter(NewNode, new KMemoryBlock( - (long)End, + EndAddr, NextPagesCount, CurrBlock.State, CurrBlock.Permission, @@ -1069,20 +1900,20 @@ namespace Ryujinx.HLE.HOS.Kernel break; } - else if (Start <= CurrStart && End < CurrEnd) + else if (BaseAddress <= CurrBaseAddr && EndAddr < CurrEndAddr) { - CurrBlock.BasePosition = (long)End; + CurrBlock.BaseAddress = EndAddr; - CurrBlock.PagesCount = (long)((CurrEnd - End) / PageSize); + CurrBlock.PagesCount = (CurrEndAddr - EndAddr) / PageSize; if (NewNode == null) { NewNode = Blocks.AddBefore(Node, Block); } } - else if (Start > CurrStart && End >= CurrEnd) + else if (BaseAddress > CurrBaseAddr && EndAddr >= CurrEndAddr) { - CurrBlock.PagesCount = (long)((Start - CurrStart) / PageSize); + CurrBlock.PagesCount = (BaseAddress - CurrBaseAddr) / PageSize; if (NewNode == null) { @@ -1109,14 +1940,15 @@ namespace Ryujinx.HLE.HOS.Kernel } MergeEqualStateNeighbours(NewNode); + + BlockAllocator.Count += Blocks.Count - OldCount; } private void MergeEqualStateNeighbours(LinkedListNode<KMemoryBlock> Node) { KMemoryBlock Block = Node.Value; - ulong Start = (ulong)Block.BasePosition; - ulong End = (ulong)Block.PagesCount * PageSize + Start; + ulong EndAddr = Block.PagesCount * PageSize + Block.BaseAddress; if (Node.Previous != null) { @@ -1126,9 +1958,7 @@ namespace Ryujinx.HLE.HOS.Kernel { Blocks.Remove(Node.Previous); - Block.BasePosition = Previous.BasePosition; - - Start = (ulong)Block.BasePosition; + Block.BaseAddress = Previous.BaseAddress; } } @@ -1140,31 +1970,84 @@ namespace Ryujinx.HLE.HOS.Kernel { Blocks.Remove(Node.Next); - End = (ulong)(Next.BasePosition + Next.PagesCount * PageSize); + EndAddr = Next.BaseAddress + Next.PagesCount * PageSize; } } - Block.PagesCount = (long)((End - Start) / PageSize); + Block.PagesCount = (EndAddr - Block.BaseAddress) / PageSize; } - private static bool BlockStateEquals(KMemoryBlock LHS, KMemoryBlock RHS) + private static bool BlockStateEquals(KMemoryBlock Lhs, KMemoryBlock Rhs) { - return LHS.State == RHS.State && - LHS.Permission == RHS.Permission && - LHS.Attribute == RHS.Attribute && - LHS.DeviceRefCount == RHS.DeviceRefCount && - LHS.IpcRefCount == RHS.IpcRefCount; + return Lhs.State == Rhs.State && + Lhs.Permission == Rhs.Permission && + Lhs.Attribute == Rhs.Attribute && + Lhs.DeviceRefCount == Rhs.DeviceRefCount && + Lhs.IpcRefCount == Rhs.IpcRefCount; } - private KMemoryBlock FindBlock(long Position) + private ulong FindFirstFit( + ulong RegionStart, + ulong RegionPagesCount, + ulong NeededPagesCount, + int Alignment, + ulong ReservedStart, + ulong ReservedPagesCount) { - return FindBlockNode(Position)?.Value; + ulong ReservedSize = ReservedPagesCount * PageSize; + + ulong TotalNeededSize = ReservedSize + NeededPagesCount * PageSize; + + ulong RegionEndAddr = RegionStart + RegionPagesCount * PageSize; + + LinkedListNode<KMemoryBlock> Node = FindBlockNode(RegionStart); + + KMemoryInfo Info = Node.Value.GetInfo(); + + while (RegionEndAddr >= Info.Address) + { + if (Info.State == MemoryState.Unmapped) + { + ulong CurrBaseAddr = Info.Address + ReservedSize; + ulong CurrEndAddr = Info.Address + Info.Size - 1; + + ulong Address = BitUtils.AlignDown(CurrBaseAddr, Alignment) + ReservedStart; + + if (CurrBaseAddr > Address) + { + Address += (ulong)Alignment; + } + + ulong AllocationEndAddr = Address + TotalNeededSize - 1; + + if (AllocationEndAddr <= RegionEndAddr && + AllocationEndAddr <= CurrEndAddr && + Address < AllocationEndAddr) + { + return Address; + } + } + + Node = Node.Next; + + if (Node == null) + { + break; + } + + Info = Node.Value.GetInfo(); + } + + return 0; } - private LinkedListNode<KMemoryBlock> FindBlockNode(long Position) + private KMemoryBlock FindBlock(ulong Address) { - ulong Addr = (ulong)Position; + return FindBlockNode(Address)?.Value; + } + private LinkedListNode<KMemoryBlock> FindBlockNode(ulong Address) + { lock (Blocks) { LinkedListNode<KMemoryBlock> Node = Blocks.First; @@ -1173,10 +2056,9 @@ namespace Ryujinx.HLE.HOS.Kernel { KMemoryBlock Block = Node.Value; - ulong Start = (ulong)Block.BasePosition; - ulong End = (ulong)Block.PagesCount * PageSize + Start; + ulong CurrEndAddr = Block.PagesCount * PageSize + Block.BaseAddress; - if (Start <= Addr && End - 1 >= Addr) + if (Block.BaseAddress <= Address && CurrEndAddr - 1 >= Address) { return Node; } @@ -1187,5 +2069,390 @@ namespace Ryujinx.HLE.HOS.Kernel return null; } + + private bool ValidateRegionForState(ulong Address, ulong Size, MemoryState State) + { + ulong EndAddr = Address + Size; + + ulong RegionBaseAddr = GetBaseAddrForState(State); + + ulong RegionEndAddr = RegionBaseAddr + GetSizeForState(State); + + bool InsideRegion() + { + return RegionBaseAddr <= Address && + EndAddr > Address && + EndAddr - 1 <= RegionEndAddr - 1; + } + + bool OutsideHeapRegion() + { + return EndAddr <= HeapRegionStart || + Address >= HeapRegionEnd; + } + + bool OutsideMapRegion() + { + return EndAddr <= AliasRegionStart || + Address >= AliasRegionEnd; + } + + switch (State) + { + case MemoryState.Io: + case MemoryState.Normal: + case MemoryState.CodeStatic: + case MemoryState.CodeMutable: + case MemoryState.SharedMemory: + case MemoryState.ModCodeStatic: + case MemoryState.ModCodeMutable: + case MemoryState.Stack: + case MemoryState.ThreadLocal: + case MemoryState.TransferMemoryIsolated: + case MemoryState.TransferMemory: + case MemoryState.ProcessMemory: + case MemoryState.CodeReadOnly: + case MemoryState.CodeWritable: + return InsideRegion() && OutsideHeapRegion() && OutsideMapRegion(); + + case MemoryState.Heap: + return InsideRegion() && OutsideMapRegion(); + + case MemoryState.IpcBuffer0: + case MemoryState.IpcBuffer1: + case MemoryState.IpcBuffer3: + return InsideRegion() && OutsideHeapRegion(); + + case MemoryState.KernelStack: + return InsideRegion(); + } + + throw new ArgumentException($"Invalid state value \"{State}\"."); + } + + private ulong GetBaseAddrForState(MemoryState State) + { + switch (State) + { + case MemoryState.Io: + case MemoryState.Normal: + case MemoryState.ThreadLocal: + return TlsIoRegionStart; + + case MemoryState.CodeStatic: + case MemoryState.CodeMutable: + case MemoryState.SharedMemory: + case MemoryState.ModCodeStatic: + case MemoryState.ModCodeMutable: + case MemoryState.TransferMemoryIsolated: + case MemoryState.TransferMemory: + case MemoryState.ProcessMemory: + case MemoryState.CodeReadOnly: + case MemoryState.CodeWritable: + return GetAddrSpaceBaseAddr(); + + case MemoryState.Heap: + return HeapRegionStart; + + case MemoryState.IpcBuffer0: + case MemoryState.IpcBuffer1: + case MemoryState.IpcBuffer3: + return AliasRegionStart; + + case MemoryState.Stack: + return StackRegionStart; + + case MemoryState.KernelStack: + return AddrSpaceStart; + } + + throw new ArgumentException($"Invalid state value \"{State}\"."); + } + + private ulong GetSizeForState(MemoryState State) + { + switch (State) + { + case MemoryState.Io: + case MemoryState.Normal: + case MemoryState.ThreadLocal: + return TlsIoRegionEnd - TlsIoRegionStart; + + case MemoryState.CodeStatic: + case MemoryState.CodeMutable: + case MemoryState.SharedMemory: + case MemoryState.ModCodeStatic: + case MemoryState.ModCodeMutable: + case MemoryState.TransferMemoryIsolated: + case MemoryState.TransferMemory: + case MemoryState.ProcessMemory: + case MemoryState.CodeReadOnly: + case MemoryState.CodeWritable: + return GetAddrSpaceSize(); + + case MemoryState.Heap: + return HeapRegionEnd - HeapRegionStart; + + case MemoryState.IpcBuffer0: + case MemoryState.IpcBuffer1: + case MemoryState.IpcBuffer3: + return AliasRegionEnd - AliasRegionStart; + + case MemoryState.Stack: + return StackRegionEnd - StackRegionStart; + + case MemoryState.KernelStack: + return AddrSpaceEnd - AddrSpaceStart; + } + + throw new ArgumentException($"Invalid state value \"{State}\"."); + } + + public ulong GetAddrSpaceBaseAddr() + { + if (AddrSpaceWidth == 36 || AddrSpaceWidth == 39) + { + return 0x8000000; + } + else if (AddrSpaceWidth == 32) + { + return 0x200000; + } + else + { + throw new InvalidOperationException("Invalid address space width!"); + } + } + + public ulong GetAddrSpaceSize() + { + if (AddrSpaceWidth == 36) + { + return 0xff8000000; + } + else if (AddrSpaceWidth == 39) + { + return 0x7ff8000000; + } + else if (AddrSpaceWidth == 32) + { + return 0xffe00000; + } + else + { + throw new InvalidOperationException("Invalid address space width!"); + } + } + + private KernelResult MapPages(ulong Address, KPageList PageList, MemoryPermission Permission) + { + ulong CurrAddr = Address; + + KernelResult Result = KernelResult.Success; + + foreach (KPageNode PageNode in PageList) + { + Result = DoMmuOperation( + CurrAddr, + PageNode.PagesCount, + PageNode.Address, + true, + Permission, + MemoryOperation.MapPa); + + if (Result != KernelResult.Success) + { + KMemoryInfo Info = FindBlock(CurrAddr).GetInfo(); + + ulong PagesCount = (Address - CurrAddr) / PageSize; + + Result = MmuUnmap(Address, PagesCount); + + break; + } + + CurrAddr += PageNode.PagesCount * PageSize; + } + + return Result; + } + + private KernelResult MmuUnmap(ulong Address, ulong PagesCount) + { + return DoMmuOperation( + Address, + PagesCount, + 0, + false, + MemoryPermission.None, + MemoryOperation.Unmap); + } + + private KernelResult MmuChangePermission(ulong Address, ulong PagesCount, MemoryPermission Permission) + { + return DoMmuOperation( + Address, + PagesCount, + 0, + false, + Permission, + MemoryOperation.ChangePermRw); + } + + private KernelResult DoMmuOperation( + ulong DstVa, + ulong PagesCount, + ulong SrcPa, + bool Map, + MemoryPermission Permission, + MemoryOperation Operation) + { + if (Map != (Operation == MemoryOperation.MapPa)) + { + throw new ArgumentException(nameof(Map) + " value is invalid for this operation."); + } + + KernelResult Result; + + switch (Operation) + { + case MemoryOperation.MapPa: + { + ulong Size = PagesCount * PageSize; + + CpuMemory.Map((long)DstVa, (long)(SrcPa - DramMemoryMap.DramBase), (long)Size); + + Result = KernelResult.Success; + + break; + } + + case MemoryOperation.Allocate: + { + KMemoryRegionManager Region = GetMemoryRegionManager(); + + Result = Region.AllocatePages(PagesCount, AslrDisabled, out KPageList PageList); + + if (Result == KernelResult.Success) + { + Result = MmuMapPages(DstVa, PageList); + } + + break; + } + + case MemoryOperation.Unmap: + { + ulong Size = PagesCount * PageSize; + + CpuMemory.Unmap((long)DstVa, (long)Size); + + Result = KernelResult.Success; + + break; + } + + case MemoryOperation.ChangePermRw: Result = KernelResult.Success; break; + case MemoryOperation.ChangePermsAndAttributes: Result = KernelResult.Success; break; + + default: throw new ArgumentException($"Invalid operation \"{Operation}\"."); + } + + return Result; + } + + private KernelResult DoMmuOperation( + ulong Address, + ulong PagesCount, + KPageList PageList, + MemoryPermission Permission, + MemoryOperation Operation) + { + if (Operation != MemoryOperation.MapVa) + { + throw new ArgumentException($"Invalid memory operation \"{Operation}\" specified."); + } + + return MmuMapPages(Address, PageList); + } + + private KMemoryRegionManager GetMemoryRegionManager() + { + return System.MemoryRegions[(int)MemRegion]; + } + + private KernelResult MmuMapPages(ulong Address, KPageList PageList) + { + foreach (KPageNode PageNode in PageList) + { + ulong Size = PageNode.PagesCount * PageSize; + + CpuMemory.Map((long)Address, (long)(PageNode.Address - DramMemoryMap.DramBase), (long)Size); + + Address += Size; + } + + return KernelResult.Success; + } + + public KernelResult ConvertVaToPa(ulong Va, out ulong Pa) + { + Pa = DramMemoryMap.DramBase + (ulong)CpuMemory.GetPhysicalAddress((long)Va); + + return KernelResult.Success; + } + + public long GetMmUsedPages() + { + lock (Blocks) + { + return BitUtils.DivRoundUp(GetMmUsedSize(), PageSize); + } + } + + private long GetMmUsedSize() + { + return Blocks.Count * KMemoryBlockSize; + } + + public bool IsInvalidRegion(ulong Address, ulong Size) + { + return Address + Size - 1 > GetAddrSpaceBaseAddr() + GetAddrSpaceSize() - 1; + } + + public bool InsideAddrSpace(ulong Address, ulong Size) + { + return AddrSpaceStart <= Address && Address + Size - 1 <= AddrSpaceEnd - 1; + } + + public bool InsideAliasRegion(ulong Address, ulong Size) + { + return Address + Size > AliasRegionStart && AliasRegionEnd > Address; + } + + public bool InsideHeapRegion(ulong Address, ulong Size) + { + return Address + Size > HeapRegionStart && HeapRegionEnd > Address; + } + + public bool InsideStackRegion(ulong Address, ulong Size) + { + return Address + Size > StackRegionStart && StackRegionEnd > Address; + } + + public bool OutsideAliasRegion(ulong Address, ulong Size) + { + return AliasRegionStart > Address || Address + Size - 1 > AliasRegionEnd - 1; + } + + public bool OutsideAddrSpace(ulong Address, ulong Size) + { + return AddrSpaceStart > Address || Address + Size - 1 > AddrSpaceEnd - 1; + } + + public bool OutsideStackRegion(ulong Address, ulong Size) + { + return StackRegionStart > Address || Address + Size - 1 > StackRegionEnd - 1; + } } }
\ No newline at end of file |