Implement support for page sizes > 4KB (#4252)

* Implement support for page sizes > 4KB

* Check and work around more alignment issues

* Was not meant to change this

* Use MemoryBlock.GetPageSize() value for signal handler code

* Do not take the path for private allocations if host supports 4KB pages

* Add Flags attribute on MemoryMapFlags

* Fix dirty region size with 16kb pages

Would accidentally report a size that was too high (generally 16k instead of 4k, uploading 4x as much data)

Co-authored-by: riperiperi <rhy3756547@hotmail.com>

6 files changed, 131 insertions, 25 deletions

diff --git a/Ryujinx.HLE/HOS/Kernel/KernelContext.cs b/Ryujinx.HLE/HOS/Kernel/KernelContext.cs
index 6c58e197..ccc5c0f0 100644
--- a/Ryujinx.HLE/HOS/Kernel/KernelContext.cs
+++ b/Ryujinx.HLE/HOS/Kernel/KernelContext.cs
@@ -84,7 +84,7 @@ namespace Ryujinx.HLE.HOS.Kernel
                 KernelConstants.UserSlabHeapItemSize,
                 KernelConstants.UserSlabHeapSize);
 
-            memory.Commit(KernelConstants.UserSlabHeapBase - DramMemoryMap.DramBase, KernelConstants.UserSlabHeapSize);
+            CommitMemory(KernelConstants.UserSlabHeapBase - DramMemoryMap.DramBase, KernelConstants.UserSlabHeapSize);
 
             CriticalSection = new KCriticalSection(this);
             Schedulers = new KScheduler[KScheduler.CpuCoresCount];
@@ -119,6 +119,17 @@ namespace Ryujinx.HLE.HOS.Kernel
             new Thread(PreemptionThreadStart) { Name = "HLE.PreemptionThread" }.Start();
         }
 
+        public void CommitMemory(ulong address, ulong size)
+        {
+            ulong alignment = MemoryBlock.GetPageSize();
+            ulong endAddress = address + size;
+
+            address &= ~(alignment - 1);
+            endAddress = (endAddress + (alignment - 1)) & ~(alignment - 1);
+
+            Memory.Commit(address, endAddress - address);
+        }
+
         public ulong NewThreadUid()
         {
             return Interlocked.Increment(ref _threadUid) - 1;
diff --git a/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryRegionManager.cs b/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryRegionManager.cs
index 5e6273b8..4596b15d 100644
--- a/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryRegionManager.cs
+++ b/Ryujinx.HLE/HOS/Kernel/Memory/KMemoryRegionManager.cs
@@ -64,7 +64,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
                 if (address != 0)
                 {
                     IncrementPagesReferenceCount(address, pagesCount);
-                    context.Memory.Commit(address - DramMemoryMap.DramBase, pagesCount * KPageTableBase.PageSize);
+                    context.CommitMemory(address - DramMemoryMap.DramBase, pagesCount * KPageTableBase.PageSize);
                 }
 
                 return address;
diff --git a/Ryujinx.HLE/HOS/Kernel/Memory/KPageTable.cs b/Ryujinx.HLE/HOS/Kernel/Memory/KPageTable.cs
index 9b7c99ba..28e9f90a 100644
--- a/Ryujinx.HLE/HOS/Kernel/Memory/KPageTable.cs
+++ b/Ryujinx.HLE/HOS/Kernel/Memory/KPageTable.cs
@@ -1,6 +1,8 @@
 using Ryujinx.Horizon.Common;
 using Ryujinx.Memory;
+using Ryujinx.Memory.Range;
 using System;
+using System.Collections.Generic;
 using System.Diagnostics;
 
 namespace Ryujinx.HLE.HOS.Kernel.Memory
@@ -9,12 +11,20 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
     {
         private readonly IVirtualMemoryManager _cpuMemory;
 
+        protected override bool Supports4KBPages => _cpuMemory.Supports4KBPages;
+
         public KPageTable(KernelContext context, IVirtualMemoryManager cpuMemory) : base(context)
         {
             _cpuMemory = cpuMemory;
         }
 
         /// <inheritdoc/>
+        protected override IEnumerable<HostMemoryRange> GetHostRegions(ulong va, ulong size)
+        {
+            return _cpuMemory.GetHostRegions(va, size);
+        }
+
+        /// <inheritdoc/>
         protected override void GetPhysicalRegions(ulong va, ulong size, KPageList pageList)
         {
             var ranges = _cpuMemory.GetPhysicalRegions(va, size);
@@ -43,7 +53,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
                 return result;
             }
 
-            result = MapPages(dst, pageList, newDstPermission, false, 0);
+            result = MapPages(dst, pageList, newDstPermission, MemoryMapFlags.Private, false, 0);
 
             if (result != Result.Success)
             {
@@ -81,7 +91,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
 
             if (result != Result.Success)
             {
-                Result mapResult = MapPages(dst, dstPageList, oldDstPermission, false, 0);
+                Result mapResult = MapPages(dst, dstPageList, oldDstPermission, MemoryMapFlags.Private, false, 0);
                 Debug.Assert(mapResult == Result.Success);
             }
 
@@ -89,13 +99,20 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
         }
 
         /// <inheritdoc/>
-        protected override Result MapPages(ulong dstVa, ulong pagesCount, ulong srcPa, KMemoryPermission permission, bool shouldFillPages, byte fillValue)
+        protected override Result MapPages(
+            ulong dstVa,
+            ulong pagesCount,
+            ulong srcPa,
+            KMemoryPermission permission,
+            MemoryMapFlags flags,
+            bool shouldFillPages,
+            byte fillValue)
         {
             ulong size = pagesCount * PageSize;
 
-            Context.Memory.Commit(srcPa - DramMemoryMap.DramBase, size);
+            Context.CommitMemory(srcPa - DramMemoryMap.DramBase, size);
 
-            _cpuMemory.Map(dstVa, srcPa - DramMemoryMap.DramBase, size);
+            _cpuMemory.Map(dstVa, srcPa - DramMemoryMap.DramBase, size, flags);
 
             if (DramMemoryMap.IsHeapPhysicalAddress(srcPa))
             {
@@ -111,7 +128,13 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
         }
 
         /// <inheritdoc/>
-        protected override Result MapPages(ulong address, KPageList pageList, KMemoryPermission permission, bool shouldFillPages, byte fillValue)
+        protected override Result MapPages(
+            ulong address,
+            KPageList pageList,
+            KMemoryPermission permission,
+            MemoryMapFlags flags,
+            bool shouldFillPages,
+            byte fillValue)
         {
             using var scopedPageList = new KScopedPageList(Context.MemoryManager, pageList);
 
@@ -122,9 +145,9 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
                 ulong addr = pageNode.Address - DramMemoryMap.DramBase;
                 ulong size = pageNode.PagesCount * PageSize;
 
-                Context.Memory.Commit(addr, size);
+                Context.CommitMemory(addr, size);
 
-                _cpuMemory.Map(currentVa, addr, size);
+                _cpuMemory.Map(currentVa, addr, size, flags);
 
                 if (shouldFillPages)
                 {
@@ -140,6 +163,21 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
         }
 
         /// <inheritdoc/>
+        protected override Result MapForeign(IEnumerable<HostMemoryRange> regions, ulong va, ulong size)
+        {
+            ulong offset = 0;
+
+            foreach (var region in regions)
+            {
+                _cpuMemory.MapForeign(va + offset, region.Address, region.Size);
+
+                offset += region.Size;
+            }
+
+            return Result.Success;
+        }
+
+        /// <inheritdoc/>
         protected override Result Unmap(ulong address, ulong pagesCount)
         {
             KPageList pagesToClose = new KPageList();
@@ -188,4 +226,4 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
             _cpuMemory.Write(va, data);
         }
     }
-}
+}
\ No newline at end of file
diff --git a/Ryujinx.HLE/HOS/Kernel/Memory/KPageTableBase.cs b/Ryujinx.HLE/HOS/Kernel/Memory/KPageTableBase.cs
index e19e22c8..bd7d5725 100644
--- a/Ryujinx.HLE/HOS/Kernel/Memory/KPageTableBase.cs
+++ b/Ryujinx.HLE/HOS/Kernel/Memory/KPageTableBase.cs
@@ -1,6 +1,8 @@
 using Ryujinx.Common;
 using Ryujinx.HLE.HOS.Kernel.Common;
 using Ryujinx.HLE.HOS.Kernel.Process;
+using Ryujinx.Memory;
+using Ryujinx.Memory.Range;
 using Ryujinx.Horizon.Common;
 using System;
 using System.Collections.Generic;
@@ -29,6 +31,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
         private const int MaxBlocksNeededForInsertion = 2;
 
         protected readonly KernelContext Context;
+        protected virtual bool Supports4KBPages => true;
 
         public ulong AddrSpaceStart { get; private set; }
         public ulong AddrSpaceEnd { get; private set; }
@@ -366,7 +369,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
                     return KernelResult.OutOfResource;
                 }
 
-                Result result = MapPages(address, pageList, permission);
+                Result result = MapPages(address, pageList, permission, MemoryMapFlags.None);
 
                 if (result == Result.Success)
                 {
@@ -502,7 +505,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
 
                 if (paIsValid)
                 {
-                    result = MapPages(address, pagesCount, srcPa, permission);
+                    result = MapPages(address, pagesCount, srcPa, permission, MemoryMapFlags.Private);
                 }
                 else
                 {
@@ -565,7 +568,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
 
             using var _ = new OnScopeExit(() => pageList.DecrementPagesReferenceCount(Context.MemoryManager));
 
-            return MapPages(address, pageList, permission);
+            return MapPages(address, pageList, permission, MemoryMapFlags.Private);
         }
 
         public Result MapProcessCodeMemory(ulong dst, ulong src, ulong size)
@@ -746,7 +749,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
                         return KernelResult.InvalidMemState;
                     }
 
-                    result = MapPages(_currentHeapAddr, pageList, KMemoryPermission.ReadAndWrite, true, (byte)_heapFillValue);
+                    result = MapPages(_currentHeapAddr, pageList, KMemoryPermission.ReadAndWrite, MemoryMapFlags.Private, true, (byte)_heapFillValue);
 
                     if (result != Result.Success)
                     {
@@ -1334,7 +1337,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
 
                         ulong currentPagesCount = Math.Min(srcPaPages, dstVaPages);
 
-                        MapPages(dstVa, currentPagesCount, srcPa, KMemoryPermission.ReadAndWrite);
+                        MapPages(dstVa, currentPagesCount, srcPa, KMemoryPermission.ReadAndWrite, MemoryMapFlags.Private);
 
                         dstVa += currentPagesCount * PageSize;
                         srcPa += currentPagesCount * PageSize;
@@ -1878,7 +1881,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
                     Context.Memory.Fill(GetDramAddressFromPa(firstPageFillAddress), unusedSizeAfter, (byte)_ipcFillValue);
                 }
 
-                Result result = MapPages(currentVa, 1, dstFirstPagePa, permission);
+                Result result = MapPages(currentVa, 1, dstFirstPagePa, permission, MemoryMapFlags.Private);
 
                 if (result != Result.Success)
                 {
@@ -1894,10 +1897,19 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
             {
                 ulong alignedSize = endAddrTruncated - addressRounded;
 
-                KPageList pageList = new KPageList();
-                srcPageTable.GetPhysicalRegions(addressRounded, alignedSize, pageList);
+                Result result;
+
+                if (srcPageTable.Supports4KBPages)
+                {
+                    KPageList pageList = new KPageList();
+                    srcPageTable.GetPhysicalRegions(addressRounded, alignedSize, pageList);
 
-                Result result = MapPages(currentVa, pageList, permission);
+                    result = MapPages(currentVa, pageList, permission, MemoryMapFlags.None);
+                }
+                else
+                {
+                    result = MapForeign(srcPageTable.GetHostRegions(addressRounded, alignedSize), currentVa, alignedSize);
+                }
 
                 if (result != Result.Success)
                 {
@@ -1932,7 +1944,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
 
                 Context.Memory.Fill(GetDramAddressFromPa(lastPageFillAddr), unusedSizeAfter, (byte)_ipcFillValue);
 
-                Result result = MapPages(currentVa, 1, dstLastPagePa, permission);
+                Result result = MapPages(currentVa, 1, dstLastPagePa, permission, MemoryMapFlags.Private);
 
                 if (result != Result.Success)
                 {
@@ -2885,6 +2897,16 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
         }
 
         /// <summary>
+        /// Gets the host regions that make up the given virtual address region.
+        /// If any part of the virtual region is unmapped, null is returned.
+        /// </summary>
+        /// <param name="va">Virtual address of the range</param>
+        /// <param name="size">Size of the range</param>
+        /// <returns>The host regions</returns>
+        /// <exception cref="Ryujinx.Memory.InvalidMemoryRegionException">Throw for unhandled invalid or unmapped memory accesses</exception>
+        protected abstract IEnumerable<HostMemoryRange> GetHostRegions(ulong va, ulong size);
+
+        /// <summary>
         /// Gets the physical regions that make up the given virtual address region.
         /// If any part of the virtual region is unmapped, null is returned.
         /// </summary>
@@ -2936,10 +2958,18 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
         /// <param name="pagesCount">Number of pages to map</param>
         /// <param name="srcPa">Physical address where the pages should be mapped. May be ignored if aliasing is not supported</param>
         /// <param name="permission">Permission of the region to be mapped</param>
+        /// <param name="flags">Flags controlling the memory map operation</param>
         /// <param name="shouldFillPages">Indicate if the pages should be filled with the <paramref name="fillValue"/> value</param>
         /// <param name="fillValue">The value used to fill pages when <paramref name="shouldFillPages"/> is set to true</param>
         /// <returns>Result of the mapping operation</returns>
-        protected abstract Result MapPages(ulong dstVa, ulong pagesCount, ulong srcPa, KMemoryPermission permission, bool shouldFillPages = false, byte fillValue = 0);
+        protected abstract Result MapPages(
+            ulong dstVa,
+            ulong pagesCount,
+            ulong srcPa,
+            KMemoryPermission permission,
+            MemoryMapFlags flags,
+            bool shouldFillPages = false,
+            byte fillValue = 0);
 
         /// <summary>
         /// Maps a region of memory into the specified physical memory region.
@@ -2947,10 +2977,26 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
         /// <param name="address">Destination virtual address that should be mapped</param>
         /// <param name="pageList">List of physical memory pages where the pages should be mapped. May be ignored if aliasing is not supported</param>
         /// <param name="permission">Permission of the region to be mapped</param>
+        /// <param name="flags">Flags controlling the memory map operation</param>
         /// <param name="shouldFillPages">Indicate if the pages should be filled with the <paramref name="fillValue"/> value</param>
         /// <param name="fillValue">The value used to fill pages when <paramref name="shouldFillPages"/> is set to true</param>
         /// <returns>Result of the mapping operation</returns>
-        protected abstract Result MapPages(ulong address, KPageList pageList, KMemoryPermission permission, bool shouldFillPages = false, byte fillValue = 0);
+        protected abstract Result MapPages(
+            ulong address,
+            KPageList pageList,
+            KMemoryPermission permission,
+            MemoryMapFlags flags,
+            bool shouldFillPages = false,
+            byte fillValue = 0);
+
+        /// <summary>
+        /// Maps pages into an arbitrary host memory location.
+        /// </summary>
+        /// <param name="regions">Host regions to be mapped into the specified virtual memory region</param>
+        /// <param name="va">Destination virtual address of the range on this page table</param>
+        /// <param name="size">Size of the range</param>
+        /// <returns>Result of the mapping operation</returns>
+        protected abstract Result MapForeign(IEnumerable<HostMemoryRange> regions, ulong va, ulong size);
 
         /// <summary>
         /// Unmaps a region of memory that was previously mapped with one of the page mapping methods.
diff --git a/Ryujinx.HLE/HOS/Kernel/Memory/KSharedMemory.cs b/Ryujinx.HLE/HOS/Kernel/Memory/KSharedMemory.cs
index 2dbaf3cd..5ec3cd72 100644
--- a/Ryujinx.HLE/HOS/Kernel/Memory/KSharedMemory.cs
+++ b/Ryujinx.HLE/HOS/Kernel/Memory/KSharedMemory.cs
@@ -2,6 +2,7 @@ using Ryujinx.Common;
 using Ryujinx.HLE.HOS.Kernel.Common;
 using Ryujinx.HLE.HOS.Kernel.Process;
 using Ryujinx.Horizon.Common;
+using Ryujinx.Memory;
 
 namespace Ryujinx.HLE.HOS.Kernel.Memory
 {
@@ -48,7 +49,17 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
                 return KernelResult.InvalidPermission;
             }
 
-            return memoryManager.MapPages(address, _pageList, MemoryState.SharedMemory, permission);
+            // On platforms with page size > 4 KB, this can fail due to the address not being page aligned,
+            // we can return an error to force the application to retry with a different address.
+
+            try
+            {
+                return memoryManager.MapPages(address, _pageList, MemoryState.SharedMemory, permission);
+            }
+            catch (InvalidMemoryRegionException)
+            {
+                return KernelResult.InvalidMemState;
+            }
         }
 
         public Result UnmapFromProcess(KPageTableBase memoryManager, ulong address, ulong size, KProcess process)
diff --git a/Ryujinx.HLE/HOS/Kernel/Memory/SharedMemoryStorage.cs b/Ryujinx.HLE/HOS/Kernel/Memory/SharedMemoryStorage.cs
index 167e0aa9..c68b7369 100644
--- a/Ryujinx.HLE/HOS/Kernel/Memory/SharedMemoryStorage.cs
+++ b/Ryujinx.HLE/HOS/Kernel/Memory/SharedMemoryStorage.cs
@@ -18,7 +18,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
             {
                 ulong address = pageNode.Address - DramMemoryMap.DramBase;
                 ulong size = pageNode.PagesCount * KPageTableBase.PageSize;
-                context.Memory.Commit(address, size);
+                context.CommitMemory(address, size);
             }
         }