misc: Implement address space size workarounds (#5191)

* misc: Implement address space size workarounds

This adds code to support userland with less than 39 bits of address
space available by testing reserving multiple sizes and reducing
guess address space when needed.

This is required for ARM64 support when the kernel is
configured to use 63..39 bits for kernel space.(meaning only 38 bits is available to userland)

* Address comments

* Fix 32 bits address space support and address more comments

2 files changed, 48 insertions, 16 deletions

diff --git a/src/Ryujinx.Cpu/AddressSpace.cs b/src/Ryujinx.Cpu/AddressSpace.cs
index 9dc32426..0e27b158 100644
--- a/src/Ryujinx.Cpu/AddressSpace.cs
+++ b/src/Ryujinx.Cpu/AddressSpace.cs
@@ -5,7 +5,7 @@ using System;
 
 namespace Ryujinx.Cpu
 {
-    class AddressSpace : IDisposable
+    public class AddressSpace : IDisposable
     {
         private const ulong PageSize = 0x1000;
 
@@ -154,7 +154,9 @@ namespace Ryujinx.Cpu
         public MemoryBlock Base { get; }
         public MemoryBlock Mirror { get; }
 
-        public AddressSpace(MemoryBlock backingMemory, ulong asSize, bool supports4KBPages)
+        public ulong AddressSpaceSize { get; }
+
+        public AddressSpace(MemoryBlock backingMemory, MemoryBlock baseMemory, MemoryBlock mirrorMemory, ulong addressSpaceSize, bool supports4KBPages)
         {
             if (!supports4KBPages)
             {
@@ -163,17 +165,48 @@ namespace Ryujinx.Cpu
                 _privateTree = new IntrusiveRedBlackTree<PrivateMapping>();
                 _treeLock = new object();
 
-                _mappingTree.Add(new Mapping(0UL, asSize, MappingType.None));
-                _privateTree.Add(new PrivateMapping(0UL, asSize, default));
+                _mappingTree.Add(new Mapping(0UL, addressSpaceSize, MappingType.None));
+                _privateTree.Add(new PrivateMapping(0UL, addressSpaceSize, default));
             }
 
             _backingMemory = backingMemory;
             _supports4KBPages = supports4KBPages;
 
+            Base = baseMemory;
+            Mirror = mirrorMemory;
+            AddressSpaceSize = addressSpaceSize;
+        }
+
+        public static bool TryCreate(MemoryBlock backingMemory, ulong asSize, bool supports4KBPages, out AddressSpace addressSpace)
+        {
+            addressSpace = null;
+
             MemoryAllocationFlags asFlags = MemoryAllocationFlags.Reserve | MemoryAllocationFlags.ViewCompatible;
 
-            Base = new MemoryBlock(asSize, asFlags);
-            Mirror = new MemoryBlock(asSize, asFlags);
+            ulong minAddressSpaceSize = Math.Min(asSize, 1UL << 36);
+
+            // Attempt to create the address space with expected size or try to reduce it until it succeed.
+            for (ulong addressSpaceSize = asSize; addressSpaceSize >= minAddressSpaceSize; addressSpaceSize >>= 1)
+            {
+                MemoryBlock baseMemory = null;
+                MemoryBlock mirrorMemory = null;
+
+                try
+                {
+                    baseMemory = new MemoryBlock(addressSpaceSize, asFlags);
+                    mirrorMemory = new MemoryBlock(addressSpaceSize, asFlags);
+                    addressSpace = new AddressSpace(backingMemory, baseMemory, mirrorMemory, addressSpaceSize, supports4KBPages);
+
+                    break;
+                }
+                catch (OutOfMemoryException)
+                {
+                    baseMemory?.Dispose();
+                    mirrorMemory?.Dispose();
+                }
+            }
+
+            return addressSpace != null;
         }
 
         public void Map(ulong va, ulong pa, ulong size, MemoryMapFlags flags)
diff --git a/src/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs b/src/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs
index 363f9000..3686eb08 100644
--- a/src/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs
+++ b/src/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs
@@ -38,7 +38,8 @@ namespace Ryujinx.Cpu.Jit
         private readonly bool _unsafeMode;
 
         private readonly AddressSpace _addressSpace;
-        private readonly ulong _addressSpaceSize;
+
+        public ulong AddressSpaceSize { get; }
 
         private readonly PageTable<ulong> _pageTable;
 
@@ -62,21 +63,21 @@ namespace Ryujinx.Cpu.Jit
         /// <summary>
         /// Creates a new instance of the host mapped memory manager.
         /// </summary>
-        /// <param name="backingMemory">Physical backing memory where virtual memory will be mapped to</param>
-        /// <param name="addressSpaceSize">Size of the address space</param>
+        /// <param name="addressSpace">Address space instance to use</param>
         /// <param name="unsafeMode">True if unmanaged access should not be masked (unsafe), false otherwise.</param>
         /// <param name="invalidAccessHandler">Optional function to handle invalid memory accesses</param>
-        public MemoryManagerHostMapped(MemoryBlock backingMemory, ulong addressSpaceSize, bool unsafeMode, InvalidAccessHandler invalidAccessHandler = null)
+        public MemoryManagerHostMapped(AddressSpace addressSpace, bool unsafeMode, InvalidAccessHandler invalidAccessHandler)
         {
+            _addressSpace = addressSpace;
             _pageTable = new PageTable<ulong>();
             _invalidAccessHandler = invalidAccessHandler;
             _unsafeMode = unsafeMode;
-            _addressSpaceSize = addressSpaceSize;
+            AddressSpaceSize = addressSpace.AddressSpaceSize;
 
             ulong asSize = PageSize;
             int asBits = PageBits;
 
-            while (asSize < addressSpaceSize)
+            while (asSize < AddressSpaceSize)
             {
                 asSize <<= 1;
                 asBits++;
@@ -86,8 +87,6 @@ namespace Ryujinx.Cpu.Jit
 
             _pageBitmap = new ulong[1 << (AddressSpaceBits - (PageBits + PageToPteShift))];
 
-            _addressSpace = new AddressSpace(backingMemory, asSize, Supports4KBPages);
-
             Tracking = new MemoryTracking(this, (int)MemoryBlock.GetPageSize(), invalidAccessHandler);
             _memoryEh = new MemoryEhMeilleure(_addressSpace.Base, _addressSpace.Mirror, Tracking);
         }
@@ -99,7 +98,7 @@ namespace Ryujinx.Cpu.Jit
         /// <returns>True if the virtual address is part of the addressable space</returns>
         private bool ValidateAddress(ulong va)
         {
-            return va < _addressSpaceSize;
+            return va < AddressSpaceSize;
         }
 
         /// <summary>
@@ -111,7 +110,7 @@ namespace Ryujinx.Cpu.Jit
         private bool ValidateAddressAndSize(ulong va, ulong size)
         {
             ulong endVa = va + size;
-            return endVa >= va && endVa >= size && endVa <= _addressSpaceSize;
+            return endVa >= va && endVa >= size && endVa <= AddressSpaceSize;
         }
 
         /// <summary>