IPC refactor part 2: Use ReplyAndReceive on HLE services and remove special handling from kernel (#1458)

* IPC refactor part 2: Use ReplyAndReceive on HLE services and remove special handling from kernel

* Fix for applet transfer memory + some nits

* Keep handles if possible to avoid server handle table exhaustion

* Fix IPC ZeroFill bug

* am: Correctly implement CreateManagedDisplayLayer and implement CreateManagedDisplaySeparableLayer

CreateManagedDisplaySeparableLayer is requires since 10.x+ when appletResourceUserId != 0

* Make it exit properly

* Make ServiceNotImplementedException show the full message again

* Allow yielding execution to avoid starving other threads

* Only wait if active

* Merge IVirtualMemoryManager and IAddressSpaceManager

* Fix Ro loading data from the wrong process

Co-authored-by: Thog <me@thog.eu>

5 files changed, 625 insertions, 10 deletions

diff --git a/Ryujinx.Memory/AddressSpaceManager.cs b/Ryujinx.Memory/AddressSpaceManager.cs
new file mode 100644
index 00000000..c6d6cab5
--- /dev/null
+++ b/Ryujinx.Memory/AddressSpaceManager.cs
@@ -0,0 +1,549 @@
+using Ryujinx.Common;
+using System;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+namespace Ryujinx.Memory
+{
+    /// <summary>
+    /// Represents a address space manager.
+    /// Supports virtual memory region mapping, address translation and read/write access to mapped regions.
+    /// </summary>
+    public sealed class AddressSpaceManager : IVirtualMemoryManager
+    {
+        public const int PageBits = 12;
+        public const int PageSize = 1 << PageBits;
+        public const int PageMask = PageSize - 1;
+
+        private const int PtLevelBits = 9; // 9 * 4 + 12 = 48 (max address space size)
+        private const int PtLevelSize = 1 << PtLevelBits;
+        private const int PtLevelMask = PtLevelSize - 1;
+
+        private const ulong Unmapped = ulong.MaxValue;
+
+        /// <summary>
+        /// Address space width in bits.
+        /// </summary>
+        public int AddressSpaceBits { get; }
+
+        private readonly ulong _addressSpaceSize;
+
+        private readonly MemoryBlock _backingMemory;
+
+        private readonly ulong[][][][] _pageTable;
+
+        /// <summary>
+        /// Creates a new instance of the memory manager.
+        /// </summary>
+        /// <param name="backingMemory">Physical backing memory where virtual memory will be mapped to</param>
+        /// <param name="addressSpaceSize">Size of the address space</param>
+        public AddressSpaceManager(MemoryBlock backingMemory, ulong addressSpaceSize)
+        {
+            ulong asSize = PageSize;
+            int asBits = PageBits;
+
+            while (asSize < addressSpaceSize)
+            {
+                asSize <<= 1;
+                asBits++;
+            }
+
+            AddressSpaceBits = asBits;
+            _addressSpaceSize = asSize;
+            _backingMemory = backingMemory;
+            _pageTable = new ulong[PtLevelSize][][][];
+        }
+
+        /// <summary>
+        /// Maps a virtual memory range into a physical memory range.
+        /// </summary>
+        /// <remarks>
+        /// Addresses and size must be page aligned.
+        /// </remarks>
+        /// <param name="va">Virtual memory address</param>
+        /// <param name="pa">Physical memory address</param>
+        /// <param name="size">Size to be mapped</param>
+        public void Map(ulong va, ulong pa, ulong size)
+        {
+            while (size != 0)
+            {
+                PtMap(va, pa);
+
+                va += PageSize;
+                pa += PageSize;
+                size -= PageSize;
+            }
+        }
+
+        /// <summary>
+        /// Unmaps a previously mapped range of virtual memory.
+        /// </summary>
+        /// <param name="va">Virtual address of the range to be unmapped</param>
+        /// <param name="size">Size of the range to be unmapped</param>
+        public void Unmap(ulong va, ulong size)
+        {
+            while (size != 0)
+            {
+                PtUnmap(va);
+
+                va += PageSize;
+                size -= PageSize;
+            }
+        }
+
+        /// <summary>
+        /// Reads data from mapped memory.
+        /// </summary>
+        /// <typeparam name="T">Type of the data being read</typeparam>
+        /// <param name="va">Virtual address of the data in memory</param>
+        /// <returns>The data</returns>
+        /// <exception cref="InvalidMemoryRegionException">Throw for unhandled invalid or unmapped memory accesses</exception>
+        public T Read<T>(ulong va) where T : unmanaged
+        {
+            return MemoryMarshal.Cast<byte, T>(GetSpan(va, Unsafe.SizeOf<T>()))[0];
+        }
+
+        /// <summary>
+        /// Reads data from mapped memory.
+        /// </summary>
+        /// <param name="va">Virtual address of the data in memory</param>
+        /// <param name="data">Span to store the data being read into</param>
+        /// <exception cref="InvalidMemoryRegionException">Throw for unhandled invalid or unmapped memory accesses</exception>
+        public void Read(ulong va, Span<byte> data)
+        {
+            ReadImpl(va, data);
+        }
+
+        /// <summary>
+        /// Writes data to mapped memory.
+        /// </summary>
+        /// <typeparam name="T">Type of the data being written</typeparam>
+        /// <param name="va">Virtual address to write the data into</param>
+        /// <param name="value">Data to be written</param>
+        /// <exception cref="InvalidMemoryRegionException">Throw for unhandled invalid or unmapped memory accesses</exception>
+        public void Write<T>(ulong va, T value) where T : unmanaged
+        {
+            Write(va, MemoryMarshal.Cast<T, byte>(MemoryMarshal.CreateSpan(ref value, 1)));
+        }
+
+        /// <summary>
+        /// Writes data to mapped memory.
+        /// </summary>
+        /// <param name="va">Virtual address to write the data into</param>
+        /// <param name="data">Data to be written</param>
+        /// <exception cref="InvalidMemoryRegionException">Throw for unhandled invalid or unmapped memory accesses</exception>
+        public void Write(ulong va, ReadOnlySpan<byte> data)
+        {
+            if (data.Length == 0)
+            {
+                return;
+            }
+
+            if (IsContiguousAndMapped(va, data.Length))
+            {
+                data.CopyTo(_backingMemory.GetSpan(GetPhysicalAddressInternal(va), data.Length));
+            }
+            else
+            {
+                int offset = 0, size;
+
+                if ((va & PageMask) != 0)
+                {
+                    ulong pa = GetPhysicalAddressInternal(va);
+
+                    size = Math.Min(data.Length, PageSize - (int)(va & PageMask));
+
+                    data.Slice(0, size).CopyTo(_backingMemory.GetSpan(pa, size));
+
+                    offset += size;
+                }
+
+                for (; offset < data.Length; offset += size)
+                {
+                    ulong pa = GetPhysicalAddressInternal(va + (ulong)offset);
+
+                    size = Math.Min(data.Length - offset, PageSize);
+
+                    data.Slice(offset, size).CopyTo(_backingMemory.GetSpan(pa, size));
+                }
+            }
+        }
+
+        /// <summary>
+        /// Gets a read-only span of data from mapped memory.
+        /// </summary>
+        /// <remarks>
+        /// This may perform a allocation if the data is not contiguous in memory.
+        /// For this reason, the span is read-only, you can't modify the data.
+        /// </remarks>
+        /// <param name="va">Virtual address of the data</param>
+        /// <param name="size">Size of the data</param>
+        /// <param name="tracked">True if read tracking is triggered on the span</param>
+        /// <returns>A read-only span of the data</returns>
+        /// <exception cref="InvalidMemoryRegionException">Throw for unhandled invalid or unmapped memory accesses</exception>
+        public ReadOnlySpan<byte> GetSpan(ulong va, int size, bool tracked = false)
+        {
+            if (size == 0)
+            {
+                return ReadOnlySpan<byte>.Empty;
+            }
+
+            if (IsContiguousAndMapped(va, size))
+            {
+                return _backingMemory.GetSpan(GetPhysicalAddressInternal(va), size);
+            }
+            else
+            {
+                Span<byte> data = new byte[size];
+
+                ReadImpl(va, data);
+
+                return data;
+            }
+        }
+
+        /// <summary>
+        /// Gets a region of memory that can be written to.
+        /// </summary>
+        /// <remarks>
+        /// If the requested region is not contiguous in physical memory,
+        /// this will perform an allocation, and flush the data (writing it
+        /// back to the backing memory) on disposal.
+        /// </remarks>
+        /// <param name="va">Virtual address of the data</param>
+        /// <param name="size">Size of the data</param>
+        /// <returns>A writable region of memory containing the data</returns>
+        /// <exception cref="InvalidMemoryRegionException">Throw for unhandled invalid or unmapped memory accesses</exception>
+        public WritableRegion GetWritableRegion(ulong va, int size)
+        {
+            if (size == 0)
+            {
+                return new WritableRegion(null, va, Memory<byte>.Empty);
+            }
+
+            if (IsContiguousAndMapped(va, size))
+            {
+                return new WritableRegion(null, va, _backingMemory.GetMemory(GetPhysicalAddressInternal(va), size));
+            }
+            else
+            {
+                Memory<byte> memory = new byte[size];
+
+                GetSpan(va, size).CopyTo(memory.Span);
+
+                return new WritableRegion(this, va, memory);
+            }
+        }
+
+        /// <summary>
+        /// Gets a reference for the given type at the specified virtual memory address.
+        /// </summary>
+        /// <remarks>
+        /// The data must be located at a contiguous memory region.
+        /// </remarks>
+        /// <typeparam name="T">Type of the data to get the reference</typeparam>
+        /// <param name="va">Virtual address of the data</param>
+        /// <returns>A reference to the data in memory</returns>
+        /// <exception cref="MemoryNotContiguousException">Throw if the specified memory region is not contiguous in physical memory</exception>
+        public ref T GetRef<T>(ulong va) where T : unmanaged
+        {
+            if (!IsContiguous(va, Unsafe.SizeOf<T>()))
+            {
+                ThrowMemoryNotContiguous();
+            }
+
+            return ref _backingMemory.GetRef<T>(GetPhysicalAddressInternal(va));
+        }
+
+        private void ThrowMemoryNotContiguous() => throw new MemoryNotContiguousException();
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private bool IsContiguousAndMapped(ulong va, int size) => IsContiguous(va, size) && IsMapped(va);
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private bool IsContiguous(ulong va, int size)
+        {
+            if (!ValidateAddress(va))
+            {
+                return false;
+            }
+
+            ulong endVa = (va + (ulong)size + PageMask) & ~(ulong)PageMask;
+
+            va &= ~(ulong)PageMask;
+
+            int pages = (int)((endVa - va) / PageSize);
+
+            for (int page = 0; page < pages - 1; page++)
+            {
+                if (!ValidateAddress(va + PageSize))
+                {
+                    return false;
+                }
+
+                if (GetPhysicalAddressInternal(va) + PageSize != GetPhysicalAddressInternal(va + PageSize))
+                {
+                    return false;
+                }
+
+                va += PageSize;
+            }
+
+            return true;
+        }
+
+        /// <summary>
+        /// Gets the physical regions that make up the given virtual address region.
+        /// If any part of the virtual region is unmapped, null is returned.
+        /// </summary>
+        /// <param name="va">Virtual address of the range</param>
+        /// <param name="size">Size of the range</param>
+        /// <returns>Array of physical regions</returns>
+        public (ulong address, ulong size)[] GetPhysicalRegions(ulong va, ulong size)
+        {
+            throw new NotImplementedException();
+        }
+
+        private void ReadImpl(ulong va, Span<byte> data)
+        {
+            if (data.Length == 0)
+            {
+                return;
+            }
+
+            int offset = 0, size;
+
+            if ((va & PageMask) != 0)
+            {
+                ulong pa = GetPhysicalAddressInternal(va);
+
+                size = Math.Min(data.Length, PageSize - (int)(va & PageMask));
+
+                _backingMemory.GetSpan(pa, size).CopyTo(data.Slice(0, size));
+
+                offset += size;
+            }
+
+            for (; offset < data.Length; offset += size)
+            {
+                ulong pa = GetPhysicalAddressInternal(va + (ulong)offset);
+
+                size = Math.Min(data.Length - offset, PageSize);
+
+                _backingMemory.GetSpan(pa, size).CopyTo(data.Slice(offset, size));
+            }
+        }
+
+        /// <summary>
+        /// Checks if the page at a given virtual address is mapped.
+        /// </summary>
+        /// <param name="va">Virtual address to check</param>
+        /// <returns>True if the address is mapped, false otherwise</returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public bool IsMapped(ulong va)
+        {
+            if (!ValidateAddress(va))
+            {
+                return false;
+            }
+
+            return PtRead(va) != Unmapped;
+        }
+
+        /// <summary>
+        /// Checks if a memory range is mapped.
+        /// </summary>
+        /// <param name="va">Virtual address of the range</param>
+        /// <param name="size">Size of the range in bytes</param>
+        /// <returns>True if the entire range is mapped, false otherwise</returns>
+        public bool IsRangeMapped(ulong va, ulong size)
+        {
+            if (size == 0UL)
+            {
+                return true;
+            }
+
+            ulong endVa = (va + size + PageMask) & ~(ulong)PageMask;
+
+            va &= ~(ulong)PageMask;
+
+            while (va < endVa)
+            {
+                if (!IsMapped(va))
+                {
+                    return false;
+                }
+
+                va += PageSize;
+            }
+
+            return true;
+        }
+
+        private bool ValidateAddress(ulong va)
+        {
+            return va < _addressSpaceSize;
+        }
+
+        /// <summary>
+        /// Performs address translation of the address inside a mapped memory range.
+        /// </summary>
+        /// <remarks>
+        /// If the address is invalid or unmapped, -1 will be returned.
+        /// </remarks>
+        /// <param name="va">Virtual address to be translated</param>
+        /// <returns>The physical address</returns>
+        public ulong GetPhysicalAddress(ulong va)
+        {
+            // We return -1L if the virtual address is invalid or unmapped.
+            if (!ValidateAddress(va) || !IsMapped(va))
+            {
+                return ulong.MaxValue;
+            }
+
+            return GetPhysicalAddressInternal(va);
+        }
+
+        private ulong GetPhysicalAddressInternal(ulong va)
+        {
+            return PtRead(va) + (va & PageMask);
+        }
+
+        /// <summary>
+        /// Reprotect a region of virtual memory for tracking. Sets software protection bits.
+        /// </summary>
+        /// <param name="va">Virtual address base</param>
+        /// <param name="size">Size of the region to protect</param>
+        /// <param name="protection">Memory protection to set</param>
+        public void TrackingReprotect(ulong va, ulong size, MemoryPermission protection)
+        {
+            throw new NotImplementedException();
+        }
+
+        private ulong PtRead(ulong va)
+        {
+            int l3 = (int)(va >> PageBits) & PtLevelMask;
+            int l2 = (int)(va >> (PageBits + PtLevelBits)) & PtLevelMask;
+            int l1 = (int)(va >> (PageBits + PtLevelBits * 2)) & PtLevelMask;
+            int l0 = (int)(va >> (PageBits + PtLevelBits * 3)) & PtLevelMask;
+
+            if (_pageTable[l0] == null)
+            {
+                return Unmapped;
+            }
+
+            if (_pageTable[l0][l1] == null)
+            {
+                return Unmapped;
+            }
+
+            if (_pageTable[l0][l1][l2] == null)
+            {
+                return Unmapped;
+            }
+
+            return _pageTable[l0][l1][l2][l3];
+        }
+
+        private void PtMap(ulong va, ulong value)
+        {
+            int l3 = (int)(va >> PageBits) & PtLevelMask;
+            int l2 = (int)(va >> (PageBits + PtLevelBits)) & PtLevelMask;
+            int l1 = (int)(va >> (PageBits + PtLevelBits * 2)) & PtLevelMask;
+            int l0 = (int)(va >> (PageBits + PtLevelBits * 3)) & PtLevelMask;
+
+            if (_pageTable[l0] == null)
+            {
+                _pageTable[l0] = new ulong[PtLevelSize][][];
+            }
+
+            if (_pageTable[l0][l1] == null)
+            {
+                _pageTable[l0][l1] = new ulong[PtLevelSize][];
+            }
+
+            if (_pageTable[l0][l1][l2] == null)
+            {
+                _pageTable[l0][l1][l2] = new ulong[PtLevelSize];
+
+                for (int i = 0; i < _pageTable[l0][l1][l2].Length; i++)
+                {
+                    _pageTable[l0][l1][l2][i] = Unmapped;
+                }
+            }
+
+            _pageTable[l0][l1][l2][l3] = value;
+        }
+
+        private void PtUnmap(ulong va)
+        {
+            int l3 = (int)(va >> PageBits) & PtLevelMask;
+            int l2 = (int)(va >> (PageBits + PtLevelBits)) & PtLevelMask;
+            int l1 = (int)(va >> (PageBits + PtLevelBits * 2)) & PtLevelMask;
+            int l0 = (int)(va >> (PageBits + PtLevelBits * 3)) & PtLevelMask;
+
+            if (_pageTable[l0] == null)
+            {
+                return;
+            }
+
+            if (_pageTable[l0][l1] == null)
+            {
+                return;
+            }
+
+            if (_pageTable[l0][l1][l2] == null)
+            {
+                return;
+            }
+
+            _pageTable[l0][l1][l2][l3] = Unmapped;
+
+            bool empty = true;
+
+            for (int i = 0; i < _pageTable[l0][l1][l2].Length; i++)
+            {
+                empty &= (_pageTable[l0][l1][l2][i] == Unmapped);
+            }
+
+            if (empty)
+            {
+                _pageTable[l0][l1][l2] = null;
+
+                RemoveIfAllNull(l0, l1);
+            }
+        }
+
+        private void RemoveIfAllNull(int l0, int l1)
+        {
+            bool empty = true;
+
+            for (int i = 0; i < _pageTable[l0][l1].Length; i++)
+            {
+                empty &= (_pageTable[l0][l1][i] == null);
+            }
+
+            if (empty)
+            {
+                _pageTable[l0][l1] = null;
+
+                RemoveIfAllNull(l0);
+            }
+        }
+
+        private void RemoveIfAllNull(int l0)
+        {
+            bool empty = true;
+
+            for (int i = 0; i < _pageTable[l0].Length; i++)
+            {
+                empty &= (_pageTable[l0][i] == null);
+            }
+
+            if (empty)
+            {
+                _pageTable[l0] = null;
+            }
+        }
+    }
+}
diff --git a/Ryujinx.Memory/IVirtualMemoryManager.cs b/Ryujinx.Memory/IVirtualMemoryManager.cs
new file mode 100644
index 00000000..8c2296e2
--- /dev/null
+++ b/Ryujinx.Memory/IVirtualMemoryManager.cs
@@ -0,0 +1,28 @@
+using System;
+
+namespace Ryujinx.Memory
+{
+    public interface IVirtualMemoryManager
+    {
+        void Map(ulong va, ulong pa, ulong size);
+        void Unmap(ulong va, ulong size);
+
+        T Read<T>(ulong va) where T : unmanaged;
+        void Read(ulong va, Span<byte> data);
+
+        void Write<T>(ulong va, T value) where T : unmanaged;
+        void Write(ulong va, ReadOnlySpan<byte> data);
+
+        ReadOnlySpan<byte> GetSpan(ulong va, int size, bool tracked = false);
+        WritableRegion GetWritableRegion(ulong va, int size);
+        ref T GetRef<T>(ulong va) where T : unmanaged;
+
+        (ulong address, ulong size)[] GetPhysicalRegions(ulong va, ulong size);
+
+        bool IsMapped(ulong va);
+        bool IsRangeMapped(ulong va, ulong size);
+        ulong GetPhysicalAddress(ulong va);
+
+        void TrackingReprotect(ulong va, ulong size, MemoryPermission protection);
+    }
+}
diff --git a/Ryujinx.Memory/MemoryNotContiguousException.cs b/Ryujinx.Memory/MemoryNotContiguousException.cs
new file mode 100644
index 00000000..3106955b
--- /dev/null
+++ b/Ryujinx.Memory/MemoryNotContiguousException.cs
@@ -0,0 +1,19 @@
+using System;
+
+namespace Ryujinx.Memory
+{
+    public class MemoryNotContiguousException : Exception
+    {
+        public MemoryNotContiguousException() : base("The specified memory region is not contiguous.")
+        {
+        }
+
+        public MemoryNotContiguousException(string message) : base(message)
+        {
+        }
+
+        public MemoryNotContiguousException(string message, Exception innerException) : base(message, innerException)
+        {
+        }
+    }
+}
\ No newline at end of file
diff --git a/Ryujinx.Memory/Tracking/IVirtualMemoryManager.cs b/Ryujinx.Memory/Tracking/IVirtualMemoryManager.cs
deleted file mode 100644
index e6d8e8c9..00000000
--- a/Ryujinx.Memory/Tracking/IVirtualMemoryManager.cs
+++ /dev/null
@@ -1,10 +0,0 @@
-namespace Ryujinx.Memory.Tracking
-{
-    public interface IVirtualMemoryManager
-    {
-        (ulong address, ulong size)[] GetPhysicalRegions(ulong va, ulong size);
-
-        bool IsRangeMapped(ulong va, ulong size);
-        void TrackingReprotect(ulong va, ulong size, MemoryPermission protection);
-    }
-}
diff --git a/Ryujinx.Memory/WritableRegion.cs b/Ryujinx.Memory/WritableRegion.cs
new file mode 100644
index 00000000..e8ea310f
--- /dev/null
+++ b/Ryujinx.Memory/WritableRegion.cs
@@ -0,0 +1,29 @@
+using System;
+
+namespace Ryujinx.Memory
+{
+    public sealed class WritableRegion : IDisposable
+    {
+        private readonly IVirtualMemoryManager _mm;
+        private readonly ulong _va;
+
+        private bool NeedsWriteback => _mm != null;
+
+        public Memory<byte> Memory { get; }
+
+        public WritableRegion(IVirtualMemoryManager mm, ulong va, Memory<byte> memory)
+        {
+            _mm = mm;
+            _va = va;
+            Memory = memory;
+        }
+
+        public void Dispose()
+        {
+            if (NeedsWriteback)
+            {
+                _mm.Write(_va, Memory.Span);
+            }
+        }
+    }
+}