Implement a new physical memory manager and replace DeviceMemory (#856)

* Implement a new physical memory manager and replace DeviceMemory

* Proper generic constraints

* Fix debug build

* Add memory tests

* New CPU memory manager and general code cleanup

* Remove host memory management from CPU project, use Ryujinx.Memory instead

* Fix tests

* Document exceptions on MemoryBlock

* Fix leak on unix memory allocation

* Proper disposal of some objects on tests

* Fix JitCache not being set as initialized

* GetRef without checks for 8-bits and 16-bits CAS

* Add MemoryBlock destructor

* Throw in separate method to improve codegen

* Address PR feedback

* QueryModified improvements

* Fix memory write tracking not marking all pages as modified in some cases

* Simplify MarkRegionAsModified

* Remove XML doc for ghost param

* Add back optimization to avoid useless buffer updates

* Add Ryujinx.Cpu project, move MemoryManager there and remove MemoryBlockWrapper

* Some nits

* Do not perform address translation when size is 0

* Address PR feedback and format NativeInterface class

* Remove ghost parameter description

* Update Ryujinx.Cpu to .NET Core 3.1

* Address PR feedback

* Fix build

* Return a well defined value for GetPhysicalAddress with invalid VA, and do not return unmapped ranges as modified

* Typo

4 files changed, 66 insertions, 46 deletions

diff --git a/Ryujinx.HLE/HOS/Services/Sockets/Bsd/IClient.cs b/Ryujinx.HLE/HOS/Services/Sockets/Bsd/IClient.cs
index b2b3d052..f4dd8ef2 100644
--- a/Ryujinx.HLE/HOS/Services/Sockets/Bsd/IClient.cs
+++ b/Ryujinx.HLE/HOS/Services/Sockets/Bsd/IClient.cs
@@ -199,21 +199,23 @@ namespace Ryujinx.HLE.HOS.Services.Sockets.Bsd
 
         private IPEndPoint ParseSockAddr(ServiceCtx context, long bufferPosition, long bufferSize)
         {
-            int size   = context.Memory.ReadByte(bufferPosition);
-            int family = context.Memory.ReadByte(bufferPosition + 1);
-            int port   = BinaryPrimitives.ReverseEndianness(context.Memory.ReadUInt16(bufferPosition + 2));
+            int size   = context.Memory.Read<byte>((ulong)bufferPosition);
+            int family = context.Memory.Read<byte>((ulong)bufferPosition + 1);
+            int port   = BinaryPrimitives.ReverseEndianness(context.Memory.Read<ushort>((ulong)bufferPosition + 2));
 
-            byte[] rawIp = context.Memory.ReadBytes(bufferPosition + 4, 4);
+            byte[] rawIp = new byte[4];
+
+            context.Memory.Read((ulong)bufferPosition + 4, rawIp);
 
             return new IPEndPoint(new IPAddress(rawIp), port);
         }
 
         private void WriteSockAddr(ServiceCtx context, long bufferPosition, IPEndPoint endPoint)
         {
-            context.Memory.WriteByte(bufferPosition, 0);
-            context.Memory.WriteByte(bufferPosition + 1, (byte)endPoint.AddressFamily);
-            context.Memory.WriteUInt16(bufferPosition + 2, BinaryPrimitives.ReverseEndianness((ushort)endPoint.Port));
-            context.Memory.WriteBytes(bufferPosition + 4, endPoint.Address.GetAddressBytes());
+            context.Memory.Write((ulong)bufferPosition, (byte)0);
+            context.Memory.Write((ulong)bufferPosition + 1, (byte)endPoint.AddressFamily);
+            context.Memory.Write((ulong)bufferPosition + 2, BinaryPrimitives.ReverseEndianness((ushort)endPoint.Port));
+            context.Memory.Write((ulong)bufferPosition + 4, endPoint.Address.GetAddressBytes());
         }
 
         private void WriteSockAddr(ServiceCtx context, long bufferPosition, BsdSocket socket, bool isRemote)
@@ -281,8 +283,11 @@ namespace Ryujinx.HLE.HOS.Services.Sockets.Bsd
 
             int flags = context.RequestData.ReadInt32();
 
-            byte[] rawPath = context.Memory.ReadBytes(bufferPosition, bufferSize);
-            string path    = Encoding.ASCII.GetString(rawPath);
+            byte[] rawPath = new byte[bufferSize];
+
+            context.Memory.Read((ulong)bufferPosition, rawPath);
+
+            string path = Encoding.ASCII.GetString(rawPath);
 
             WriteBsdResult(context, -1, LinuxError.EOPNOTSUPP);
 
@@ -321,7 +326,7 @@ namespace Ryujinx.HLE.HOS.Services.Sockets.Bsd
 
             for (int i = 0; i < fdsCount; i++)
             {
-                int socketFd = context.Memory.ReadInt32(bufferPosition + i * 8);
+                int socketFd = context.Memory.Read<int>((ulong)(bufferPosition + i * 8));
 
                 BsdSocket socket = RetrieveSocket(socketFd);
 
@@ -329,8 +334,8 @@ namespace Ryujinx.HLE.HOS.Services.Sockets.Bsd
                 {
                     return WriteBsdResult(context, -1, LinuxError.EBADF);}
 
-                PollEvent.EventTypeMask inputEvents  = (PollEvent.EventTypeMask)context.Memory.ReadInt16(bufferPosition + i * 8 + 4);
-                PollEvent.EventTypeMask outputEvents = (PollEvent.EventTypeMask)context.Memory.ReadInt16(bufferPosition + i * 8 + 6);
+                PollEvent.EventTypeMask inputEvents  = (PollEvent.EventTypeMask)context.Memory.Read<short>((ulong)(bufferPosition + i * 8 + 4));
+                PollEvent.EventTypeMask outputEvents = (PollEvent.EventTypeMask)context.Memory.Read<short>((ulong)(bufferPosition + i * 8 + 6));
 
                 events[i] = new PollEvent(socketFd, socket, inputEvents, outputEvents);
             }
@@ -405,8 +410,8 @@ namespace Ryujinx.HLE.HOS.Services.Sockets.Bsd
             for (int i = 0; i < fdsCount; i++)
             {
                 PollEvent Event = events[i];
-                context.Memory.WriteInt32(bufferPosition + i * 8, Event.SocketFd);
-                context.Memory.WriteInt16(bufferPosition + i * 8 + 4, (short)Event.InputEvents);
+                context.Memory.Write((ulong)(bufferPosition + i * 8), Event.SocketFd);
+                context.Memory.Write((ulong)(bufferPosition + i * 8 + 4), (short)Event.InputEvents);
 
                 PollEvent.EventTypeMask outputEvents = 0;
 
@@ -435,7 +440,7 @@ namespace Ryujinx.HLE.HOS.Services.Sockets.Bsd
                     outputEvents |= PollEvent.EventTypeMask.Output;
                 }
 
-                context.Memory.WriteInt16(bufferPosition + i * 8 + 6, (short)outputEvents);
+                context.Memory.Write((ulong)(bufferPosition + i * 8 + 6), (short)outputEvents);
             }
 
             return WriteBsdResult(context, readEvents.Count + writeEvents.Count + errorEvents.Count, LinuxError.SUCCESS);
@@ -481,7 +486,7 @@ namespace Ryujinx.HLE.HOS.Services.Sockets.Bsd
                     result = socket.Handle.Receive(receivedBuffer, socketFlags);
                     errno  = SetResultErrno(socket.Handle, result);
 
-                    context.Memory.WriteBytes(receivePosition, receivedBuffer);
+                    context.Memory.Write((ulong)receivePosition, receivedBuffer);
                 }
                 catch (SocketException exception)
                 {
@@ -524,7 +529,7 @@ namespace Ryujinx.HLE.HOS.Services.Sockets.Bsd
                     result = socket.Handle.ReceiveFrom(receivedBuffer, receivedBuffer.Length, socketFlags, ref endPoint);
                     errno  = SetResultErrno(socket.Handle, result);
 
-                    context.Memory.WriteBytes(receivePosition, receivedBuffer);
+                    context.Memory.Write((ulong)receivePosition, receivedBuffer);
                     WriteSockAddr(context, sockAddrOutPosition, (IPEndPoint)endPoint);
                 }
                 catch (SocketException exception)
@@ -559,7 +564,9 @@ namespace Ryujinx.HLE.HOS.Services.Sockets.Bsd
                     return WriteBsdResult(context, -1, LinuxError.EOPNOTSUPP);
                 }
 
-                byte[] sendBuffer = context.Memory.ReadBytes(sendPosition, sendSize);
+                byte[] sendBuffer = new byte[sendSize];
+
+                context.Memory.Read((ulong)sendPosition, sendBuffer);
 
                 try
                 {
@@ -600,8 +607,11 @@ namespace Ryujinx.HLE.HOS.Services.Sockets.Bsd
                     return WriteBsdResult(context, -1, LinuxError.EOPNOTSUPP);
                 }
 
-                byte[]   sendBuffer = context.Memory.ReadBytes(sendPosition, sendSize);
-                EndPoint endPoint   = ParseSockAddr(context, bufferPosition, bufferSize);
+                byte[] sendBuffer = new byte[sendSize];
+
+                context.Memory.Read((ulong)sendPosition, sendBuffer);
+
+                EndPoint endPoint = ParseSockAddr(context, bufferPosition, bufferSize);
 
                 try
                 {
@@ -856,7 +866,7 @@ namespace Ryujinx.HLE.HOS.Services.Sockets.Bsd
                         (long bufferPosition, long bufferSize) = context.Request.GetBufferType0x22();
 
                         // FIXME: OOB not implemented.
-                        context.Memory.WriteInt32(bufferPosition, 0);
+                        context.Memory.Write((ulong)bufferPosition, 0);
                         break;
 
                     default:
@@ -925,13 +935,13 @@ namespace Ryujinx.HLE.HOS.Services.Sockets.Bsd
                     case SocketOptionName.Type:
                     case SocketOptionName.Linger:
                         socket.Handle.GetSocketOption(SocketOptionLevel.Socket, optionName, optionValue);
-                        context.Memory.WriteBytes(optionValuePosition, optionValue);
+                        context.Memory.Write((ulong)optionValuePosition, optionValue);
 
                         return LinuxError.SUCCESS;
 
                     case (SocketOptionName)0x200:
                         socket.Handle.GetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, optionValue);
-                        context.Memory.WriteBytes(optionValuePosition, optionValue);
+                        context.Memory.Write((ulong)optionValuePosition, optionValue);
 
                         return LinuxError.SUCCESS;
 
@@ -965,18 +975,18 @@ namespace Ryujinx.HLE.HOS.Services.Sockets.Bsd
                     case SocketOptionName.SendTimeout:
                     case SocketOptionName.Type:
                     case SocketOptionName.ReuseAddress:
-                        socket.Handle.SetSocketOption(SocketOptionLevel.Socket, optionName, context.Memory.ReadInt32(optionValuePosition));
+                        socket.Handle.SetSocketOption(SocketOptionLevel.Socket, optionName, context.Memory.Read<int>((ulong)optionValuePosition));
 
                         return LinuxError.SUCCESS;
 
                     case (SocketOptionName)0x200:
-                        socket.Handle.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, context.Memory.ReadInt32(optionValuePosition));
+                        socket.Handle.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, context.Memory.Read<int>((ulong)optionValuePosition));
 
                         return LinuxError.SUCCESS;
 
                     case SocketOptionName.Linger:
                         socket.Handle.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.Linger,
-                            new LingerOption(context.Memory.ReadInt32(optionValuePosition) != 0, context.Memory.ReadInt32(optionValuePosition + 4)));
+                            new LingerOption(context.Memory.Read<int>((ulong)optionValuePosition) != 0, context.Memory.Read<int>((ulong)optionValuePosition + 4)));
 
                         return LinuxError.SUCCESS;
 
@@ -1100,7 +1110,9 @@ namespace Ryujinx.HLE.HOS.Services.Sockets.Bsd
 
             if (socket != null)
             {
-                byte[] sendBuffer = context.Memory.ReadBytes(sendPosition, sendSize);
+                byte[] sendBuffer = new byte[sendSize];
+
+                context.Memory.Read((ulong)sendPosition, sendBuffer);
 
                 try
                 {
diff --git a/Ryujinx.HLE/HOS/Services/Sockets/Nsd/IManager.cs b/Ryujinx.HLE/HOS/Services/Sockets/Nsd/IManager.cs
index 277bc374..c5ceec5b 100644
--- a/Ryujinx.HLE/HOS/Services/Sockets/Nsd/IManager.cs
+++ b/Ryujinx.HLE/HOS/Services/Sockets/Nsd/IManager.cs
@@ -44,7 +44,7 @@ namespace Ryujinx.HLE.HOS.Services.Sockets.Nsd
             {
                 byte[] settingNameBuffer = Encoding.UTF8.GetBytes(settingName + '\0');
 
-                context.Memory.WriteBytes(outputPosition, settingNameBuffer);
+                context.Memory.Write((ulong)outputPosition, settingNameBuffer);
             }
 
             return result;
@@ -62,7 +62,7 @@ namespace Ryujinx.HLE.HOS.Services.Sockets.Nsd
             {
                 byte[] identifierBuffer = Encoding.UTF8.GetBytes(identifier + '\0');
 
-                context.Memory.WriteBytes(outputPosition, identifierBuffer);
+                context.Memory.Write((ulong)outputPosition, identifierBuffer);
             }
 
             return result;
@@ -138,7 +138,7 @@ namespace Ryujinx.HLE.HOS.Services.Sockets.Nsd
 
             byte[] resolvedAddressBuffer = Encoding.UTF8.GetBytes(resolvedAddress + '\0');
 
-            context.Memory.WriteBytes(outputPosition, resolvedAddressBuffer);
+            context.Memory.Write((ulong)outputPosition, resolvedAddressBuffer);
 
             return result;
         }
@@ -153,7 +153,7 @@ namespace Ryujinx.HLE.HOS.Services.Sockets.Nsd
 
             byte[] resolvedAddressBuffer = Encoding.UTF8.GetBytes(resolvedAddress + '\0');
 
-            context.Memory.WriteBytes(outputPosition, resolvedAddressBuffer);
+            context.Memory.Write((ulong)outputPosition, resolvedAddressBuffer);
 
             context.ResponseData.Write((int)errorCode);
 
diff --git a/Ryujinx.HLE/HOS/Services/Sockets/Nsd/Manager/FqdnResolver.cs b/Ryujinx.HLE/HOS/Services/Sockets/Nsd/Manager/FqdnResolver.cs
index aaab7b3b..246c92bd 100644
--- a/Ryujinx.HLE/HOS/Services/Sockets/Nsd/Manager/FqdnResolver.cs
+++ b/Ryujinx.HLE/HOS/Services/Sockets/Nsd/Manager/FqdnResolver.cs
@@ -108,8 +108,11 @@ namespace Ryujinx.HLE.HOS.Services.Sockets.Nsd.Manager
         {
             (long inputPosition, long inputSize)  = context.Request.GetBufferType0x21();
 
-            byte[] addressBuffer = context.Memory.ReadBytes(inputPosition, inputSize);
-            string address       = Encoding.UTF8.GetString(addressBuffer);
+            byte[] addressBuffer = new byte[inputSize];
+
+            context.Memory.Read((ulong)inputPosition, addressBuffer);
+
+            string address = Encoding.UTF8.GetString(addressBuffer);
 
             resultCode = Resolve(context, address, out resolvedAddress);
 
diff --git a/Ryujinx.HLE/HOS/Services/Sockets/Sfdnsres/IResolver.cs b/Ryujinx.HLE/HOS/Services/Sockets/Sfdnsres/IResolver.cs
index 1cf2aa1c..a5e4c11b 100644
--- a/Ryujinx.HLE/HOS/Services/Sockets/Sfdnsres/IResolver.cs
+++ b/Ryujinx.HLE/HOS/Services/Sockets/Sfdnsres/IResolver.cs
@@ -21,37 +21,37 @@ namespace Ryujinx.HLE.HOS.Services.Sockets.Sfdnsres
             string hostName = hostEntry.HostName + '\0';
 
             // h_name
-            context.Memory.WriteBytes(bufferPosition, Encoding.ASCII.GetBytes(hostName));
+            context.Memory.Write((ulong)bufferPosition, Encoding.ASCII.GetBytes(hostName));
             bufferPosition += hostName.Length;
 
             // h_aliases list size
-            context.Memory.WriteInt32(bufferPosition, IPAddress.HostToNetworkOrder(hostEntry.Aliases.Length));
+            context.Memory.Write((ulong)bufferPosition, IPAddress.HostToNetworkOrder(hostEntry.Aliases.Length));
             bufferPosition += 4;
 
             // Actual aliases
             foreach (string alias in hostEntry.Aliases)
             {
-                context.Memory.WriteBytes(bufferPosition, Encoding.ASCII.GetBytes(alias + '\0'));
+                context.Memory.Write((ulong)bufferPosition, Encoding.ASCII.GetBytes(alias + '\0'));
                 bufferPosition += alias.Length + 1;
             }
 
             // h_addrtype but it's a short (also only support IPv4)
-            context.Memory.WriteInt16(bufferPosition, IPAddress.HostToNetworkOrder((short)2));
+            context.Memory.Write((ulong)bufferPosition, IPAddress.HostToNetworkOrder((short)2));
             bufferPosition += 2;
 
             // h_length but it's a short
-            context.Memory.WriteInt16(bufferPosition, IPAddress.HostToNetworkOrder((short)4));
+            context.Memory.Write((ulong)bufferPosition, IPAddress.HostToNetworkOrder((short)4));
             bufferPosition += 2;
 
             // Ip address count, we can only support ipv4 (blame Nintendo)
-            context.Memory.WriteInt32(bufferPosition, addresses != null ? IPAddress.HostToNetworkOrder(addresses.Count) : 0);
+            context.Memory.Write((ulong)bufferPosition, addresses != null ? IPAddress.HostToNetworkOrder(addresses.Count) : 0);
             bufferPosition += 4;
 
             if (addresses != null)
             {
                 foreach (IPAddress ip in addresses)
                 {
-                    context.Memory.WriteInt32(bufferPosition, IPAddress.HostToNetworkOrder(BitConverter.ToInt32(ip.GetAddressBytes(), 0)));
+                    context.Memory.Write((ulong)bufferPosition, IPAddress.HostToNetworkOrder(BitConverter.ToInt32(ip.GetAddressBytes(), 0)));
                     bufferPosition += 4;
                 }
             }
@@ -168,8 +168,11 @@ namespace Ryujinx.HLE.HOS.Services.Sockets.Sfdnsres
         // GetHostByName(u8, u32, u64, pid, buffer<unknown, 5, 0>) -> (u32, u32, u32, buffer<unknown, 6, 0>)
         public ResultCode GetHostByName(ServiceCtx context)
         {
-            byte[] rawName = context.Memory.ReadBytes(context.Request.SendBuff[0].Position, context.Request.SendBuff[0].Size);
-            string name    = Encoding.ASCII.GetString(rawName).TrimEnd('\0');
+            byte[] rawName = new byte[context.Request.SendBuff[0].Size];
+
+            context.Memory.Read((ulong)context.Request.SendBuff[0].Position, rawName);
+
+            string name = Encoding.ASCII.GetString(rawName).TrimEnd('\0');
 
             // TODO: use params
             bool  enableNsdResolve = context.RequestData.ReadInt32() == 1;
@@ -248,7 +251,9 @@ namespace Ryujinx.HLE.HOS.Services.Sockets.Sfdnsres
         // GetHostByAddr(u32, u32, u32, u64, pid, buffer<unknown, 5, 0>) -> (u32, u32, u32, buffer<unknown, 6, 0>)
         public ResultCode GetHostByAddress(ServiceCtx context)
         {
-            byte[] rawIp = context.Memory.ReadBytes(context.Request.SendBuff[0].Position, context.Request.SendBuff[0].Size);
+            byte[] rawIp = new byte[context.Request.SendBuff[0].Size];
+
+            context.Memory.Read((ulong)context.Request.SendBuff[0].Position, rawIp);
 
             // TODO: use params
             uint  socketLength   = context.RequestData.ReadUInt32();
@@ -325,7 +330,7 @@ namespace Ryujinx.HLE.HOS.Services.Sockets.Sfdnsres
             if (errorString.Length + 1 <= context.Request.ReceiveBuff[0].Size)
             {
                 resultCode = 0;
-                context.Memory.WriteBytes(context.Request.ReceiveBuff[0].Position, Encoding.ASCII.GetBytes(errorString + '\0'));
+                context.Memory.Write((ulong)context.Request.ReceiveBuff[0].Position, Encoding.ASCII.GetBytes(errorString + '\0'));
             }
 
             return resultCode;
@@ -342,7 +347,7 @@ namespace Ryujinx.HLE.HOS.Services.Sockets.Sfdnsres
             if (errorString.Length + 1 <= context.Request.ReceiveBuff[0].Size)
             {
                 resultCode = 0;
-                context.Memory.WriteBytes(context.Request.ReceiveBuff[0].Position, Encoding.ASCII.GetBytes(errorString + '\0'));
+                context.Memory.Write((ulong)context.Request.ReceiveBuff[0].Position, Encoding.ASCII.GetBytes(errorString + '\0'));
             }
 
             return resultCode;