Improve kernel IPC implementation (#550)

* Implement some IPC related kernel SVCs properly

* Fix BLZ decompression when the segment also has a uncompressed chunck

* Set default cpu core on process start from ProgramLoader, remove debug message

* Load process capabilities properly on KIPs

* Fix a copy/paste error in UnmapPhysicalMemory64

* Implement smarter switching between old and new IPC system to support the old HLE services implementation without the manual switch

* Implement RegisterService on sm and AcceptSession (partial)

* Misc fixes and improvements on new IPC methods

* Move IPC related SVCs into a separate file, and logging on RegisterService (sm)

* Some small fixes related to receive list buffers and error cases

* Load NSOs using the correct pool partition

* Fix corner case on GetMaskFromMinMax where range is 64, doesn't happen in pratice however

* Fix send static buffer copy

* Session release, implement closing requests on client disconnect

* Implement ConnectToPort SVC

* KLightSession init

1 files changed, 1262 insertions, 0 deletions

diff --git a/Ryujinx.HLE/HOS/Kernel/Ipc/KServerSession.cs b/Ryujinx.HLE/HOS/Kernel/Ipc/KServerSession.cs
new file mode 100644
index 00000000..5a45ff4a
--- /dev/null
+++ b/Ryujinx.HLE/HOS/Kernel/Ipc/KServerSession.cs
@@ -0,0 +1,1262 @@
+using Ryujinx.Common;
+using Ryujinx.HLE.HOS.Kernel.Common;
+using Ryujinx.HLE.HOS.Kernel.Memory;
+using Ryujinx.HLE.HOS.Kernel.Process;
+using Ryujinx.HLE.HOS.Kernel.Threading;
+using System.Collections.Generic;
+
+namespace Ryujinx.HLE.HOS.Kernel.Ipc
+{
+    class KServerSession : KSynchronizationObject
+    {
+        private static readonly MemoryState[] IpcMemoryStates = new MemoryState[]
+        {
+            MemoryState.IpcBuffer3,
+            MemoryState.IpcBuffer0,
+            MemoryState.IpcBuffer1,
+            (MemoryState)0xfffce5d4 //This is invalid, shouldn't be accessed.
+        };
+
+        private struct Message
+        {
+            public ulong Address     { get; }
+            public ulong DramAddress { get; }
+            public ulong Size        { get; }
+            public bool  IsCustom    { get; }
+
+            public Message(KThread thread, ulong customCmdBuffAddress, ulong customCmdBuffSize)
+            {
+                IsCustom = customCmdBuffAddress != 0;
+
+                if (IsCustom)
+                {
+                    Address = customCmdBuffAddress;
+                    Size    = customCmdBuffSize;
+
+                    KProcess process = thread.Owner;
+
+                    DramAddress = process.MemoryManager.GetDramAddressFromVa(Address);
+                }
+                else
+                {
+                    Address     = thread.TlsAddress;
+                    DramAddress = thread.TlsDramAddress;
+                    Size        = 0x100;
+                }
+            }
+
+            public Message(KSessionRequest request) : this(
+                request.ClientThread,
+                request.CustomCmdBuffAddr,
+                request.CustomCmdBuffSize) { }
+        }
+
+        private struct MessageHeader
+        {
+            public uint Word0 { get; }
+            public uint Word1 { get; }
+            public uint Word2 { get; }
+
+            public uint PointerBuffersCount  { get; }
+            public uint SendBuffersCount     { get; }
+            public uint ReceiveBuffersCount  { get; }
+            public uint ExchangeBuffersCount { get; }
+
+            public uint RawDataSizeInWords { get; }
+
+            public uint ReceiveListType { get; }
+
+            public uint MessageSizeInWords       { get; }
+            public uint ReceiveListOffsetInWords { get; }
+            public uint ReceiveListOffset        { get; }
+
+            public bool HasHandles { get; }
+
+            public bool HasPid { get; }
+
+            public uint CopyHandlesCount { get; }
+            public uint MoveHandlesCount { get; }
+
+            public MessageHeader(uint word0, uint word1, uint word2)
+            {
+                Word0 = word0;
+                Word1 = word1;
+                Word2 = word2;
+
+                HasHandles = word1 >> 31 != 0;
+
+                uint handleDescSizeInWords = 0;
+
+                if (HasHandles)
+                {
+                    uint pidSize = (word2 & 1) * 8;
+
+                    HasPid = pidSize != 0;
+
+                    CopyHandlesCount = (word2 >> 1) & 0xf;
+                    MoveHandlesCount = (word2 >> 5) & 0xf;
+
+                    handleDescSizeInWords = (pidSize + CopyHandlesCount * 4 + MoveHandlesCount * 4) / 4;
+                }
+                else
+                {
+                    HasPid = false;
+
+                    CopyHandlesCount = 0;
+                    MoveHandlesCount = 0;
+                }
+
+                PointerBuffersCount  = (word0 >> 16) & 0xf;
+                SendBuffersCount     = (word0 >> 20) & 0xf;
+                ReceiveBuffersCount  = (word0 >> 24) & 0xf;
+                ExchangeBuffersCount =  word0 >> 28;
+
+                uint pointerDescSizeInWords  = PointerBuffersCount  * 2;
+                uint sendDescSizeInWords     = SendBuffersCount     * 3;
+                uint receiveDescSizeInWords  = ReceiveBuffersCount  * 3;
+                uint exchangeDescSizeInWords = ExchangeBuffersCount * 3;
+
+                RawDataSizeInWords = word1 & 0x3ff;
+
+                ReceiveListType = (word1 >> 10) & 0xf;
+
+                ReceiveListOffsetInWords = (word1 >> 20) & 0x7ff;
+
+                uint paddingSizeInWords = HasHandles ? 3u : 2u;
+
+                MessageSizeInWords = pointerDescSizeInWords  +
+                                     sendDescSizeInWords     +
+                                     receiveDescSizeInWords  +
+                                     exchangeDescSizeInWords +
+                                     RawDataSizeInWords      +
+                                     paddingSizeInWords      +
+                                     handleDescSizeInWords;
+
+                if (ReceiveListOffsetInWords == 0)
+                {
+                    ReceiveListOffsetInWords = MessageSizeInWords;
+                }
+
+                ReceiveListOffset = ReceiveListOffsetInWords * 4;
+            }
+        }
+
+        private struct PointerBufferDesc
+        {
+            public uint ReceiveIndex { get; }
+
+            public uint  BufferSize    { get; }
+            public ulong BufferAddress { get; set; }
+
+            public PointerBufferDesc(ulong dword)
+            {
+                ReceiveIndex = (uint)dword & 0xf;
+                BufferSize   = (uint)dword >> 16;
+
+                BufferAddress  = (dword >> 2)  & 0x70;
+                BufferAddress |= (dword >> 12) & 0xf;
+
+                BufferAddress = (BufferAddress << 32) | (dword >> 32);
+            }
+
+            public ulong Pack()
+            {
+                ulong dword = (ReceiveIndex & 0xf) | ((BufferSize & 0xffff) << 16);
+
+                dword |=  BufferAddress << 32;
+                dword |= (BufferAddress >> 20) & 0xf000;
+                dword |= (BufferAddress >> 30) & 0xffc0;
+
+                return dword;
+            }
+        }
+
+        private KSession _parent;
+
+        private LinkedList<KSessionRequest> _requests;
+
+        private KSessionRequest _activeRequest;
+
+        public KServerSession(Horizon system, KSession parent) : base(system)
+        {
+            _parent = parent;
+
+            _requests = new LinkedList<KSessionRequest>();
+        }
+
+        public KernelResult EnqueueRequest(KSessionRequest request)
+        {
+            if (_parent.ClientSession.State != ChannelState.Open)
+            {
+                return KernelResult.PortRemoteClosed;
+            }
+
+            if (request.AsyncEvent == null)
+            {
+                if (request.ClientThread.ShallBeTerminated ||
+                    request.ClientThread.SchedFlags == ThreadSchedState.TerminationPending)
+                {
+                    return KernelResult.ThreadTerminating;
+                }
+
+                request.ClientThread.Reschedule(ThreadSchedState.Paused);
+            }
+
+            _requests.AddLast(request);
+
+            if (_requests.Count == 1)
+            {
+                Signal();
+            }
+
+            return KernelResult.Success;
+        }
+
+        public KernelResult Receive(ulong customCmdBuffAddr = 0, ulong customCmdBuffSize = 0)
+        {
+            KThread  serverThread  = System.Scheduler.GetCurrentThread();
+            KProcess serverProcess = serverThread.Owner;
+
+            System.CriticalSection.Enter();
+
+            if (_parent.ClientSession.State != ChannelState.Open)
+            {
+                System.CriticalSection.Leave();
+
+                return KernelResult.PortRemoteClosed;
+            }
+
+            if (_activeRequest != null || !DequeueRequest(out KSessionRequest request))
+            {
+                System.CriticalSection.Leave();
+
+                return KernelResult.NotFound;
+            }
+
+            if (request.ClientThread == null)
+            {
+                System.CriticalSection.Leave();
+
+                return KernelResult.PortRemoteClosed;
+            }
+
+            KThread  clientThread  = request.ClientThread;
+            KProcess clientProcess = clientThread.Owner;
+
+            System.CriticalSection.Leave();
+
+            _activeRequest = request;
+
+            request.ServerProcess = serverProcess;
+
+            Message clientMsg = new Message(request);
+            Message serverMsg = new Message(serverThread, customCmdBuffAddr, customCmdBuffSize);
+
+            MessageHeader clientHeader = GetClientMessageHeader(clientMsg);
+            MessageHeader serverHeader = GetServerMessageHeader(serverMsg);
+
+            KernelResult serverResult = KernelResult.NotFound;
+            KernelResult clientResult = KernelResult.Success;
+
+            void CleanUpForError()
+            {
+                if (request.BufferDescriptorTable.UnmapServerBuffers(serverProcess.MemoryManager) == KernelResult.Success)
+                {
+                    request.BufferDescriptorTable.RestoreClientBuffers(clientProcess.MemoryManager);
+                }
+
+                CloseAllHandles(serverMsg, clientHeader, serverProcess);
+
+                System.CriticalSection.Enter();
+
+                _activeRequest = null;
+
+                if (_requests.Count != 0)
+                {
+                    Signal();
+                }
+
+                System.CriticalSection.Leave();
+
+                WakeClientThread(request, clientResult);
+            }
+
+            if (clientHeader.ReceiveListType < 2 &&
+                clientHeader.ReceiveListOffset > clientMsg.Size)
+            {
+                CleanUpForError();
+
+                return KernelResult.InvalidCombination;
+            }
+            else if (clientHeader.ReceiveListType == 2 &&
+                     clientHeader.ReceiveListOffset + 8 > clientMsg.Size)
+            {
+                CleanUpForError();
+
+                return KernelResult.InvalidCombination;
+            }
+            else if (clientHeader.ReceiveListType > 2 &&
+                     clientHeader.ReceiveListType * 8 - 0x10 + clientHeader.ReceiveListOffset > clientMsg.Size)
+            {
+                CleanUpForError();
+
+                return KernelResult.InvalidCombination;
+            }
+
+            if (clientHeader.ReceiveListOffsetInWords < clientHeader.MessageSizeInWords)
+            {
+                CleanUpForError();
+
+                return KernelResult.InvalidCombination;
+            }
+
+            if (clientHeader.MessageSizeInWords * 4 > clientMsg.Size)
+            {
+                CleanUpForError();
+
+                return KernelResult.CmdBufferTooSmall;
+            }
+
+            ulong[] receiveList = GetReceiveList(
+                serverMsg,
+                serverHeader.ReceiveListType,
+                serverHeader.ReceiveListOffset);
+
+            serverProcess.CpuMemory.WriteUInt32((long)serverMsg.Address + 0, clientHeader.Word0);
+            serverProcess.CpuMemory.WriteUInt32((long)serverMsg.Address + 4, clientHeader.Word1);
+
+            uint offset;
+
+            //Copy handles.
+            if (clientHeader.HasHandles)
+            {
+                if (clientHeader.MoveHandlesCount != 0)
+                {
+                    CleanUpForError();
+
+                    return KernelResult.InvalidCombination;
+                }
+
+                serverProcess.CpuMemory.WriteUInt32((long)serverMsg.Address + 8, clientHeader.Word2);
+
+                offset = 3;
+
+                if (clientHeader.HasPid)
+                {
+                    serverProcess.CpuMemory.WriteInt64((long)serverMsg.Address + offset * 4, clientProcess.Pid);
+
+                    offset += 2;
+                }
+
+                for (int index = 0; index < clientHeader.CopyHandlesCount; index++)
+                {
+                    int newHandle = 0;
+
+                    int handle = System.Device.Memory.ReadInt32((long)clientMsg.DramAddress + offset * 4);
+
+                    if (clientResult == KernelResult.Success && handle != 0)
+                    {
+                        clientResult = GetCopyObjectHandle(clientThread, serverProcess, handle, out newHandle);
+                    }
+
+                    serverProcess.CpuMemory.WriteInt32((long)serverMsg.Address + offset * 4, newHandle);
+
+                    offset++;
+                }
+
+                for (int index = 0; index < clientHeader.MoveHandlesCount; index++)
+                {
+                    int newHandle = 0;
+
+                    int handle = System.Device.Memory.ReadInt32((long)clientMsg.DramAddress + offset * 4);
+
+                    if (handle != 0)
+                    {
+                        if (clientResult == KernelResult.Success)
+                        {
+                            clientResult = GetMoveObjectHandle(clientProcess, serverProcess, handle, out newHandle);
+                        }
+                        else
+                        {
+                            clientProcess.HandleTable.CloseHandle(handle);
+                        }
+                    }
+
+                    serverProcess.CpuMemory.WriteInt32((long)serverMsg.Address + offset * 4, newHandle);
+
+                    offset++;
+                }
+
+                if (clientResult != KernelResult.Success)
+                {
+                    CleanUpForError();
+
+                    return serverResult;
+                }
+            }
+            else
+            {
+                offset = 2;
+            }
+
+            //Copy pointer/receive list buffers.
+            uint recvListDstOffset = 0;
+
+            for (int index = 0; index < clientHeader.PointerBuffersCount; index++)
+            {
+                ulong pointerDesc = System.Device.Memory.ReadUInt64((long)clientMsg.DramAddress + offset * 4);
+
+                PointerBufferDesc descriptor = new PointerBufferDesc(pointerDesc);
+
+                if (descriptor.BufferSize != 0)
+                {
+                    clientResult = GetReceiveListAddress(
+                        descriptor,
+                        serverMsg,
+                        serverHeader.ReceiveListType,
+                        clientHeader.MessageSizeInWords,
+                        receiveList,
+                        ref       recvListDstOffset,
+                        out ulong recvListBufferAddress);
+
+                    if (clientResult != KernelResult.Success)
+                    {
+                        CleanUpForError();
+
+                        return serverResult;
+                    }
+
+                    clientResult = clientProcess.MemoryManager.CopyDataToCurrentProcess(
+                        recvListBufferAddress,
+                        descriptor.BufferSize,
+                        descriptor.BufferAddress,
+                        MemoryState.IsPoolAllocated,
+                        MemoryState.IsPoolAllocated,
+                        MemoryPermission.Read,
+                        MemoryAttribute.Uncached,
+                        MemoryAttribute.None);
+
+                    if (clientResult != KernelResult.Success)
+                    {
+                        CleanUpForError();
+
+                        return serverResult;
+                    }
+
+                    descriptor.BufferAddress = recvListBufferAddress;
+                }
+                else
+                {
+                    descriptor.BufferAddress = 0;
+                }
+
+                serverProcess.CpuMemory.WriteUInt64((long)serverMsg.Address + offset * 4, descriptor.Pack());
+
+                offset += 2;
+            }
+
+            //Copy send, receive and exchange buffers.
+            uint totalBuffersCount =
+                clientHeader.SendBuffersCount    +
+                clientHeader.ReceiveBuffersCount +
+                clientHeader.ExchangeBuffersCount;
+
+            for (int index = 0; index < totalBuffersCount; index++)
+            {
+                long clientDescAddress = (long)clientMsg.DramAddress + offset * 4;
+
+                uint descWord0 = System.Device.Memory.ReadUInt32(clientDescAddress + 0);
+                uint descWord1 = System.Device.Memory.ReadUInt32(clientDescAddress + 4);
+                uint descWord2 = System.Device.Memory.ReadUInt32(clientDescAddress + 8);
+
+                bool isSendDesc     = index <  clientHeader.SendBuffersCount;
+                bool isExchangeDesc = index >= clientHeader.SendBuffersCount + clientHeader.ReceiveBuffersCount;
+
+                bool notReceiveDesc = isSendDesc || isExchangeDesc;
+                bool isReceiveDesc  = !notReceiveDesc;
+
+                MemoryPermission permission = index >= clientHeader.SendBuffersCount
+                    ? MemoryPermission.ReadAndWrite
+                    : MemoryPermission.Read;
+
+                uint sizeHigh4 = (descWord2 >> 24) & 0xf;
+
+                ulong bufferSize = descWord0 | (ulong)sizeHigh4 << 32;
+
+                ulong dstAddress = 0;
+
+                if (bufferSize != 0)
+                {
+                    ulong bufferAddress;
+
+                    bufferAddress  =   descWord2 >> 28;
+                    bufferAddress |= ((descWord2 >> 2) & 7) << 4;
+
+                    bufferAddress = (bufferAddress << 32) | descWord1;
+
+                    MemoryState state = IpcMemoryStates[(descWord2 + 1) & 3];
+
+                    clientResult = serverProcess.MemoryManager.MapBufferFromClientProcess(
+                        bufferSize,
+                        bufferAddress,
+                        clientProcess.MemoryManager,
+                        permission,
+                        state,
+                        notReceiveDesc,
+                        out dstAddress);
+
+                    if (clientResult != KernelResult.Success)
+                    {
+                        CleanUpForError();
+
+                        return serverResult;
+                    }
+
+                    if (isSendDesc)
+                    {
+                        clientResult = request.BufferDescriptorTable.AddSendBuffer(bufferAddress, dstAddress, bufferSize, state);
+                    }
+                    else if (isReceiveDesc)
+                    {
+                        clientResult = request.BufferDescriptorTable.AddReceiveBuffer(bufferAddress, dstAddress, bufferSize, state);
+                    }
+                    else /* if (isExchangeDesc) */
+                    {
+                        clientResult = request.BufferDescriptorTable.AddExchangeBuffer(bufferAddress, dstAddress, bufferSize, state);
+                    }
+
+                    if (clientResult != KernelResult.Success)
+                    {
+                        CleanUpForError();
+
+                        return serverResult;
+                    }
+                }
+
+                descWord1 = (uint)dstAddress;
+
+                descWord2 &= 3;
+
+                descWord2 |= sizeHigh4 << 24;
+
+                descWord2 |= (uint)(dstAddress >> 34) & 0x3ffffffc;
+                descWord2 |= (uint)(dstAddress >> 4)  & 0xf0000000;
+
+                long serverDescAddress = (long)serverMsg.Address + offset * 4;
+
+                serverProcess.CpuMemory.WriteUInt32(serverDescAddress + 0, descWord0);
+                serverProcess.CpuMemory.WriteUInt32(serverDescAddress + 4, descWord1);
+                serverProcess.CpuMemory.WriteUInt32(serverDescAddress + 8, descWord2);
+
+                offset += 3;
+            }
+
+            //Copy raw data.
+            if (clientHeader.RawDataSizeInWords != 0)
+            {
+                ulong copySrc = clientMsg.Address + offset * 4;
+                ulong copyDst = serverMsg.Address + offset * 4;
+
+                ulong copySize = clientHeader.RawDataSizeInWords * 4;
+
+                if (serverMsg.IsCustom || clientMsg.IsCustom)
+                {
+                    MemoryPermission permission = clientMsg.IsCustom
+                        ? MemoryPermission.None
+                        : MemoryPermission.Read;
+
+                    clientResult = clientProcess.MemoryManager.CopyDataToCurrentProcess(
+                        copyDst,
+                        copySize,
+                        copySrc,
+                        MemoryState.IsPoolAllocated,
+                        MemoryState.IsPoolAllocated,
+                        permission,
+                        MemoryAttribute.Uncached,
+                        MemoryAttribute.None);
+                }
+                else
+                {
+                    copySrc = clientProcess.MemoryManager.GetDramAddressFromVa(copySrc);
+                    copyDst = serverProcess.MemoryManager.GetDramAddressFromVa(copyDst);
+
+                    System.Device.Memory.Copy(copyDst, copySrc, copySize);
+                }
+
+                if (clientResult != KernelResult.Success)
+                {
+                    CleanUpForError();
+
+                    return serverResult;
+                }
+            }
+
+            return KernelResult.Success;
+        }
+
+        public KernelResult Reply(ulong customCmdBuffAddr = 0, ulong customCmdBuffSize = 0)
+        {
+            KThread  serverThread  = System.Scheduler.GetCurrentThread();
+            KProcess serverProcess = serverThread.Owner;
+
+            System.CriticalSection.Enter();
+
+            if (_activeRequest == null)
+            {
+                System.CriticalSection.Leave();
+
+                return KernelResult.InvalidState;
+            }
+
+            KSessionRequest request = _activeRequest;
+
+            _activeRequest = null;
+
+            if (_requests.Count != 0)
+            {
+                Signal();
+            }
+
+            System.CriticalSection.Leave();
+
+            KThread  clientThread  = request.ClientThread;
+            KProcess clientProcess = clientThread.Owner;
+
+            Message clientMsg = new Message(request);
+            Message serverMsg = new Message(serverThread, customCmdBuffAddr, customCmdBuffSize);
+
+            MessageHeader clientHeader = GetClientMessageHeader(clientMsg);
+            MessageHeader serverHeader = GetServerMessageHeader(serverMsg);
+
+            KernelResult clientResult = KernelResult.Success;
+            KernelResult serverResult = KernelResult.Success;
+
+            void CleanUpForError()
+            {
+                CloseAllHandles(clientMsg, serverHeader, clientProcess);
+
+                CancelRequest(request, clientResult);
+            }
+
+            if (clientHeader.ReceiveListType < 2 &&
+                clientHeader.ReceiveListOffset > clientMsg.Size)
+            {
+                CleanUpForError();
+
+                return KernelResult.InvalidCombination;
+            }
+            else if (clientHeader.ReceiveListType == 2 &&
+                     clientHeader.ReceiveListOffset + 8 > clientMsg.Size)
+            {
+                CleanUpForError();
+
+                return KernelResult.InvalidCombination;
+            }
+            else if (clientHeader.ReceiveListType > 2 &&
+                     clientHeader.ReceiveListType * 8 - 0x10 + clientHeader.ReceiveListOffset > clientMsg.Size)
+            {
+                CleanUpForError();
+
+                return KernelResult.InvalidCombination;
+            }
+
+            if (clientHeader.ReceiveListOffsetInWords < clientHeader.MessageSizeInWords)
+            {
+                CleanUpForError();
+
+                return KernelResult.InvalidCombination;
+            }
+
+            if (serverHeader.MessageSizeInWords * 4 > clientMsg.Size)
+            {
+                CleanUpForError();
+
+                return KernelResult.CmdBufferTooSmall;
+            }
+
+            if (serverHeader.SendBuffersCount     != 0 ||
+                serverHeader.ReceiveBuffersCount  != 0 ||
+                serverHeader.ExchangeBuffersCount != 0)
+            {
+                CleanUpForError();
+
+                return KernelResult.InvalidCombination;
+            }
+
+            //Read receive list.
+            ulong[] receiveList = GetReceiveList(
+                clientMsg,
+                clientHeader.ReceiveListType,
+                clientHeader.ReceiveListOffset);
+
+            //Copy receive and exchange buffers.
+            clientResult = request.BufferDescriptorTable.CopyBuffersToClient(clientProcess.MemoryManager);
+
+            if (clientResult != KernelResult.Success)
+            {
+                CleanUpForError();
+
+                return serverResult;
+            }
+
+            //Copy header.
+            System.Device.Memory.WriteUInt32((long)clientMsg.DramAddress + 0, serverHeader.Word0);
+            System.Device.Memory.WriteUInt32((long)clientMsg.DramAddress + 4, serverHeader.Word1);
+
+            //Copy handles.
+            uint offset;
+
+            if (serverHeader.HasHandles)
+            {
+                offset = 3;
+
+                System.Device.Memory.WriteUInt32((long)clientMsg.DramAddress + 8, serverHeader.Word2);
+
+                if (serverHeader.HasPid)
+                {
+                    System.Device.Memory.WriteInt64((long)clientMsg.DramAddress + offset * 4, serverProcess.Pid);
+
+                    offset += 2;
+                }
+
+                for (int index = 0; index < serverHeader.CopyHandlesCount; index++)
+                {
+                    int newHandle = 0;
+
+                    int handle = serverProcess.CpuMemory.ReadInt32((long)serverMsg.Address + offset * 4);
+
+                    if (handle != 0)
+                    {
+                        GetCopyObjectHandle(serverThread, clientProcess, handle, out newHandle);
+                    }
+
+                    System.Device.Memory.WriteInt32((long)clientMsg.DramAddress + offset * 4, newHandle);
+
+                    offset++;
+                }
+
+                for (int index = 0; index < serverHeader.MoveHandlesCount; index++)
+                {
+                    int newHandle = 0;
+
+                    int handle = serverProcess.CpuMemory.ReadInt32((long)serverMsg.Address + offset * 4);
+
+                    if (handle != 0)
+                    {
+                        if (clientResult == KernelResult.Success)
+                        {
+                            clientResult = GetMoveObjectHandle(serverProcess, clientProcess, handle, out newHandle);
+                        }
+                        else
+                        {
+                            serverProcess.HandleTable.CloseHandle(handle);
+                        }
+                    }
+
+                    System.Device.Memory.WriteInt32((long)clientMsg.DramAddress + offset * 4, newHandle);
+
+                    offset++;
+                }
+            }
+            else
+            {
+                offset = 2;
+            }
+
+            //Copy pointer/receive list buffers.
+            uint recvListDstOffset = 0;
+
+            for (int index = 0; index < serverHeader.PointerBuffersCount; index++)
+            {
+                ulong pointerDesc = serverProcess.CpuMemory.ReadUInt64((long)serverMsg.Address + offset * 4);
+
+                PointerBufferDesc descriptor = new PointerBufferDesc(pointerDesc);
+
+                if (descriptor.BufferSize != 0)
+                {
+                    clientResult = GetReceiveListAddress(
+                        descriptor,
+                        clientMsg,
+                        clientHeader.ReceiveListType,
+                        serverHeader.MessageSizeInWords,
+                        receiveList,
+                        ref       recvListDstOffset,
+                        out ulong recvListBufferAddress);
+
+                    if (clientResult != KernelResult.Success)
+                    {
+                        CleanUpForError();
+
+                        return serverResult;
+                    }
+
+                    clientResult = clientProcess.MemoryManager.CopyDataFromCurrentProcess(
+                        recvListBufferAddress,
+                        descriptor.BufferSize,
+                        MemoryState.IsPoolAllocated,
+                        MemoryState.IsPoolAllocated,
+                        MemoryPermission.Read,
+                        MemoryAttribute.Uncached,
+                        MemoryAttribute.None,
+                        descriptor.BufferAddress);
+
+                    if (clientResult != KernelResult.Success)
+                    {
+                        CleanUpForError();
+
+                        return serverResult;
+                    }
+                }
+
+                offset += 2;
+            }
+
+            //Set send, receive and exchange buffer descriptors to zero.
+            uint totalBuffersCount =
+                serverHeader.SendBuffersCount    +
+                serverHeader.ReceiveBuffersCount +
+                serverHeader.ExchangeBuffersCount;
+
+            for (int index = 0; index < totalBuffersCount; index++)
+            {
+                long dstDescAddress = (long)clientMsg.DramAddress + offset * 4;
+
+                System.Device.Memory.WriteUInt32(dstDescAddress + 0, 0);
+                System.Device.Memory.WriteUInt32(dstDescAddress + 4, 0);
+                System.Device.Memory.WriteUInt32(dstDescAddress + 8, 0);
+
+                offset += 3;
+            }
+
+            //Copy raw data.
+            if (serverHeader.RawDataSizeInWords != 0)
+            {
+                ulong copyDst = clientMsg.Address + offset * 4;
+                ulong copySrc = serverMsg.Address + offset * 4;
+
+                ulong copySize = serverHeader.RawDataSizeInWords * 4;
+
+                if (serverMsg.IsCustom || clientMsg.IsCustom)
+                {
+                    MemoryPermission permission = clientMsg.IsCustom
+                        ? MemoryPermission.None
+                        : MemoryPermission.Read;
+
+                    clientResult = clientProcess.MemoryManager.CopyDataFromCurrentProcess(
+                        copyDst,
+                        copySize,
+                        MemoryState.IsPoolAllocated,
+                        MemoryState.IsPoolAllocated,
+                        permission,
+                        MemoryAttribute.Uncached,
+                        MemoryAttribute.None,
+                        copySrc);
+                }
+                else
+                {
+                    copyDst = clientProcess.MemoryManager.GetDramAddressFromVa(copyDst);
+                    copySrc = serverProcess.MemoryManager.GetDramAddressFromVa(copySrc);
+
+                    System.Device.Memory.Copy(copyDst, copySrc, copySize);
+                }
+            }
+
+            //Unmap buffers from server.
+            clientResult = request.BufferDescriptorTable.UnmapServerBuffers(serverProcess.MemoryManager);
+
+            if (clientResult != KernelResult.Success)
+            {
+                CleanUpForError();
+
+                return serverResult;
+            }
+
+            WakeClientThread(request, clientResult);
+
+            return serverResult;
+        }
+
+        private MessageHeader GetClientMessageHeader(Message clientMsg)
+        {
+            uint word0 = System.Device.Memory.ReadUInt32((long)clientMsg.DramAddress + 0);
+            uint word1 = System.Device.Memory.ReadUInt32((long)clientMsg.DramAddress + 4);
+            uint word2 = System.Device.Memory.ReadUInt32((long)clientMsg.DramAddress + 8);
+
+            return new MessageHeader(word0, word1, word2);
+        }
+
+        private MessageHeader GetServerMessageHeader(Message serverMsg)
+        {
+            KProcess currentProcess = System.Scheduler.GetCurrentProcess();
+
+            uint word0 = currentProcess.CpuMemory.ReadUInt32((long)serverMsg.Address + 0);
+            uint word1 = currentProcess.CpuMemory.ReadUInt32((long)serverMsg.Address + 4);
+            uint word2 = currentProcess.CpuMemory.ReadUInt32((long)serverMsg.Address + 8);
+
+            return new MessageHeader(word0, word1, word2);
+        }
+
+        private KernelResult GetCopyObjectHandle(
+            KThread  srcThread,
+            KProcess dstProcess,
+            int      srcHandle,
+            out int  dstHandle)
+        {
+            dstHandle = 0;
+
+            KProcess srcProcess = srcThread.Owner;
+
+            KAutoObject obj;
+
+            if (srcHandle == KHandleTable.SelfProcessHandle)
+            {
+                obj = srcProcess;
+            }
+            else if (srcHandle == KHandleTable.SelfThreadHandle)
+            {
+                obj = srcThread;
+            }
+            else
+            {
+                obj = srcProcess.HandleTable.GetObject<KAutoObject>(srcHandle);
+            }
+
+            if (obj != null)
+            {
+                return dstProcess.HandleTable.GenerateHandle(obj, out dstHandle);
+            }
+            else
+            {
+                return KernelResult.InvalidHandle;
+            }
+        }
+
+        private KernelResult GetMoveObjectHandle(
+            KProcess srcProcess,
+            KProcess dstProcess,
+            int      srcHandle,
+            out int  dstHandle)
+        {
+            dstHandle = 0;
+
+            KAutoObject obj = srcProcess.HandleTable.GetObject<KAutoObject>(srcHandle);
+
+            if (obj != null)
+            {
+                KernelResult result = dstProcess.HandleTable.GenerateHandle(obj, out dstHandle);
+
+                srcProcess.HandleTable.CloseHandle(srcHandle);
+
+                return result;
+            }
+            else
+            {
+                return KernelResult.InvalidHandle;
+            }
+        }
+
+        private ulong[] GetReceiveList(Message message, uint recvListType, uint recvListOffset)
+        {
+            int recvListSize = 0;
+
+            if (recvListType >= 3)
+            {
+                recvListSize = (int)recvListType - 2;
+            }
+            else if (recvListType == 2)
+            {
+                recvListSize = 1;
+            }
+
+            ulong[] receiveList = new ulong[recvListSize];
+
+            long recvListAddress = (long)message.DramAddress + recvListOffset;
+
+            for (int index = 0; index < recvListSize; index++)
+            {
+                receiveList[index] = System.Device.Memory.ReadUInt64(recvListAddress + index * 8);
+            }
+
+            return receiveList;
+        }
+
+        private KernelResult GetReceiveListAddress(
+            PointerBufferDesc descriptor,
+            Message           message,
+            uint              recvListType,
+            uint              messageSizeInWords,
+            ulong[]           receiveList,
+            ref uint          dstOffset,
+            out ulong         address)
+        {
+            ulong recvListBufferAddress = address = 0;
+
+            if (recvListType == 0)
+            {
+                return KernelResult.OutOfResource;
+            }
+            else if (recvListType == 1 || recvListType == 2)
+            {
+                ulong recvListBaseAddr;
+                ulong recvListEndAddr;
+
+                if (recvListType == 1)
+                {
+                    recvListBaseAddr = message.Address + messageSizeInWords * 4;
+                    recvListEndAddr  = message.Address + message.Size;
+                }
+                else /* if (recvListType == 2) */
+                {
+                    ulong packed = receiveList[0];
+
+                    recvListBaseAddr = packed & 0x7fffffffff;
+
+                    uint size = (uint)(packed >> 48);
+
+                    if (size == 0)
+                    {
+                        return KernelResult.OutOfResource;
+                    }
+
+                    recvListEndAddr = recvListBaseAddr + size;
+                }
+
+                recvListBufferAddress = BitUtils.AlignUp(recvListBaseAddr + dstOffset, 0x10);
+
+                ulong endAddress = recvListBufferAddress + descriptor.BufferSize;
+
+                dstOffset = (uint)endAddress - (uint)recvListBaseAddr;
+
+                if (recvListBufferAddress + descriptor.BufferSize <= recvListBufferAddress ||
+                    recvListBufferAddress + descriptor.BufferSize >  recvListEndAddr)
+                {
+                    return KernelResult.OutOfResource;
+                }
+            }
+            else /* if (recvListType > 2) */
+            {
+                if (descriptor.ReceiveIndex >= receiveList.Length)
+                {
+                    return KernelResult.OutOfResource;
+                }
+
+                ulong packed = receiveList[descriptor.ReceiveIndex];
+
+                recvListBufferAddress = packed & 0x7fffffffff;
+
+                uint size = (uint)(packed >> 48);
+
+                if (recvListBufferAddress == 0 || size == 0 || size < descriptor.BufferSize)
+                {
+                    return KernelResult.OutOfResource;
+                }
+            }
+
+            address = recvListBufferAddress;
+
+            return KernelResult.Success;
+        }
+
+        private void CloseAllHandles(Message message, MessageHeader header, KProcess process)
+        {
+            if (header.HasHandles)
+            {
+                uint totalHandeslCount = header.CopyHandlesCount + header.MoveHandlesCount;
+
+                uint offset = 3;
+
+                if (header.HasPid)
+                {
+                    process.CpuMemory.WriteInt64((long)message.Address + offset * 4, 0);
+
+                    offset += 2;
+                }
+
+                for (int index = 0; index < totalHandeslCount; index++)
+                {
+                    int handle = process.CpuMemory.ReadInt32((long)message.Address + offset * 4);
+
+                    if (handle != 0)
+                    {
+                        process.HandleTable.CloseHandle(handle);
+
+                        process.CpuMemory.WriteInt32((long)message.Address + offset * 4, 0);
+                    }
+
+                    offset++;
+                }
+            }
+        }
+
+        public override bool IsSignaled()
+        {
+            if (_parent.ClientSession.State != ChannelState.Open)
+            {
+                return true;
+            }
+
+            return _requests.Count != 0 && _activeRequest == null;
+        }
+
+        protected override void Destroy()
+        {
+            _parent.DisconnectServer();
+
+            CancelAllRequestsServerDisconnected();
+
+            _parent.DecrementReferenceCount();
+        }
+
+        private void CancelAllRequestsServerDisconnected()
+        {
+            foreach (KSessionRequest request in IterateWithRemovalOfAllRequests())
+            {
+                CancelRequest(request, KernelResult.PortRemoteClosed);
+            }
+        }
+
+        public void CancelAllRequestsClientDisconnected()
+        {
+            foreach (KSessionRequest request in IterateWithRemovalOfAllRequests())
+            {
+                if (request.ClientThread.ShallBeTerminated ||
+                    request.ClientThread.SchedFlags == ThreadSchedState.TerminationPending)
+                {
+                    continue;
+                }
+
+                //Client sessions can only be disconnected on async requests (because
+                //the client would be otherwise blocked waiting for the response), so
+                //we only need to handle the async case here.
+                if (request.AsyncEvent != null)
+                {
+                    SendResultToAsyncRequestClient(request, KernelResult.PortRemoteClosed);
+                }
+            }
+
+            WakeServerThreads(KernelResult.PortRemoteClosed);
+        }
+
+        private IEnumerable<KSessionRequest> IterateWithRemovalOfAllRequests()
+        {
+            System.CriticalSection.Enter();
+
+            if (_activeRequest != null)
+            {
+                KSessionRequest request = _activeRequest;
+
+                _activeRequest = null;
+
+                System.CriticalSection.Leave();
+
+                yield return request;
+            }
+            else
+            {
+                System.CriticalSection.Leave();
+            }
+
+            while (DequeueRequest(out KSessionRequest request))
+            {
+                yield return request;
+            }
+        }
+
+        private bool DequeueRequest(out KSessionRequest request)
+        {
+            request = null;
+
+            System.CriticalSection.Enter();
+
+            bool hasRequest = _requests.First != null;
+
+            if (hasRequest)
+            {
+                request = _requests.First.Value;
+
+                _requests.RemoveFirst();
+            }
+
+            System.CriticalSection.Leave();
+
+            return hasRequest;
+        }
+
+        private void CancelRequest(KSessionRequest request, KernelResult result)
+        {
+            KProcess clientProcess = request.ClientThread.Owner;
+            KProcess serverProcess = request.ServerProcess;
+
+            KernelResult unmapResult = KernelResult.Success;
+
+            if (serverProcess != null)
+            {
+                unmapResult = request.BufferDescriptorTable.UnmapServerBuffers(serverProcess.MemoryManager);
+            }
+
+            if (unmapResult == KernelResult.Success)
+            {
+                request.BufferDescriptorTable.RestoreClientBuffers(clientProcess.MemoryManager);
+            }
+
+            WakeClientThread(request, result);
+        }
+
+        private void WakeClientThread(KSessionRequest request, KernelResult result)
+        {
+            //Wait client thread waiting for a response for the given request.
+            if (request.AsyncEvent != null)
+            {
+                SendResultToAsyncRequestClient(request, result);
+            }
+            else
+            {
+                System.CriticalSection.Enter();
+
+                WakeAndSetResult(request.ClientThread, result);
+
+                System.CriticalSection.Leave();
+            }
+        }
+
+        private void SendResultToAsyncRequestClient(KSessionRequest request, KernelResult result)
+        {
+            KProcess clientProcess = request.ClientThread.Owner;
+
+            ulong address = clientProcess.MemoryManager.GetDramAddressFromVa(request.CustomCmdBuffAddr);
+
+            System.Device.Memory.WriteInt64((long)address + 0, 0);
+            System.Device.Memory.WriteInt32((long)address + 8, (int)result);
+
+            clientProcess.MemoryManager.UnborrowIpcBuffer(
+                request.CustomCmdBuffAddr,
+                request.CustomCmdBuffSize);
+
+            request.AsyncEvent.Signal();
+        }
+
+        private void WakeServerThreads(KernelResult result)
+        {
+            //Wake all server threads waiting for requests.
+            System.CriticalSection.Enter();
+
+            foreach (KThread thread in WaitingThreads)
+            {
+                WakeAndSetResult(thread, result);
+            }
+
+            System.CriticalSection.Leave();
+        }
+
+        private void WakeAndSetResult(KThread thread, KernelResult result)
+        {
+            if ((thread.SchedFlags & ThreadSchedState.LowMask) == ThreadSchedState.Paused)
+            {
+                thread.SignaledObj   = null;
+                thread.ObjSyncResult = result;
+
+                thread.Reschedule(ThreadSchedState.Running);
+            }
+        }
+    }
+}
\ No newline at end of file