Move solution and projects to src

5 files changed, 588 insertions, 0 deletions

diff --git a/src/ARMeilleure/Translation/Cache/CacheEntry.cs b/src/ARMeilleure/Translation/Cache/CacheEntry.cs
new file mode 100644
index 00000000..dc5503b1
--- /dev/null
+++ b/src/ARMeilleure/Translation/Cache/CacheEntry.cs
@@ -0,0 +1,26 @@
+using ARMeilleure.CodeGen.Unwinding;
+using System;
+using System.Diagnostics.CodeAnalysis;
+
+namespace ARMeilleure.Translation.Cache
+{
+    readonly struct CacheEntry : IComparable<CacheEntry>
+    {
+        public int Offset { get; }
+        public int Size   { get; }
+
+        public UnwindInfo UnwindInfo { get; }
+
+        public CacheEntry(int offset, int size, UnwindInfo unwindInfo)
+        {
+            Offset     = offset;
+            Size       = size;
+            UnwindInfo = unwindInfo;
+        }
+
+        public int CompareTo([AllowNull] CacheEntry other)
+        {
+            return Offset.CompareTo(other.Offset);
+        }
+    }
+}
\ No newline at end of file
diff --git a/src/ARMeilleure/Translation/Cache/CacheMemoryAllocator.cs b/src/ARMeilleure/Translation/Cache/CacheMemoryAllocator.cs
new file mode 100644
index 00000000..4c22de40
--- /dev/null
+++ b/src/ARMeilleure/Translation/Cache/CacheMemoryAllocator.cs
@@ -0,0 +1,96 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+
+namespace ARMeilleure.Translation.Cache
+{
+    class CacheMemoryAllocator
+    {
+        private readonly struct MemoryBlock : IComparable<MemoryBlock>
+        {
+            public int Offset { get; }
+            public int Size { get; }
+
+            public MemoryBlock(int offset, int size)
+            {
+                Offset = offset;
+                Size = size;
+            }
+
+            public int CompareTo([AllowNull] MemoryBlock other)
+            {
+                return Offset.CompareTo(other.Offset);
+            }
+        }
+
+        private readonly List<MemoryBlock> _blocks = new List<MemoryBlock>();
+
+        public CacheMemoryAllocator(int capacity)
+        {
+            _blocks.Add(new MemoryBlock(0, capacity));
+        }
+
+        public int Allocate(int size)
+        {
+            for (int i = 0; i < _blocks.Count; i++)
+            {
+                MemoryBlock block = _blocks[i];
+
+                if (block.Size > size)
+                {
+                    _blocks[i] = new MemoryBlock(block.Offset + size, block.Size - size);
+                    return block.Offset;
+                }
+                else if (block.Size == size)
+                {
+                    _blocks.RemoveAt(i);
+                    return block.Offset;
+                }
+            }
+
+            // We don't have enough free memory to perform the allocation.
+            return -1;
+        }
+
+        public void Free(int offset, int size)
+        {
+            Insert(new MemoryBlock(offset, size));
+        }
+
+        private void Insert(MemoryBlock block)
+        {
+            int index = _blocks.BinarySearch(block);
+
+            if (index < 0)
+            {
+                index = ~index;
+            }
+
+            if (index < _blocks.Count)
+            {
+                MemoryBlock next = _blocks[index];
+
+                int endOffs = block.Offset + block.Size;
+
+                if (next.Offset == endOffs)
+                {
+                    block = new MemoryBlock(block.Offset, block.Size + next.Size);
+                    _blocks.RemoveAt(index);
+                }
+            }
+
+            if (index > 0)
+            {
+                MemoryBlock prev = _blocks[index - 1];
+
+                if (prev.Offset + prev.Size == block.Offset)
+                {
+                    block = new MemoryBlock(block.Offset - prev.Size, block.Size + prev.Size);
+                    _blocks.RemoveAt(--index);
+                }
+            }
+
+            _blocks.Insert(index, block);
+        }
+    }
+}
diff --git a/src/ARMeilleure/Translation/Cache/JitCache.cs b/src/ARMeilleure/Translation/Cache/JitCache.cs
new file mode 100644
index 00000000..f496a8e9
--- /dev/null
+++ b/src/ARMeilleure/Translation/Cache/JitCache.cs
@@ -0,0 +1,198 @@
+using ARMeilleure.CodeGen;
+using ARMeilleure.CodeGen.Unwinding;
+using ARMeilleure.Memory;
+using ARMeilleure.Native;
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Runtime.InteropServices;
+
+namespace ARMeilleure.Translation.Cache
+{
+    static class JitCache
+    {
+        private const int PageSize = 4 * 1024;
+        private const int PageMask = PageSize - 1;
+
+        private const int CodeAlignment = 4; // Bytes.
+        private const int CacheSize = 2047 * 1024 * 1024;
+
+        private static ReservedRegion _jitRegion;
+        private static JitCacheInvalidation _jitCacheInvalidator;
+
+        private static CacheMemoryAllocator _cacheAllocator;
+
+        private static readonly List<CacheEntry> _cacheEntries = new List<CacheEntry>();
+
+        private static readonly object _lock = new object();
+        private static bool _initialized;
+
+        public static void Initialize(IJitMemoryAllocator allocator)
+        {
+            if (_initialized) return;
+
+            lock (_lock)
+            {
+                if (_initialized) return;
+
+                _jitRegion = new ReservedRegion(allocator, CacheSize);
+                _jitCacheInvalidator = new JitCacheInvalidation(allocator);
+
+                _cacheAllocator = new CacheMemoryAllocator(CacheSize);
+
+                if (OperatingSystem.IsWindows())
+                {
+                    JitUnwindWindows.InstallFunctionTableHandler(_jitRegion.Pointer, CacheSize, _jitRegion.Pointer + Allocate(PageSize));
+                }
+
+                _initialized = true;
+            }
+        }
+
+        public static IntPtr Map(CompiledFunction func)
+        {
+            byte[] code = func.Code;
+
+            lock (_lock)
+            {
+                Debug.Assert(_initialized);
+
+                int funcOffset = Allocate(code.Length);
+
+                IntPtr funcPtr = _jitRegion.Pointer + funcOffset;
+
+                if (OperatingSystem.IsMacOS() && RuntimeInformation.ProcessArchitecture == Architecture.Arm64)
+                {
+                    unsafe
+                    {
+                        fixed (byte *codePtr = code)
+                        {
+                            JitSupportDarwin.Copy(funcPtr, (IntPtr)codePtr, (ulong)code.Length);
+                        }
+                    }
+                }
+                else
+                {
+                    ReprotectAsWritable(funcOffset, code.Length);
+                    Marshal.Copy(code, 0, funcPtr, code.Length);
+                    ReprotectAsExecutable(funcOffset, code.Length);
+
+                    _jitCacheInvalidator.Invalidate(funcPtr, (ulong)code.Length);
+                }
+
+                Add(funcOffset, code.Length, func.UnwindInfo);
+
+                return funcPtr;
+            }
+        }
+
+        public static void Unmap(IntPtr pointer)
+        {
+            lock (_lock)
+            {
+                Debug.Assert(_initialized);
+
+                int funcOffset = (int)(pointer.ToInt64() - _jitRegion.Pointer.ToInt64());
+
+                bool result = TryFind(funcOffset, out CacheEntry entry);
+                Debug.Assert(result);
+
+                _cacheAllocator.Free(funcOffset, AlignCodeSize(entry.Size));
+
+                Remove(funcOffset);
+            }
+        }
+
+        private static void ReprotectAsWritable(int offset, int size)
+        {
+            int endOffs = offset + size;
+
+            int regionStart = offset & ~PageMask;
+            int regionEnd = (endOffs + PageMask) & ~PageMask;
+
+            _jitRegion.Block.MapAsRwx((ulong)regionStart, (ulong)(regionEnd - regionStart));
+        }
+
+        private static void ReprotectAsExecutable(int offset, int size)
+        {
+            int endOffs = offset + size;
+
+            int regionStart = offset & ~PageMask;
+            int regionEnd = (endOffs + PageMask) & ~PageMask;
+
+            _jitRegion.Block.MapAsRx((ulong)regionStart, (ulong)(regionEnd - regionStart));
+        }
+
+        private static int Allocate(int codeSize)
+        {
+            codeSize = AlignCodeSize(codeSize);
+
+            int allocOffset = _cacheAllocator.Allocate(codeSize);
+
+            if (allocOffset < 0)
+            {
+                throw new OutOfMemoryException("JIT Cache exhausted.");
+            }
+
+            _jitRegion.ExpandIfNeeded((ulong)allocOffset + (ulong)codeSize);
+
+            return allocOffset;
+        }
+
+        private static int AlignCodeSize(int codeSize)
+        {
+            return checked(codeSize + (CodeAlignment - 1)) & ~(CodeAlignment - 1);
+        }
+
+        private static void Add(int offset, int size, UnwindInfo unwindInfo)
+        {
+            CacheEntry entry = new CacheEntry(offset, size, unwindInfo);
+
+            int index = _cacheEntries.BinarySearch(entry);
+
+            if (index < 0)
+            {
+                index = ~index;
+            }
+
+            _cacheEntries.Insert(index, entry);
+        }
+
+        private static void Remove(int offset)
+        {
+            int index = _cacheEntries.BinarySearch(new CacheEntry(offset, 0, default));
+
+            if (index < 0)
+            {
+                index = ~index - 1;
+            }
+
+            if (index >= 0)
+            {
+                _cacheEntries.RemoveAt(index);
+            }
+        }
+
+        public static bool TryFind(int offset, out CacheEntry entry)
+        {
+            lock (_lock)
+            {
+                int index = _cacheEntries.BinarySearch(new CacheEntry(offset, 0, default));
+
+                if (index < 0)
+                {
+                    index = ~index - 1;
+                }
+
+                if (index >= 0)
+                {
+                    entry = _cacheEntries[index];
+                    return true;
+                }
+            }
+
+            entry = default;
+            return false;
+        }
+    }
+}
\ No newline at end of file
diff --git a/src/ARMeilleure/Translation/Cache/JitCacheInvalidation.cs b/src/ARMeilleure/Translation/Cache/JitCacheInvalidation.cs
new file mode 100644
index 00000000..ec2ae73b
--- /dev/null
+++ b/src/ARMeilleure/Translation/Cache/JitCacheInvalidation.cs
@@ -0,0 +1,79 @@
+using ARMeilleure.Memory;
+using System;
+using System.Runtime.InteropServices;
+
+namespace ARMeilleure.Translation.Cache
+{
+    class JitCacheInvalidation
+    {
+        private static int[] _invalidationCode = new int[]
+        {
+            unchecked((int)0xd53b0022), // mrs  x2, ctr_el0
+            unchecked((int)0xd3504c44), // ubfx x4, x2, #16, #4
+            unchecked((int)0x52800083), // mov  w3, #0x4
+            unchecked((int)0x12000c45), // and  w5, w2, #0xf
+            unchecked((int)0x1ac42064), // lsl  w4, w3, w4
+            unchecked((int)0x51000482), // sub  w2, w4, #0x1
+            unchecked((int)0x8a220002), // bic  x2, x0, x2
+            unchecked((int)0x1ac52063), // lsl  w3, w3, w5
+            unchecked((int)0xeb01005f), // cmp  x2, x1
+            unchecked((int)0x93407c84), // sxtw x4, w4
+            unchecked((int)0x540000a2), // b.cs 3c <do_ic_clear>
+            unchecked((int)0xd50b7b22), // dc   cvau, x2
+            unchecked((int)0x8b040042), // add  x2, x2, x4
+            unchecked((int)0xeb02003f), // cmp  x1, x2
+            unchecked((int)0x54ffffa8), // b.hi 2c <dc_clear_loop>
+            unchecked((int)0xd5033b9f), // dsb  ish
+            unchecked((int)0x51000462), // sub  w2, w3, #0x1
+            unchecked((int)0x93407c63), // sxtw x3, w3
+            unchecked((int)0x8a220000), // bic  x0, x0, x2
+            unchecked((int)0xeb00003f), // cmp  x1, x0
+            unchecked((int)0x540000a9), // b.ls 64 <exit>
+            unchecked((int)0xd50b7520), // ic   ivau, x0
+            unchecked((int)0x8b030000), // add  x0, x0, x3
+            unchecked((int)0xeb00003f), // cmp  x1, x0
+            unchecked((int)0x54ffffa8), // b.hi 54 <ic_clear_loop>
+            unchecked((int)0xd5033b9f), // dsb  ish
+            unchecked((int)0xd5033fdf), // isb
+            unchecked((int)0xd65f03c0), // ret
+        };
+
+        private delegate void InvalidateCache(ulong start, ulong end);
+
+        private InvalidateCache _invalidateCache;
+        private ReservedRegion _invalidateCacheCodeRegion;
+
+        private readonly bool _needsInvalidation;
+
+        public JitCacheInvalidation(IJitMemoryAllocator allocator)
+        {
+            // On macOS, a different path is used to write to the JIT cache, which does the invalidation.
+            if (!OperatingSystem.IsMacOS() && RuntimeInformation.ProcessArchitecture == Architecture.Arm64)
+            {
+                ulong size = (ulong)_invalidationCode.Length * sizeof(int);
+                ulong mask = (ulong)ReservedRegion.DefaultGranularity - 1;
+
+                size = (size + mask) & ~mask;
+
+                _invalidateCacheCodeRegion = new ReservedRegion(allocator, size);
+                _invalidateCacheCodeRegion.ExpandIfNeeded(size);
+
+                Marshal.Copy(_invalidationCode, 0, _invalidateCacheCodeRegion.Pointer, _invalidationCode.Length);
+
+                _invalidateCacheCodeRegion.Block.MapAsRx(0, size);
+
+                _invalidateCache = Marshal.GetDelegateForFunctionPointer<InvalidateCache>(_invalidateCacheCodeRegion.Pointer);
+
+                _needsInvalidation = true;
+            }
+        }
+
+        public void Invalidate(IntPtr basePointer, ulong size)
+        {
+            if (_needsInvalidation)
+            {
+                _invalidateCache((ulong)basePointer, (ulong)basePointer + size);
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/src/ARMeilleure/Translation/Cache/JitUnwindWindows.cs b/src/ARMeilleure/Translation/Cache/JitUnwindWindows.cs
new file mode 100644
index 00000000..77727bf1
--- /dev/null
+++ b/src/ARMeilleure/Translation/Cache/JitUnwindWindows.cs
@@ -0,0 +1,189 @@
+// https://github.com/MicrosoftDocs/cpp-docs/blob/master/docs/build/exception-handling-x64.md
+
+using ARMeilleure.CodeGen.Unwinding;
+using System;
+using System.Diagnostics;
+using System.Runtime.InteropServices;
+
+namespace ARMeilleure.Translation.Cache
+{
+    static partial class JitUnwindWindows
+    {
+        private const int MaxUnwindCodesArraySize = 32; // Must be an even value.
+
+        private struct RuntimeFunction
+        {
+            public uint BeginAddress;
+            public uint EndAddress;
+            public uint UnwindData;
+        }
+
+        private struct UnwindInfo
+        {
+            public byte VersionAndFlags;
+            public byte SizeOfProlog;
+            public byte CountOfUnwindCodes;
+            public byte FrameRegister;
+            public unsafe fixed ushort UnwindCodes[MaxUnwindCodesArraySize];
+        }
+
+        private enum UnwindOp
+        {
+            PushNonvol    = 0,
+            AllocLarge    = 1,
+            AllocSmall    = 2,
+            SetFpreg      = 3,
+            SaveNonvol    = 4,
+            SaveNonvolFar = 5,
+            SaveXmm128    = 8,
+            SaveXmm128Far = 9,
+            PushMachframe = 10
+        }
+
+        private unsafe delegate RuntimeFunction* GetRuntimeFunctionCallback(ulong controlPc, IntPtr context);
+
+        [LibraryImport("kernel32.dll")]
+        [return: MarshalAs(UnmanagedType.Bool)]
+        private static unsafe partial bool RtlInstallFunctionTableCallback(
+            ulong tableIdentifier,
+            ulong baseAddress,
+            uint length,
+            GetRuntimeFunctionCallback callback,
+            IntPtr context,
+            [MarshalAs(UnmanagedType.LPWStr)] string outOfProcessCallbackDll);
+
+        private static GetRuntimeFunctionCallback _getRuntimeFunctionCallback;
+
+        private static int _sizeOfRuntimeFunction;
+
+        private unsafe static RuntimeFunction* _runtimeFunction;
+
+        private unsafe static UnwindInfo* _unwindInfo;
+
+        public static void InstallFunctionTableHandler(IntPtr codeCachePointer, uint codeCacheLength, IntPtr workBufferPtr)
+        {
+            ulong codeCachePtr = (ulong)codeCachePointer.ToInt64();
+
+            _sizeOfRuntimeFunction = Marshal.SizeOf<RuntimeFunction>();
+
+            bool result;
+
+            unsafe
+            {
+                _runtimeFunction = (RuntimeFunction*)workBufferPtr;
+
+                _unwindInfo = (UnwindInfo*)(workBufferPtr + _sizeOfRuntimeFunction);
+
+                _getRuntimeFunctionCallback = new GetRuntimeFunctionCallback(FunctionTableHandler);
+
+                result = RtlInstallFunctionTableCallback(
+                    codeCachePtr | 3,
+                    codeCachePtr,
+                    codeCacheLength,
+                    _getRuntimeFunctionCallback,
+                    codeCachePointer,
+                    null);
+            }
+
+            if (!result)
+            {
+                throw new InvalidOperationException("Failure installing function table callback.");
+            }
+        }
+
+        private static unsafe RuntimeFunction* FunctionTableHandler(ulong controlPc, IntPtr context)
+        {
+            int offset = (int)((long)controlPc - context.ToInt64());
+
+            if (!JitCache.TryFind(offset, out CacheEntry funcEntry))
+            {
+                return null; // Not found.
+            }
+
+            var unwindInfo = funcEntry.UnwindInfo;
+
+            int codeIndex = 0;
+
+            for (int index = unwindInfo.PushEntries.Length - 1; index >= 0; index--)
+            {
+                var entry = unwindInfo.PushEntries[index];
+
+                switch (entry.PseudoOp)
+                {
+                    case UnwindPseudoOp.SaveXmm128:
+                    {
+                        int stackOffset = entry.StackOffsetOrAllocSize;
+
+                        Debug.Assert(stackOffset % 16 == 0);
+
+                        if (stackOffset <= 0xFFFF0)
+                        {
+                            _unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOp.SaveXmm128, entry.PrologOffset, entry.RegIndex);
+                            _unwindInfo->UnwindCodes[codeIndex++] = (ushort)(stackOffset / 16);
+                        }
+                        else
+                        {
+                            _unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOp.SaveXmm128Far, entry.PrologOffset, entry.RegIndex);
+                            _unwindInfo->UnwindCodes[codeIndex++] = (ushort)(stackOffset >> 0);
+                            _unwindInfo->UnwindCodes[codeIndex++] = (ushort)(stackOffset >> 16);
+                        }
+
+                        break;
+                    }
+
+                    case UnwindPseudoOp.AllocStack:
+                    {
+                        int allocSize = entry.StackOffsetOrAllocSize;
+
+                        Debug.Assert(allocSize % 8 == 0);
+
+                        if (allocSize <= 128)
+                        {
+                            _unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOp.AllocSmall, entry.PrologOffset, (allocSize / 8) - 1);
+                        }
+                        else if (allocSize <= 0x7FFF8)
+                        {
+                            _unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOp.AllocLarge, entry.PrologOffset, 0);
+                            _unwindInfo->UnwindCodes[codeIndex++] = (ushort)(allocSize / 8);
+                        }
+                        else
+                        {
+                            _unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOp.AllocLarge, entry.PrologOffset, 1);
+                            _unwindInfo->UnwindCodes[codeIndex++] = (ushort)(allocSize >> 0);
+                            _unwindInfo->UnwindCodes[codeIndex++] = (ushort)(allocSize >> 16);
+                        }
+
+                        break;
+                    }
+
+                    case UnwindPseudoOp.PushReg:
+                    {
+                        _unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOp.PushNonvol, entry.PrologOffset, entry.RegIndex);
+
+                        break;
+                    }
+
+                    default: throw new NotImplementedException($"({nameof(entry.PseudoOp)} = {entry.PseudoOp})");
+                }
+            }
+
+            Debug.Assert(codeIndex <= MaxUnwindCodesArraySize);
+
+            _unwindInfo->VersionAndFlags    = 1; // Flags: The function has no handler.
+            _unwindInfo->SizeOfProlog       = (byte)unwindInfo.PrologSize;
+            _unwindInfo->CountOfUnwindCodes = (byte)codeIndex;
+            _unwindInfo->FrameRegister      = 0;
+
+            _runtimeFunction->BeginAddress = (uint)funcEntry.Offset;
+            _runtimeFunction->EndAddress   = (uint)(funcEntry.Offset + funcEntry.Size);
+            _runtimeFunction->UnwindData   = (uint)_sizeOfRuntimeFunction;
+
+            return _runtimeFunction;
+        }
+
+        private static ushort PackUnwindOp(UnwindOp op, int prologOffset, int opInfo)
+        {
+            return (ushort)(prologOffset | ((int)op << 8) | (opInfo << 12));
+        }
+    }
+}
\ No newline at end of file