diff options
Diffstat (limited to 'ARMeilleure/Translation')
22 files changed, 2085 insertions, 228 deletions
diff --git a/ARMeilleure/Translation/ArmEmitterContext.cs b/ARMeilleure/Translation/ArmEmitterContext.cs index a905c722..7ce6a3f4 100644 --- a/ARMeilleure/Translation/ArmEmitterContext.cs +++ b/ARMeilleure/Translation/ArmEmitterContext.cs @@ -11,7 +11,7 @@ namespace ARMeilleure.Translation { class ArmEmitterContext : EmitterContext { - private Dictionary<ulong, Operand> _labels; + private readonly Dictionary<ulong, Operand> _labels; private OpCode _optOpLastCompare; private OpCode _optOpLastFlagSet; diff --git a/ARMeilleure/Translation/Compiler.cs b/ARMeilleure/Translation/Compiler.cs index ec2f2968..934c0db6 100644 --- a/ARMeilleure/Translation/Compiler.cs +++ b/ARMeilleure/Translation/Compiler.cs @@ -7,18 +7,30 @@ using System.Runtime.InteropServices; namespace ARMeilleure.Translation { + using PTC; + static class Compiler { - public static T Compile<T>(ControlFlowGraph cfg, OperandType[] argTypes, OperandType retType, CompilerOptions options) + public static T Compile<T>( + ControlFlowGraph cfg, + OperandType[] argTypes, + OperandType retType, + CompilerOptions options, + PtcInfo ptcInfo = null) { - CompiledFunction func = Compile(cfg, argTypes, retType, options); + CompiledFunction func = Compile(cfg, argTypes, retType, options, ptcInfo); IntPtr codePtr = JitCache.Map(func); return Marshal.GetDelegateForFunctionPointer<T>(codePtr); } - public static CompiledFunction Compile(ControlFlowGraph cfg, OperandType[] argTypes, OperandType retType, CompilerOptions options) + public static CompiledFunction Compile( + ControlFlowGraph cfg, + OperandType[] argTypes, + OperandType retType, + CompilerOptions options, + PtcInfo ptcInfo = null) { Logger.StartPass(PassName.Dominance); @@ -45,7 +57,7 @@ namespace ARMeilleure.Translation CompilerContext cctx = new CompilerContext(cfg, argTypes, retType, options); - return CodeGenerator.Generate(cctx); + return CodeGenerator.Generate(cctx, ptcInfo); } } }
\ No newline at end of file diff --git a/ARMeilleure/Translation/DelegateCache.cs b/ARMeilleure/Translation/DelegateCache.cs deleted file mode 100644 index 7328c61a..00000000 --- a/ARMeilleure/Translation/DelegateCache.cs +++ /dev/null @@ -1,26 +0,0 @@ -using System; -using System.Collections.Concurrent; -using System.Reflection; - -namespace ARMeilleure.Translation -{ - static class DelegateCache - { - private static ConcurrentDictionary<string, Delegate> _delegates; - - static DelegateCache() - { - _delegates = new ConcurrentDictionary<string, Delegate>(); - } - - public static Delegate GetOrAdd(Delegate dlg) - { - return _delegates.GetOrAdd(GetKey(dlg.Method), (key) => dlg); - } - - private static string GetKey(MethodInfo info) - { - return $"{info.DeclaringType.FullName}.{info.Name}"; - } - } -}
\ No newline at end of file diff --git a/ARMeilleure/Translation/DelegateHelper.cs b/ARMeilleure/Translation/DelegateHelper.cs new file mode 100644 index 00000000..f021d116 --- /dev/null +++ b/ARMeilleure/Translation/DelegateHelper.cs @@ -0,0 +1,107 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Reflection.Emit; + +namespace ARMeilleure.Translation +{ + static class DelegateHelper + { + private const string DelegateTypesAssemblyName = "JitDelegateTypes"; + + private static readonly ModuleBuilder _modBuilder; + + private static readonly Dictionary<string, Type> _delegateTypesCache; + + static DelegateHelper() + { + AssemblyBuilder asmBuilder = AssemblyBuilder.DefineDynamicAssembly(new AssemblyName(DelegateTypesAssemblyName), AssemblyBuilderAccess.Run); + + _modBuilder = asmBuilder.DefineDynamicModule(DelegateTypesAssemblyName); + + _delegateTypesCache = new Dictionary<string, Type>(); + } + + public static Delegate GetDelegate(MethodInfo info) + { + if (info == null) + { + throw new ArgumentNullException(nameof(info)); + } + + Type[] parameters = info.GetParameters().Select(pI => pI.ParameterType).ToArray(); + Type returnType = info.ReturnType; + + Type delegateType = GetDelegateType(parameters, returnType); + + return Delegate.CreateDelegate(delegateType, info); + } + + private static Type GetDelegateType(Type[] parameters, Type returnType) + { + string key = GetFunctionSignatureKey(parameters, returnType); + + if (!_delegateTypesCache.TryGetValue(key, out Type delegateType)) + { + delegateType = MakeDelegateType(parameters, returnType, key); + + _delegateTypesCache.TryAdd(key, delegateType); + } + + return delegateType; + } + + private static string GetFunctionSignatureKey(Type[] parameters, Type returnType) + { + string sig = GetTypeName(returnType); + + foreach (Type type in parameters) + { + sig += '_' + GetTypeName(type); + } + + return sig; + } + + private static string GetTypeName(Type type) + { + return type.FullName.Replace(".", string.Empty); + } + + private const MethodAttributes CtorAttributes = + MethodAttributes.RTSpecialName | + MethodAttributes.HideBySig | + MethodAttributes.Public; + + private const TypeAttributes DelegateTypeAttributes = + TypeAttributes.Class | + TypeAttributes.Public | + TypeAttributes.Sealed | + TypeAttributes.AnsiClass | + TypeAttributes.AutoClass; + + private const MethodImplAttributes ImplAttributes = + MethodImplAttributes.Runtime | + MethodImplAttributes.Managed; + + private const MethodAttributes InvokeAttributes = + MethodAttributes.Public | + MethodAttributes.HideBySig | + MethodAttributes.NewSlot | + MethodAttributes.Virtual; + + private static readonly Type[] _delegateCtorSignature = { typeof(object), typeof(IntPtr) }; + + private static Type MakeDelegateType(Type[] parameters, Type returnType, string name) + { + TypeBuilder builder = _modBuilder.DefineType(name, DelegateTypeAttributes, typeof(MulticastDelegate)); + + builder.DefineConstructor(CtorAttributes, CallingConventions.Standard, _delegateCtorSignature).SetImplementationFlags(ImplAttributes); + + builder.DefineMethod("Invoke", InvokeAttributes, returnType, parameters).SetImplementationFlags(ImplAttributes); + + return builder.CreateTypeInfo(); + } + } +} diff --git a/ARMeilleure/Translation/DelegateInfo.cs b/ARMeilleure/Translation/DelegateInfo.cs new file mode 100644 index 00000000..e68cfc1b --- /dev/null +++ b/ARMeilleure/Translation/DelegateInfo.cs @@ -0,0 +1,19 @@ +using System; +using System.Runtime.InteropServices; + +namespace ARMeilleure.Translation +{ + sealed class DelegateInfo + { + private readonly Delegate _dlg; // Ensure that this delegate will not be garbage collected. + + public IntPtr FuncPtr { get; } + + public DelegateInfo(Delegate dlg) + { + _dlg = dlg; + + FuncPtr = Marshal.GetFunctionPointerForDelegate<Delegate>(dlg); + } + } +} diff --git a/ARMeilleure/Translation/Delegates.cs b/ARMeilleure/Translation/Delegates.cs new file mode 100644 index 00000000..addb15f6 --- /dev/null +++ b/ARMeilleure/Translation/Delegates.cs @@ -0,0 +1,305 @@ +using ARMeilleure.Instructions; +using System; +using System.Collections.Generic; +using System.Reflection; + +namespace ARMeilleure.Translation +{ + static class Delegates + { + public static bool TryGetDelegateFuncPtrByIndex(int index, out IntPtr funcPtr) + { + if (index >= 0 && index < _delegates.Count) + { + funcPtr = _delegates.Values[index].FuncPtr; // O(1). + + return true; + } + else + { + funcPtr = default; + + return false; + } + } + + public static IntPtr GetDelegateFuncPtrByIndex(int index) + { + if (index < 0 || index >= _delegates.Count) + { + throw new ArgumentOutOfRangeException($"({nameof(index)} = {index})"); + } + + return _delegates.Values[index].FuncPtr; // O(1). + } + + public static IntPtr GetDelegateFuncPtr(MethodInfo info) + { + if (info == null) + { + throw new ArgumentNullException(nameof(info)); + } + + string key = GetKey(info); + + if (!_delegates.TryGetValue(key, out DelegateInfo dlgInfo)) // O(log(n)). + { + throw new KeyNotFoundException($"({nameof(key)} = {key})"); + } + + return dlgInfo.FuncPtr; + } + + public static int GetDelegateIndex(MethodInfo info) + { + if (info == null) + { + throw new ArgumentNullException(nameof(info)); + } + + string key = GetKey(info); + + int index = _delegates.IndexOfKey(key); // O(log(n)). + + if (index == -1) + { + throw new KeyNotFoundException($"({nameof(key)} = {key})"); + } + + return index; + } + + private static void SetDelegateInfo(MethodInfo info) + { + string key = GetKey(info); + + Delegate dlg = DelegateHelper.GetDelegate(info); + + _delegates.Add(key, new DelegateInfo(dlg)); // ArgumentException (key). + } + + private static string GetKey(MethodInfo info) + { + return $"{info.DeclaringType.Name}.{info.Name}"; + } + + private static readonly SortedList<string, DelegateInfo> _delegates; + + static Delegates() + { + _delegates = new SortedList<string, DelegateInfo>(); + + SetDelegateInfo(typeof(Math).GetMethod(nameof(Math.Abs), new Type[] { typeof(double) })); + SetDelegateInfo(typeof(Math).GetMethod(nameof(Math.Ceiling), new Type[] { typeof(double) })); + SetDelegateInfo(typeof(Math).GetMethod(nameof(Math.Floor), new Type[] { typeof(double) })); + SetDelegateInfo(typeof(Math).GetMethod(nameof(Math.Round), new Type[] { typeof(double), typeof(MidpointRounding) })); + SetDelegateInfo(typeof(Math).GetMethod(nameof(Math.Truncate), new Type[] { typeof(double) })); + + SetDelegateInfo(typeof(MathF).GetMethod(nameof(MathF.Abs), new Type[] { typeof(float) })); + SetDelegateInfo(typeof(MathF).GetMethod(nameof(MathF.Ceiling), new Type[] { typeof(float) })); + SetDelegateInfo(typeof(MathF).GetMethod(nameof(MathF.Floor), new Type[] { typeof(float) })); + SetDelegateInfo(typeof(MathF).GetMethod(nameof(MathF.Round), new Type[] { typeof(float), typeof(MidpointRounding) })); + SetDelegateInfo(typeof(MathF).GetMethod(nameof(MathF.Truncate), new Type[] { typeof(float) })); + + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.Break))); + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.CheckSynchronization))); + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.ClearExclusive))); + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetCntfrqEl0))); + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetCntpctEl0))); + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetCntvctEl0))); + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetCtrEl0))); + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetDczidEl0))); + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFpcr))); + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFpscr))); // A32 only. + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFpsr))); + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFunctionAddress))); + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetIndirectFunctionAddress))); + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetTpidr))); + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetTpidr32))); // A32 only. + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetTpidrEl0))); + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetTpidrEl032))); // A32 only. + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.ReadByte))); + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.ReadByteExclusive))); + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.ReadUInt16))); + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.ReadUInt16Exclusive))); + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.ReadUInt32))); + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.ReadUInt32Exclusive))); + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.ReadUInt64))); + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.ReadUInt64Exclusive))); + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.ReadVector128))); + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.ReadVector128Exclusive))); + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.SetFpcr))); + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.SetFpscr))); // A32 only. + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.SetFpsr))); + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.SetTpidrEl0))); + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.SetTpidrEl032))); // A32 only. + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.SupervisorCall))); + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.Undefined))); + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteByte))); + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteByteExclusive))); + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteUInt16))); + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteUInt16Exclusive))); + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteUInt32))); + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteUInt32Exclusive))); + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteUInt64))); + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteUInt64Exclusive))); + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteVector128))); + SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteVector128Exclusive))); + + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.BinarySignedSatQAcc))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.BinarySignedSatQAdd))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.BinarySignedSatQSub))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.BinaryUnsignedSatQAcc))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.BinaryUnsignedSatQAdd))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.BinaryUnsignedSatQSub))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.CountLeadingSigns))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.CountLeadingZeros))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.CountSetBits8))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.Crc32b))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.Crc32cb))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.Crc32ch))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.Crc32cw))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.Crc32cx))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.Crc32h))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.Crc32w))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.Crc32x))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.Decrypt))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.DoubleToInt32))); // A32 only. + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.DoubleToUInt32))); // A32 only. + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.Encrypt))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.FixedRotate))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.FloatToInt32))); // A32 only. + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.FloatToUInt32))); // A32 only. + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.HashChoose))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.HashLower))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.HashMajority))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.HashParity))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.HashUpper))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.InverseMixColumns))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.MixColumns))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.Round))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.RoundF))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.SatF32ToS32))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.SatF32ToS64))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.SatF32ToU32))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.SatF32ToU64))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.SatF64ToS32))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.SatF64ToS64))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.SatF64ToU32))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.SatF64ToU64))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.Sha1SchedulePart1))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.Sha1SchedulePart2))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.Sha256SchedulePart1))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.Sha256SchedulePart2))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.SignedShlReg))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.SignedShlRegSatQ))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.SignedShrImm64))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.SignedSrcSignedDstSatQ))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.SignedSrcUnsignedDstSatQ))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.Tbl1))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.Tbl2))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.Tbl3))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.Tbl4))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.Tbx1))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.Tbx2))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.Tbx3))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.Tbx4))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.UnarySignedSatQAbsOrNeg))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.UnsignedShlReg))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.UnsignedShlRegSatQ))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.UnsignedShrImm64))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.UnsignedSrcSignedDstSatQ))); + SetDelegateInfo(typeof(SoftFallback).GetMethod(nameof(SoftFallback.UnsignedSrcUnsignedDstSatQ))); + + SetDelegateInfo(typeof(SoftFloat16_32).GetMethod(nameof(SoftFloat16_32.FPConvert))); + + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPAdd))); + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPAddFpscr))); // A32 only. + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPCompare))); + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPCompareEQ))); + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPCompareEQFpscr))); // A32 only. + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPCompareGE))); + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPCompareGEFpscr))); // A32 only. + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPCompareGT))); + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPCompareGTFpscr))); // A32 only. + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPCompareLE))); + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPCompareLEFpscr))); // A32 only. + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPCompareLT))); + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPCompareLTFpscr))); // A32 only. + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPDiv))); + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPMax))); + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPMaxFpscr))); // A32 only. + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPMaxNum))); + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPMaxNumFpscr))); // A32 only. + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPMin))); + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPMinFpscr))); // A32 only. + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPMinNum))); + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPMinNumFpscr))); // A32 only. + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPMul))); + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPMulFpscr))); // A32 only. + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPMulAdd))); + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPMulAddFpscr))); // A32 only. + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPMulSub))); + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPMulSubFpscr))); // A32 only. + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPMulX))); + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPNegMulAdd))); + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPNegMulSub))); + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPRecipEstimate))); + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPRecipEstimateFpscr))); // A32 only. + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPRecipStep))); // A32 only. + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPRecipStepFused))); + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPRecpX))); + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPRSqrtEstimate))); + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPRSqrtEstimateFpscr))); // A32 only. + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPRSqrtStep))); // A32 only. + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPRSqrtStepFused))); + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPSqrt))); + SetDelegateInfo(typeof(SoftFloat32).GetMethod(nameof(SoftFloat32.FPSub))); + + SetDelegateInfo(typeof(SoftFloat32_16).GetMethod(nameof(SoftFloat32_16.FPConvert))); + + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPAdd))); + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPAddFpscr))); // A32 only. + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPCompare))); + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPCompareEQ))); + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPCompareEQFpscr))); // A32 only. + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPCompareGE))); + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPCompareGEFpscr))); // A32 only. + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPCompareGT))); + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPCompareGTFpscr))); // A32 only. + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPCompareLE))); + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPCompareLEFpscr))); // A32 only. + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPCompareLT))); + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPCompareLTFpscr))); // A32 only. + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPDiv))); + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPMax))); + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPMaxFpscr))); // A32 only. + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPMaxNum))); + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPMaxNumFpscr))); // A32 only. + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPMin))); + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPMinFpscr))); // A32 only. + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPMinNum))); + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPMinNumFpscr))); // A32 only. + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPMul))); + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPMulFpscr))); // A32 only. + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPMulAdd))); + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPMulAddFpscr))); // A32 only. + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPMulSub))); + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPMulSubFpscr))); // A32 only. + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPMulX))); + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPNegMulAdd))); + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPNegMulSub))); + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPRecipEstimate))); + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPRecipEstimateFpscr))); // A32 only. + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPRecipStep))); // A32 only. + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPRecipStepFused))); + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPRecpX))); + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPRSqrtEstimate))); + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPRSqrtEstimateFpscr))); // A32 only. + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPRSqrtStep))); // A32 only. + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPRSqrtStepFused))); + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPSqrt))); + SetDelegateInfo(typeof(SoftFloat64).GetMethod(nameof(SoftFloat64.FPSub))); + } + } +} diff --git a/ARMeilleure/Translation/DirectCallStubs.cs b/ARMeilleure/Translation/DirectCallStubs.cs index 42a78c71..7c11fdb2 100644 --- a/ARMeilleure/Translation/DirectCallStubs.cs +++ b/ARMeilleure/Translation/DirectCallStubs.cs @@ -2,6 +2,7 @@ using ARMeilleure.IntermediateRepresentation; using ARMeilleure.State; using System; +using System.Diagnostics; using System.Runtime.InteropServices; using static ARMeilleure.IntermediateRepresentation.OperandHelper; @@ -12,10 +13,10 @@ namespace ARMeilleure.Translation { private delegate long GuestFunction(IntPtr nativeContextPtr); - private static GuestFunction _directCallStub; - private static GuestFunction _directTailCallStub; - private static GuestFunction _indirectCallStub; - private static GuestFunction _indirectTailCallStub; + private static IntPtr _directCallStubPtr; + private static IntPtr _directTailCallStubPtr; + private static IntPtr _indirectCallStubPtr; + private static IntPtr _indirectTailCallStubPtr; private static readonly object _lock = new object(); private static bool _initialized; @@ -23,25 +24,32 @@ namespace ARMeilleure.Translation public static void InitializeStubs() { if (_initialized) return; + lock (_lock) { if (_initialized) return; - _directCallStub = GenerateDirectCallStub(false); - _directTailCallStub = GenerateDirectCallStub(true); - _indirectCallStub = GenerateIndirectCallStub(false); - _indirectTailCallStub = GenerateIndirectCallStub(true); + + _directCallStubPtr = Marshal.GetFunctionPointerForDelegate<GuestFunction>(GenerateDirectCallStub(false)); + _directTailCallStubPtr = Marshal.GetFunctionPointerForDelegate<GuestFunction>(GenerateDirectCallStub(true)); + _indirectCallStubPtr = Marshal.GetFunctionPointerForDelegate<GuestFunction>(GenerateIndirectCallStub(false)); + _indirectTailCallStubPtr = Marshal.GetFunctionPointerForDelegate<GuestFunction>(GenerateIndirectCallStub(true)); + _initialized = true; } } public static IntPtr DirectCallStub(bool tailCall) { - return Marshal.GetFunctionPointerForDelegate(tailCall ? _directTailCallStub : _directCallStub); + Debug.Assert(_initialized); + + return tailCall ? _directTailCallStubPtr : _directCallStubPtr; } public static IntPtr IndirectCallStub(bool tailCall) { - return Marshal.GetFunctionPointerForDelegate(tailCall ? _indirectTailCallStub : _indirectCallStub); + Debug.Assert(_initialized); + + return tailCall ? _indirectTailCallStubPtr : _indirectCallStubPtr; } private static void EmitCall(EmitterContext context, Operand address, bool tailCall) @@ -70,21 +78,18 @@ namespace ARMeilleure.Translation Operand address = context.Load(OperandType.I64, context.Add(nativeContextPtr, Const((long)NativeContext.GetCallAddressOffset()))); address = context.BitwiseOr(address, Const(address.Type, 1)); // Set call flag. - Operand functionAddr = context.Call(new _U64_U64(NativeInterface.GetFunctionAddress), address); + Operand functionAddr = context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFunctionAddress)), address); EmitCall(context, functionAddr, tailCall); ControlFlowGraph cfg = context.GetControlFlowGraph(); - OperandType[] argTypes = new OperandType[] - { - OperandType.I64 - }; + OperandType[] argTypes = new OperandType[] { OperandType.I64 }; return Compiler.Compile<GuestFunction>(cfg, argTypes, OperandType.I64, CompilerOptions.HighCq); } /// <summary> - /// Generates a stub that is used to find function addresses and add them to an indirect table. + /// Generates a stub that is used to find function addresses and add them to an indirect table. /// Used for indirect calls entries (already claimed) when their jump table does not have the host address yet. /// Takes a NativeContext like a translated guest function, and extracts the target indirect table entry from the NativeContext. /// If the function we find is highCq, the entry in the table is updated to point to that function rather than this stub. @@ -100,17 +105,14 @@ namespace ARMeilleure.Translation // We need to find the missing function. If the function is HighCq, then it replaces this stub in the indirect table. // Either way, we call it afterwards. - Operand functionAddr = context.Call(new _U64_U64_U64(NativeInterface.GetIndirectFunctionAddress), address, entryAddress); + Operand functionAddr = context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetIndirectFunctionAddress)), address, entryAddress); // Call and save the function. EmitCall(context, functionAddr, tailCall); ControlFlowGraph cfg = context.GetControlFlowGraph(); - OperandType[] argTypes = new OperandType[] - { - OperandType.I64 - }; + OperandType[] argTypes = new OperandType[] { OperandType.I64 }; return Compiler.Compile<GuestFunction>(cfg, argTypes, OperandType.I64, CompilerOptions.HighCq); } diff --git a/ARMeilleure/Translation/EmitterContext.cs b/ARMeilleure/Translation/EmitterContext.cs index f14c920e..656f1704 100644 --- a/ARMeilleure/Translation/EmitterContext.cs +++ b/ARMeilleure/Translation/EmitterContext.cs @@ -3,12 +3,14 @@ using ARMeilleure.IntermediateRepresentation; using ARMeilleure.State; using System; using System.Collections.Generic; -using System.Runtime.InteropServices; +using System.Reflection; using static ARMeilleure.IntermediateRepresentation.OperandHelper; namespace ARMeilleure.Translation { + using PTC; + class EmitterContext { private Dictionary<Operand, BasicBlock> _irLabels; @@ -79,42 +81,52 @@ namespace ARMeilleure.Translation return Add(Instruction.ByteSwap, Local(op1.Type), op1); } - public Operand Call(Delegate func, params Operand[] callArgs) + public Operand Call(MethodInfo info, params Operand[] callArgs) { - // Add the delegate to the cache to ensure it will not be garbage collected. - func = DelegateCache.GetOrAdd(func); + if (Ptc.State == PtcState.Disabled) + { + IntPtr funcPtr = Delegates.GetDelegateFuncPtr(info); - IntPtr ptr = Marshal.GetFunctionPointerForDelegate<Delegate>(func); + OperandType returnType = GetOperandType(info.ReturnType); - Symbols.Add((ulong)ptr.ToInt64(), func.Method.Name); + Symbols.Add((ulong)funcPtr.ToInt64(), info.Name); - OperandType returnType = GetOperandType(func.Method.ReturnType); + return Call(Const(funcPtr.ToInt64()), returnType, callArgs); + } + else + { + int index = Delegates.GetDelegateIndex(info); - return Call(Const(ptr.ToInt64()), returnType, callArgs); - } + IntPtr funcPtr = Delegates.GetDelegateFuncPtrByIndex(index); - private static Dictionary<TypeCode, OperandType> _typeCodeToOperandTypeMap = - new Dictionary<TypeCode, OperandType>() - { - { TypeCode.Boolean, OperandType.I32 }, - { TypeCode.Byte, OperandType.I32 }, - { TypeCode.Char, OperandType.I32 }, - { TypeCode.Double, OperandType.FP64 }, - { TypeCode.Int16, OperandType.I32 }, - { TypeCode.Int32, OperandType.I32 }, - { TypeCode.Int64, OperandType.I64 }, - { TypeCode.SByte, OperandType.I32 }, - { TypeCode.Single, OperandType.FP32 }, - { TypeCode.UInt16, OperandType.I32 }, - { TypeCode.UInt32, OperandType.I32 }, - { TypeCode.UInt64, OperandType.I64 } - }; + OperandType returnType = GetOperandType(info.ReturnType); + + Symbols.Add((ulong)funcPtr.ToInt64(), info.Name); + + return Call(Const(funcPtr.ToInt64(), true, index), returnType, callArgs); + } + } private static OperandType GetOperandType(Type type) { - if (_typeCodeToOperandTypeMap.TryGetValue(Type.GetTypeCode(type), out OperandType ot)) + if (type == typeof(bool) || type == typeof(byte) || + type == typeof(char) || type == typeof(short) || + type == typeof(int) || type == typeof(sbyte) || + type == typeof(ushort) || type == typeof(uint)) { - return ot; + return OperandType.I32; + } + else if (type == typeof(long) || type == typeof(ulong)) + { + return OperandType.I64; + } + else if (type == typeof(double)) + { + return OperandType.FP64; + } + else if (type == typeof(float)) + { + return OperandType.FP32; } else if (type == typeof(V128)) { @@ -124,8 +136,10 @@ namespace ARMeilleure.Translation { return OperandType.None; } - - throw new ArgumentException($"Invalid type \"{type.Name}\"."); + else + { + throw new ArgumentException($"Invalid type \"{type.Name}\"."); + } } public Operand Call(Operand address, OperandType returnType, params Operand[] callArgs) @@ -615,4 +629,4 @@ namespace ARMeilleure.Translation return new ControlFlowGraph(_irBlocks.First, _irBlocks); } } -}
\ No newline at end of file +} diff --git a/ARMeilleure/Translation/JitCache.cs b/ARMeilleure/Translation/JitCache.cs index 32d40c20..a828b4ed 100644 --- a/ARMeilleure/Translation/JitCache.cs +++ b/ARMeilleure/Translation/JitCache.cs @@ -12,11 +12,12 @@ namespace ARMeilleure.Translation private const int PageSize = 4 * 1024; private const int PageMask = PageSize - 1; - private const int CodeAlignment = 4; // Bytes + private const int CodeAlignment = 4; // Bytes. private const int CacheSize = 2047 * 1024 * 1024; private static ReservedRegion _jitRegion; private static int _offset; + private static readonly List<JitCacheEntry> _cacheEntries = new List<JitCacheEntry>(); private static readonly object _lock = new object(); @@ -25,19 +26,23 @@ namespace ARMeilleure.Translation public static void Initialize(IJitMemoryAllocator allocator) { if (_initialized) return; + lock (_lock) { if (_initialized) return; + _jitRegion = new ReservedRegion(allocator, CacheSize); if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { - _jitRegion.ExpandIfNeeded(PageSize); + _jitRegion.ExpandIfNeeded((ulong)PageSize); + JitUnwindWindows.InstallFunctionTableHandler(_jitRegion.Pointer, CacheSize); // The first page is used for the table based SEH structs. _offset = PageSize; } + _initialized = true; } } @@ -97,13 +102,13 @@ namespace ARMeilleure.Translation _offset += codeSize; - _jitRegion.ExpandIfNeeded((ulong)_offset); - - if ((ulong)(uint)_offset > CacheSize) + if (_offset > CacheSize) { - throw new OutOfMemoryException(); + throw new OutOfMemoryException("JIT Cache exhausted."); } + _jitRegion.ExpandIfNeeded((ulong)_offset); + return allocOffset; } diff --git a/ARMeilleure/Translation/JitUnwindWindows.cs b/ARMeilleure/Translation/JitUnwindWindows.cs index 3f5b3282..e118d129 100644 --- a/ARMeilleure/Translation/JitUnwindWindows.cs +++ b/ARMeilleure/Translation/JitUnwindWindows.cs @@ -27,7 +27,7 @@ namespace ARMeilleure.Translation public unsafe fixed ushort UnwindCodes[MaxUnwindCodesArraySize]; } - private enum UnwindOperation + private enum UnwindOp { PushNonvol = 0, AllocLarge = 1, @@ -117,12 +117,12 @@ namespace ARMeilleure.Translation if (stackOffset <= 0xFFFF0) { - _unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOperation.SaveXmm128, entry.PrologOffset, entry.RegIndex); + _unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOp.SaveXmm128, entry.PrologOffset, entry.RegIndex); _unwindInfo->UnwindCodes[codeIndex++] = (ushort)(stackOffset / 16); } else { - _unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOperation.SaveXmm128Far, entry.PrologOffset, entry.RegIndex); + _unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOp.SaveXmm128Far, entry.PrologOffset, entry.RegIndex); _unwindInfo->UnwindCodes[codeIndex++] = (ushort)(stackOffset >> 0); _unwindInfo->UnwindCodes[codeIndex++] = (ushort)(stackOffset >> 16); } @@ -138,16 +138,16 @@ namespace ARMeilleure.Translation if (allocSize <= 128) { - _unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOperation.AllocSmall, entry.PrologOffset, (allocSize / 8) - 1); + _unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOp.AllocSmall, entry.PrologOffset, (allocSize / 8) - 1); } else if (allocSize <= 0x7FFF8) { - _unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOperation.AllocLarge, entry.PrologOffset, 0); + _unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOp.AllocLarge, entry.PrologOffset, 0); _unwindInfo->UnwindCodes[codeIndex++] = (ushort)(allocSize / 8); } else { - _unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOperation.AllocLarge, entry.PrologOffset, 1); + _unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOp.AllocLarge, entry.PrologOffset, 1); _unwindInfo->UnwindCodes[codeIndex++] = (ushort)(allocSize >> 0); _unwindInfo->UnwindCodes[codeIndex++] = (ushort)(allocSize >> 16); } @@ -157,7 +157,7 @@ namespace ARMeilleure.Translation case UnwindPseudoOp.PushReg: { - _unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOperation.PushNonvol, entry.PrologOffset, entry.RegIndex); + _unwindInfo->UnwindCodes[codeIndex++] = PackUnwindOp(UnwindOp.PushNonvol, entry.PrologOffset, entry.RegIndex); break; } @@ -180,7 +180,7 @@ namespace ARMeilleure.Translation return _runtimeFunction; } - private static ushort PackUnwindOp(UnwindOperation op, int prologOffset, int opInfo) + private static ushort PackUnwindOp(UnwindOp op, int prologOffset, int opInfo) { return (ushort)(prologOffset | ((int)op << 8) | (opInfo << 12)); } diff --git a/ARMeilleure/Translation/JumpTable.cs b/ARMeilleure/Translation/JumpTable.cs index 40ea0fce..fe7a6ec3 100644 --- a/ARMeilleure/Translation/JumpTable.cs +++ b/ARMeilleure/Translation/JumpTable.cs @@ -3,11 +3,14 @@ using ARMeilleure.Memory; using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Diagnostics; using System.Runtime.InteropServices; using System.Threading; namespace ARMeilleure.Translation { + using PTC; + class JumpTable { // The jump table is a block of (guestAddress, hostAddress) function mappings. @@ -15,10 +18,9 @@ namespace ARMeilleure.Translation // reserved specifically for each call. // The _dependants dictionary can be used to update the hostAddress for any functions that change. - public const int JumpTableStride = 16; // 8 byte guest address, 8 byte host address + public const int JumpTableStride = 16; // 8 byte guest address, 8 byte host address. private const int JumpTableSize = 1048576; - private const int JumpTableByteSize = JumpTableSize * JumpTableStride; // The dynamic table is also a block of (guestAddress, hostAddress) function mappings. @@ -32,74 +34,125 @@ namespace ARMeilleure.Translation // If it is 0, NativeInterface is called to find the rejited address of the call. // If none is found, the hostAddress entry stays at 0. Otherwise, the new address is placed in the entry. - // If the table size is exhausted and we didn't find our desired address, we fall back to requesting + // If the table size is exhausted and we didn't find our desired address, we fall back to requesting // the function from the JIT. - private const int DynamicTableSize = 1048576; - public const int DynamicTableElems = 1; public const int DynamicTableStride = DynamicTableElems * JumpTableStride; - private const int DynamicTableByteSize = DynamicTableSize * JumpTableStride * DynamicTableElems; + private const int DynamicTableSize = 1048576; + private const int DynamicTableByteSize = DynamicTableSize * DynamicTableStride; - private int _tableEnd = 0; - private int _dynTableEnd = 0; + private readonly ReservedRegion _jumpRegion; + private readonly ReservedRegion _dynamicRegion; - private ConcurrentDictionary<ulong, TranslatedFunction> _targets; - private ConcurrentDictionary<ulong, LinkedList<int>> _dependants; // TODO: Attach to TranslatedFunction or a wrapper class. + private int _tableEnd = 0; + private int _dynTableEnd = 0; - private ReservedRegion _jumpRegion; - private ReservedRegion _dynamicRegion; - public IntPtr JumpPointer => _jumpRegion.Pointer; + public IntPtr JumpPointer => _jumpRegion.Pointer; public IntPtr DynamicPointer => _dynamicRegion.Pointer; + public int TableEnd => _tableEnd; + public int DynTableEnd => _dynTableEnd; + + public ConcurrentDictionary<ulong, TranslatedFunction> Targets { get; } + public ConcurrentDictionary<ulong, LinkedList<int>> Dependants { get; } // TODO: Attach to TranslatedFunction or a wrapper class. + public JumpTable(IJitMemoryAllocator allocator) { - _jumpRegion = new ReservedRegion(allocator, JumpTableByteSize); + _jumpRegion = new ReservedRegion(allocator, JumpTableByteSize); _dynamicRegion = new ReservedRegion(allocator, DynamicTableByteSize); - _targets = new ConcurrentDictionary<ulong, TranslatedFunction>(); - _dependants = new ConcurrentDictionary<ulong, LinkedList<int>>(); + Targets = new ConcurrentDictionary<ulong, TranslatedFunction>(); + Dependants = new ConcurrentDictionary<ulong, LinkedList<int>>(); Symbols.Add((ulong)_jumpRegion.Pointer.ToInt64(), JumpTableByteSize, JumpTableStride, "JMP_TABLE"); Symbols.Add((ulong)_dynamicRegion.Pointer.ToInt64(), DynamicTableByteSize, DynamicTableStride, "DYN_TABLE"); } + public void Initialize(PtcJumpTable ptcJumpTable, ConcurrentDictionary<ulong, TranslatedFunction> funcs) + { + _tableEnd = ptcJumpTable.TableEnd; + _dynTableEnd = ptcJumpTable.DynTableEnd; + + foreach (ulong guestAddress in ptcJumpTable.Targets) + { + if (funcs.TryGetValue(guestAddress, out TranslatedFunction func)) + { + Targets.TryAdd(guestAddress, func); + } + else + { + throw new KeyNotFoundException($"({nameof(guestAddress)} = 0x{guestAddress:X16})"); + } + } + + foreach (var item in ptcJumpTable.Dependants) + { + Dependants.TryAdd(item.Key, new LinkedList<int>(item.Value)); + } + } + public void RegisterFunction(ulong address, TranslatedFunction func) { address &= ~3UL; - _targets.AddOrUpdate(address, func, (key, oldFunc) => func); - long funcPtr = func.GetPointer().ToInt64(); + Targets.AddOrUpdate(address, func, (key, oldFunc) => func); + long funcPtr = func.FuncPtr.ToInt64(); // Update all jump table entries that target this address. - if (_dependants.TryGetValue(address, out LinkedList<int> myDependants)) + if (Dependants.TryGetValue(address, out LinkedList<int> myDependants)) { lock (myDependants) { - foreach (var entry in myDependants) + foreach (int entry in myDependants) { - IntPtr addr = _jumpRegion.Pointer + entry * JumpTableStride; + IntPtr addr = GetEntryAddressJumpTable(entry); + Marshal.WriteInt64(addr, 8, funcPtr); } } } } - public int ReserveDynamicEntry(bool isJump) + public int ReserveTableEntry(long ownerAddress, long address, bool isJump) { - int entry = Interlocked.Increment(ref _dynTableEnd); - if (entry >= DynamicTableSize) + int entry = Interlocked.Increment(ref _tableEnd); + + ExpandIfNeededJumpTable(entry); + + // Is the address we have already registered? If so, put the function address in the jump table. + // If not, it will point to the direct call stub. + long value = DirectCallStubs.DirectCallStub(isJump).ToInt64(); + if (Targets.TryGetValue((ulong)address, out TranslatedFunction func)) { - throw new OutOfMemoryException("JIT Dynamic Jump Table exhausted."); + value = func.FuncPtr.ToInt64(); + } + + // Make sure changes to the function at the target address update this jump table entry. + LinkedList<int> targetDependants = Dependants.GetOrAdd((ulong)address, (addr) => new LinkedList<int>()); + lock (targetDependants) + { + targetDependants.AddLast(entry); } - _dynamicRegion.ExpandIfNeeded((ulong)((entry + 1) * DynamicTableStride)); + IntPtr addr = GetEntryAddressJumpTable(entry); - // Initialize all host function pointers to the indirect call stub. + Marshal.WriteInt64(addr, 0, address); + Marshal.WriteInt64(addr, 8, value); + + return entry; + } + + public int ReserveDynamicEntry(bool isJump) + { + int entry = Interlocked.Increment(ref _dynTableEnd); - IntPtr addr = _dynamicRegion.Pointer + entry * DynamicTableStride; - long stubPtr = (long)DirectCallStubs.IndirectCallStub(isJump); + ExpandIfNeededDynamicTable(entry); + + // Initialize all host function pointers to the indirect call stub. + IntPtr addr = GetEntryAddressDynamicTable(entry); + long stubPtr = DirectCallStubs.IndirectCallStub(isJump).ToInt64(); for (int i = 0; i < DynamicTableElems; i++) { @@ -109,37 +162,46 @@ namespace ARMeilleure.Translation return entry; } - public int ReserveTableEntry(long ownerAddress, long address, bool isJump) + public void ExpandIfNeededJumpTable(int entries) { - int entry = Interlocked.Increment(ref _tableEnd); - if (entry >= JumpTableSize) + Debug.Assert(entries > 0); + + if (entries < JumpTableSize) + { + _jumpRegion.ExpandIfNeeded((ulong)((entries + 1) * JumpTableStride)); + } + else { throw new OutOfMemoryException("JIT Direct Jump Table exhausted."); } + } - _jumpRegion.ExpandIfNeeded((ulong)((entry + 1) * JumpTableStride)); + public void ExpandIfNeededDynamicTable(int entries) + { + Debug.Assert(entries > 0); - // Is the address we have already registered? If so, put the function address in the jump table. - // If not, it will point to the direct call stub. - long value = (long)DirectCallStubs.DirectCallStub(isJump); - if (_targets.TryGetValue((ulong)address, out TranslatedFunction func)) + if (entries < DynamicTableSize) { - value = func.GetPointer().ToInt64(); + _dynamicRegion.ExpandIfNeeded((ulong)((entries + 1) * DynamicTableStride)); } - - // Make sure changes to the function at the target address update this jump table entry. - LinkedList<int> targetDependants = _dependants.GetOrAdd((ulong)address, (addr) => new LinkedList<int>()); - lock (targetDependants) + else { - targetDependants.AddLast(entry); + throw new OutOfMemoryException("JIT Dynamic Jump Table exhausted."); } + } - IntPtr addr = _jumpRegion.Pointer + entry * JumpTableStride; + public IntPtr GetEntryAddressJumpTable(int entry) + { + Debug.Assert(entry >= 1 && entry <= _tableEnd); - Marshal.WriteInt64(addr, 0, address); - Marshal.WriteInt64(addr, 8, value); + return _jumpRegion.Pointer + entry * JumpTableStride; + } - return entry; + public IntPtr GetEntryAddressDynamicTable(int entry) + { + Debug.Assert(entry >= 1 && entry <= _dynTableEnd); + + return _dynamicRegion.Pointer + entry * DynamicTableStride; } } } diff --git a/ARMeilleure/Translation/PTC/EncodingCache.cs b/ARMeilleure/Translation/PTC/EncodingCache.cs new file mode 100644 index 00000000..b87e0d7a --- /dev/null +++ b/ARMeilleure/Translation/PTC/EncodingCache.cs @@ -0,0 +1,9 @@ +using System.Text; + +namespace ARMeilleure.Translation.PTC +{ + internal static class EncodingCache + { + internal static readonly Encoding UTF8NoBOM = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true); + } +}
\ No newline at end of file diff --git a/ARMeilleure/Translation/PTC/Ptc.cs b/ARMeilleure/Translation/PTC/Ptc.cs new file mode 100644 index 00000000..76d3d7e1 --- /dev/null +++ b/ARMeilleure/Translation/PTC/Ptc.cs @@ -0,0 +1,768 @@ +using ARMeilleure.CodeGen; +using ARMeilleure.CodeGen.Unwinding; +using ARMeilleure.Memory; +using Ryujinx.Common.Logging; +using System; +using System.Buffers.Binary; +using System.Collections.Concurrent; +using System.Diagnostics; +using System.IO; +using System.IO.Compression; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics.X86; +using System.Runtime.Serialization.Formatters.Binary; +using System.Threading; +using System.Threading.Tasks; + +namespace ARMeilleure.Translation.PTC +{ + public static class Ptc + { + private const string HeaderMagic = "PTChd"; + + private const int InternalVersion = 0; //! To be incremented manually for each change to the ARMeilleure project. + + private const string BaseDir = "Ryujinx"; + + private const string ActualDir = "0"; + private const string BackupDir = "1"; + + private const string TitleIdTextDefault = "0000000000000000"; + private const string DisplayVersionDefault = "0"; + + internal const int PageTablePointerIndex = -1; // Must be a negative value. + internal const int JumpPointerIndex = -2; // Must be a negative value. + internal const int DynamicPointerIndex = -3; // Must be a negative value. + + private const CompressionLevel SaveCompressionLevel = CompressionLevel.Fastest; + + private static readonly MemoryStream _infosStream; + private static readonly MemoryStream _codesStream; + private static readonly MemoryStream _relocsStream; + private static readonly MemoryStream _unwindInfosStream; + + private static readonly BinaryWriter _infosWriter; + + private static readonly BinaryFormatter _binaryFormatter; + + private static readonly ManualResetEvent _waitEvent; + + private static readonly AutoResetEvent _loggerEvent; + + private static readonly string _basePath; + + private static readonly object _lock; + + private static bool _disposed; + + private static volatile int _translateCount; + private static volatile int _rejitCount; + + internal static PtcJumpTable PtcJumpTable { get; private set; } + + internal static string TitleIdText { get; private set; } + internal static string DisplayVersion { get; private set; } + + internal static string CachePathActual { get; private set; } + internal static string CachePathBackup { get; private set; } + + internal static PtcState State { get; private set; } + + static Ptc() + { + _infosStream = new MemoryStream(); + _codesStream = new MemoryStream(); + _relocsStream = new MemoryStream(); + _unwindInfosStream = new MemoryStream(); + + _infosWriter = new BinaryWriter(_infosStream, EncodingCache.UTF8NoBOM, true); + + _binaryFormatter = new BinaryFormatter(); + + _waitEvent = new ManualResetEvent(true); + + _loggerEvent = new AutoResetEvent(false); + + _basePath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), BaseDir); + + _lock = new object(); + + _disposed = false; + + PtcJumpTable = new PtcJumpTable(); + + TitleIdText = TitleIdTextDefault; + DisplayVersion = DisplayVersionDefault; + + CachePathActual = string.Empty; + CachePathBackup = string.Empty; + + Disable(); + + AppDomain.CurrentDomain.UnhandledException += CurrentDomain_UnhandledException; + AppDomain.CurrentDomain.ProcessExit += CurrentDomain_ProcessExit; + } + + public static void Initialize(string titleIdText, string displayVersion, bool enabled) + { + Wait(); + ClearMemoryStreams(); + PtcJumpTable.Clear(); + + PtcProfiler.Stop(); + PtcProfiler.Wait(); + PtcProfiler.ClearEntries(); + + if (String.IsNullOrEmpty(titleIdText) || titleIdText == TitleIdTextDefault) + { + TitleIdText = TitleIdTextDefault; + DisplayVersion = DisplayVersionDefault; + + CachePathActual = string.Empty; + CachePathBackup = string.Empty; + + Disable(); + + return; + } + + Logger.PrintInfo(LogClass.Ptc, $"Initializing Profiled Persistent Translation Cache (enabled: {enabled})."); + + TitleIdText = titleIdText; + DisplayVersion = !String.IsNullOrEmpty(displayVersion) ? displayVersion : DisplayVersionDefault; + + if (enabled) + { + string workPathActual = Path.Combine(_basePath, "games", TitleIdText, "cache", "cpu", ActualDir); + string workPathBackup = Path.Combine(_basePath, "games", TitleIdText, "cache", "cpu", BackupDir); + + if (!Directory.Exists(workPathActual)) + { + Directory.CreateDirectory(workPathActual); + } + + if (!Directory.Exists(workPathBackup)) + { + Directory.CreateDirectory(workPathBackup); + } + + CachePathActual = Path.Combine(workPathActual, DisplayVersion); + CachePathBackup = Path.Combine(workPathBackup, DisplayVersion); + + Enable(); + + PreLoad(); + PtcProfiler.PreLoad(); + } + else + { + CachePathActual = string.Empty; + CachePathBackup = string.Empty; + + Disable(); + } + } + + internal static void ClearMemoryStreams() + { + _infosStream.SetLength(0L); + _codesStream.SetLength(0L); + _relocsStream.SetLength(0L); + _unwindInfosStream.SetLength(0L); + } + + private static void PreLoad() + { + string fileNameActual = String.Concat(CachePathActual, ".cache"); + string fileNameBackup = String.Concat(CachePathBackup, ".cache"); + + FileInfo fileInfoActual = new FileInfo(fileNameActual); + FileInfo fileInfoBackup = new FileInfo(fileNameBackup); + + if (fileInfoActual.Exists && fileInfoActual.Length != 0L) + { + if (!Load(fileNameActual)) + { + if (fileInfoBackup.Exists && fileInfoBackup.Length != 0L) + { + Load(fileNameBackup); + } + } + } + else if (fileInfoBackup.Exists && fileInfoBackup.Length != 0L) + { + Load(fileNameBackup); + } + } + + private static bool Load(string fileName) + { + using (FileStream compressedStream = new FileStream(fileName, FileMode.Open)) + using (DeflateStream deflateStream = new DeflateStream(compressedStream, CompressionMode.Decompress, true)) + using (MemoryStream stream = new MemoryStream()) + using (System.Security.Cryptography.MD5 md5 = System.Security.Cryptography.MD5.Create()) + { + int hashSize = md5.HashSize / 8; + + deflateStream.CopyTo(stream); + + stream.Seek(0L, SeekOrigin.Begin); + + byte[] currentHash = new byte[hashSize]; + stream.Read(currentHash, 0, hashSize); + + byte[] expectedHash = md5.ComputeHash(stream); + + if (!CompareHash(currentHash, expectedHash)) + { + InvalidateCompressedStream(compressedStream); + + return false; + } + + stream.Seek((long)hashSize, SeekOrigin.Begin); + + Header header = ReadHeader(stream); + + if (header.Magic != HeaderMagic) + { + InvalidateCompressedStream(compressedStream); + + return false; + } + + if (header.CacheFileVersion != InternalVersion) + { + InvalidateCompressedStream(compressedStream); + + return false; + } + + if (header.FeatureInfo != GetFeatureInfo()) + { + InvalidateCompressedStream(compressedStream); + + return false; + } + + if (header.InfosLen % InfoEntry.Stride != 0) + { + InvalidateCompressedStream(compressedStream); + + return false; + } + + byte[] infosBuf = new byte[header.InfosLen]; + byte[] codesBuf = new byte[header.CodesLen]; + byte[] relocsBuf = new byte[header.RelocsLen]; + byte[] unwindInfosBuf = new byte[header.UnwindInfosLen]; + + stream.Read(infosBuf, 0, header.InfosLen); + stream.Read(codesBuf, 0, header.CodesLen); + stream.Read(relocsBuf, 0, header.RelocsLen); + stream.Read(unwindInfosBuf, 0, header.UnwindInfosLen); + + try + { + PtcJumpTable = (PtcJumpTable)_binaryFormatter.Deserialize(stream); + } + catch + { + PtcJumpTable = new PtcJumpTable(); + + InvalidateCompressedStream(compressedStream); + + return false; + } + + _infosStream.Write(infosBuf, 0, header.InfosLen); + _codesStream.Write(codesBuf, 0, header.CodesLen); + _relocsStream.Write(relocsBuf, 0, header.RelocsLen); + _unwindInfosStream.Write(unwindInfosBuf, 0, header.UnwindInfosLen); + + return true; + } + } + + private static bool CompareHash(ReadOnlySpan<byte> currentHash, ReadOnlySpan<byte> expectedHash) + { + return currentHash.SequenceEqual(expectedHash); + } + + private static Header ReadHeader(MemoryStream stream) + { + using (BinaryReader headerReader = new BinaryReader(stream, EncodingCache.UTF8NoBOM, true)) + { + Header header = new Header(); + + header.Magic = headerReader.ReadString(); + + header.CacheFileVersion = headerReader.ReadInt32(); + header.FeatureInfo = headerReader.ReadUInt64(); + + header.InfosLen = headerReader.ReadInt32(); + header.CodesLen = headerReader.ReadInt32(); + header.RelocsLen = headerReader.ReadInt32(); + header.UnwindInfosLen = headerReader.ReadInt32(); + + return header; + } + } + + private static void InvalidateCompressedStream(FileStream compressedStream) + { + compressedStream.SetLength(0L); + } + + private static void PreSave(object state) + { + _waitEvent.Reset(); + + string fileNameActual = String.Concat(CachePathActual, ".cache"); + string fileNameBackup = String.Concat(CachePathBackup, ".cache"); + + FileInfo fileInfoActual = new FileInfo(fileNameActual); + + if (fileInfoActual.Exists && fileInfoActual.Length != 0L) + { + File.Copy(fileNameActual, fileNameBackup, true); + } + + Save(fileNameActual); + + _waitEvent.Set(); + } + + private static void Save(string fileName) + { + using (MemoryStream stream = new MemoryStream()) + using (System.Security.Cryptography.MD5 md5 = System.Security.Cryptography.MD5.Create()) + { + int hashSize = md5.HashSize / 8; + + stream.Seek((long)hashSize, SeekOrigin.Begin); + + WriteHeader(stream); + + _infosStream.WriteTo(stream); + _codesStream.WriteTo(stream); + _relocsStream.WriteTo(stream); + _unwindInfosStream.WriteTo(stream); + + _binaryFormatter.Serialize(stream, PtcJumpTable); + + stream.Seek((long)hashSize, SeekOrigin.Begin); + byte[] hash = md5.ComputeHash(stream); + + stream.Seek(0L, SeekOrigin.Begin); + stream.Write(hash, 0, hashSize); + + using (FileStream compressedStream = new FileStream(fileName, FileMode.OpenOrCreate)) + using (DeflateStream deflateStream = new DeflateStream(compressedStream, SaveCompressionLevel, true)) + { + try + { + stream.WriteTo(deflateStream); + } + catch + { + compressedStream.Position = 0L; + } + + if (compressedStream.Position < compressedStream.Length) + { + compressedStream.SetLength(compressedStream.Position); + } + } + } + } + + private static void WriteHeader(MemoryStream stream) + { + using (BinaryWriter headerWriter = new BinaryWriter(stream, EncodingCache.UTF8NoBOM, true)) + { + headerWriter.Write((string)HeaderMagic); // Header.Magic + + headerWriter.Write((int)InternalVersion); // Header.CacheFileVersion + headerWriter.Write((ulong)GetFeatureInfo()); // Header.FeatureInfo + + headerWriter.Write((int)_infosStream.Length); // Header.InfosLen + headerWriter.Write((int)_codesStream.Length); // Header.CodesLen + headerWriter.Write((int)_relocsStream.Length); // Header.RelocsLen + headerWriter.Write((int)_unwindInfosStream.Length); // Header.UnwindInfosLen + } + } + + internal static void LoadTranslations(ConcurrentDictionary<ulong, TranslatedFunction> funcs, IntPtr pageTablePointer, JumpTable jumpTable) + { + if ((int)_infosStream.Length == 0 || + (int)_codesStream.Length == 0 || + (int)_relocsStream.Length == 0 || + (int)_unwindInfosStream.Length == 0) + { + return; + } + + Debug.Assert(funcs.Count == 0); + + _infosStream.Seek(0L, SeekOrigin.Begin); + _codesStream.Seek(0L, SeekOrigin.Begin); + _relocsStream.Seek(0L, SeekOrigin.Begin); + _unwindInfosStream.Seek(0L, SeekOrigin.Begin); + + using (BinaryReader infosReader = new BinaryReader(_infosStream, EncodingCache.UTF8NoBOM, true)) + using (BinaryReader codesReader = new BinaryReader(_codesStream, EncodingCache.UTF8NoBOM, true)) + using (BinaryReader relocsReader = new BinaryReader(_relocsStream, EncodingCache.UTF8NoBOM, true)) + using (BinaryReader unwindInfosReader = new BinaryReader(_unwindInfosStream, EncodingCache.UTF8NoBOM, true)) + { + int infosEntriesCount = (int)_infosStream.Length / InfoEntry.Stride; + + for (int i = 0; i < infosEntriesCount; i++) + { + InfoEntry infoEntry = ReadInfo(infosReader); + + byte[] code = ReadCode(codesReader, infoEntry.CodeLen); + + if (infoEntry.RelocEntriesCount != 0) + { + RelocEntry[] relocEntries = GetRelocEntries(relocsReader, infoEntry.RelocEntriesCount); + + PatchCode(code, relocEntries, pageTablePointer, jumpTable); + } + + UnwindInfo unwindInfo = ReadUnwindInfo(unwindInfosReader); + + TranslatedFunction func = FastTranslate(code, unwindInfo, infoEntry.HighCq); + + funcs.AddOrUpdate((ulong)infoEntry.Address, func, (key, oldFunc) => func.HighCq && !oldFunc.HighCq ? func : oldFunc); + } + } + + if (_infosStream.Position < _infosStream.Length || + _codesStream.Position < _codesStream.Length || + _relocsStream.Position < _relocsStream.Length || + _unwindInfosStream.Position < _unwindInfosStream.Length) + { + throw new Exception("Could not reach the end of one or more memory streams."); + } + + jumpTable.Initialize(PtcJumpTable, funcs); + + PtcJumpTable.WriteJumpTable(jumpTable, funcs); + PtcJumpTable.WriteDynamicTable(jumpTable); + } + + private static InfoEntry ReadInfo(BinaryReader infosReader) + { + InfoEntry infoEntry = new InfoEntry(); + + infoEntry.Address = infosReader.ReadInt64(); + infoEntry.HighCq = infosReader.ReadBoolean(); + infoEntry.CodeLen = infosReader.ReadInt32(); + infoEntry.RelocEntriesCount = infosReader.ReadInt32(); + + return infoEntry; + } + + private static byte[] ReadCode(BinaryReader codesReader, int codeLen) + { + byte[] codeBuf = new byte[codeLen]; + + codesReader.Read(codeBuf, 0, codeLen); + + return codeBuf; + } + + private static RelocEntry[] GetRelocEntries(BinaryReader relocsReader, int relocEntriesCount) + { + RelocEntry[] relocEntries = new RelocEntry[relocEntriesCount]; + + for (int i = 0; i < relocEntriesCount; i++) + { + int position = relocsReader.ReadInt32(); + int index = relocsReader.ReadInt32(); + + relocEntries[i] = new RelocEntry(position, index); + } + + return relocEntries; + } + + private static void PatchCode(Span<byte> code, RelocEntry[] relocEntries, IntPtr pageTablePointer, JumpTable jumpTable) + { + foreach (RelocEntry relocEntry in relocEntries) + { + ulong imm; + + if (relocEntry.Index == PageTablePointerIndex) + { + imm = (ulong)pageTablePointer.ToInt64(); + } + else if (relocEntry.Index == JumpPointerIndex) + { + imm = (ulong)jumpTable.JumpPointer.ToInt64(); + } + else if (relocEntry.Index == DynamicPointerIndex) + { + imm = (ulong)jumpTable.DynamicPointer.ToInt64(); + } + else if (Delegates.TryGetDelegateFuncPtrByIndex(relocEntry.Index, out IntPtr funcPtr)) + { + imm = (ulong)funcPtr.ToInt64(); + } + else + { + throw new Exception($"Unexpected reloc entry {relocEntry}."); + } + + BinaryPrimitives.WriteUInt64LittleEndian(code.Slice(relocEntry.Position, 8), imm); + } + } + + private static UnwindInfo ReadUnwindInfo(BinaryReader unwindInfosReader) + { + int pushEntriesLength = unwindInfosReader.ReadInt32(); + + UnwindPushEntry[] pushEntries = new UnwindPushEntry[pushEntriesLength]; + + for (int i = 0; i < pushEntriesLength; i++) + { + int pseudoOp = unwindInfosReader.ReadInt32(); + int prologOffset = unwindInfosReader.ReadInt32(); + int regIndex = unwindInfosReader.ReadInt32(); + int stackOffsetOrAllocSize = unwindInfosReader.ReadInt32(); + + pushEntries[i] = new UnwindPushEntry((UnwindPseudoOp)pseudoOp, prologOffset, regIndex, stackOffsetOrAllocSize); + } + + int prologueSize = unwindInfosReader.ReadInt32(); + + return new UnwindInfo(pushEntries, prologueSize); + } + + private static TranslatedFunction FastTranslate(byte[] code, UnwindInfo unwindInfo, bool highCq) + { + CompiledFunction cFunc = new CompiledFunction(code, unwindInfo); + + IntPtr codePtr = JitCache.Map(cFunc); + + GuestFunction gFunc = Marshal.GetDelegateForFunctionPointer<GuestFunction>(codePtr); + + TranslatedFunction tFunc = new TranslatedFunction(gFunc, highCq); + + return tFunc; + } + + internal static void MakeAndSaveTranslations(ConcurrentDictionary<ulong, TranslatedFunction> funcs, IMemoryManager memory, JumpTable jumpTable) + { + if (PtcProfiler.ProfiledFuncs.Count == 0) + { + return; + } + + _translateCount = 0; + _rejitCount = 0; + + ThreadPool.QueueUserWorkItem(TranslationLogger, (funcs.Count, PtcProfiler.ProfiledFuncs.Count)); + + int maxDegreeOfParallelism = (Environment.ProcessorCount * 3) / 4; + + Parallel.ForEach(PtcProfiler.ProfiledFuncs, new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, (item, state) => + { + ulong address = item.Key; + + Debug.Assert(PtcProfiler.IsAddressInStaticCodeRange(address)); + + if (!funcs.ContainsKey(address)) + { + TranslatedFunction func = Translator.Translate(memory, jumpTable, address, item.Value.mode, item.Value.highCq); + + funcs.TryAdd(address, func); + + if (func.HighCq) + { + jumpTable.RegisterFunction(address, func); + } + + Interlocked.Increment(ref _translateCount); + } + else if (item.Value.highCq && !funcs[address].HighCq) + { + TranslatedFunction func = Translator.Translate(memory, jumpTable, address, item.Value.mode, highCq: true); + + funcs[address] = func; + + jumpTable.RegisterFunction(address, func); + + Interlocked.Increment(ref _rejitCount); + } + + if (State != PtcState.Enabled) + { + state.Stop(); + } + }); + + _loggerEvent.Set(); + + if (_translateCount != 0 || _rejitCount != 0) + { + PtcJumpTable.Initialize(jumpTable); + + PtcJumpTable.ReadJumpTable(jumpTable); + PtcJumpTable.ReadDynamicTable(jumpTable); + + ThreadPool.QueueUserWorkItem(PreSave); + } + } + + private static void TranslationLogger(object state) + { + const int refreshRate = 1; // Seconds. + + (int funcsCount, int ProfiledFuncsCount) = ((int, int))state; + + do + { + Logger.PrintInfo(LogClass.Ptc, $"{funcsCount + _translateCount} of {ProfiledFuncsCount} functions to translate - {_rejitCount} functions rejited"); + } + while (!_loggerEvent.WaitOne(refreshRate * 1000)); + + Logger.PrintInfo(LogClass.Ptc, $"{funcsCount + _translateCount} of {ProfiledFuncsCount} functions to translate - {_rejitCount} functions rejited"); + } + + internal static void WriteInfoCodeReloc(long address, bool highCq, PtcInfo ptcInfo) + { + lock (_lock) + { + // WriteInfo. + _infosWriter.Write((long)address); // InfoEntry.Address + _infosWriter.Write((bool)highCq); // InfoEntry.HighCq + _infosWriter.Write((int)ptcInfo.CodeStream.Length); // InfoEntry.CodeLen + _infosWriter.Write((int)ptcInfo.RelocEntriesCount); // InfoEntry.RelocEntriesCount + + // WriteCode. + ptcInfo.CodeStream.WriteTo(_codesStream); + + // WriteReloc. + ptcInfo.RelocStream.WriteTo(_relocsStream); + + // WriteUnwindInfo. + ptcInfo.UnwindInfoStream.WriteTo(_unwindInfosStream); + } + } + + private static ulong GetFeatureInfo() + { + ulong featureInfo = 0ul; + + featureInfo |= (Sse3.IsSupported ? 1ul : 0ul) << 0; + featureInfo |= (Pclmulqdq.IsSupported ? 1ul : 0ul) << 1; + featureInfo |= (Ssse3.IsSupported ? 1ul : 0ul) << 9; + featureInfo |= (Fma.IsSupported ? 1ul : 0ul) << 12; + featureInfo |= (Sse41.IsSupported ? 1ul : 0ul) << 19; + featureInfo |= (Sse42.IsSupported ? 1ul : 0ul) << 20; + featureInfo |= (Popcnt.IsSupported ? 1ul : 0ul) << 23; + featureInfo |= (Aes.IsSupported ? 1ul : 0ul) << 25; + featureInfo |= (Avx.IsSupported ? 1ul : 0ul) << 28; + featureInfo |= (Sse.IsSupported ? 1ul : 0ul) << 57; + featureInfo |= (Sse2.IsSupported ? 1ul : 0ul) << 58; + + return featureInfo; + } + + private struct Header + { + public string Magic; + + public int CacheFileVersion; + public ulong FeatureInfo; + + public int InfosLen; + public int CodesLen; + public int RelocsLen; + public int UnwindInfosLen; + } + + private struct InfoEntry + { + public const int Stride = 17; // Bytes. + + public long Address; + public bool HighCq; + public int CodeLen; + public int RelocEntriesCount; + } + + private static void Enable() + { + State = PtcState.Enabled; + } + + public static void Continue() + { + if (State == PtcState.Enabled) + { + State = PtcState.Continuing; + } + } + + public static void Close() + { + if (State == PtcState.Enabled || + State == PtcState.Continuing) + { + State = PtcState.Closing; + } + } + + internal static void Disable() + { + State = PtcState.Disabled; + } + + private static void Wait() + { + _waitEvent.WaitOne(); + } + + public static void Dispose() + { + if (!_disposed) + { + _disposed = true; + + AppDomain.CurrentDomain.UnhandledException -= CurrentDomain_UnhandledException; + AppDomain.CurrentDomain.ProcessExit -= CurrentDomain_ProcessExit; + + Wait(); + _waitEvent.Dispose(); + + _infosWriter.Dispose(); + + _infosStream.Dispose(); + _codesStream.Dispose(); + _relocsStream.Dispose(); + _unwindInfosStream.Dispose(); + } + } + + private static void CurrentDomain_UnhandledException(object sender, UnhandledExceptionEventArgs e) + { + Close(); + PtcProfiler.Stop(); + + if (e.IsTerminating) + { + Dispose(); + PtcProfiler.Dispose(); + } + } + + private static void CurrentDomain_ProcessExit(object sender, EventArgs e) + { + Dispose(); + PtcProfiler.Dispose(); + } + } +}
\ No newline at end of file diff --git a/ARMeilleure/Translation/PTC/PtcInfo.cs b/ARMeilleure/Translation/PTC/PtcInfo.cs new file mode 100644 index 00000000..f03eb6ba --- /dev/null +++ b/ARMeilleure/Translation/PTC/PtcInfo.cs @@ -0,0 +1,68 @@ +using ARMeilleure.CodeGen.Unwinding; +using System; +using System.IO; + +namespace ARMeilleure.Translation.PTC +{ + sealed class PtcInfo : IDisposable + { + private readonly BinaryWriter _relocWriter; + private readonly BinaryWriter _unwindInfoWriter; + + public MemoryStream CodeStream { get; } + public MemoryStream RelocStream { get; } + public MemoryStream UnwindInfoStream { get; } + + public int RelocEntriesCount { get; private set; } + + public PtcInfo() + { + CodeStream = new MemoryStream(); + RelocStream = new MemoryStream(); + UnwindInfoStream = new MemoryStream(); + + _relocWriter = new BinaryWriter(RelocStream, EncodingCache.UTF8NoBOM, true); + _unwindInfoWriter = new BinaryWriter(UnwindInfoStream, EncodingCache.UTF8NoBOM, true); + + RelocEntriesCount = 0; + } + + public void WriteCode(MemoryStream codeStream) + { + codeStream.WriteTo(CodeStream); + } + + public void WriteRelocEntry(RelocEntry relocEntry) + { + _relocWriter.Write((int)relocEntry.Position); + _relocWriter.Write((int)relocEntry.Index); + + RelocEntriesCount++; + } + + public void WriteUnwindInfo(UnwindInfo unwindInfo) + { + _unwindInfoWriter.Write((int)unwindInfo.PushEntries.Length); + + foreach (UnwindPushEntry unwindPushEntry in unwindInfo.PushEntries) + { + _unwindInfoWriter.Write((int)unwindPushEntry.PseudoOp); + _unwindInfoWriter.Write((int)unwindPushEntry.PrologOffset); + _unwindInfoWriter.Write((int)unwindPushEntry.RegIndex); + _unwindInfoWriter.Write((int)unwindPushEntry.StackOffsetOrAllocSize); + } + + _unwindInfoWriter.Write((int)unwindInfo.PrologSize); + } + + public void Dispose() + { + _relocWriter.Dispose(); + _unwindInfoWriter.Dispose(); + + CodeStream.Dispose(); + RelocStream.Dispose(); + UnwindInfoStream.Dispose(); + } + } +} diff --git a/ARMeilleure/Translation/PTC/PtcJumpTable.cs b/ARMeilleure/Translation/PTC/PtcJumpTable.cs new file mode 100644 index 00000000..0a3ae240 --- /dev/null +++ b/ARMeilleure/Translation/PTC/PtcJumpTable.cs @@ -0,0 +1,222 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Runtime.InteropServices; + +namespace ARMeilleure.Translation.PTC +{ + [Serializable] + class PtcJumpTable + { + private readonly List<KeyValuePair<long, DirectHostAddress>> _jumpTable; + private readonly List<KeyValuePair<long, IndirectHostAddress>> _dynamicTable; + + private readonly List<ulong> _targets; + private readonly Dictionary<ulong, LinkedList<int>> _dependants; + + public int TableEnd => _jumpTable.Count; + public int DynTableEnd => _dynamicTable.Count; + + public List<ulong> Targets => _targets; + public Dictionary<ulong, LinkedList<int>> Dependants => _dependants; + + public PtcJumpTable() + { + _jumpTable = new List<KeyValuePair<long, DirectHostAddress>>(); + _dynamicTable = new List<KeyValuePair<long, IndirectHostAddress>>(); + + _targets = new List<ulong>(); + _dependants = new Dictionary<ulong, LinkedList<int>>(); + } + + public void Initialize(JumpTable jumpTable) + { + _targets.Clear(); + + foreach (ulong guestAddress in jumpTable.Targets.Keys) + { + _targets.Add(guestAddress); + } + + _dependants.Clear(); + + foreach (var item in jumpTable.Dependants) + { + _dependants.Add(item.Key, new LinkedList<int>(item.Value)); + } + } + + public void Clear() + { + _jumpTable.Clear(); + _dynamicTable.Clear(); + + _targets.Clear(); + _dependants.Clear(); + } + + public void WriteJumpTable(JumpTable jumpTable, ConcurrentDictionary<ulong, TranslatedFunction> funcs) + { + jumpTable.ExpandIfNeededJumpTable(TableEnd); + + int entry = 0; + + foreach (var item in _jumpTable) + { + entry += 1; + + long guestAddress = item.Key; + DirectHostAddress directHostAddress = item.Value; + + long hostAddress; + + if (directHostAddress == DirectHostAddress.CallStub) + { + hostAddress = DirectCallStubs.DirectCallStub(false).ToInt64(); + } + else if (directHostAddress == DirectHostAddress.TailCallStub) + { + hostAddress = DirectCallStubs.DirectCallStub(true).ToInt64(); + } + else if (directHostAddress == DirectHostAddress.Host) + { + if (funcs.TryGetValue((ulong)guestAddress, out TranslatedFunction func)) + { + hostAddress = func.FuncPtr.ToInt64(); + } + else + { + throw new KeyNotFoundException($"({nameof(guestAddress)} = 0x{(ulong)guestAddress:X16})"); + } + } + else + { + throw new InvalidOperationException(nameof(directHostAddress)); + } + + IntPtr addr = jumpTable.GetEntryAddressJumpTable(entry); + + Marshal.WriteInt64(addr, 0, guestAddress); + Marshal.WriteInt64(addr, 8, hostAddress); + } + } + + public void WriteDynamicTable(JumpTable jumpTable) + { + if (JumpTable.DynamicTableElems > 1) + { + throw new NotSupportedException(); + } + + jumpTable.ExpandIfNeededDynamicTable(DynTableEnd); + + int entry = 0; + + foreach (var item in _dynamicTable) + { + entry += 1; + + long guestAddress = item.Key; + IndirectHostAddress indirectHostAddress = item.Value; + + long hostAddress; + + if (indirectHostAddress == IndirectHostAddress.CallStub) + { + hostAddress = DirectCallStubs.IndirectCallStub(false).ToInt64(); + } + else if (indirectHostAddress == IndirectHostAddress.TailCallStub) + { + hostAddress = DirectCallStubs.IndirectCallStub(true).ToInt64(); + } + else + { + throw new InvalidOperationException(nameof(indirectHostAddress)); + } + + IntPtr addr = jumpTable.GetEntryAddressDynamicTable(entry); + + Marshal.WriteInt64(addr, 0, guestAddress); + Marshal.WriteInt64(addr, 8, hostAddress); + } + } + + public void ReadJumpTable(JumpTable jumpTable) + { + _jumpTable.Clear(); + + for (int entry = 1; entry <= jumpTable.TableEnd; entry++) + { + IntPtr addr = jumpTable.GetEntryAddressJumpTable(entry); + + long guestAddress = Marshal.ReadInt64(addr, 0); + long hostAddress = Marshal.ReadInt64(addr, 8); + + DirectHostAddress directHostAddress; + + if (hostAddress == DirectCallStubs.DirectCallStub(false).ToInt64()) + { + directHostAddress = DirectHostAddress.CallStub; + } + else if (hostAddress == DirectCallStubs.DirectCallStub(true).ToInt64()) + { + directHostAddress = DirectHostAddress.TailCallStub; + } + else + { + directHostAddress = DirectHostAddress.Host; + } + + _jumpTable.Add(new KeyValuePair<long, DirectHostAddress>(guestAddress, directHostAddress)); + } + } + + public void ReadDynamicTable(JumpTable jumpTable) + { + if (JumpTable.DynamicTableElems > 1) + { + throw new NotSupportedException(); + } + + _dynamicTable.Clear(); + + for (int entry = 1; entry <= jumpTable.DynTableEnd; entry++) + { + IntPtr addr = jumpTable.GetEntryAddressDynamicTable(entry); + + long guestAddress = Marshal.ReadInt64(addr, 0); + long hostAddress = Marshal.ReadInt64(addr, 8); + + IndirectHostAddress indirectHostAddress; + + if (hostAddress == DirectCallStubs.IndirectCallStub(false).ToInt64()) + { + indirectHostAddress = IndirectHostAddress.CallStub; + } + else if (hostAddress == DirectCallStubs.IndirectCallStub(true).ToInt64()) + { + indirectHostAddress = IndirectHostAddress.TailCallStub; + } + else + { + throw new InvalidOperationException($"({nameof(hostAddress)} = 0x{hostAddress:X16})"); + } + + _dynamicTable.Add(new KeyValuePair<long, IndirectHostAddress>(guestAddress, indirectHostAddress)); + } + } + + private enum DirectHostAddress + { + CallStub, + TailCallStub, + Host + } + + private enum IndirectHostAddress + { + CallStub, + TailCallStub + } + } +}
\ No newline at end of file diff --git a/ARMeilleure/Translation/PTC/PtcProfiler.cs b/ARMeilleure/Translation/PTC/PtcProfiler.cs new file mode 100644 index 00000000..dcc31275 --- /dev/null +++ b/ARMeilleure/Translation/PTC/PtcProfiler.cs @@ -0,0 +1,267 @@ +using ARMeilleure.State; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.IO.Compression; +using System.Runtime.Serialization.Formatters.Binary; +using System.Security.Cryptography; +using System.Threading; + +namespace ARMeilleure.Translation.PTC +{ + public static class PtcProfiler + { + private const int SaveInterval = 30; // Seconds. + + private const CompressionLevel SaveCompressionLevel = CompressionLevel.Fastest; + + private static readonly BinaryFormatter _binaryFormatter; + + private static readonly System.Timers.Timer _timer; + + private static readonly ManualResetEvent _waitEvent; + + private static readonly object _lock; + + private static bool _disposed; + + internal static Dictionary<ulong, (ExecutionMode mode, bool highCq)> ProfiledFuncs { get; private set; } //! Not to be modified. + + internal static bool Enabled { get; private set; } + + public static ulong StaticCodeStart { internal get; set; } + public static int StaticCodeSize { internal get; set; } + + static PtcProfiler() + { + _binaryFormatter = new BinaryFormatter(); + + _timer = new System.Timers.Timer((double)SaveInterval * 1000d); + _timer.Elapsed += PreSave; + + _waitEvent = new ManualResetEvent(true); + + _lock = new object(); + + _disposed = false; + + ProfiledFuncs = new Dictionary<ulong, (ExecutionMode, bool)>(); + + Enabled = false; + } + + internal static void AddEntry(ulong address, ExecutionMode mode, bool highCq) + { + if (IsAddressInStaticCodeRange(address)) + { + lock (_lock) + { + Debug.Assert(!highCq && !ProfiledFuncs.ContainsKey(address)); + + ProfiledFuncs.TryAdd(address, (mode, highCq)); + } + } + } + + internal static void UpdateEntry(ulong address, ExecutionMode mode, bool highCq) + { + if (IsAddressInStaticCodeRange(address)) + { + lock (_lock) + { + Debug.Assert(highCq && ProfiledFuncs.ContainsKey(address)); + + ProfiledFuncs[address] = (mode, highCq); + } + } + } + + internal static bool IsAddressInStaticCodeRange(ulong address) + { + return address >= StaticCodeStart && address < StaticCodeStart + (ulong)StaticCodeSize; + } + + internal static void ClearEntries() + { + ProfiledFuncs.Clear(); + } + + internal static void PreLoad() + { + string fileNameActual = String.Concat(Ptc.CachePathActual, ".info"); + string fileNameBackup = String.Concat(Ptc.CachePathBackup, ".info"); + + FileInfo fileInfoActual = new FileInfo(fileNameActual); + FileInfo fileInfoBackup = new FileInfo(fileNameBackup); + + if (fileInfoActual.Exists && fileInfoActual.Length != 0L) + { + if (!Load(fileNameActual)) + { + if (fileInfoBackup.Exists && fileInfoBackup.Length != 0L) + { + Load(fileNameBackup); + } + } + } + else if (fileInfoBackup.Exists && fileInfoBackup.Length != 0L) + { + Load(fileNameBackup); + } + } + + private static bool Load(string fileName) + { + using (FileStream compressedStream = new FileStream(fileName, FileMode.Open)) + using (DeflateStream deflateStream = new DeflateStream(compressedStream, CompressionMode.Decompress, true)) + using (MemoryStream stream = new MemoryStream()) + using (MD5 md5 = MD5.Create()) + { + int hashSize = md5.HashSize / 8; + + deflateStream.CopyTo(stream); + + stream.Seek(0L, SeekOrigin.Begin); + + byte[] currentHash = new byte[hashSize]; + stream.Read(currentHash, 0, hashSize); + + byte[] expectedHash = md5.ComputeHash(stream); + + if (!CompareHash(currentHash, expectedHash)) + { + InvalidateCompressedStream(compressedStream); + + return false; + } + + stream.Seek((long)hashSize, SeekOrigin.Begin); + + try + { + ProfiledFuncs = (Dictionary<ulong, (ExecutionMode, bool)>)_binaryFormatter.Deserialize(stream); + } + catch + { + ProfiledFuncs = new Dictionary<ulong, (ExecutionMode, bool)>(); + + InvalidateCompressedStream(compressedStream); + + return false; + } + + return true; + } + } + + private static bool CompareHash(ReadOnlySpan<byte> currentHash, ReadOnlySpan<byte> expectedHash) + { + return currentHash.SequenceEqual(expectedHash); + } + + private static void InvalidateCompressedStream(FileStream compressedStream) + { + compressedStream.SetLength(0L); + } + + private static void PreSave(object source, System.Timers.ElapsedEventArgs e) + { + _waitEvent.Reset(); + + string fileNameActual = String.Concat(Ptc.CachePathActual, ".info"); + string fileNameBackup = String.Concat(Ptc.CachePathBackup, ".info"); + + FileInfo fileInfoActual = new FileInfo(fileNameActual); + + if (fileInfoActual.Exists && fileInfoActual.Length != 0L) + { + File.Copy(fileNameActual, fileNameBackup, true); + } + + Save(fileNameActual); + + _waitEvent.Set(); + } + + private static void Save(string fileName) + { + using (MemoryStream stream = new MemoryStream()) + using (MD5 md5 = MD5.Create()) + { + int hashSize = md5.HashSize / 8; + + stream.Seek((long)hashSize, SeekOrigin.Begin); + + lock (_lock) + { + _binaryFormatter.Serialize(stream, ProfiledFuncs); + } + + stream.Seek((long)hashSize, SeekOrigin.Begin); + byte[] hash = md5.ComputeHash(stream); + + stream.Seek(0L, SeekOrigin.Begin); + stream.Write(hash, 0, hashSize); + + using (FileStream compressedStream = new FileStream(fileName, FileMode.OpenOrCreate)) + using (DeflateStream deflateStream = new DeflateStream(compressedStream, SaveCompressionLevel, true)) + { + try + { + stream.WriteTo(deflateStream); + } + catch + { + compressedStream.Position = 0L; + } + + if (compressedStream.Position < compressedStream.Length) + { + compressedStream.SetLength(compressedStream.Position); + } + } + } + } + + internal static void Start() + { + if (Ptc.State == PtcState.Enabled || + Ptc.State == PtcState.Continuing) + { + Enabled = true; + + _timer.Enabled = true; + } + } + + public static void Stop() + { + Enabled = false; + + if (!_disposed) + { + _timer.Enabled = false; + } + } + + internal static void Wait() + { + _waitEvent.WaitOne(); + } + + public static void Dispose() + { + if (!_disposed) + { + _disposed = true; + + _timer.Elapsed -= PreSave; + _timer.Dispose(); + + Wait(); + _waitEvent.Dispose(); + } + } + } +}
\ No newline at end of file diff --git a/ARMeilleure/Translation/PTC/PtcState.cs b/ARMeilleure/Translation/PTC/PtcState.cs new file mode 100644 index 00000000..ca4f4108 --- /dev/null +++ b/ARMeilleure/Translation/PTC/PtcState.cs @@ -0,0 +1,10 @@ +namespace ARMeilleure.Translation.PTC +{ + enum PtcState + { + Enabled, + Continuing, + Closing, + Disabled + } +}
\ No newline at end of file diff --git a/ARMeilleure/Translation/PTC/RelocEntry.cs b/ARMeilleure/Translation/PTC/RelocEntry.cs new file mode 100644 index 00000000..3d729fbb --- /dev/null +++ b/ARMeilleure/Translation/PTC/RelocEntry.cs @@ -0,0 +1,19 @@ +namespace ARMeilleure.Translation.PTC +{ + struct RelocEntry + { + public int Position; + public int Index; + + public RelocEntry(int position, int index) + { + Position = position; + Index = index; + } + + public override string ToString() + { + return $"({nameof(Position)} = {Position}, {nameof(Index)} = {Index})"; + } + } +} diff --git a/ARMeilleure/Translation/PriorityQueue.cs b/ARMeilleure/Translation/PriorityQueue.cs deleted file mode 100644 index 000a5009..00000000 --- a/ARMeilleure/Translation/PriorityQueue.cs +++ /dev/null @@ -1,39 +0,0 @@ -using System.Collections.Concurrent; - -namespace ARMeilleure.Translation -{ - class PriorityQueue<T> - { - private ConcurrentStack<T>[] _queues; - - public PriorityQueue(int priorities) - { - _queues = new ConcurrentStack<T>[priorities]; - - for (int index = 0; index < priorities; index++) - { - _queues[index] = new ConcurrentStack<T>(); - } - } - - public void Enqueue(int priority, T value) - { - _queues[priority].Push(value); - } - - public bool TryDequeue(out T value) - { - for (int index = 0; index < _queues.Length; index++) - { - if (_queues[index].TryPop(out value)) - { - return true; - } - } - - value = default(T); - - return false; - } - } -}
\ No newline at end of file diff --git a/ARMeilleure/Translation/RejitRequest.cs b/ARMeilleure/Translation/RejitRequest.cs index e0b0e0b9..1bed5c0a 100644 --- a/ARMeilleure/Translation/RejitRequest.cs +++ b/ARMeilleure/Translation/RejitRequest.cs @@ -1,4 +1,4 @@ -using ARMeilleure.State; +using ARMeilleure.State; namespace ARMeilleure.Translation { diff --git a/ARMeilleure/Translation/TranslatedFunction.cs b/ARMeilleure/Translation/TranslatedFunction.cs index f1dc6dee..36fae50a 100644 --- a/ARMeilleure/Translation/TranslatedFunction.cs +++ b/ARMeilleure/Translation/TranslatedFunction.cs @@ -4,22 +4,23 @@ using System.Threading; namespace ARMeilleure.Translation { - class TranslatedFunction + sealed class TranslatedFunction { private const int MinCallsForRejit = 100; - private GuestFunction _func; - private IntPtr _funcPtr; + private readonly GuestFunction _func; // Ensure that this delegate will not be garbage collected. - private bool _rejit; - private int _callCount; + private int _callCount = 0; - public bool HighCq => !_rejit; + public bool HighCq { get; } + public IntPtr FuncPtr { get; } - public TranslatedFunction(GuestFunction func, bool rejit) + public TranslatedFunction(GuestFunction func, bool highCq) { - _func = func; - _rejit = rejit; + _func = func; + + HighCq = highCq; + FuncPtr = Marshal.GetFunctionPointerForDelegate<GuestFunction>(func); } public ulong Execute(State.ExecutionContext context) @@ -29,17 +30,7 @@ namespace ARMeilleure.Translation public bool ShouldRejit() { - return _rejit && Interlocked.Increment(ref _callCount) == MinCallsForRejit; - } - - public IntPtr GetPointer() - { - if (_funcPtr == IntPtr.Zero) - { - _funcPtr = Marshal.GetFunctionPointerForDelegate(_func); - } - - return _funcPtr; + return !HighCq && Interlocked.Increment(ref _callCount) == MinCallsForRejit; } } }
\ No newline at end of file diff --git a/ARMeilleure/Translation/Translator.cs b/ARMeilleure/Translation/Translator.cs index 700b54c2..1c2ead4f 100644 --- a/ARMeilleure/Translation/Translator.cs +++ b/ARMeilleure/Translation/Translator.cs @@ -13,22 +13,22 @@ using static ARMeilleure.IntermediateRepresentation.OperationHelper; namespace ARMeilleure.Translation { + using PTC; + public class Translator { private const ulong CallFlag = InstEmitFlowHelper.CallFlag; - private const bool AlwaysTranslateFunctions = true; // If false, only translates a single block for lowCq. - private readonly IMemoryManager _memory; private readonly ConcurrentDictionary<ulong, TranslatedFunction> _funcs; - private readonly JumpTable _jumpTable; - - private readonly PriorityQueue<RejitRequest> _backgroundQueue; + private readonly ConcurrentStack<RejitRequest> _backgroundStack; private readonly AutoResetEvent _backgroundTranslatorEvent; + private readonly JumpTable _jumpTable; + private volatile int _threadCount; public Translator(IJitMemoryAllocator allocator, IMemoryManager memory) @@ -37,32 +37,45 @@ namespace ARMeilleure.Translation _funcs = new ConcurrentDictionary<ulong, TranslatedFunction>(); - _jumpTable = new JumpTable(allocator); - - _backgroundQueue = new PriorityQueue<RejitRequest>(2); + _backgroundStack = new ConcurrentStack<RejitRequest>(); _backgroundTranslatorEvent = new AutoResetEvent(false); + _jumpTable = new JumpTable(allocator); + JitCache.Initialize(allocator); + DirectCallStubs.InitializeStubs(); + + if (Ptc.State == PtcState.Enabled) + { + Ptc.LoadTranslations(_funcs, memory.PageTablePointer, _jumpTable); + } } - private void TranslateQueuedSubs() + private void TranslateStackedSubs() { while (_threadCount != 0) { - if (_backgroundQueue.TryDequeue(out RejitRequest request)) + if (_backgroundStack.TryPop(out RejitRequest request)) { - TranslatedFunction func = Translate(request.Address, request.Mode, highCq: true); + TranslatedFunction func = Translate(_memory, _jumpTable, request.Address, request.Mode, highCq: true); _funcs.AddOrUpdate(request.Address, func, (key, oldFunc) => func); + _jumpTable.RegisterFunction(request.Address, func); + + if (PtcProfiler.Enabled) + { + PtcProfiler.UpdateEntry(request.Address, request.Mode, highCq: true); + } } else { _backgroundTranslatorEvent.WaitOne(); } } + _backgroundTranslatorEvent.Set(); // Wake up any other background translator threads, to encourage them to exit. } @@ -70,16 +83,27 @@ namespace ARMeilleure.Translation { if (Interlocked.Increment(ref _threadCount) == 1) { + if (Ptc.State == PtcState.Enabled) + { + Ptc.MakeAndSaveTranslations(_funcs, _memory, _jumpTable); + } + + PtcProfiler.Start(); + + Ptc.Disable(); + // Simple heuristic, should be user configurable in future. (1 for 4 core/ht or less, 2 for 6 core+ht etc). // All threads are normal priority except from the last, which just fills as much of the last core as the os lets it with a low priority. // If we only have one rejit thread, it should be normal priority as highCq code is performance critical. // TODO: Use physical cores rather than logical. This only really makes sense for processors with hyperthreading. Requires OS specific code. int unboundedThreadCount = Math.Max(1, (Environment.ProcessorCount - 6) / 3); - int threadCount = Math.Min(4, unboundedThreadCount); + int threadCount = Math.Min(4, unboundedThreadCount); + for (int i = 0; i < threadCount; i++) { bool last = i != 0 && i == unboundedThreadCount - 1; - Thread backgroundTranslatorThread = new Thread(TranslateQueuedSubs) + + Thread backgroundTranslatorThread = new Thread(TranslateStackedSubs) { Name = "CPU.BackgroundTranslatorThread." + i, Priority = last ? ThreadPriority.Lowest : ThreadPriority.Normal @@ -130,13 +154,19 @@ namespace ARMeilleure.Translation if (!_funcs.TryGetValue(address, out TranslatedFunction func)) { - func = Translate(address, mode, highCq: false); + func = Translate(_memory, _jumpTable, address, mode, highCq: false); _funcs.TryAdd(address, func); + + if (PtcProfiler.Enabled) + { + PtcProfiler.AddEntry(address, mode, highCq: false); + } } - else if (isCallTarget && func.ShouldRejit()) + + if (isCallTarget && func.ShouldRejit()) { - _backgroundQueue.Enqueue(0, new RejitRequest(address, mode)); + _backgroundStack.Push(new RejitRequest(address, mode)); _backgroundTranslatorEvent.Set(); } @@ -144,18 +174,16 @@ namespace ARMeilleure.Translation return func; } - private TranslatedFunction Translate(ulong address, ExecutionMode mode, bool highCq) + internal static TranslatedFunction Translate(IMemoryManager memory, JumpTable jumpTable, ulong address, ExecutionMode mode, bool highCq) { - ArmEmitterContext context = new ArmEmitterContext(_memory, _jumpTable, (long)address, highCq, Aarch32Mode.User); + ArmEmitterContext context = new ArmEmitterContext(memory, jumpTable, (long)address, highCq, Aarch32Mode.User); PrepareOperandPool(highCq); PrepareOperationPool(highCq); Logger.StartPass(PassName.Decoding); - Block[] blocks = AlwaysTranslateFunctions - ? Decoder.DecodeFunction (_memory, address, mode, highCq) - : Decoder.DecodeBasicBlock(_memory, address, mode); + Block[] blocks = Decoder.DecodeFunction(memory, address, mode, highCq); Logger.EndPass(PassName.Decoding); @@ -182,12 +210,26 @@ namespace ARMeilleure.Translation CompilerOptions options = highCq ? CompilerOptions.HighCq : CompilerOptions.None; - GuestFunction func = Compiler.Compile<GuestFunction>(cfg, argTypes, OperandType.I64, options); + GuestFunction func; + + if (Ptc.State == PtcState.Disabled) + { + func = Compiler.Compile<GuestFunction>(cfg, argTypes, OperandType.I64, options); + } + else + { + using (PtcInfo ptcInfo = new PtcInfo()) + { + func = Compiler.Compile<GuestFunction>(cfg, argTypes, OperandType.I64, options, ptcInfo); + + Ptc.WriteInfoCodeReloc((long)address, highCq, ptcInfo); + } + } ResetOperandPool(highCq); ResetOperationPool(highCq); - return new TranslatedFunction(func, rejit: !highCq); + return new TranslatedFunction(func, highCq); } private static ControlFlowGraph EmitAndGetCFG(ArmEmitterContext context, Block[] blocks) @@ -264,7 +306,7 @@ namespace ARMeilleure.Translation context.BranchIfTrue(lblNonZero, count); - Operand running = context.Call(new _Bool(NativeInterface.CheckSynchronization)); + Operand running = context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.CheckSynchronization))); context.BranchIfTrue(lblExit, running); @@ -281,4 +323,4 @@ namespace ARMeilleure.Translation context.MarkLabel(lblExit); } } -}
\ No newline at end of file +} |