aboutsummaryrefslogtreecommitdiff
path: root/ARMeilleure/CodeGen/Arm64
diff options
context:
space:
mode:
authorTSR Berry <20988865+TSRBerry@users.noreply.github.com>2023-04-08 01:22:00 +0200
committerMary <thog@protonmail.com>2023-04-27 23:51:14 +0200
commitcee712105850ac3385cd0091a923438167433f9f (patch)
tree4a5274b21d8b7f938c0d0ce18736d3f2993b11b1 /ARMeilleure/CodeGen/Arm64
parentcd124bda587ef09668a971fa1cac1c3f0cfc9f21 (diff)
Move solution and projects to src
Diffstat (limited to 'ARMeilleure/CodeGen/Arm64')
-rw-r--r--ARMeilleure/CodeGen/Arm64/Arm64Optimizer.cs270
-rw-r--r--ARMeilleure/CodeGen/Arm64/ArmCondition.cs47
-rw-r--r--ARMeilleure/CodeGen/Arm64/ArmExtensionType.cs14
-rw-r--r--ARMeilleure/CodeGen/Arm64/ArmShiftType.cs11
-rw-r--r--ARMeilleure/CodeGen/Arm64/Assembler.cs1160
-rw-r--r--ARMeilleure/CodeGen/Arm64/CallingConvention.cs96
-rw-r--r--ARMeilleure/CodeGen/Arm64/CodeGenCommon.cs91
-rw-r--r--ARMeilleure/CodeGen/Arm64/CodeGenContext.cs287
-rw-r--r--ARMeilleure/CodeGen/Arm64/CodeGenerator.cs1580
-rw-r--r--ARMeilleure/CodeGen/Arm64/CodeGeneratorIntrinsic.cs662
-rw-r--r--ARMeilleure/CodeGen/Arm64/HardwareCapabilities.cs185
-rw-r--r--ARMeilleure/CodeGen/Arm64/IntrinsicInfo.cs14
-rw-r--r--ARMeilleure/CodeGen/Arm64/IntrinsicTable.cs463
-rw-r--r--ARMeilleure/CodeGen/Arm64/IntrinsicType.cs59
-rw-r--r--ARMeilleure/CodeGen/Arm64/PreAllocator.cs892
15 files changed, 0 insertions, 5831 deletions
diff --git a/ARMeilleure/CodeGen/Arm64/Arm64Optimizer.cs b/ARMeilleure/CodeGen/Arm64/Arm64Optimizer.cs
deleted file mode 100644
index fdd4d024..00000000
--- a/ARMeilleure/CodeGen/Arm64/Arm64Optimizer.cs
+++ /dev/null
@@ -1,270 +0,0 @@
-using ARMeilleure.CodeGen.Optimizations;
-using ARMeilleure.IntermediateRepresentation;
-using ARMeilleure.Translation;
-using System.Collections.Generic;
-using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
-using static ARMeilleure.IntermediateRepresentation.Operation.Factory;
-
-namespace ARMeilleure.CodeGen.Arm64
-{
- static class Arm64Optimizer
- {
- private const int MaxConstantUses = 10000;
-
- public static void RunPass(ControlFlowGraph cfg)
- {
- var constants = new Dictionary<ulong, Operand>();
-
- Operand GetConstantCopy(BasicBlock block, Operation operation, Operand source)
- {
- // If the constant has many uses, we also force a new constant mov to be added, in order
- // to avoid overflow of the counts field (that is limited to 16 bits).
- if (!constants.TryGetValue(source.Value, out var constant) || constant.UsesCount > MaxConstantUses)
- {
- constant = Local(source.Type);
-
- Operation copyOp = Operation(Instruction.Copy, constant, source);
-
- block.Operations.AddBefore(operation, copyOp);
-
- constants[source.Value] = constant;
- }
-
- return constant;
- }
-
- for (BasicBlock block = cfg.Blocks.First; block != null; block = block.ListNext)
- {
- constants.Clear();
-
- Operation nextNode;
-
- for (Operation node = block.Operations.First; node != default; node = nextNode)
- {
- nextNode = node.ListNext;
-
- // Insert copies for constants that can't fit on a 32-bit immediate.
- // Doing this early unblocks a few optimizations.
- if (node.Instruction == Instruction.Add)
- {
- Operand src1 = node.GetSource(0);
- Operand src2 = node.GetSource(1);
-
- if (src1.Kind == OperandKind.Constant && (src1.Relocatable || ConstTooLong(src1, OperandType.I32)))
- {
- node.SetSource(0, GetConstantCopy(block, node, src1));
- }
-
- if (src2.Kind == OperandKind.Constant && (src2.Relocatable || ConstTooLong(src2, OperandType.I32)))
- {
- node.SetSource(1, GetConstantCopy(block, node, src2));
- }
- }
-
- // Try to fold something like:
- // lsl x1, x1, #2
- // add x0, x0, x1
- // ldr x0, [x0]
- // add x2, x2, #16
- // ldr x2, [x2]
- // Into:
- // ldr x0, [x0, x1, lsl #2]
- // ldr x2, [x2, #16]
- if (IsMemoryLoadOrStore(node.Instruction))
- {
- OperandType type;
-
- if (node.Destination != default)
- {
- type = node.Destination.Type;
- }
- else
- {
- type = node.GetSource(1).Type;
- }
-
- Operand memOp = GetMemoryOperandOrNull(node.GetSource(0), type);
-
- if (memOp != default)
- {
- node.SetSource(0, memOp);
- }
- }
- }
- }
-
- Optimizer.RemoveUnusedNodes(cfg);
- }
-
- private static Operand GetMemoryOperandOrNull(Operand addr, OperandType type)
- {
- Operand baseOp = addr;
-
- // First we check if the address is the result of a local X with immediate
- // addition. If that is the case, then the baseOp is X, and the memory operand immediate
- // becomes the addition immediate. Otherwise baseOp keeps being the address.
- int imm = GetConstOp(ref baseOp, type);
- if (imm != 0)
- {
- return MemoryOp(type, baseOp, default, Multiplier.x1, imm);
- }
-
- // Now we check if the baseOp is the result of a local Y with a local Z addition.
- // If that is the case, we now set baseOp to Y and indexOp to Z. We further check
- // if Z is the result of a left shift of local W by a value == 0 or == Log2(AccessSize),
- // if that is the case, we set indexOp to W and adjust the scale value of the memory operand
- // to match that of the left shift.
- // There is one missed case, which is the address being a shift result, but this is
- // probably not worth optimizing as it should never happen.
- (Operand indexOp, Multiplier scale) = GetIndexOp(ref baseOp, type);
-
- // If baseOp is still equal to address, then there's nothing that can be optimized.
- if (baseOp == addr)
- {
- return default;
- }
-
- return MemoryOp(type, baseOp, indexOp, scale, 0);
- }
-
- private static int GetConstOp(ref Operand baseOp, OperandType accessType)
- {
- Operation operation = GetAsgOpWithInst(baseOp, Instruction.Add);
-
- if (operation == default)
- {
- return 0;
- }
-
- Operand src1 = operation.GetSource(0);
- Operand src2 = operation.GetSource(1);
-
- Operand constOp;
- Operand otherOp;
-
- if (src1.Kind == OperandKind.Constant && src2.Kind == OperandKind.LocalVariable)
- {
- constOp = src1;
- otherOp = src2;
- }
- else if (src1.Kind == OperandKind.LocalVariable && src2.Kind == OperandKind.Constant)
- {
- constOp = src2;
- otherOp = src1;
- }
- else
- {
- return 0;
- }
-
- // If we have addition by a constant that we can't encode on the instruction,
- // then we can't optimize it further.
- if (ConstTooLong(constOp, accessType))
- {
- return 0;
- }
-
- baseOp = otherOp;
-
- return constOp.AsInt32();
- }
-
- private static (Operand, Multiplier) GetIndexOp(ref Operand baseOp, OperandType accessType)
- {
- Operand indexOp = default;
-
- Multiplier scale = Multiplier.x1;
-
- Operation addOp = GetAsgOpWithInst(baseOp, Instruction.Add);
-
- if (addOp == default)
- {
- return (indexOp, scale);
- }
-
- Operand src1 = addOp.GetSource(0);
- Operand src2 = addOp.GetSource(1);
-
- if (src1.Kind != OperandKind.LocalVariable || src2.Kind != OperandKind.LocalVariable)
- {
- return (indexOp, scale);
- }
-
- baseOp = src1;
- indexOp = src2;
-
- Operation shlOp = GetAsgOpWithInst(src1, Instruction.ShiftLeft);
-
- bool indexOnSrc2 = false;
-
- if (shlOp == default)
- {
- shlOp = GetAsgOpWithInst(src2, Instruction.ShiftLeft);
-
- indexOnSrc2 = true;
- }
-
- if (shlOp != default)
- {
- Operand shSrc = shlOp.GetSource(0);
- Operand shift = shlOp.GetSource(1);
-
- int maxShift = Assembler.GetScaleForType(accessType);
-
- if (shSrc.Kind == OperandKind.LocalVariable &&
- shift.Kind == OperandKind.Constant &&
- (shift.Value == 0 || shift.Value == (ulong)maxShift))
- {
- scale = shift.Value switch
- {
- 1 => Multiplier.x2,
- 2 => Multiplier.x4,
- 3 => Multiplier.x8,
- 4 => Multiplier.x16,
- _ => Multiplier.x1
- };
-
- baseOp = indexOnSrc2 ? src1 : src2;
- indexOp = shSrc;
- }
- }
-
- return (indexOp, scale);
- }
-
- private static Operation GetAsgOpWithInst(Operand op, Instruction inst)
- {
- // If we have multiple assignments, folding is not safe
- // as the value may be different depending on the
- // control flow path.
- if (op.AssignmentsCount != 1)
- {
- return default;
- }
-
- Operation asgOp = op.Assignments[0];
-
- if (asgOp.Instruction != inst)
- {
- return default;
- }
-
- return asgOp;
- }
-
- private static bool IsMemoryLoadOrStore(Instruction inst)
- {
- return inst == Instruction.Load || inst == Instruction.Store;
- }
-
- private static bool ConstTooLong(Operand constOp, OperandType accessType)
- {
- if ((uint)constOp.Value != constOp.Value)
- {
- return true;
- }
-
- return !CodeGenCommon.ConstFitsOnUImm12(constOp.AsInt32(), accessType);
- }
- }
-}
diff --git a/ARMeilleure/CodeGen/Arm64/ArmCondition.cs b/ARMeilleure/CodeGen/Arm64/ArmCondition.cs
deleted file mode 100644
index db27a810..00000000
--- a/ARMeilleure/CodeGen/Arm64/ArmCondition.cs
+++ /dev/null
@@ -1,47 +0,0 @@
-using ARMeilleure.IntermediateRepresentation;
-using System;
-
-namespace ARMeilleure.CodeGen.Arm64
-{
- enum ArmCondition
- {
- Eq = 0,
- Ne = 1,
- GeUn = 2,
- LtUn = 3,
- Mi = 4,
- Pl = 5,
- Vs = 6,
- Vc = 7,
- GtUn = 8,
- LeUn = 9,
- Ge = 10,
- Lt = 11,
- Gt = 12,
- Le = 13,
- Al = 14,
- Nv = 15
- }
-
- static class ComparisonArm64Extensions
- {
- public static ArmCondition ToArmCondition(this Comparison comp)
- {
- return comp switch
- {
- Comparison.Equal => ArmCondition.Eq,
- Comparison.NotEqual => ArmCondition.Ne,
- Comparison.Greater => ArmCondition.Gt,
- Comparison.LessOrEqual => ArmCondition.Le,
- Comparison.GreaterUI => ArmCondition.GtUn,
- Comparison.LessOrEqualUI => ArmCondition.LeUn,
- Comparison.GreaterOrEqual => ArmCondition.Ge,
- Comparison.Less => ArmCondition.Lt,
- Comparison.GreaterOrEqualUI => ArmCondition.GeUn,
- Comparison.LessUI => ArmCondition.LtUn,
-
- _ => throw new ArgumentException(null, nameof(comp))
- };
- }
- }
-}
diff --git a/ARMeilleure/CodeGen/Arm64/ArmExtensionType.cs b/ARMeilleure/CodeGen/Arm64/ArmExtensionType.cs
deleted file mode 100644
index 062a6d0b..00000000
--- a/ARMeilleure/CodeGen/Arm64/ArmExtensionType.cs
+++ /dev/null
@@ -1,14 +0,0 @@
-namespace ARMeilleure.CodeGen.Arm64
-{
- enum ArmExtensionType
- {
- Uxtb = 0,
- Uxth = 1,
- Uxtw = 2,
- Uxtx = 3,
- Sxtb = 4,
- Sxth = 5,
- Sxtw = 6,
- Sxtx = 7
- }
-}
diff --git a/ARMeilleure/CodeGen/Arm64/ArmShiftType.cs b/ARMeilleure/CodeGen/Arm64/ArmShiftType.cs
deleted file mode 100644
index d223a146..00000000
--- a/ARMeilleure/CodeGen/Arm64/ArmShiftType.cs
+++ /dev/null
@@ -1,11 +0,0 @@
-
-namespace ARMeilleure.CodeGen.Arm64
-{
- enum ArmShiftType
- {
- Lsl = 0,
- Lsr = 1,
- Asr = 2,
- Ror = 3
- }
-} \ No newline at end of file
diff --git a/ARMeilleure/CodeGen/Arm64/Assembler.cs b/ARMeilleure/CodeGen/Arm64/Assembler.cs
deleted file mode 100644
index 0ec0be7c..00000000
--- a/ARMeilleure/CodeGen/Arm64/Assembler.cs
+++ /dev/null
@@ -1,1160 +0,0 @@
-using ARMeilleure.IntermediateRepresentation;
-using System;
-using System.Diagnostics;
-using System.IO;
-using static ARMeilleure.IntermediateRepresentation.Operand;
-
-namespace ARMeilleure.CodeGen.Arm64
-{
- class Assembler
- {
- public const uint SfFlag = 1u << 31;
-
- private const int SpRegister = 31;
- private const int ZrRegister = 31;
-
- private readonly Stream _stream;
-
- public Assembler(Stream stream)
- {
- _stream = stream;
- }
-
- public void Add(Operand rd, Operand rn, Operand rm, ArmExtensionType extensionType, int shiftAmount = 0)
- {
- WriteInstructionAuto(0x0b200000u, rd, rn, rm, extensionType, shiftAmount);
- }
-
- public void Add(Operand rd, Operand rn, Operand rm, ArmShiftType shiftType = ArmShiftType.Lsl, int shiftAmount = 0, bool immForm = false)
- {
- WriteInstructionAuto(0x11000000u, 0x0b000000u, rd, rn, rm, shiftType, shiftAmount, immForm);
- }
-
- public void And(Operand rd, Operand rn, Operand rm, ArmShiftType shiftType = ArmShiftType.Lsl, int shiftAmount = 0)
- {
- WriteInstructionBitwiseAuto(0x12000000u, 0x0a000000u, rd, rn, rm, shiftType, shiftAmount);
- }
-
- public void Ands(Operand rd, Operand rn, Operand rm, ArmShiftType shiftType = ArmShiftType.Lsl, int shiftAmount = 0)
- {
- WriteInstructionBitwiseAuto(0x72000000u, 0x6a000000u, rd, rn, rm, shiftType, shiftAmount);
- }
-
- public void Asr(Operand rd, Operand rn, Operand rm)
- {
- if (rm.Kind == OperandKind.Constant)
- {
- int shift = rm.AsInt32();
- int mask = rd.Type == OperandType.I64 ? 63 : 31;
- shift &= mask;
- Sbfm(rd, rn, shift, mask);
- }
- else
- {
- Asrv(rd, rn, rm);
- }
- }
-
- public void Asrv(Operand rd, Operand rn, Operand rm)
- {
- WriteInstructionBitwiseAuto(0x1ac02800u, rd, rn, rm);
- }
-
- public void B(int imm)
- {
- WriteUInt32(0x14000000u | EncodeSImm26_2(imm));
- }
-
- public void B(ArmCondition condition, int imm)
- {
- WriteUInt32(0x54000000u | (uint)condition | (EncodeSImm19_2(imm) << 5));
- }
-
- public void Blr(Operand rn)
- {
- WriteUInt32(0xd63f0000u | (EncodeReg(rn) << 5));
- }
-
- public void Br(Operand rn)
- {
- WriteUInt32(0xd61f0000u | (EncodeReg(rn) << 5));
- }
-
- public void Brk()
- {
- WriteUInt32(0xd4200000u);
- }
-
- public void Cbz(Operand rt, int imm)
- {
- WriteInstructionAuto(0x34000000u | (EncodeSImm19_2(imm) << 5), rt);
- }
-
- public void Cbnz(Operand rt, int imm)
- {
- WriteInstructionAuto(0x35000000u | (EncodeSImm19_2(imm) << 5), rt);
- }
-
- public void Clrex(int crm = 15)
- {
- WriteUInt32(0xd503305fu | (EncodeUImm4(crm) << 8));
- }
-
- public void Clz(Operand rd, Operand rn)
- {
- WriteInstructionAuto(0x5ac01000u, rd, rn);
- }
-
- public void CmeqVector(Operand rd, Operand rn, Operand rm, int size, bool q = true)
- {
- Debug.Assert((uint)size < 4);
- WriteSimdInstruction(0x2e208c00u | ((uint)size << 22), rd, rn, rm, q);
- }
-
- public void Cmp(Operand rn, Operand rm, ArmShiftType shiftType = ArmShiftType.Lsl, int shiftAmount = 0)
- {
- Subs(Factory.Register(ZrRegister, RegisterType.Integer, rn.Type), rn, rm, shiftType, shiftAmount);
- }
-
- public void Csel(Operand rd, Operand rn, Operand rm, ArmCondition condition)
- {
- WriteInstructionBitwiseAuto(0x1a800000u | ((uint)condition << 12), rd, rn, rm);
- }
-
- public void Cset(Operand rd, ArmCondition condition)
- {
- var zr = Factory.Register(ZrRegister, RegisterType.Integer, rd.Type);
- Csinc(rd, zr, zr, (ArmCondition)((int)condition ^ 1));
- }
-
- public void Csinc(Operand rd, Operand rn, Operand rm, ArmCondition condition)
- {
- WriteInstructionBitwiseAuto(0x1a800400u | ((uint)condition << 12), rd, rn, rm);
- }
-
- public void Dmb(uint option)
- {
- WriteUInt32(0xd50330bfu | (option << 8));
- }
-
- public void DupScalar(Operand rd, Operand rn, int index, int size)
- {
- WriteInstruction(0x5e000400u | (EncodeIndexSizeImm5(index, size) << 16), rd, rn);
- }
-
- public void Eor(Operand rd, Operand rn, Operand rm, ArmShiftType shiftType = ArmShiftType.Lsl, int shiftAmount = 0)
- {
- WriteInstructionBitwiseAuto(0x52000000u, 0x4a000000u, rd, rn, rm, shiftType, shiftAmount);
- }
-
- public void EorVector(Operand rd, Operand rn, Operand rm, bool q = true)
- {
- WriteSimdInstruction(0x2e201c00u, rd, rn, rm, q);
- }
-
- public void Extr(Operand rd, Operand rn, Operand rm, int imms)
- {
- uint n = rd.Type == OperandType.I64 ? 1u << 22 : 0u;
- WriteInstructionBitwiseAuto(0x13800000u | n | (EncodeUImm6(imms) << 10), rd, rn, rm);
- }
-
- public void FaddScalar(Operand rd, Operand rn, Operand rm)
- {
- WriteFPInstructionAuto(0x1e202800u, rd, rn, rm);
- }
-
- public void FcvtScalar(Operand rd, Operand rn)
- {
- uint instruction = 0x1e224000u | (rd.Type == OperandType.FP64 ? 1u << 15 : 1u << 22);
- WriteUInt32(instruction | EncodeReg(rd) | (EncodeReg(rn) << 5));
- }
-
- public void FdivScalar(Operand rd, Operand rn, Operand rm)
- {
- WriteFPInstructionAuto(0x1e201800u, rd, rn, rm);
- }
-
- public void Fmov(Operand rd, Operand rn)
- {
- WriteFPInstructionAuto(0x1e204000u, rd, rn);
- }
-
- public void Fmov(Operand rd, Operand rn, bool topHalf)
- {
- Debug.Assert(rd.Type.IsInteger() != rn.Type.IsInteger());
- Debug.Assert(rd.Type == OperandType.I64 || rn.Type == OperandType.I64 || !topHalf);
-
- uint opcode = rd.Type.IsInteger() ? 0b110u : 0b111u;
-
- uint rmode = topHalf ? 1u << 19 : 0u;
- uint ftype = rd.Type == OperandType.FP64 || rn.Type == OperandType.FP64 ? 1u << 22 : 0u;
- uint sf = rd.Type == OperandType.I64 || rn.Type == OperandType.I64 ? SfFlag : 0u;
-
- WriteUInt32(0x1e260000u | (opcode << 16) | rmode | ftype | sf | EncodeReg(rd) | (EncodeReg(rn) << 5));
- }
-
- public void FmulScalar(Operand rd, Operand rn, Operand rm)
- {
- WriteFPInstructionAuto(0x1e200800u, rd, rn, rm);
- }
-
- public void FnegScalar(Operand rd, Operand rn)
- {
- WriteFPInstructionAuto(0x1e214000u, rd, rn);
- }
-
- public void FsubScalar(Operand rd, Operand rn, Operand rm)
- {
- WriteFPInstructionAuto(0x1e203800u, rd, rn, rm);
- }
-
- public void Ins(Operand rd, Operand rn, int index, int size)
- {
- WriteInstruction(0x4e001c00u | (EncodeIndexSizeImm5(index, size) << 16), rd, rn);
- }
-
- public void Ins(Operand rd, Operand rn, int srcIndex, int dstIndex, int size)
- {
- uint imm4 = (uint)srcIndex << size;
- Debug.Assert((uint)srcIndex < (16u >> size));
- WriteInstruction(0x6e000400u | (imm4 << 11) | (EncodeIndexSizeImm5(dstIndex, size) << 16), rd, rn);
- }
-
- public void Ldaxp(Operand rt, Operand rt2, Operand rn)
- {
- WriteInstruction(0x887f8000u | ((rt.Type == OperandType.I64 ? 3u : 2u) << 30), rt, rn, rt2);
- }
-
- public void Ldaxr(Operand rt, Operand rn)
- {
- WriteInstruction(0x085ffc00u | ((rt.Type == OperandType.I64 ? 3u : 2u) << 30), rt, rn);
- }
-
- public void Ldaxrb(Operand rt, Operand rn)
- {
- WriteInstruction(0x085ffc00u, rt, rn);
- }
-
- public void Ldaxrh(Operand rt, Operand rn)
- {
- WriteInstruction(0x085ffc00u | (1u << 30), rt, rn);
- }
-
- public void LdpRiPost(Operand rt, Operand rt2, Operand rn, int imm)
- {
- uint instruction = GetLdpStpInstruction(0x28c00000u, 0x2cc00000u, imm, rt.Type);
- WriteInstruction(instruction, rt, rn, rt2);
- }
-
- public void LdpRiPre(Operand rt, Operand rt2, Operand rn, int imm)
- {
- uint instruction = GetLdpStpInstruction(0x29c00000u, 0x2dc00000u, imm, rt.Type);
- WriteInstruction(instruction, rt, rn, rt2);
- }
-
- public void LdpRiUn(Operand rt, Operand rt2, Operand rn, int imm)
- {
- uint instruction = GetLdpStpInstruction(0x29400000u, 0x2d400000u, imm, rt.Type);
- WriteInstruction(instruction, rt, rn, rt2);
- }
-
- public void Ldr(Operand rt, Operand rn)
- {
- if (rn.Kind == OperandKind.Memory)
- {
- MemoryOperand memOp = rn.GetMemory();
-
- if (memOp.Index != default)
- {
- Debug.Assert(memOp.Displacement == 0);
- Debug.Assert(memOp.Scale == Multiplier.x1 || (int)memOp.Scale == GetScaleForType(rt.Type));
- LdrRr(rt, memOp.BaseAddress, memOp.Index, ArmExtensionType.Uxtx, memOp.Scale != Multiplier.x1);
- }
- else
- {
- LdrRiUn(rt, memOp.BaseAddress, memOp.Displacement);
- }
- }
- else
- {
- LdrRiUn(rt, rn, 0);
- }
- }
-
- public void LdrLit(Operand rt, int offset)
- {
- uint instruction = 0x18000000u | (EncodeSImm19_2(offset) << 5);
-
- if (rt.Type == OperandType.I64)
- {
- instruction |= 1u << 30;
- }
-
- WriteInstruction(instruction, rt);
- }
-
- public void LdrRiPost(Operand rt, Operand rn, int imm)
- {
- uint instruction = GetLdrStrInstruction(0xb8400400u, 0x3c400400u, rt.Type) | (EncodeSImm9(imm) << 12);
- WriteInstruction(instruction, rt, rn);
- }
-
- public void LdrRiPre(Operand rt, Operand rn, int imm)
- {
- uint instruction = GetLdrStrInstruction(0xb8400c00u, 0x3c400c00u, rt.Type) | (EncodeSImm9(imm) << 12);
- WriteInstruction(instruction, rt, rn);
- }
-
- public void LdrRiUn(Operand rt, Operand rn, int imm)
- {
- uint instruction = GetLdrStrInstruction(0xb9400000u, 0x3d400000u, rt.Type) | (EncodeUImm12(imm, rt.Type) << 10);
- WriteInstruction(instruction, rt, rn);
- }
-
- public void LdrRr(Operand rt, Operand rn, Operand rm, ArmExtensionType extensionType, bool shift)
- {
- uint instruction = GetLdrStrInstruction(0xb8600800u, 0x3ce00800u, rt.Type);
- WriteInstructionLdrStrAuto(instruction, rt, rn, rm, extensionType, shift);
- }
-
- public void LdrbRiPost(Operand rt, Operand rn, int imm)
- {
- uint instruction = 0x38400400u | (EncodeSImm9(imm) << 12);
- WriteInstruction(instruction, rt, rn);
- }
-
- public void LdrbRiPre(Operand rt, Operand rn, int imm)
- {
- uint instruction = 0x38400c00u | (EncodeSImm9(imm) << 12);
- WriteInstruction(instruction, rt, rn);
- }
-
- public void LdrbRiUn(Operand rt, Operand rn, int imm)
- {
- uint instruction = 0x39400000u | (EncodeUImm12(imm, 0) << 10);
- WriteInstruction(instruction, rt, rn);
- }
-
- public void LdrhRiPost(Operand rt, Operand rn, int imm)
- {
- uint instruction = 0x78400400u | (EncodeSImm9(imm) << 12);
- WriteInstruction(instruction, rt, rn);
- }
-
- public void LdrhRiPre(Operand rt, Operand rn, int imm)
- {
- uint instruction = 0x78400c00u | (EncodeSImm9(imm) << 12);
- WriteInstruction(instruction, rt, rn);
- }
-
- public void LdrhRiUn(Operand rt, Operand rn, int imm)
- {
- uint instruction = 0x79400000u | (EncodeUImm12(imm, 1) << 10);
- WriteInstruction(instruction, rt, rn);
- }
-
- public void Ldur(Operand rt, Operand rn, int imm)
- {
- uint instruction = GetLdrStrInstruction(0xb8400000u, 0x3c400000u, rt.Type) | (EncodeSImm9(imm) << 12);
- WriteInstruction(instruction, rt, rn);
- }
-
- public void Lsl(Operand rd, Operand rn, Operand rm)
- {
- if (rm.Kind == OperandKind.Constant)
- {
- int shift = rm.AsInt32();
- int mask = rd.Type == OperandType.I64 ? 63 : 31;
- shift &= mask;
- Ubfm(rd, rn, -shift & mask, mask - shift);
- }
- else
- {
- Lslv(rd, rn, rm);
- }
- }
-
- public void Lslv(Operand rd, Operand rn, Operand rm)
- {
- WriteInstructionBitwiseAuto(0x1ac02000u, rd, rn, rm);
- }
-
- public void Lsr(Operand rd, Operand rn, Operand rm)
- {
- if (rm.Kind == OperandKind.Constant)
- {
- int shift = rm.AsInt32();
- int mask = rd.Type == OperandType.I64 ? 63 : 31;
- shift &= mask;
- Ubfm(rd, rn, shift, mask);
- }
- else
- {
- Lsrv(rd, rn, rm);
- }
- }
-
- public void Lsrv(Operand rd, Operand rn, Operand rm)
- {
- WriteInstructionBitwiseAuto(0x1ac02400u, rd, rn, rm);
- }
-
- public void Madd(Operand rd, Operand rn, Operand rm, Operand ra)
- {
- WriteInstructionAuto(0x1b000000u, rd, rn, rm, ra);
- }
-
- public void Mul(Operand rd, Operand rn, Operand rm)
- {
- Madd(rd, rn, rm, Factory.Register(ZrRegister, RegisterType.Integer, rd.Type));
- }
-
- public void Mov(Operand rd, Operand rn)
- {
- if (rd.Type.IsInteger())
- {
- Orr(rd, Factory.Register(ZrRegister, RegisterType.Integer, rd.Type), rn);
- }
- else
- {
- OrrVector(rd, rn, rn);
- }
- }
-
- public void MovSp(Operand rd, Operand rn)
- {
- if (rd.GetRegister().Index == SpRegister ||
- rn.GetRegister().Index == SpRegister)
- {
- Add(rd, rn, Factory.Const(rd.Type, 0), immForm: true);
- }
- else
- {
- Mov(rd, rn);
- }
- }
-
- public void Mov(Operand rd, int imm)
- {
- Movz(rd, imm, 0);
- }
-
- public void Movz(Operand rd, int imm, int hw)
- {
- Debug.Assert((hw & (rd.Type == OperandType.I64 ? 3 : 1)) == hw);
- WriteInstructionAuto(0x52800000u | (EncodeUImm16(imm) << 5) | ((uint)hw << 21), rd);
- }
-
- public void Movk(Operand rd, int imm, int hw)
- {
- Debug.Assert((hw & (rd.Type == OperandType.I64 ? 3 : 1)) == hw);
- WriteInstructionAuto(0x72800000u | (EncodeUImm16(imm) << 5) | ((uint)hw << 21), rd);
- }
-
- public void Mrs(Operand rt, uint o0, uint op1, uint crn, uint crm, uint op2)
- {
- uint instruction = 0xd5300000u;
-
- instruction |= (op2 & 7) << 5;
- instruction |= (crm & 15) << 8;
- instruction |= (crn & 15) << 12;
- instruction |= (op1 & 7) << 16;
- instruction |= (o0 & 1) << 19;
-
- WriteInstruction(instruction, rt);
- }
-
- public void Mvn(Operand rd, Operand rn, ArmShiftType shiftType = ArmShiftType.Lsl, int shiftAmount = 0)
- {
- Orn(rd, Factory.Register(ZrRegister, RegisterType.Integer, rd.Type), rn, shiftType, shiftAmount);
- }
-
- public void Neg(Operand rd, Operand rn, ArmShiftType shiftType = ArmShiftType.Lsl, int shiftAmount = 0)
- {
- Sub(rd, Factory.Register(ZrRegister, RegisterType.Integer, rd.Type), rn, shiftType, shiftAmount);
- }
-
- public void Orn(Operand rd, Operand rn, Operand rm, ArmShiftType shiftType = ArmShiftType.Lsl, int shiftAmount = 0)
- {
- WriteInstructionBitwiseAuto(0x2a200000u, rd, rn, rm, shiftType, shiftAmount);
- }
-
- public void Orr(Operand rd, Operand rn, Operand rm, ArmShiftType shiftType = ArmShiftType.Lsl, int shiftAmount = 0)
- {
- WriteInstructionBitwiseAuto(0x32000000u, 0x2a000000u, rd, rn, rm, shiftType, shiftAmount);
- }
-
- public void OrrVector(Operand rd, Operand rn, Operand rm, bool q = true)
- {
- WriteSimdInstruction(0x0ea01c00u, rd, rn, rm, q);
- }
-
- public void Ret(Operand rn)
- {
- WriteUInt32(0xd65f0000u | (EncodeReg(rn) << 5));
- }
-
- public void Rev(Operand rd, Operand rn)
- {
- uint opc0 = rd.Type == OperandType.I64 ? 1u << 10 : 0u;
- WriteInstructionAuto(0x5ac00800u | opc0, rd, rn);
- }
-
- public void Ror(Operand rd, Operand rn, Operand rm)
- {
- if (rm.Kind == OperandKind.Constant)
- {
- int shift = rm.AsInt32();
- int mask = rd.Type == OperandType.I64 ? 63 : 31;
- shift &= mask;
- Extr(rd, rn, rn, shift);
- }
- else
- {
- Rorv(rd, rn, rm);
- }
- }
-
- public void Rorv(Operand rd, Operand rn, Operand rm)
- {
- WriteInstructionBitwiseAuto(0x1ac02c00u, rd, rn, rm);
- }
-
- public void Sbfm(Operand rd, Operand rn, int immr, int imms)
- {
- uint n = rd.Type == OperandType.I64 ? 1u << 22 : 0u;
- WriteInstructionAuto(0x13000000u | n | (EncodeUImm6(imms) << 10) | (EncodeUImm6(immr) << 16), rd, rn);
- }
-
- public void ScvtfScalar(Operand rd, Operand rn)
- {
- uint instruction = 0x1e220000u;
-
- if (rn.Type == OperandType.I64)
- {
- instruction |= SfFlag;
- }
-
- WriteFPInstructionAuto(instruction, rd, rn);
- }
-
- public void Sdiv(Operand rd, Operand rn, Operand rm)
- {
- WriteInstructionRm16Auto(0x1ac00c00u, rd, rn, rm);
- }
-
- public void Smulh(Operand rd, Operand rn, Operand rm)
- {
- WriteInstructionRm16(0x9b407c00u, rd, rn, rm);
- }
-
- public void Stlxp(Operand rt, Operand rt2, Operand rn, Operand rs)
- {
- WriteInstruction(0x88208000u | ((rt.Type == OperandType.I64 ? 3u : 2u) << 30), rt, rn, rs, rt2);
- }
-
- public void Stlxr(Operand rt, Operand rn, Operand rs)
- {
- WriteInstructionRm16(0x0800fc00u | ((rt.Type == OperandType.I64 ? 3u : 2u) << 30), rt, rn, rs);
- }
-
- public void Stlxrb(Operand rt, Operand rn, Operand rs)
- {
- WriteInstructionRm16(0x0800fc00u, rt, rn, rs);
- }
-
- public void Stlxrh(Operand rt, Operand rn, Operand rs)
- {
- WriteInstructionRm16(0x0800fc00u | (1u << 30), rt, rn, rs);
- }
-
- public void StpRiPost(Operand rt, Operand rt2, Operand rn, int imm)
- {
- uint instruction = GetLdpStpInstruction(0x28800000u, 0x2c800000u, imm, rt.Type);
- WriteInstruction(instruction, rt, rn, rt2);
- }
-
- public void StpRiPre(Operand rt, Operand rt2, Operand rn, int imm)
- {
- uint instruction = GetLdpStpInstruction(0x29800000u, 0x2d800000u, imm, rt.Type);
- WriteInstruction(instruction, rt, rn, rt2);
- }
-
- public void StpRiUn(Operand rt, Operand rt2, Operand rn, int imm)
- {
- uint instruction = GetLdpStpInstruction(0x29000000u, 0x2d000000u, imm, rt.Type);
- WriteInstruction(instruction, rt, rn, rt2);
- }
-
- public void Str(Operand rt, Operand rn)
- {
- if (rn.Kind == OperandKind.Memory)
- {
- MemoryOperand memOp = rn.GetMemory();
-
- if (memOp.Index != default)
- {
- Debug.Assert(memOp.Displacement == 0);
- Debug.Assert(memOp.Scale == Multiplier.x1 || (int)memOp.Scale == GetScaleForType(rt.Type));
- StrRr(rt, memOp.BaseAddress, memOp.Index, ArmExtensionType.Uxtx, memOp.Scale != Multiplier.x1);
- }
- else
- {
- StrRiUn(rt, memOp.BaseAddress, memOp.Displacement);
- }
- }
- else
- {
- StrRiUn(rt, rn, 0);
- }
- }
-
- public void StrRiPost(Operand rt, Operand rn, int imm)
- {
- uint instruction = GetLdrStrInstruction(0xb8000400u, 0x3c000400u, rt.Type) | (EncodeSImm9(imm) << 12);
- WriteInstruction(instruction, rt, rn);
- }
-
- public void StrRiPre(Operand rt, Operand rn, int imm)
- {
- uint instruction = GetLdrStrInstruction(0xb8000c00u, 0x3c000c00u, rt.Type) | (EncodeSImm9(imm) << 12);
- WriteInstruction(instruction, rt, rn);
- }
-
- public void StrRiUn(Operand rt, Operand rn, int imm)
- {
- uint instruction = GetLdrStrInstruction(0xb9000000u, 0x3d000000u, rt.Type) | (EncodeUImm12(imm, rt.Type) << 10);
- WriteInstruction(instruction, rt, rn);
- }
-
- public void StrRr(Operand rt, Operand rn, Operand rm, ArmExtensionType extensionType, bool shift)
- {
- uint instruction = GetLdrStrInstruction(0xb8200800u, 0x3ca00800u, rt.Type);
- WriteInstructionLdrStrAuto(instruction, rt, rn, rm, extensionType, shift);
- }
-
- public void StrbRiPost(Operand rt, Operand rn, int imm)
- {
- uint instruction = 0x38000400u | (EncodeSImm9(imm) << 12);
- WriteInstruction(instruction, rt, rn);
- }
-
- public void StrbRiPre(Operand rt, Operand rn, int imm)
- {
- uint instruction = 0x38000c00u | (EncodeSImm9(imm) << 12);
- WriteInstruction(instruction, rt, rn);
- }
-
- public void StrbRiUn(Operand rt, Operand rn, int imm)
- {
- uint instruction = 0x39000000u | (EncodeUImm12(imm, 0) << 10);
- WriteInstruction(instruction, rt, rn);
- }
-
- public void StrhRiPost(Operand rt, Operand rn, int imm)
- {
- uint instruction = 0x78000400u | (EncodeSImm9(imm) << 12);
- WriteInstruction(instruction, rt, rn);
- }
-
- public void StrhRiPre(Operand rt, Operand rn, int imm)
- {
- uint instruction = 0x78000c00u | (EncodeSImm9(imm) << 12);
- WriteInstruction(instruction, rt, rn);
- }
-
- public void StrhRiUn(Operand rt, Operand rn, int imm)
- {
- uint instruction = 0x79000000u | (EncodeUImm12(imm, 1) << 10);
- WriteInstruction(instruction, rt, rn);
- }
-
- public void Stur(Operand rt, Operand rn, int imm)
- {
- uint instruction = GetLdrStrInstruction(0xb8000000u, 0x3c000000u, rt.Type) | (EncodeSImm9(imm) << 12);
- WriteInstruction(instruction, rt, rn);
- }
-
- public void Sub(Operand rd, Operand rn, Operand rm, ArmExtensionType extensionType, int shiftAmount = 0)
- {
- WriteInstructionAuto(0x4b200000u, rd, rn, rm, extensionType, shiftAmount);
- }
-
- public void Sub(Operand rd, Operand rn, Operand rm, ArmShiftType shiftType = ArmShiftType.Lsl, int shiftAmount = 0)
- {
- WriteInstructionAuto(0x51000000u, 0x4b000000u, rd, rn, rm, shiftType, shiftAmount);
- }
-
- public void Subs(Operand rd, Operand rn, Operand rm, ArmShiftType shiftType = ArmShiftType.Lsl, int shiftAmount = 0)
- {
- WriteInstructionAuto(0x71000000u, 0x6b000000u, rd, rn, rm, shiftType, shiftAmount);
- }
-
- public void Sxtb(Operand rd, Operand rn)
- {
- Sbfm(rd, rn, 0, 7);
- }
-
- public void Sxth(Operand rd, Operand rn)
- {
- Sbfm(rd, rn, 0, 15);
- }
-
- public void Sxtw(Operand rd, Operand rn)
- {
- Sbfm(rd, rn, 0, 31);
- }
-
- public void Tst(Operand rn, Operand rm, ArmShiftType shiftType = ArmShiftType.Lsl, int shiftAmount = 0)
- {
- Ands(Factory.Register(ZrRegister, RegisterType.Integer, rn.Type), rn, rm, shiftType, shiftAmount);
- }
-
- public void Ubfm(Operand rd, Operand rn, int immr, int imms)
- {
- uint n = rd.Type == OperandType.I64 ? 1u << 22 : 0u;
- WriteInstructionAuto(0x53000000u | n | (EncodeUImm6(imms) << 10) | (EncodeUImm6(immr) << 16), rd, rn);
- }
-
- public void UcvtfScalar(Operand rd, Operand rn)
- {
- uint instruction = 0x1e230000u;
-
- if (rn.Type == OperandType.I64)
- {
- instruction |= SfFlag;
- }
-
- WriteFPInstructionAuto(instruction, rd, rn);
- }
-
- public void Udiv(Operand rd, Operand rn, Operand rm)
- {
- WriteInstructionRm16Auto(0x1ac00800u, rd, rn, rm);
- }
-
- public void Umov(Operand rd, Operand rn, int index, int size)
- {
- uint q = size == 3 ? 1u << 30 : 0u;
- WriteInstruction(0x0e003c00u | (EncodeIndexSizeImm5(index, size) << 16) | q, rd, rn);
- }
-
- public void Umulh(Operand rd, Operand rn, Operand rm)
- {
- WriteInstructionRm16(0x9bc07c00u, rd, rn, rm);
- }
-
- public void Uxtb(Operand rd, Operand rn)
- {
- Ubfm(rd, rn, 0, 7);
- }
-
- public void Uxth(Operand rd, Operand rn)
- {
- Ubfm(rd, rn, 0, 15);
- }
-
- private void WriteInstructionAuto(
- uint instI,
- uint instR,
- Operand rd,
- Operand rn,
- Operand rm,
- ArmShiftType shiftType = ArmShiftType.Lsl,
- int shiftAmount = 0,
- bool immForm = false)
- {
- if (rm.Kind == OperandKind.Constant && (rm.Value != 0 || immForm))
- {
- Debug.Assert(shiftAmount == 0);
- int imm = rm.AsInt32();
- Debug.Assert((uint)imm == rm.Value);
- if (imm != 0 && (imm & 0xfff) == 0)
- {
- instI |= 1 << 22; // sh flag
- imm >>= 12;
- }
- WriteInstructionAuto(instI | (EncodeUImm12(imm, 0) << 10), rd, rn);
- }
- else
- {
- instR |= EncodeUImm6(shiftAmount) << 10;
- instR |= (uint)shiftType << 22;
-
- WriteInstructionRm16Auto(instR, rd, rn, rm);
- }
- }
-
- private void WriteInstructionAuto(
- uint instruction,
- Operand rd,
- Operand rn,
- Operand rm,
- ArmExtensionType extensionType,
- int shiftAmount = 0)
- {
- Debug.Assert((uint)shiftAmount <= 4);
-
- instruction |= (uint)shiftAmount << 10;
- instruction |= (uint)extensionType << 13;
-
- WriteInstructionRm16Auto(instruction, rd, rn, rm);
- }
-
- private void WriteInstructionBitwiseAuto(
- uint instI,
- uint instR,
- Operand rd,
- Operand rn,
- Operand rm,
- ArmShiftType shiftType = ArmShiftType.Lsl,
- int shiftAmount = 0)
- {
- if (rm.Kind == OperandKind.Constant && rm.Value != 0)
- {
- Debug.Assert(shiftAmount == 0);
- bool canEncode = CodeGenCommon.TryEncodeBitMask(rm, out int immN, out int immS, out int immR);
- Debug.Assert(canEncode);
- uint instruction = instI | ((uint)immS << 10) | ((uint)immR << 16) | ((uint)immN << 22);
-
- WriteInstructionAuto(instruction, rd, rn);
- }
- else
- {
- WriteInstructionBitwiseAuto(instR, rd, rn, rm, shiftType, shiftAmount);
- }
- }
-
- private void WriteInstructionBitwiseAuto(
- uint instruction,
- Operand rd,
- Operand rn,
- Operand rm,
- ArmShiftType shiftType = ArmShiftType.Lsl,
- int shiftAmount = 0)
- {
- if (rd.Type == OperandType.I64)
- {
- instruction |= SfFlag;
- }
-
- instruction |= EncodeUImm6(shiftAmount) << 10;
- instruction |= (uint)shiftType << 22;
-
- WriteInstructionRm16(instruction, rd, rn, rm);
- }
-
- private void WriteInstructionLdrStrAuto(
- uint instruction,
- Operand rd,
- Operand rn,
- Operand rm,
- ArmExtensionType extensionType,
- bool shift)
- {
- if (shift)
- {
- instruction |= 1u << 12;
- }
-
- instruction |= (uint)extensionType << 13;
-
- if (rd.Type == OperandType.I64)
- {
- instruction |= 1u << 30;
- }
-
- WriteInstructionRm16(instruction, rd, rn, rm);
- }
-
- private void WriteInstructionAuto(uint instruction, Operand rd)
- {
- if (rd.Type == OperandType.I64)
- {
- instruction |= SfFlag;
- }
-
- WriteInstruction(instruction, rd);
- }
-
- public void WriteInstructionAuto(uint instruction, Operand rd, Operand rn)
- {
- if (rd.Type == OperandType.I64)
- {
- instruction |= SfFlag;
- }
-
- WriteInstruction(instruction, rd, rn);
- }
-
- private void WriteInstructionAuto(uint instruction, Operand rd, Operand rn, Operand rm, Operand ra)
- {
- if (rd.Type == OperandType.I64)
- {
- instruction |= SfFlag;
- }
-
- WriteInstruction(instruction, rd, rn, rm, ra);
- }
-
- public void WriteInstruction(uint instruction, Operand rd)
- {
- WriteUInt32(instruction | EncodeReg(rd));
- }
-
- public void WriteInstruction(uint instruction, Operand rd, Operand rn)
- {
- WriteUInt32(instruction | EncodeReg(rd) | (EncodeReg(rn) << 5));
- }
-
- public void WriteInstruction(uint instruction, Operand rd, Operand rn, Operand rm)
- {
- WriteUInt32(instruction | EncodeReg(rd) | (EncodeReg(rn) << 5) | (EncodeReg(rm) << 10));
- }
-
- public void WriteInstruction(uint instruction, Operand rd, Operand rn, Operand rm, Operand ra)
- {
- WriteUInt32(instruction | EncodeReg(rd) | (EncodeReg(rn) << 5) | (EncodeReg(ra) << 10) | (EncodeReg(rm) << 16));
- }
-
- private void WriteFPInstructionAuto(uint instruction, Operand rd, Operand rn)
- {
- if (rd.Type == OperandType.FP64)
- {
- instruction |= 1u << 22;
- }
-
- WriteUInt32(instruction | EncodeReg(rd) | (EncodeReg(rn) << 5));
- }
-
- private void WriteFPInstructionAuto(uint instruction, Operand rd, Operand rn, Operand rm)
- {
- if (rd.Type == OperandType.FP64)
- {
- instruction |= 1u << 22;
- }
-
- WriteInstructionRm16(instruction, rd, rn, rm);
- }
-
- private void WriteSimdInstruction(uint instruction, Operand rd, Operand rn, Operand rm, bool q = true)
- {
- if (q)
- {
- instruction |= 1u << 30;
- }
-
- WriteInstructionRm16(instruction, rd, rn, rm);
- }
-
- private void WriteInstructionRm16Auto(uint instruction, Operand rd, Operand rn, Operand rm)
- {
- if (rd.Type == OperandType.I64)
- {
- instruction |= SfFlag;
- }
-
- WriteInstructionRm16(instruction, rd, rn, rm);
- }
-
- public void WriteInstructionRm16(uint instruction, Operand rd, Operand rn, Operand rm)
- {
- WriteUInt32(instruction | EncodeReg(rd) | (EncodeReg(rn) << 5) | (EncodeReg(rm) << 16));
- }
-
- public void WriteInstructionRm16NoRet(uint instruction, Operand rn, Operand rm)
- {
- WriteUInt32(instruction | (EncodeReg(rn) << 5) | (EncodeReg(rm) << 16));
- }
-
- private static uint GetLdpStpInstruction(uint intInst, uint vecInst, int imm, OperandType type)
- {
- uint instruction;
- int scale;
-
- if (type.IsInteger())
- {
- instruction = intInst;
-
- if (type == OperandType.I64)
- {
- instruction |= SfFlag;
- scale = 3;
- }
- else
- {
- scale = 2;
- }
- }
- else
- {
- int opc = type switch
- {
- OperandType.FP32 => 0,
- OperandType.FP64 => 1,
- _ => 2
- };
-
- instruction = vecInst | ((uint)opc << 30);
- scale = 2 + opc;
- }
-
- instruction |= (EncodeSImm7(imm, scale) << 15);
-
- return instruction;
- }
-
- private static uint GetLdrStrInstruction(uint intInst, uint vecInst, OperandType type)
- {
- uint instruction;
-
- if (type.IsInteger())
- {
- instruction = intInst;
-
- if (type == OperandType.I64)
- {
- instruction |= 1 << 30;
- }
- }
- else
- {
- instruction = vecInst;
-
- if (type == OperandType.V128)
- {
- instruction |= 1u << 23;
- }
- else
- {
- instruction |= type == OperandType.FP32 ? 2u << 30 : 3u << 30;
- }
- }
-
- return instruction;
- }
-
- private static uint EncodeIndexSizeImm5(int index, int size)
- {
- Debug.Assert((uint)size < 4);
- Debug.Assert((uint)index < (16u >> size), $"Invalid index {index} and size {size} combination.");
- return ((uint)index << (size + 1)) | (1u << size);
- }
-
- private static uint EncodeSImm7(int value, int scale)
- {
- uint imm = (uint)(value >> scale) & 0x7f;
- Debug.Assert(((int)imm << 25) >> (25 - scale) == value, $"Failed to encode constant 0x{value:X} with scale {scale}.");
- return imm;
- }
-
- private static uint EncodeSImm9(int value)
- {
- uint imm = (uint)value & 0x1ff;
- Debug.Assert(((int)imm << 23) >> 23 == value, $"Failed to encode constant 0x{value:X}.");
- return imm;
- }
-
- private static uint EncodeSImm19_2(int value)
- {
- uint imm = (uint)(value >> 2) & 0x7ffff;
- Debug.Assert(((int)imm << 13) >> 11 == value, $"Failed to encode constant 0x{value:X}.");
- return imm;
- }
-
- private static uint EncodeSImm26_2(int value)
- {
- uint imm = (uint)(value >> 2) & 0x3ffffff;
- Debug.Assert(((int)imm << 6) >> 4 == value, $"Failed to encode constant 0x{value:X}.");
- return imm;
- }
-
- private static uint EncodeUImm4(int value)
- {
- uint imm = (uint)value & 0xf;
- Debug.Assert((int)imm == value, $"Failed to encode constant 0x{value:X}.");
- return imm;
- }
-
- private static uint EncodeUImm6(int value)
- {
- uint imm = (uint)value & 0x3f;
- Debug.Assert((int)imm == value, $"Failed to encode constant 0x{value:X}.");
- return imm;
- }
-
- private static uint EncodeUImm12(int value, OperandType type)
- {
- return EncodeUImm12(value, GetScaleForType(type));
- }
-
- private static uint EncodeUImm12(int value, int scale)
- {
- uint imm = (uint)(value >> scale) & 0xfff;
- Debug.Assert((int)imm << scale == value, $"Failed to encode constant 0x{value:X} with scale {scale}.");
- return imm;
- }
-
- private static uint EncodeUImm16(int value)
- {
- uint imm = (uint)value & 0xffff;
- Debug.Assert((int)imm == value, $"Failed to encode constant 0x{value:X}.");
- return imm;
- }
-
- private static uint EncodeReg(Operand reg)
- {
- if (reg.Kind == OperandKind.Constant && reg.Value == 0)
- {
- return ZrRegister;
- }
-
- uint regIndex = (uint)reg.GetRegister().Index;
- Debug.Assert(reg.Kind == OperandKind.Register);
- Debug.Assert(regIndex < 32);
- return regIndex;
- }
-
- public static int GetScaleForType(OperandType type)
- {
- return type switch
- {
- OperandType.I32 => 2,
- OperandType.I64 => 3,
- OperandType.FP32 => 2,
- OperandType.FP64 => 3,
- OperandType.V128 => 4,
- _ => throw new ArgumentException($"Invalid type {type}.")
- };
- }
-
- private void WriteInt16(short value)
- {
- WriteUInt16((ushort)value);
- }
-
- private void WriteInt32(int value)
- {
- WriteUInt32((uint)value);
- }
-
- private void WriteByte(byte value)
- {
- _stream.WriteByte(value);
- }
-
- private void WriteUInt16(ushort value)
- {
- _stream.WriteByte((byte)(value >> 0));
- _stream.WriteByte((byte)(value >> 8));
- }
-
- private void WriteUInt32(uint value)
- {
- _stream.WriteByte((byte)(value >> 0));
- _stream.WriteByte((byte)(value >> 8));
- _stream.WriteByte((byte)(value >> 16));
- _stream.WriteByte((byte)(value >> 24));
- }
- }
-}
diff --git a/ARMeilleure/CodeGen/Arm64/CallingConvention.cs b/ARMeilleure/CodeGen/Arm64/CallingConvention.cs
deleted file mode 100644
index fda8d786..00000000
--- a/ARMeilleure/CodeGen/Arm64/CallingConvention.cs
+++ /dev/null
@@ -1,96 +0,0 @@
-using System;
-
-namespace ARMeilleure.CodeGen.Arm64
-{
- static class CallingConvention
- {
- private const int RegistersMask = unchecked((int)0xffffffff);
-
- // Some of those register have specific roles and can't be used as general purpose registers.
- // X18 - Reserved for platform specific usage.
- // X29 - Frame pointer.
- // X30 - Return address.
- // X31 - Not an actual register, in some cases maps to SP, and in others to ZR.
- private const int ReservedRegsMask = (1 << CodeGenCommon.ReservedRegister) | (1 << 18) | (1 << 29) | (1 << 30) | (1 << 31);
-
- public static int GetIntAvailableRegisters()
- {
- return RegistersMask & ~ReservedRegsMask;
- }
-
- public static int GetVecAvailableRegisters()
- {
- return RegistersMask;
- }
-
- public static int GetIntCallerSavedRegisters()
- {
- return (GetIntCalleeSavedRegisters() ^ RegistersMask) & ~ReservedRegsMask;
- }
-
- public static int GetFpCallerSavedRegisters()
- {
- return GetFpCalleeSavedRegisters() ^ RegistersMask;
- }
-
- public static int GetVecCallerSavedRegisters()
- {
- return GetVecCalleeSavedRegisters() ^ RegistersMask;
- }
-
- public static int GetIntCalleeSavedRegisters()
- {
- return 0x1ff80000; // X19 to X28
- }
-
- public static int GetFpCalleeSavedRegisters()
- {
- return 0xff00; // D8 to D15
- }
-
- public static int GetVecCalleeSavedRegisters()
- {
- return 0;
- }
-
- public static int GetArgumentsOnRegsCount()
- {
- return 8;
- }
-
- public static int GetIntArgumentRegister(int index)
- {
- if ((uint)index < (uint)GetArgumentsOnRegsCount())
- {
- return index;
- }
-
- throw new ArgumentOutOfRangeException(nameof(index));
- }
-
- public static int GetVecArgumentRegister(int index)
- {
- if ((uint)index < (uint)GetArgumentsOnRegsCount())
- {
- return index;
- }
-
- throw new ArgumentOutOfRangeException(nameof(index));
- }
-
- public static int GetIntReturnRegister()
- {
- return 0;
- }
-
- public static int GetIntReturnRegisterHigh()
- {
- return 1;
- }
-
- public static int GetVecReturnRegister()
- {
- return 0;
- }
- }
-} \ No newline at end of file
diff --git a/ARMeilleure/CodeGen/Arm64/CodeGenCommon.cs b/ARMeilleure/CodeGen/Arm64/CodeGenCommon.cs
deleted file mode 100644
index 8d1e597b..00000000
--- a/ARMeilleure/CodeGen/Arm64/CodeGenCommon.cs
+++ /dev/null
@@ -1,91 +0,0 @@
-using ARMeilleure.IntermediateRepresentation;
-using System.Numerics;
-
-namespace ARMeilleure.CodeGen.Arm64
-{
- static class CodeGenCommon
- {
- public const int TcAddressRegister = 8;
- public const int ReservedRegister = 17;
-
- public static bool ConstFitsOnSImm7(int value, int scale)
- {
- return (((value >> scale) << 25) >> (25 - scale)) == value;
- }
-
- public static bool ConstFitsOnSImm9(int value)
- {
- return ((value << 23) >> 23) == value;
- }
-
- public static bool ConstFitsOnUImm12(int value)
- {
- return (value & 0xfff) == value;
- }
-
- public static bool ConstFitsOnUImm12(int value, OperandType type)
- {
- int scale = Assembler.GetScaleForType(type);
- return (((value >> scale) & 0xfff) << scale) == value;
- }
-
- public static bool TryEncodeBitMask(Operand operand, out int immN, out int immS, out int immR)
- {
- return TryEncodeBitMask(operand.Type, operand.Value, out immN, out immS, out immR);
- }
-
- public static bool TryEncodeBitMask(OperandType type, ulong value, out int immN, out int immS, out int immR)
- {
- if (type == OperandType.I32)
- {
- value |= value << 32;
- }
-
- return TryEncodeBitMask(value, out immN, out immS, out immR);
- }
-
- public static bool TryEncodeBitMask(ulong value, out int immN, out int immS, out int immR)
- {
- // Some special values also can't be encoded:
- // 0 can't be encoded because we need to subtract 1 from onesCount (which would became negative if 0).
- // A value with all bits set can't be encoded because it is reserved according to the spec, because:
- // Any value AND all ones will be equal itself, so it's effectively a no-op.
- // Any value OR all ones will be equal all ones, so one can just use MOV.
- // Any value XOR all ones will be equal its inverse, so one can just use MVN.
- if (value == 0 || value == ulong.MaxValue)
- {
- immN = 0;
- immS = 0;
- immR = 0;
-
- return false;
- }
-
- // Normalize value, rotating it such that the LSB is 1: Ensures we get a complete element that has not
- // been cut-in-half across the word boundary.
- int rotation = BitOperations.TrailingZeroCount(value & (value + 1));
- ulong rotatedValue = ulong.RotateRight(value, rotation);
-
- // Now that we have a complete element in the LSB with the LSB = 1, determine size and number of ones
- // in element.
- int elementSize = BitOperations.TrailingZeroCount(rotatedValue & (rotatedValue + 1));
- int onesInElement = BitOperations.TrailingZeroCount(~rotatedValue);
-
- // Check the value is repeating; also ensures element size is a power of two.
- if (ulong.RotateRight(value, elementSize) != value)
- {
- immN = 0;
- immS = 0;
- immR = 0;
-
- return false;
- }
-
- immN = (elementSize >> 6) & 1;
- immS = (((~elementSize + 1) << 1) | (onesInElement - 1)) & 0x3f;
- immR = (elementSize - rotation) & (elementSize - 1);
-
- return true;
- }
- }
-} \ No newline at end of file
diff --git a/ARMeilleure/CodeGen/Arm64/CodeGenContext.cs b/ARMeilleure/CodeGen/Arm64/CodeGenContext.cs
deleted file mode 100644
index 0dd5355f..00000000
--- a/ARMeilleure/CodeGen/Arm64/CodeGenContext.cs
+++ /dev/null
@@ -1,287 +0,0 @@
-using ARMeilleure.CodeGen.Linking;
-using ARMeilleure.CodeGen.RegisterAllocators;
-using ARMeilleure.IntermediateRepresentation;
-using Ryujinx.Common.Memory;
-using System;
-using System.Collections.Generic;
-using System.IO;
-
-namespace ARMeilleure.CodeGen.Arm64
-{
- class CodeGenContext
- {
- private const int BccInstLength = 4;
- private const int CbnzInstLength = 4;
- private const int LdrLitInstLength = 4;
-
- private Stream _stream;
-
- public int StreamOffset => (int)_stream.Length;
-
- public AllocationResult AllocResult { get; }
-
- public Assembler Assembler { get; }
-
- public BasicBlock CurrBlock { get; private set; }
-
- public bool HasCall { get; }
-
- public int CallArgsRegionSize { get; }
- public int FpLrSaveRegionSize { get; }
-
- private readonly Dictionary<BasicBlock, long> _visitedBlocks;
- private readonly Dictionary<BasicBlock, List<(ArmCondition Condition, long BranchPos)>> _pendingBranches;
-
- private struct ConstantPoolEntry
- {
- public readonly int Offset;
- public readonly Symbol Symbol;
- public readonly List<(Operand, int)> LdrOffsets;
-
- public ConstantPoolEntry(int offset, Symbol symbol)
- {
- Offset = offset;
- Symbol = symbol;
- LdrOffsets = new List<(Operand, int)>();
- }
- }
-
- private readonly Dictionary<ulong, ConstantPoolEntry> _constantPool;
-
- private bool _constantPoolWritten;
- private long _constantPoolOffset;
-
- private ArmCondition _jNearCondition;
- private Operand _jNearValue;
-
- private long _jNearPosition;
-
- private readonly bool _relocatable;
-
- public CodeGenContext(AllocationResult allocResult, int maxCallArgs, int blocksCount, bool relocatable)
- {
- _stream = MemoryStreamManager.Shared.GetStream();
-
- AllocResult = allocResult;
-
- Assembler = new Assembler(_stream);
-
- bool hasCall = maxCallArgs >= 0;
-
- HasCall = hasCall;
-
- if (maxCallArgs < 0)
- {
- maxCallArgs = 0;
- }
-
- CallArgsRegionSize = maxCallArgs * 16;
- FpLrSaveRegionSize = hasCall ? 16 : 0;
-
- _visitedBlocks = new Dictionary<BasicBlock, long>();
- _pendingBranches = new Dictionary<BasicBlock, List<(ArmCondition, long)>>();
- _constantPool = new Dictionary<ulong, ConstantPoolEntry>();
-
- _relocatable = relocatable;
- }
-
- public void EnterBlock(BasicBlock block)
- {
- CurrBlock = block;
-
- long target = _stream.Position;
-
- if (_pendingBranches.TryGetValue(block, out var list))
- {
- foreach (var tuple in list)
- {
- _stream.Seek(tuple.BranchPos, SeekOrigin.Begin);
- WriteBranch(tuple.Condition, target);
- }
-
- _stream.Seek(target, SeekOrigin.Begin);
- _pendingBranches.Remove(block);
- }
-
- _visitedBlocks.Add(block, target);
- }
-
- public void JumpTo(BasicBlock target)
- {
- JumpTo(ArmCondition.Al, target);
- }
-
- public void JumpTo(ArmCondition condition, BasicBlock target)
- {
- if (_visitedBlocks.TryGetValue(target, out long offset))
- {
- WriteBranch(condition, offset);
- }
- else
- {
- if (!_pendingBranches.TryGetValue(target, out var list))
- {
- list = new List<(ArmCondition, long)>();
- _pendingBranches.Add(target, list);
- }
-
- list.Add((condition, _stream.Position));
-
- _stream.Seek(BccInstLength, SeekOrigin.Current);
- }
- }
-
- private void WriteBranch(ArmCondition condition, long to)
- {
- int imm = checked((int)(to - _stream.Position));
-
- if (condition != ArmCondition.Al)
- {
- Assembler.B(condition, imm);
- }
- else
- {
- Assembler.B(imm);
- }
- }
-
- public void JumpToNear(ArmCondition condition)
- {
- _jNearCondition = condition;
- _jNearPosition = _stream.Position;
-
- _stream.Seek(BccInstLength, SeekOrigin.Current);
- }
-
- public void JumpToNearIfNotZero(Operand value)
- {
- _jNearValue = value;
- _jNearPosition = _stream.Position;
-
- _stream.Seek(CbnzInstLength, SeekOrigin.Current);
- }
-
- public void JumpHere()
- {
- long currentPosition = _stream.Position;
- long offset = currentPosition - _jNearPosition;
-
- _stream.Seek(_jNearPosition, SeekOrigin.Begin);
-
- if (_jNearValue != default)
- {
- Assembler.Cbnz(_jNearValue, checked((int)offset));
- _jNearValue = default;
- }
- else
- {
- Assembler.B(_jNearCondition, checked((int)offset));
- }
-
- _stream.Seek(currentPosition, SeekOrigin.Begin);
- }
-
- public void ReserveRelocatableConstant(Operand rt, Symbol symbol, ulong value)
- {
- if (!_constantPool.TryGetValue(value, out ConstantPoolEntry cpe))
- {
- cpe = new ConstantPoolEntry(_constantPool.Count * sizeof(ulong), symbol);
- _constantPool.Add(value, cpe);
- }
-
- cpe.LdrOffsets.Add((rt, (int)_stream.Position));
- _stream.Seek(LdrLitInstLength, SeekOrigin.Current);
- }
-
- private long WriteConstantPool()
- {
- if (_constantPoolWritten)
- {
- return _constantPoolOffset;
- }
-
- long constantPoolBaseOffset = _stream.Position;
-
- foreach (ulong value in _constantPool.Keys)
- {
- WriteUInt64(value);
- }
-
- foreach (ConstantPoolEntry cpe in _constantPool.Values)
- {
- foreach ((Operand rt, int ldrOffset) in cpe.LdrOffsets)
- {
- _stream.Seek(ldrOffset, SeekOrigin.Begin);
-
- int absoluteOffset = checked((int)(constantPoolBaseOffset + cpe.Offset));
- int pcRelativeOffset = absoluteOffset - ldrOffset;
-
- Assembler.LdrLit(rt, pcRelativeOffset);
- }
- }
-
- _stream.Seek(constantPoolBaseOffset + _constantPool.Count * sizeof(ulong), SeekOrigin.Begin);
-
- _constantPoolOffset = constantPoolBaseOffset;
- _constantPoolWritten = true;
-
- return constantPoolBaseOffset;
- }
-
- public (byte[], RelocInfo) GetCode()
- {
- long constantPoolBaseOffset = WriteConstantPool();
-
- byte[] code = new byte[_stream.Length];
-
- long originalPosition = _stream.Position;
-
- _stream.Seek(0, SeekOrigin.Begin);
- _stream.Read(code, 0, code.Length);
- _stream.Seek(originalPosition, SeekOrigin.Begin);
-
- RelocInfo relocInfo;
-
- if (_relocatable)
- {
- RelocEntry[] relocs = new RelocEntry[_constantPool.Count];
-
- int index = 0;
-
- foreach (ConstantPoolEntry cpe in _constantPool.Values)
- {
- if (cpe.Symbol.Type != SymbolType.None)
- {
- int absoluteOffset = checked((int)(constantPoolBaseOffset + cpe.Offset));
- relocs[index++] = new RelocEntry(absoluteOffset, cpe.Symbol);
- }
- }
-
- if (index != relocs.Length)
- {
- Array.Resize(ref relocs, index);
- }
-
- relocInfo = new RelocInfo(relocs);
- }
- else
- {
- relocInfo = new RelocInfo(Array.Empty<RelocEntry>());
- }
-
- return (code, relocInfo);
- }
-
- private void WriteUInt64(ulong value)
- {
- _stream.WriteByte((byte)(value >> 0));
- _stream.WriteByte((byte)(value >> 8));
- _stream.WriteByte((byte)(value >> 16));
- _stream.WriteByte((byte)(value >> 24));
- _stream.WriteByte((byte)(value >> 32));
- _stream.WriteByte((byte)(value >> 40));
- _stream.WriteByte((byte)(value >> 48));
- _stream.WriteByte((byte)(value >> 56));
- }
- }
-} \ No newline at end of file
diff --git a/ARMeilleure/CodeGen/Arm64/CodeGenerator.cs b/ARMeilleure/CodeGen/Arm64/CodeGenerator.cs
deleted file mode 100644
index fc4fa976..00000000
--- a/ARMeilleure/CodeGen/Arm64/CodeGenerator.cs
+++ /dev/null
@@ -1,1580 +0,0 @@
-using ARMeilleure.CodeGen.Linking;
-using ARMeilleure.CodeGen.Optimizations;
-using ARMeilleure.CodeGen.RegisterAllocators;
-using ARMeilleure.CodeGen.Unwinding;
-using ARMeilleure.Common;
-using ARMeilleure.Diagnostics;
-using ARMeilleure.IntermediateRepresentation;
-using ARMeilleure.Translation;
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.Numerics;
-
-using static ARMeilleure.IntermediateRepresentation.Operand;
-using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
-
-namespace ARMeilleure.CodeGen.Arm64
-{
- static class CodeGenerator
- {
- private const int DWordScale = 3;
-
- private const int RegistersCount = 32;
-
- private const int FpRegister = 29;
- private const int LrRegister = 30;
- private const int SpRegister = 31;
- private const int ZrRegister = 31;
-
- private enum AccessSize
- {
- Byte,
- Hword,
- Auto
- }
-
- private static Action<CodeGenContext, Operation>[] _instTable;
-
- static CodeGenerator()
- {
- _instTable = new Action<CodeGenContext, Operation>[EnumUtils.GetCount(typeof(Instruction))];
-
- Add(Instruction.Add, GenerateAdd);
- Add(Instruction.BitwiseAnd, GenerateBitwiseAnd);
- Add(Instruction.BitwiseExclusiveOr, GenerateBitwiseExclusiveOr);
- Add(Instruction.BitwiseNot, GenerateBitwiseNot);
- Add(Instruction.BitwiseOr, GenerateBitwiseOr);
- Add(Instruction.BranchIf, GenerateBranchIf);
- Add(Instruction.ByteSwap, GenerateByteSwap);
- Add(Instruction.Call, GenerateCall);
- //Add(Instruction.Clobber, GenerateClobber);
- Add(Instruction.Compare, GenerateCompare);
- Add(Instruction.CompareAndSwap, GenerateCompareAndSwap);
- Add(Instruction.CompareAndSwap16, GenerateCompareAndSwap16);
- Add(Instruction.CompareAndSwap8, GenerateCompareAndSwap8);
- Add(Instruction.ConditionalSelect, GenerateConditionalSelect);
- Add(Instruction.ConvertI64ToI32, GenerateConvertI64ToI32);
- Add(Instruction.ConvertToFP, GenerateConvertToFP);
- Add(Instruction.ConvertToFPUI, GenerateConvertToFPUI);
- Add(Instruction.Copy, GenerateCopy);
- Add(Instruction.CountLeadingZeros, GenerateCountLeadingZeros);
- Add(Instruction.Divide, GenerateDivide);
- Add(Instruction.DivideUI, GenerateDivideUI);
- Add(Instruction.Fill, GenerateFill);
- Add(Instruction.Load, GenerateLoad);
- Add(Instruction.Load16, GenerateLoad16);
- Add(Instruction.Load8, GenerateLoad8);
- Add(Instruction.MemoryBarrier, GenerateMemoryBarrier);
- Add(Instruction.Multiply, GenerateMultiply);
- Add(Instruction.Multiply64HighSI, GenerateMultiply64HighSI);
- Add(Instruction.Multiply64HighUI, GenerateMultiply64HighUI);
- Add(Instruction.Negate, GenerateNegate);
- Add(Instruction.Return, GenerateReturn);
- Add(Instruction.RotateRight, GenerateRotateRight);
- Add(Instruction.ShiftLeft, GenerateShiftLeft);
- Add(Instruction.ShiftRightSI, GenerateShiftRightSI);
- Add(Instruction.ShiftRightUI, GenerateShiftRightUI);
- Add(Instruction.SignExtend16, GenerateSignExtend16);
- Add(Instruction.SignExtend32, GenerateSignExtend32);
- Add(Instruction.SignExtend8, GenerateSignExtend8);
- Add(Instruction.Spill, GenerateSpill);
- Add(Instruction.SpillArg, GenerateSpillArg);
- Add(Instruction.StackAlloc, GenerateStackAlloc);
- Add(Instruction.Store, GenerateStore);
- Add(Instruction.Store16, GenerateStore16);
- Add(Instruction.Store8, GenerateStore8);
- Add(Instruction.Subtract, GenerateSubtract);
- Add(Instruction.Tailcall, GenerateTailcall);
- Add(Instruction.VectorCreateScalar, GenerateVectorCreateScalar);
- Add(Instruction.VectorExtract, GenerateVectorExtract);
- Add(Instruction.VectorExtract16, GenerateVectorExtract16);
- Add(Instruction.VectorExtract8, GenerateVectorExtract8);
- Add(Instruction.VectorInsert, GenerateVectorInsert);
- Add(Instruction.VectorInsert16, GenerateVectorInsert16);
- Add(Instruction.VectorInsert8, GenerateVectorInsert8);
- Add(Instruction.VectorOne, GenerateVectorOne);
- Add(Instruction.VectorZero, GenerateVectorZero);
- Add(Instruction.VectorZeroUpper64, GenerateVectorZeroUpper64);
- Add(Instruction.VectorZeroUpper96, GenerateVectorZeroUpper96);
- Add(Instruction.ZeroExtend16, GenerateZeroExtend16);
- Add(Instruction.ZeroExtend32, GenerateZeroExtend32);
- Add(Instruction.ZeroExtend8, GenerateZeroExtend8);
-
- static void Add(Instruction inst, Action<CodeGenContext, Operation> func)
- {
- _instTable[(int)inst] = func;
- }
- }
-
- public static CompiledFunction Generate(CompilerContext cctx)
- {
- ControlFlowGraph cfg = cctx.Cfg;
-
- Logger.StartPass(PassName.Optimization);
-
- if (cctx.Options.HasFlag(CompilerOptions.Optimize))
- {
- if (cctx.Options.HasFlag(CompilerOptions.SsaForm))
- {
- Optimizer.RunPass(cfg);
- }
-
- BlockPlacement.RunPass(cfg);
- }
-
- Arm64Optimizer.RunPass(cfg);
-
- Logger.EndPass(PassName.Optimization, cfg);
-
- Logger.StartPass(PassName.PreAllocation);
-
- StackAllocator stackAlloc = new();
-
- PreAllocator.RunPass(cctx, stackAlloc, out int maxCallArgs);
-
- Logger.EndPass(PassName.PreAllocation, cfg);
-
- Logger.StartPass(PassName.RegisterAllocation);
-
- if (cctx.Options.HasFlag(CompilerOptions.SsaForm))
- {
- Ssa.Deconstruct(cfg);
- }
-
- IRegisterAllocator regAlloc;
-
- if (cctx.Options.HasFlag(CompilerOptions.Lsra))
- {
- regAlloc = new LinearScanAllocator();
- }
- else
- {
- regAlloc = new HybridAllocator();
- }
-
- RegisterMasks regMasks = new(
- CallingConvention.GetIntAvailableRegisters(),
- CallingConvention.GetVecAvailableRegisters(),
- CallingConvention.GetIntCallerSavedRegisters(),
- CallingConvention.GetVecCallerSavedRegisters(),
- CallingConvention.GetIntCalleeSavedRegisters(),
- CallingConvention.GetVecCalleeSavedRegisters(),
- RegistersCount);
-
- AllocationResult allocResult = regAlloc.RunPass(cfg, stackAlloc, regMasks);
-
- Logger.EndPass(PassName.RegisterAllocation, cfg);
-
- Logger.StartPass(PassName.CodeGeneration);
-
- //Console.Error.WriteLine(IRDumper.GetDump(cfg));
-
- bool relocatable = (cctx.Options & CompilerOptions.Relocatable) != 0;
-
- CodeGenContext context = new(allocResult, maxCallArgs, cfg.Blocks.Count, relocatable);
-
- UnwindInfo unwindInfo = WritePrologue(context);
-
- for (BasicBlock block = cfg.Blocks.First; block != null; block = block.ListNext)
- {
- context.EnterBlock(block);
-
- for (Operation node = block.Operations.First; node != default;)
- {
- node = GenerateOperation(context, node);
- }
-
- if (block.SuccessorsCount == 0)
- {
- // The only blocks which can have 0 successors are exit blocks.
- Operation last = block.Operations.Last;
-
- Debug.Assert(last.Instruction == Instruction.Tailcall ||
- last.Instruction == Instruction.Return);
- }
- else
- {
- BasicBlock succ = block.GetSuccessor(0);
-
- if (succ != block.ListNext)
- {
- context.JumpTo(succ);
- }
- }
- }
-
- (byte[] code, RelocInfo relocInfo) = context.GetCode();
-
- Logger.EndPass(PassName.CodeGeneration);
-
- return new CompiledFunction(code, unwindInfo, relocInfo);
- }
-
- private static Operation GenerateOperation(CodeGenContext context, Operation operation)
- {
- if (operation.Instruction == Instruction.Extended)
- {
- CodeGeneratorIntrinsic.GenerateOperation(context, operation);
- }
- else
- {
- if (IsLoadOrStore(operation) &&
- operation.ListNext != default &&
- operation.ListNext.Instruction == operation.Instruction &&
- TryPairMemoryOp(context, operation, operation.ListNext))
- {
- // Skip next operation if we managed to pair them.
- return operation.ListNext.ListNext;
- }
-
- Action<CodeGenContext, Operation> func = _instTable[(int)operation.Instruction];
-
- if (func != null)
- {
- func(context, operation);
- }
- else
- {
- throw new ArgumentException($"Invalid instruction \"{operation.Instruction}\".");
- }
- }
-
- return operation.ListNext;
- }
-
- private static void GenerateAdd(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination;
- Operand src1 = operation.GetSource(0);
- Operand src2 = operation.GetSource(1);
-
- // ValidateBinOp(dest, src1, src2);
-
- if (dest.Type.IsInteger())
- {
- context.Assembler.Add(dest, src1, src2);
- }
- else
- {
- context.Assembler.FaddScalar(dest, src1, src2);
- }
- }
-
- private static void GenerateBitwiseAnd(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination;
- Operand src1 = operation.GetSource(0);
- Operand src2 = operation.GetSource(1);
-
- ValidateBinOp(dest, src1, src2);
-
- Debug.Assert(dest.Type.IsInteger());
-
- context.Assembler.And(dest, src1, src2);
- }
-
- private static void GenerateBitwiseExclusiveOr(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination;
- Operand src1 = operation.GetSource(0);
- Operand src2 = operation.GetSource(1);
-
- ValidateBinOp(dest, src1, src2);
-
- if (dest.Type.IsInteger())
- {
- context.Assembler.Eor(dest, src1, src2);
- }
- else
- {
- context.Assembler.EorVector(dest, src1, src2);
- }
- }
-
- private static void GenerateBitwiseNot(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination;
- Operand source = operation.GetSource(0);
-
- ValidateUnOp(dest, source);
-
- Debug.Assert(dest.Type.IsInteger());
-
- context.Assembler.Mvn(dest, source);
- }
-
- private static void GenerateBitwiseOr(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination;
- Operand src1 = operation.GetSource(0);
- Operand src2 = operation.GetSource(1);
-
- ValidateBinOp(dest, src1, src2);
-
- Debug.Assert(dest.Type.IsInteger());
-
- context.Assembler.Orr(dest, src1, src2);
- }
-
- private static void GenerateBranchIf(CodeGenContext context, Operation operation)
- {
- Operand comp = operation.GetSource(2);
-
- Debug.Assert(comp.Kind == OperandKind.Constant);
-
- var cond = ((Comparison)comp.AsInt32()).ToArmCondition();
-
- GenerateCompareCommon(context, operation);
-
- context.JumpTo(cond, context.CurrBlock.GetSuccessor(1));
- }
-
- private static void GenerateByteSwap(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination;
- Operand source = operation.GetSource(0);
-
- ValidateUnOp(dest, source);
-
- Debug.Assert(dest.Type.IsInteger());
-
- context.Assembler.Rev(dest, source);
- }
-
- private static void GenerateCall(CodeGenContext context, Operation operation)
- {
- context.Assembler.Blr(operation.GetSource(0));
- }
-
- private static void GenerateCompare(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination;
- Operand comp = operation.GetSource(2);
-
- Debug.Assert(dest.Type == OperandType.I32);
- Debug.Assert(comp.Kind == OperandKind.Constant);
-
- var cond = ((Comparison)comp.AsInt32()).ToArmCondition();
-
- GenerateCompareCommon(context, operation);
-
- context.Assembler.Cset(dest, cond);
- }
-
- private static void GenerateCompareAndSwap(CodeGenContext context, Operation operation)
- {
- if (operation.SourcesCount == 5) // CompareAndSwap128 has 5 sources, compared to CompareAndSwap64/32's 3.
- {
- Operand actualLow = operation.GetDestination(0);
- Operand actualHigh = operation.GetDestination(1);
- Operand temp0 = operation.GetDestination(2);
- Operand temp1 = operation.GetDestination(3);
- Operand address = operation.GetSource(0);
- Operand expectedLow = operation.GetSource(1);
- Operand expectedHigh = operation.GetSource(2);
- Operand desiredLow = operation.GetSource(3);
- Operand desiredHigh = operation.GetSource(4);
-
- GenerateAtomicDcas(
- context,
- address,
- expectedLow,
- expectedHigh,
- desiredLow,
- desiredHigh,
- actualLow,
- actualHigh,
- temp0,
- temp1);
- }
- else
- {
- Operand actual = operation.GetDestination(0);
- Operand result = operation.GetDestination(1);
- Operand address = operation.GetSource(0);
- Operand expected = operation.GetSource(1);
- Operand desired = operation.GetSource(2);
-
- GenerateAtomicCas(context, address, expected, desired, actual, result, AccessSize.Auto);
- }
- }
-
- private static void GenerateCompareAndSwap16(CodeGenContext context, Operation operation)
- {
- Operand actual = operation.GetDestination(0);
- Operand result = operation.GetDestination(1);
- Operand address = operation.GetSource(0);
- Operand expected = operation.GetSource(1);
- Operand desired = operation.GetSource(2);
-
- GenerateAtomicCas(context, address, expected, desired, actual, result, AccessSize.Hword);
- }
-
- private static void GenerateCompareAndSwap8(CodeGenContext context, Operation operation)
- {
- Operand actual = operation.GetDestination(0);
- Operand result = operation.GetDestination(1);
- Operand address = operation.GetSource(0);
- Operand expected = operation.GetSource(1);
- Operand desired = operation.GetSource(2);
-
- GenerateAtomicCas(context, address, expected, desired, actual, result, AccessSize.Byte);
- }
-
- private static void GenerateCompareCommon(CodeGenContext context, Operation operation)
- {
- Operand src1 = operation.GetSource(0);
- Operand src2 = operation.GetSource(1);
-
- EnsureSameType(src1, src2);
-
- Debug.Assert(src1.Type.IsInteger());
-
- context.Assembler.Cmp(src1, src2);
- }
-
- private static void GenerateConditionalSelect(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination;
- Operand src1 = operation.GetSource(0);
- Operand src2 = operation.GetSource(1);
- Operand src3 = operation.GetSource(2);
-
- EnsureSameType(dest, src2, src3);
-
- Debug.Assert(dest.Type.IsInteger());
- Debug.Assert(src1.Type == OperandType.I32);
-
- context.Assembler.Cmp (src1, Const(src1.Type, 0));
- context.Assembler.Csel(dest, src2, src3, ArmCondition.Ne);
- }
-
- private static void GenerateConvertI64ToI32(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination;
- Operand source = operation.GetSource(0);
-
- Debug.Assert(dest.Type == OperandType.I32 && source.Type == OperandType.I64);
-
- context.Assembler.Mov(dest, Register(source, OperandType.I32));
- }
-
- private static void GenerateConvertToFP(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination;
- Operand source = operation.GetSource(0);
-
- Debug.Assert(dest.Type == OperandType.FP32 || dest.Type == OperandType.FP64);
- Debug.Assert(dest.Type != source.Type);
- Debug.Assert(source.Type != OperandType.V128);
-
- if (source.Type.IsInteger())
- {
- context.Assembler.ScvtfScalar(dest, source);
- }
- else
- {
- context.Assembler.FcvtScalar(dest, source);
- }
- }
-
- private static void GenerateConvertToFPUI(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination;
- Operand source = operation.GetSource(0);
-
- Debug.Assert(dest.Type == OperandType.FP32 || dest.Type == OperandType.FP64);
- Debug.Assert(dest.Type != source.Type);
- Debug.Assert(source.Type.IsInteger());
-
- context.Assembler.UcvtfScalar(dest, source);
- }
-
- private static void GenerateCopy(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination;
- Operand source = operation.GetSource(0);
-
- EnsureSameType(dest, source);
-
- Debug.Assert(dest.Type.IsInteger() || source.Kind != OperandKind.Constant);
-
- // Moves to the same register are useless.
- if (dest.Kind == source.Kind && dest.Value == source.Value)
- {
- return;
- }
-
- if (dest.Kind == OperandKind.Register && source.Kind == OperandKind.Constant)
- {
- if (source.Relocatable)
- {
- context.ReserveRelocatableConstant(dest, source.Symbol, source.Value);
- }
- else
- {
- GenerateConstantCopy(context, dest, source.Value);
- }
- }
- else
- {
- context.Assembler.Mov(dest, source);
- }
- }
-
- private static void GenerateCountLeadingZeros(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination;
- Operand source = operation.GetSource(0);
-
- EnsureSameType(dest, source);
-
- Debug.Assert(dest.Type.IsInteger());
-
- context.Assembler.Clz(dest, source);
- }
-
- private static void GenerateDivide(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination;
- Operand dividend = operation.GetSource(0);
- Operand divisor = operation.GetSource(1);
-
- ValidateBinOp(dest, dividend, divisor);
-
- if (dest.Type.IsInteger())
- {
- context.Assembler.Sdiv(dest, dividend, divisor);
- }
- else
- {
- context.Assembler.FdivScalar(dest, dividend, divisor);
- }
- }
-
- private static void GenerateDivideUI(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination;
- Operand dividend = operation.GetSource(0);
- Operand divisor = operation.GetSource(1);
-
- ValidateBinOp(dest, dividend, divisor);
-
- context.Assembler.Udiv(dest, dividend, divisor);
- }
-
- private static void GenerateLoad(CodeGenContext context, Operation operation)
- {
- Operand value = operation.Destination;
- Operand address = operation.GetSource(0);
-
- context.Assembler.Ldr(value, address);
- }
-
- private static void GenerateLoad16(CodeGenContext context, Operation operation)
- {
- Operand value = operation.Destination;
- Operand address = operation.GetSource(0);
-
- Debug.Assert(value.Type.IsInteger());
-
- context.Assembler.LdrhRiUn(value, address, 0);
- }
-
- private static void GenerateLoad8(CodeGenContext context, Operation operation)
- {
- Operand value = operation.Destination;
- Operand address = operation.GetSource(0);
-
- Debug.Assert(value.Type.IsInteger());
-
- context.Assembler.LdrbRiUn(value, address, 0);
- }
-
- private static void GenerateMemoryBarrier(CodeGenContext context, Operation operation)
- {
- context.Assembler.Dmb(0xf);
- }
-
- private static void GenerateMultiply(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination;
- Operand src1 = operation.GetSource(0);
- Operand src2 = operation.GetSource(1);
-
- EnsureSameType(dest, src1, src2);
-
- if (dest.Type.IsInteger())
- {
- context.Assembler.Mul(dest, src1, src2);
- }
- else
- {
- context.Assembler.FmulScalar(dest, src1, src2);
- }
- }
-
- private static void GenerateMultiply64HighSI(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination;
- Operand src1 = operation.GetSource(0);
- Operand src2 = operation.GetSource(1);
-
- EnsureSameType(dest, src1, src2);
-
- Debug.Assert(dest.Type == OperandType.I64);
-
- context.Assembler.Smulh(dest, src1, src2);
- }
-
- private static void GenerateMultiply64HighUI(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination;
- Operand src1 = operation.GetSource(0);
- Operand src2 = operation.GetSource(1);
-
- EnsureSameType(dest, src1, src2);
-
- Debug.Assert(dest.Type == OperandType.I64);
-
- context.Assembler.Umulh(dest, src1, src2);
- }
-
- private static void GenerateNegate(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination;
- Operand source = operation.GetSource(0);
-
- ValidateUnOp(dest, source);
-
- if (dest.Type.IsInteger())
- {
- context.Assembler.Neg(dest, source);
- }
- else
- {
- context.Assembler.FnegScalar(dest, source);
- }
- }
-
- private static void GenerateLoad(CodeGenContext context, Operand value, Operand address, int offset)
- {
- if (CodeGenCommon.ConstFitsOnUImm12(offset, value.Type))
- {
- context.Assembler.LdrRiUn(value, address, offset);
- }
- else if (CodeGenCommon.ConstFitsOnSImm9(offset))
- {
- context.Assembler.Ldur(value, address, offset);
- }
- else
- {
- Operand tempAddress = Register(CodeGenCommon.ReservedRegister);
- GenerateConstantCopy(context, tempAddress, (ulong)offset);
- context.Assembler.Add(tempAddress, address, tempAddress, ArmExtensionType.Uxtx); // Address might be SP and must be the first input.
- context.Assembler.LdrRiUn(value, tempAddress, 0);
- }
- }
-
- private static void GenerateReturn(CodeGenContext context, Operation operation)
- {
- WriteEpilogue(context);
-
- context.Assembler.Ret(Register(LrRegister));
- }
-
- private static void GenerateRotateRight(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination;
- Operand src1 = operation.GetSource(0);
- Operand src2 = operation.GetSource(1);
-
- ValidateShift(dest, src1, src2);
-
- context.Assembler.Ror(dest, src1, src2);
- }
-
- private static void GenerateShiftLeft(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination;
- Operand src1 = operation.GetSource(0);
- Operand src2 = operation.GetSource(1);
-
- ValidateShift(dest, src1, src2);
-
- context.Assembler.Lsl(dest, src1, src2);
- }
-
- private static void GenerateShiftRightSI(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination;
- Operand src1 = operation.GetSource(0);
- Operand src2 = operation.GetSource(1);
-
- ValidateShift(dest, src1, src2);
-
- context.Assembler.Asr(dest, src1, src2);
- }
-
- private static void GenerateShiftRightUI(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination;
- Operand src1 = operation.GetSource(0);
- Operand src2 = operation.GetSource(1);
-
- ValidateShift(dest, src1, src2);
-
- context.Assembler.Lsr(dest, src1, src2);
- }
-
- private static void GenerateSignExtend16(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination;
- Operand source = operation.GetSource(0);
-
- Debug.Assert(dest.Type.IsInteger() && source.Type.IsInteger());
-
- context.Assembler.Sxth(dest, source);
- }
-
- private static void GenerateSignExtend32(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination;
- Operand source = operation.GetSource(0);
-
- Debug.Assert(dest.Type.IsInteger() && source.Type.IsInteger());
-
- context.Assembler.Sxtw(dest, source);
- }
-
- private static void GenerateSignExtend8(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination;
- Operand source = operation.GetSource(0);
-
- Debug.Assert(dest.Type.IsInteger() && source.Type.IsInteger());
-
- context.Assembler.Sxtb(dest, source);
- }
-
- private static void GenerateFill(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination;
- Operand offset = operation.GetSource(0);
-
- Debug.Assert(offset.Kind == OperandKind.Constant);
-
- int offs = offset.AsInt32() + context.CallArgsRegionSize + context.FpLrSaveRegionSize;
-
- GenerateLoad(context, dest, Register(SpRegister), offs);
- }
-
- private static void GenerateStore(CodeGenContext context, Operand value, Operand address, int offset)
- {
- if (CodeGenCommon.ConstFitsOnUImm12(offset, value.Type))
- {
- context.Assembler.StrRiUn(value, address, offset);
- }
- else if (CodeGenCommon.ConstFitsOnSImm9(offset))
- {
- context.Assembler.Stur(value, address, offset);
- }
- else
- {
- Operand tempAddress = Register(CodeGenCommon.ReservedRegister);
- GenerateConstantCopy(context, tempAddress, (ulong)offset);
- context.Assembler.Add(tempAddress, address, tempAddress, ArmExtensionType.Uxtx); // Address might be SP and must be the first input.
- context.Assembler.StrRiUn(value, tempAddress, 0);
- }
- }
-
- private static void GenerateSpill(CodeGenContext context, Operation operation)
- {
- GenerateSpill(context, operation, context.CallArgsRegionSize + context.FpLrSaveRegionSize);
- }
-
- private static void GenerateSpillArg(CodeGenContext context, Operation operation)
- {
- GenerateSpill(context, operation, 0);
- }
-
- private static void GenerateStackAlloc(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination;
- Operand offset = operation.GetSource(0);
-
- Debug.Assert(offset.Kind == OperandKind.Constant);
-
- int offs = offset.AsInt32() + context.CallArgsRegionSize + context.FpLrSaveRegionSize;
-
- context.Assembler.Add(dest, Register(SpRegister), Const(dest.Type, offs));
- }
-
- private static void GenerateStore(CodeGenContext context, Operation operation)
- {
- Operand value = operation.GetSource(1);
- Operand address = operation.GetSource(0);
-
- context.Assembler.Str(value, address);
- }
-
- private static void GenerateStore16(CodeGenContext context, Operation operation)
- {
- Operand value = operation.GetSource(1);
- Operand address = operation.GetSource(0);
-
- Debug.Assert(value.Type.IsInteger());
-
- context.Assembler.StrhRiUn(value, address, 0);
- }
-
- private static void GenerateStore8(CodeGenContext context, Operation operation)
- {
- Operand value = operation.GetSource(1);
- Operand address = operation.GetSource(0);
-
- Debug.Assert(value.Type.IsInteger());
-
- context.Assembler.StrbRiUn(value, address, 0);
- }
-
- private static void GenerateSpill(CodeGenContext context, Operation operation, int baseOffset)
- {
- Operand offset = operation.GetSource(0);
- Operand source = operation.GetSource(1);
-
- Debug.Assert(offset.Kind == OperandKind.Constant);
-
- int offs = offset.AsInt32() + baseOffset;
-
- GenerateStore(context, source, Register(SpRegister), offs);
- }
-
- private static void GenerateSubtract(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination;
- Operand src1 = operation.GetSource(0);
- Operand src2 = operation.GetSource(1);
-
- // ValidateBinOp(dest, src1, src2);
-
- if (dest.Type.IsInteger())
- {
- context.Assembler.Sub(dest, src1, src2);
- }
- else
- {
- context.Assembler.FsubScalar(dest, src1, src2);
- }
- }
-
- private static void GenerateTailcall(CodeGenContext context, Operation operation)
- {
- WriteEpilogue(context);
-
- context.Assembler.Br(operation.GetSource(0));
- }
-
- private static void GenerateVectorCreateScalar(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination;
- Operand source = operation.GetSource(0);
-
- if (dest != default)
- {
- Debug.Assert(!dest.Type.IsInteger() && source.Type.IsInteger());
-
- OperandType destType = source.Type == OperandType.I64 ? OperandType.FP64 : OperandType.FP32;
-
- context.Assembler.Fmov(Register(dest, destType), source, topHalf: false);
- }
- }
-
- private static void GenerateVectorExtract(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination; // Value
- Operand src1 = operation.GetSource(0); // Vector
- Operand src2 = operation.GetSource(1); // Index
-
- Debug.Assert(src1.Type == OperandType.V128);
- Debug.Assert(src2.Kind == OperandKind.Constant);
-
- byte index = src2.AsByte();
-
- Debug.Assert(index < OperandType.V128.GetSizeInBytes() / dest.Type.GetSizeInBytes());
-
- if (dest.Type.IsInteger())
- {
- context.Assembler.Umov(dest, src1, index, dest.Type == OperandType.I64 ? 3 : 2);
- }
- else
- {
- context.Assembler.DupScalar(dest, src1, index, dest.Type == OperandType.FP64 ? 3 : 2);
- }
- }
-
- private static void GenerateVectorExtract16(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination; // Value
- Operand src1 = operation.GetSource(0); // Vector
- Operand src2 = operation.GetSource(1); // Index
-
- Debug.Assert(src1.Type == OperandType.V128);
- Debug.Assert(src2.Kind == OperandKind.Constant);
-
- byte index = src2.AsByte();
-
- Debug.Assert(index < 8);
-
- context.Assembler.Umov(dest, src1, index, 1);
- }
-
- private static void GenerateVectorExtract8(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination; // Value
- Operand src1 = operation.GetSource(0); // Vector
- Operand src2 = operation.GetSource(1); // Index
-
- Debug.Assert(src1.Type == OperandType.V128);
- Debug.Assert(src2.Kind == OperandKind.Constant);
-
- byte index = src2.AsByte();
-
- Debug.Assert(index < 16);
-
- context.Assembler.Umov(dest, src1, index, 0);
- }
-
- private static void GenerateVectorInsert(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination;
- Operand src1 = operation.GetSource(0); // Vector
- Operand src2 = operation.GetSource(1); // Value
- Operand src3 = operation.GetSource(2); // Index
-
- EnsureSameReg(dest, src1);
-
- Debug.Assert(src1.Type == OperandType.V128);
- Debug.Assert(src3.Kind == OperandKind.Constant);
-
- byte index = src3.AsByte();
-
- if (src2.Type.IsInteger())
- {
- context.Assembler.Ins(dest, src2, index, src2.Type == OperandType.I64 ? 3 : 2);
- }
- else
- {
- context.Assembler.Ins(dest, src2, 0, index, src2.Type == OperandType.FP64 ? 3 : 2);
- }
- }
-
- private static void GenerateVectorInsert16(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination;
- Operand src1 = operation.GetSource(0); // Vector
- Operand src2 = operation.GetSource(1); // Value
- Operand src3 = operation.GetSource(2); // Index
-
- EnsureSameReg(dest, src1);
-
- Debug.Assert(src1.Type == OperandType.V128);
- Debug.Assert(src3.Kind == OperandKind.Constant);
-
- byte index = src3.AsByte();
-
- context.Assembler.Ins(dest, src2, index, 1);
- }
-
- private static void GenerateVectorInsert8(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination;
- Operand src1 = operation.GetSource(0); // Vector
- Operand src2 = operation.GetSource(1); // Value
- Operand src3 = operation.GetSource(2); // Index
-
- EnsureSameReg(dest, src1);
-
- Debug.Assert(src1.Type == OperandType.V128);
- Debug.Assert(src3.Kind == OperandKind.Constant);
-
- byte index = src3.AsByte();
-
- context.Assembler.Ins(dest, src2, index, 0);
- }
-
- private static void GenerateVectorOne(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination;
-
- Debug.Assert(!dest.Type.IsInteger());
-
- context.Assembler.CmeqVector(dest, dest, dest, 2);
- }
-
- private static void GenerateVectorZero(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination;
-
- Debug.Assert(!dest.Type.IsInteger());
-
- context.Assembler.EorVector(dest, dest, dest);
- }
-
- private static void GenerateVectorZeroUpper64(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination;
- Operand source = operation.GetSource(0);
-
- Debug.Assert(dest.Type == OperandType.V128 && source.Type == OperandType.V128);
-
- context.Assembler.Fmov(Register(dest, OperandType.FP64), Register(source, OperandType.FP64));
- }
-
- private static void GenerateVectorZeroUpper96(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination;
- Operand source = operation.GetSource(0);
-
- Debug.Assert(dest.Type == OperandType.V128 && source.Type == OperandType.V128);
-
- context.Assembler.Fmov(Register(dest, OperandType.FP32), Register(source, OperandType.FP32));
- }
-
- private static void GenerateZeroExtend16(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination;
- Operand source = operation.GetSource(0);
-
- Debug.Assert(dest.Type.IsInteger() && source.Type.IsInteger());
-
- context.Assembler.Uxth(dest, source);
- }
-
- private static void GenerateZeroExtend32(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination;
- Operand source = operation.GetSource(0);
-
- Debug.Assert(dest.Type.IsInteger() && source.Type.IsInteger());
-
- // We can eliminate the move if source is already 32-bit and the registers are the same.
- if (dest.Value == source.Value && source.Type == OperandType.I32)
- {
- return;
- }
-
- context.Assembler.Mov(Register(dest.GetRegister().Index, OperandType.I32), source);
- }
-
- private static void GenerateZeroExtend8(CodeGenContext context, Operation operation)
- {
- Operand dest = operation.Destination;
- Operand source = operation.GetSource(0);
-
- Debug.Assert(dest.Type.IsInteger() && source.Type.IsInteger());
-
- context.Assembler.Uxtb(dest, source);
- }
-
- private static UnwindInfo WritePrologue(CodeGenContext context)
- {
- List<UnwindPushEntry> pushEntries = new List<UnwindPushEntry>();
-
- Operand rsp = Register(SpRegister);
-
- int intMask = CallingConvention.GetIntCalleeSavedRegisters() & context.AllocResult.IntUsedRegisters;
- int vecMask = CallingConvention.GetFpCalleeSavedRegisters() & context.AllocResult.VecUsedRegisters;
-
- int intCalleeSavedRegsCount = BitOperations.PopCount((uint)intMask);
- int vecCalleeSavedRegsCount = BitOperations.PopCount((uint)vecMask);
-
- int calleeSaveRegionSize = Align16(intCalleeSavedRegsCount * 8 + vecCalleeSavedRegsCount * 8);
-
- int offset = 0;
-
- WritePrologueCalleeSavesPreIndexed(context, pushEntries, ref intMask, ref offset, calleeSaveRegionSize, OperandType.I64);
- WritePrologueCalleeSavesPreIndexed(context, pushEntries, ref vecMask, ref offset, calleeSaveRegionSize, OperandType.FP64);
-
- int localSize = Align16(context.AllocResult.SpillRegionSize + context.FpLrSaveRegionSize);
- int outArgsSize = context.CallArgsRegionSize;
-
- if (CodeGenCommon.ConstFitsOnSImm7(localSize, DWordScale))
- {
- if (context.HasCall)
- {
- context.Assembler.StpRiPre(Register(FpRegister), Register(LrRegister), rsp, -localSize);
- context.Assembler.MovSp(Register(FpRegister), rsp);
- }
-
- if (outArgsSize != 0)
- {
- context.Assembler.Sub(rsp, rsp, Const(OperandType.I64, outArgsSize));
- }
- }
- else
- {
- int frameSize = localSize + outArgsSize;
- if (frameSize != 0)
- {
- if (CodeGenCommon.ConstFitsOnUImm12(frameSize))
- {
- context.Assembler.Sub(rsp, rsp, Const(OperandType.I64, frameSize));
- }
- else
- {
- Operand tempSize = Register(CodeGenCommon.ReservedRegister);
- GenerateConstantCopy(context, tempSize, (ulong)frameSize);
- context.Assembler.Sub(rsp, rsp, tempSize, ArmExtensionType.Uxtx);
- }
- }
-
- context.Assembler.StpRiUn(Register(FpRegister), Register(LrRegister), rsp, outArgsSize);
-
- if (outArgsSize != 0)
- {
- context.Assembler.Add(Register(FpRegister), Register(SpRegister), Const(OperandType.I64, outArgsSize));
- }
- else
- {
- context.Assembler.MovSp(Register(FpRegister), Register(SpRegister));
- }
- }
-
- return new UnwindInfo(pushEntries.ToArray(), context.StreamOffset);
- }
-
- private static void WritePrologueCalleeSavesPreIndexed(
- CodeGenContext context,
- List<UnwindPushEntry> pushEntries,
- ref int mask,
- ref int offset,
- int calleeSaveRegionSize,
- OperandType type)
- {
- if ((BitOperations.PopCount((uint)mask) & 1) != 0)
- {
- int reg = BitOperations.TrailingZeroCount(mask);
-
- pushEntries.Add(new UnwindPushEntry(UnwindPseudoOp.PushReg, context.StreamOffset, regIndex: reg));
-
- mask &= ~(1 << reg);
-
- if (offset != 0)
- {
- context.Assembler.StrRiUn(Register(reg, type), Register(SpRegister), offset);
- }
- else
- {
- context.Assembler.StrRiPre(Register(reg, type), Register(SpRegister), -calleeSaveRegionSize);
- }
-
- offset += type.GetSizeInBytes();
- }
-
- while (mask != 0)
- {
- int reg = BitOperations.TrailingZeroCount(mask);
-
- pushEntries.Add(new UnwindPushEntry(UnwindPseudoOp.PushReg, context.StreamOffset, regIndex: reg));
-
- mask &= ~(1 << reg);
-
- int reg2 = BitOperations.TrailingZeroCount(mask);
-
- pushEntries.Add(new UnwindPushEntry(UnwindPseudoOp.PushReg, context.StreamOffset, regIndex: reg2));
-
- mask &= ~(1 << reg2);
-
- if (offset != 0)
- {
- context.Assembler.StpRiUn(Register(reg, type), Register(reg2, type), Register(SpRegister), offset);
- }
- else
- {
- context.Assembler.StpRiPre(Register(reg, type), Register(reg2, type), Register(SpRegister), -calleeSaveRegionSize);
- }
-
- offset += type.GetSizeInBytes() * 2;
- }
- }
-
- private static void WriteEpilogue(CodeGenContext context)
- {
- Operand rsp = Register(SpRegister);
-
- int localSize = Align16(context.AllocResult.SpillRegionSize + context.FpLrSaveRegionSize);
- int outArgsSize = context.CallArgsRegionSize;
-
- if (CodeGenCommon.ConstFitsOnSImm7(localSize, DWordScale))
- {
- if (outArgsSize != 0)
- {
- context.Assembler.Add(rsp, rsp, Const(OperandType.I64, outArgsSize));
- }
-
- if (context.HasCall)
- {
- context.Assembler.LdpRiPost(Register(FpRegister), Register(LrRegister), rsp, localSize);
- }
- }
- else
- {
- if (context.HasCall)
- {
- context.Assembler.LdpRiUn(Register(FpRegister), Register(LrRegister), rsp, outArgsSize);
- }
-
- int frameSize = localSize + outArgsSize;
- if (frameSize != 0)
- {
- if (CodeGenCommon.ConstFitsOnUImm12(frameSize))
- {
- context.Assembler.Add(rsp, rsp, Const(OperandType.I64, frameSize));
- }
- else
- {
- Operand tempSize = Register(CodeGenCommon.ReservedRegister);
- GenerateConstantCopy(context, tempSize, (ulong)frameSize);
- context.Assembler.Add(rsp, rsp, tempSize, ArmExtensionType.Uxtx);
- }
- }
- }
-
- int intMask = CallingConvention.GetIntCalleeSavedRegisters() & context.AllocResult.IntUsedRegisters;
- int vecMask = CallingConvention.GetFpCalleeSavedRegisters() & context.AllocResult.VecUsedRegisters;
-
- int intCalleeSavedRegsCount = BitOperations.PopCount((uint)intMask);
- int vecCalleeSavedRegsCount = BitOperations.PopCount((uint)vecMask);
-
- int offset = intCalleeSavedRegsCount * 8 + vecCalleeSavedRegsCount * 8;
- int calleeSaveRegionSize = Align16(offset);
-
- WriteEpilogueCalleeSavesPostIndexed(context, ref vecMask, ref offset, calleeSaveRegionSize, OperandType.FP64);
- WriteEpilogueCalleeSavesPostIndexed(context, ref intMask, ref offset, calleeSaveRegionSize, OperandType.I64);
- }
-
- private static void WriteEpilogueCalleeSavesPostIndexed(
- CodeGenContext context,
- ref int mask,
- ref int offset,
- int calleeSaveRegionSize,
- OperandType type)
- {
- while (mask != 0)
- {
- int reg = BitUtils.HighestBitSet(mask);
-
- mask &= ~(1 << reg);
-
- if (mask != 0)
- {
- int reg2 = BitUtils.HighestBitSet(mask);
-
- mask &= ~(1 << reg2);
-
- offset -= type.GetSizeInBytes() * 2;
-
- if (offset != 0)
- {
- context.Assembler.LdpRiUn(Register(reg2, type), Register(reg, type), Register(SpRegister), offset);
- }
- else
- {
- context.Assembler.LdpRiPost(Register(reg2, type), Register(reg, type), Register(SpRegister), calleeSaveRegionSize);
- }
- }
- else
- {
- offset -= type.GetSizeInBytes();
-
- if (offset != 0)
- {
- context.Assembler.LdrRiUn(Register(reg, type), Register(SpRegister), offset);
- }
- else
- {
- context.Assembler.LdrRiPost(Register(reg, type), Register(SpRegister), calleeSaveRegionSize);
- }
- }
- }
- }
-
- private static void GenerateConstantCopy(CodeGenContext context, Operand dest, ulong value)
- {
- if (value == 0)
- {
- context.Assembler.Mov(dest, Register(ZrRegister, dest.Type));
- }
- else if (CodeGenCommon.TryEncodeBitMask(dest.Type, value, out _, out _, out _))
- {
- context.Assembler.Orr(dest, Register(ZrRegister, dest.Type), Const(dest.Type, (long)value));
- }
- else
- {
- int hw = 0;
- bool first = true;
-
- while (value != 0)
- {
- int valueLow = (ushort)value;
- if (valueLow != 0)
- {
- if (first)
- {
- context.Assembler.Movz(dest, valueLow, hw);
- first = false;
- }
- else
- {
- context.Assembler.Movk(dest, valueLow, hw);
- }
- }
-
- hw++;
- value >>= 16;
- }
- }
- }
-
- private static void GenerateAtomicCas(
- CodeGenContext context,
- Operand address,
- Operand expected,
- Operand desired,
- Operand actual,
- Operand result,
- AccessSize accessSize)
- {
- int startOffset = context.StreamOffset;
-
- switch (accessSize)
- {
- case AccessSize.Byte:
- context.Assembler.Ldaxrb(actual, address);
- break;
- case AccessSize.Hword:
- context.Assembler.Ldaxrh(actual, address);
- break;
- default:
- context.Assembler.Ldaxr(actual, address);
- break;
- }
-
- context.Assembler.Cmp(actual, expected);
-
- context.JumpToNear(ArmCondition.Ne);
-
- switch (accessSize)
- {
- case AccessSize.Byte:
- context.Assembler.Stlxrb(desired, address, result);
- break;
- case AccessSize.Hword:
- context.Assembler.Stlxrh(desired, address, result);
- break;
- default:
- context.Assembler.Stlxr(desired, address, result);
- break;
- }
-
- context.Assembler.Cbnz(result, startOffset - context.StreamOffset); // Retry if store failed.
-
- context.JumpHere();
-
- context.Assembler.Clrex();
- }
-
- private static void GenerateAtomicDcas(
- CodeGenContext context,
- Operand address,
- Operand expectedLow,
- Operand expectedHigh,
- Operand desiredLow,
- Operand desiredHigh,
- Operand actualLow,
- Operand actualHigh,
- Operand temp0,
- Operand temp1)
- {
- int startOffset = context.StreamOffset;
-
- context.Assembler.Ldaxp(actualLow, actualHigh, address);
- context.Assembler.Eor(temp0, actualHigh, expectedHigh);
- context.Assembler.Eor(temp1, actualLow, expectedLow);
- context.Assembler.Orr(temp0, temp1, temp0);
-
- context.JumpToNearIfNotZero(temp0);
-
- Operand result = Register(temp0, OperandType.I32);
-
- context.Assembler.Stlxp(desiredLow, desiredHigh, address, result);
- context.Assembler.Cbnz(result, startOffset - context.StreamOffset); // Retry if store failed.
-
- context.JumpHere();
-
- context.Assembler.Clrex();
- }
-
- private static bool TryPairMemoryOp(CodeGenContext context, Operation currentOp, Operation nextOp)
- {
- if (!TryGetMemOpBaseAndOffset(currentOp, out Operand op1Base, out int op1Offset))
- {
- return false;
- }
-
- if (!TryGetMemOpBaseAndOffset(nextOp, out Operand op2Base, out int op2Offset))
- {
- return false;
- }
-
- if (op1Base != op2Base)
- {
- return false;
- }
-
- OperandType valueType = GetMemOpValueType(currentOp);
-
- if (valueType != GetMemOpValueType(nextOp) || op1Offset + valueType.GetSizeInBytes() != op2Offset)
- {
- return false;
- }
-
- if (!CodeGenCommon.ConstFitsOnSImm7(op1Offset, valueType.GetSizeInBytesLog2()))
- {
- return false;
- }
-
- if (currentOp.Instruction == Instruction.Load)
- {
- context.Assembler.LdpRiUn(currentOp.Destination, nextOp.Destination, op1Base, op1Offset);
- }
- else if (currentOp.Instruction == Instruction.Store)
- {
- context.Assembler.StpRiUn(currentOp.GetSource(1), nextOp.GetSource(1), op1Base, op1Offset);
- }
- else
- {
- return false;
- }
-
- return true;
- }
-
- private static bool IsLoadOrStore(Operation operation)
- {
- return operation.Instruction == Instruction.Load || operation.Instruction == Instruction.Store;
- }
-
- private static OperandType GetMemOpValueType(Operation operation)
- {
- if (operation.Destination != default)
- {
- return operation.Destination.Type;
- }
-
- return operation.GetSource(1).Type;
- }
-
- private static bool TryGetMemOpBaseAndOffset(Operation operation, out Operand baseAddress, out int offset)
- {
- baseAddress = default;
- offset = 0;
- Operand address = operation.GetSource(0);
-
- if (address.Kind != OperandKind.Memory)
- {
- return false;
- }
-
- MemoryOperand memOp = address.GetMemory();
- Operand baseOp = memOp.BaseAddress;
-
- if (baseOp == default)
- {
- baseOp = memOp.Index;
-
- if (baseOp == default || memOp.Scale != Multiplier.x1)
- {
- return false;
- }
- }
- if (memOp.Index != default)
- {
- return false;
- }
-
- baseAddress = memOp.BaseAddress;
- offset = memOp.Displacement;
-
- return true;
- }
-
- private static Operand Register(Operand operand, OperandType type = OperandType.I64)
- {
- return Register(operand.GetRegister().Index, type);
- }
-
- private static Operand Register(int register, OperandType type = OperandType.I64)
- {
- return Factory.Register(register, RegisterType.Integer, type);
- }
-
- private static int Align16(int value)
- {
- return (value + 0xf) & ~0xf;
- }
-
- [Conditional("DEBUG")]
- private static void ValidateUnOp(Operand dest, Operand source)
- {
- // Destination and source aren't forced to be equals
- // EnsureSameReg (dest, source);
- EnsureSameType(dest, source);
- }
-
- [Conditional("DEBUG")]
- private static void ValidateBinOp(Operand dest, Operand src1, Operand src2)
- {
- // Destination and source aren't forced to be equals
- // EnsureSameReg (dest, src1);
- EnsureSameType(dest, src1, src2);
- }
-
- [Conditional("DEBUG")]
- private static void ValidateShift(Operand dest, Operand src1, Operand src2)
- {
- // Destination and source aren't forced to be equals
- // EnsureSameReg (dest, src1);
- EnsureSameType(dest, src1);
-
- Debug.Assert(dest.Type.IsInteger() && src2.Type == OperandType.I32);
- }
-
- private static void EnsureSameReg(Operand op1, Operand op2)
- {
- Debug.Assert(op1.Kind == OperandKind.Register || op1.Kind == OperandKind.Memory);
- Debug.Assert(op1.Kind == op2.Kind);
- Debug.Assert(op1.Value == op2.Value);
- }
-
- private static void EnsureSameType(Operand op1, Operand op2)
- {
- Debug.Assert(op1.Type == op2.Type);
- }
-
- private static void EnsureSameType(Operand op1, Operand op2, Operand op3)
- {
- Debug.Assert(op1.Type == op2.Type);
- Debug.Assert(op1.Type == op3.Type);
- }
-
- private static void EnsureSameType(Operand op1, Operand op2, Operand op3, Operand op4)
- {
- Debug.Assert(op1.Type == op2.Type);
- Debug.Assert(op1.Type == op3.Type);
- Debug.Assert(op1.Type == op4.Type);
- }
- }
-} \ No newline at end of file
diff --git a/ARMeilleure/CodeGen/Arm64/CodeGeneratorIntrinsic.cs b/ARMeilleure/CodeGen/Arm64/CodeGeneratorIntrinsic.cs
deleted file mode 100644
index aaa00bb6..00000000
--- a/ARMeilleure/CodeGen/Arm64/CodeGeneratorIntrinsic.cs
+++ /dev/null
@@ -1,662 +0,0 @@
-using ARMeilleure.IntermediateRepresentation;
-using System;
-using System.Diagnostics;
-
-namespace ARMeilleure.CodeGen.Arm64
-{
- static class CodeGeneratorIntrinsic
- {
- public static void GenerateOperation(CodeGenContext context, Operation operation)
- {
- Intrinsic intrin = operation.Intrinsic;
-
- IntrinsicInfo info = IntrinsicTable.GetInfo(intrin & ~(Intrinsic.Arm64VTypeMask | Intrinsic.Arm64VSizeMask));
-
- switch (info.Type)
- {
- case IntrinsicType.ScalarUnary:
- GenerateVectorUnary(
- context,
- 0,
- (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
- info.Inst,
- operation.Destination,
- operation.GetSource(0));
- break;
- case IntrinsicType.ScalarUnaryByElem:
- Debug.Assert(operation.GetSource(1).Kind == OperandKind.Constant);
-
- GenerateVectorUnaryByElem(
- context,
- 0,
- (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
- info.Inst,
- (uint)operation.GetSource(1).AsInt32(),
- operation.Destination,
- operation.GetSource(0));
- break;
- case IntrinsicType.ScalarBinary:
- GenerateVectorBinary(
- context,
- 0,
- (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
- info.Inst,
- operation.Destination,
- operation.GetSource(0),
- operation.GetSource(1));
- break;
- case IntrinsicType.ScalarBinaryFPByElem:
- Debug.Assert(operation.GetSource(2).Kind == OperandKind.Constant);
-
- GenerateVectorBinaryFPByElem(
- context,
- 0,
- (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
- info.Inst,
- (uint)operation.GetSource(2).AsInt32(),
- operation.Destination,
- operation.GetSource(0),
- operation.GetSource(1));
- break;
- case IntrinsicType.ScalarBinaryRd:
- GenerateVectorUnary(
- context,
- 0,
- (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
- info.Inst,
- operation.Destination,
- operation.GetSource(1));
- break;
- case IntrinsicType.ScalarBinaryShl:
- Debug.Assert(operation.GetSource(1).Kind == OperandKind.Constant);
-
- GenerateVectorBinaryShlImm(
- context,
- 0,
- (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
- info.Inst,
- operation.Destination,
- operation.GetSource(0),
- (uint)operation.GetSource(1).AsInt32());
- break;
- case IntrinsicType.ScalarBinaryShr:
- Debug.Assert(operation.GetSource(1).Kind == OperandKind.Constant);
-
- GenerateVectorBinaryShrImm(
- context,
- 0,
- (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
- info.Inst,
- operation.Destination,
- operation.GetSource(0),
- (uint)operation.GetSource(1).AsInt32());
- break;
- case IntrinsicType.ScalarFPCompare:
- GenerateScalarFPCompare(
- context,
- (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
- info.Inst,
- operation.Destination,
- operation.GetSource(0),
- operation.GetSource(1));
- break;
- case IntrinsicType.ScalarFPConvFixed:
- Debug.Assert(operation.GetSource(1).Kind == OperandKind.Constant);
-
- GenerateVectorBinaryShrImm(
- context,
- 0,
- ((uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift) + 2u,
- info.Inst,
- operation.Destination,
- operation.GetSource(0),
- (uint)operation.GetSource(1).AsInt32());
- break;
- case IntrinsicType.ScalarFPConvFixedGpr:
- Debug.Assert(operation.GetSource(1).Kind == OperandKind.Constant);
-
- GenerateScalarFPConvGpr(
- context,
- (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
- info.Inst,
- operation.Destination,
- operation.GetSource(0),
- (uint)operation.GetSource(1).AsInt32());
- break;
- case IntrinsicType.ScalarFPConvGpr:
- GenerateScalarFPConvGpr(
- context,
- (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
- info.Inst,
- operation.Destination,
- operation.GetSource(0));
- break;
- case IntrinsicType.ScalarTernary:
- GenerateScalarTernary(
- context,
- (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
- info.Inst,
- operation.Destination,
- operation.GetSource(1),
- operation.GetSource(2),
- operation.GetSource(0));
- break;
- case IntrinsicType.ScalarTernaryFPRdByElem:
- Debug.Assert(operation.GetSource(3).Kind == OperandKind.Constant);
-
- GenerateVectorBinaryFPByElem(
- context,
- 0,
- (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
- info.Inst,
- (uint)operation.GetSource(3).AsInt32(),
- operation.Destination,
- operation.GetSource(1),
- operation.GetSource(2));
- break;
- case IntrinsicType.ScalarTernaryShlRd:
- Debug.Assert(operation.GetSource(2).Kind == OperandKind.Constant);
-
- GenerateVectorBinaryShlImm(
- context,
- 0,
- (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
- info.Inst,
- operation.Destination,
- operation.GetSource(1),
- (uint)operation.GetSource(2).AsInt32());
- break;
- case IntrinsicType.ScalarTernaryShrRd:
- Debug.Assert(operation.GetSource(2).Kind == OperandKind.Constant);
-
- GenerateVectorBinaryShrImm(
- context,
- 0,
- (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
- info.Inst,
- operation.Destination,
- operation.GetSource(1),
- (uint)operation.GetSource(2).AsInt32());
- break;
-
- case IntrinsicType.VectorUnary:
- GenerateVectorUnary(
- context,
- (uint)(intrin & Intrinsic.Arm64VTypeMask) >> (int)Intrinsic.Arm64VTypeShift,
- (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
- info.Inst,
- operation.Destination,
- operation.GetSource(0));
- break;
- case IntrinsicType.VectorUnaryByElem:
- Debug.Assert(operation.GetSource(1).Kind == OperandKind.Constant);
-
- GenerateVectorUnaryByElem(
- context,
- (uint)(intrin & Intrinsic.Arm64VTypeMask) >> (int)Intrinsic.Arm64VTypeShift,
- (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
- info.Inst,
- (uint)operation.GetSource(1).AsInt32(),
- operation.Destination,
- operation.GetSource(0));
- break;
- case IntrinsicType.VectorBinary:
- GenerateVectorBinary(
- context,
- (uint)(intrin & Intrinsic.Arm64VTypeMask) >> (int)Intrinsic.Arm64VTypeShift,
- (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
- info.Inst,
- operation.Destination,
- operation.GetSource(0),
- operation.GetSource(1));
- break;
- case IntrinsicType.VectorBinaryBitwise:
- GenerateVectorBinary(
- context,
- (uint)(intrin & Intrinsic.Arm64VTypeMask) >> (int)Intrinsic.Arm64VTypeShift,
- info.Inst,
- operation.Destination,
- operation.GetSource(0),
- operation.GetSource(1));
- break;
- case IntrinsicType.VectorBinaryByElem:
- Debug.Assert(operation.GetSource(2).Kind == OperandKind.Constant);
-
- GenerateVectorBinaryByElem(
- context,
- (uint)(intrin & Intrinsic.Arm64VTypeMask) >> (int)Intrinsic.Arm64VTypeShift,
- (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
- info.Inst,
- (uint)operation.GetSource(2).AsInt32(),
- operation.Destination,
- operation.GetSource(0),
- operation.GetSource(1));
- break;
- case IntrinsicType.VectorBinaryFPByElem:
- Debug.Assert(operation.GetSource(2).Kind == OperandKind.Constant);
-
- GenerateVectorBinaryFPByElem(
- context,
- (uint)(intrin & Intrinsic.Arm64VTypeMask) >> (int)Intrinsic.Arm64VTypeShift,
- (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
- info.Inst,
- (uint)operation.GetSource(2).AsInt32(),
- operation.Destination,
- operation.GetSource(0),
- operation.GetSource(1));
- break;
- case IntrinsicType.VectorBinaryRd:
- GenerateVectorUnary(
- context,
- (uint)(intrin & Intrinsic.Arm64VTypeMask) >> (int)Intrinsic.Arm64VTypeShift,
- (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
- info.Inst,
- operation.Destination,
- operation.GetSource(1));
- break;
- case IntrinsicType.VectorBinaryShl:
- Debug.Assert(operation.GetSource(1).Kind == OperandKind.Constant);
-
- GenerateVectorBinaryShlImm(
- context,
- (uint)(intrin & Intrinsic.Arm64VTypeMask) >> (int)Intrinsic.Arm64VTypeShift,
- (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
- info.Inst,
- operation.Destination,
- operation.GetSource(0),
- (uint)operation.GetSource(1).AsInt32());
- break;
- case IntrinsicType.VectorBinaryShr:
- Debug.Assert(operation.GetSource(1).Kind == OperandKind.Constant);
-
- GenerateVectorBinaryShrImm(
- context,
- (uint)(intrin & Intrinsic.Arm64VTypeMask) >> (int)Intrinsic.Arm64VTypeShift,
- (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
- info.Inst,
- operation.Destination,
- operation.GetSource(0),
- (uint)operation.GetSource(1).AsInt32());
- break;
- case IntrinsicType.VectorFPConvFixed:
- Debug.Assert(operation.GetSource(1).Kind == OperandKind.Constant);
-
- GenerateVectorBinaryShrImm(
- context,
- (uint)(intrin & Intrinsic.Arm64VTypeMask) >> (int)Intrinsic.Arm64VTypeShift,
- ((uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift) + 2u,
- info.Inst,
- operation.Destination,
- operation.GetSource(0),
- (uint)operation.GetSource(1).AsInt32());
- break;
- case IntrinsicType.VectorInsertByElem:
- Debug.Assert(operation.GetSource(1).Kind == OperandKind.Constant);
- Debug.Assert(operation.GetSource(3).Kind == OperandKind.Constant);
-
- GenerateVectorInsertByElem(
- context,
- (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
- info.Inst,
- (uint)operation.GetSource(3).AsInt32(),
- (uint)operation.GetSource(1).AsInt32(),
- operation.Destination,
- operation.GetSource(2));
- break;
- case IntrinsicType.VectorLookupTable:
- Debug.Assert((uint)(operation.SourcesCount - 2) <= 3);
-
- for (int i = 1; i < operation.SourcesCount - 1; i++)
- {
- Register currReg = operation.GetSource(i).GetRegister();
- Register prevReg = operation.GetSource(i - 1).GetRegister();
-
- Debug.Assert(prevReg.Index + 1 == currReg.Index && currReg.Type == RegisterType.Vector);
- }
-
- GenerateVectorBinary(
- context,
- (uint)(intrin & Intrinsic.Arm64VTypeMask) >> (int)Intrinsic.Arm64VTypeShift,
- info.Inst | ((uint)(operation.SourcesCount - 2) << 13),
- operation.Destination,
- operation.GetSource(0),
- operation.GetSource(operation.SourcesCount - 1));
- break;
- case IntrinsicType.VectorTernaryFPRdByElem:
- Debug.Assert(operation.GetSource(3).Kind == OperandKind.Constant);
-
- GenerateVectorBinaryFPByElem(
- context,
- (uint)(intrin & Intrinsic.Arm64VTypeMask) >> (int)Intrinsic.Arm64VTypeShift,
- (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
- info.Inst,
- (uint)operation.GetSource(3).AsInt32(),
- operation.Destination,
- operation.GetSource(1),
- operation.GetSource(2));
- break;
- case IntrinsicType.VectorTernaryRd:
- GenerateVectorBinary(
- context,
- (uint)(intrin & Intrinsic.Arm64VTypeMask) >> (int)Intrinsic.Arm64VTypeShift,
- (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
- info.Inst,
- operation.Destination,
- operation.GetSource(1),
- operation.GetSource(2));
- break;
- case IntrinsicType.VectorTernaryRdBitwise:
- GenerateVectorBinary(
- context,
- (uint)(intrin & Intrinsic.Arm64VTypeMask) >> (int)Intrinsic.Arm64VTypeShift,
- info.Inst,
- operation.Destination,
- operation.GetSource(1),
- operation.GetSource(2));
- break;
- case IntrinsicType.VectorTernaryRdByElem:
- Debug.Assert(operation.GetSource(3).Kind == OperandKind.Constant);
-
- GenerateVectorBinaryByElem(
- context,
- (uint)(intrin & Intrinsic.Arm64VTypeMask) >> (int)Intrinsic.Arm64VTypeShift,
- (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
- info.Inst,
- (uint)operation.GetSource(3).AsInt32(),
- operation.Destination,
- operation.GetSource(1),
- operation.GetSource(2));
- break;
- case IntrinsicType.VectorTernaryShlRd:
- Debug.Assert(operation.GetSource(2).Kind == OperandKind.Constant);
-
- GenerateVectorBinaryShlImm(
- context,
- (uint)(intrin & Intrinsic.Arm64VTypeMask) >> (int)Intrinsic.Arm64VTypeShift,
- (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
- info.Inst,
- operation.Destination,
- operation.GetSource(1),
- (uint)operation.GetSource(2).AsInt32());
- break;
- case IntrinsicType.VectorTernaryShrRd:
- Debug.Assert(operation.GetSource(2).Kind == OperandKind.Constant);
-
- GenerateVectorBinaryShrImm(
- context,
- (uint)(intrin & Intrinsic.Arm64VTypeMask) >> (int)Intrinsic.Arm64VTypeShift,
- (uint)(intrin & Intrinsic.Arm64VSizeMask) >> (int)Intrinsic.Arm64VSizeShift,
- info.Inst,
- operation.Destination,
- operation.GetSource(1),
- (uint)operation.GetSource(2).AsInt32());
- break;
-
- case IntrinsicType.GetRegister:
- context.Assembler.WriteInstruction(info.Inst, operation.Destination);
- break;
- case IntrinsicType.SetRegister:
- context.Assembler.WriteInstruction(info.Inst, operation.GetSource(0));
- break;
-
- default:
- throw new NotImplementedException(info.Type.ToString());
- }
- }
-
- private static void GenerateScalarFPCompare(
- CodeGenContext context,
- uint sz,
- uint instruction,
- Operand dest,
- Operand rn,
- Operand rm)
- {
- instruction |= (sz << 22);
-
- if (rm.Kind == OperandKind.Constant && rm.Value == 0)
- {
- instruction |= 0b1000;
- rm = rn;
- }
-
- context.Assembler.WriteInstructionRm16NoRet(instruction, rn, rm);
- context.Assembler.Mrs(dest, 1, 3, 4, 2, 0);
- }
-
- private static void GenerateScalarFPConvGpr(
- CodeGenContext context,
- uint sz,
- uint instruction,
- Operand rd,
- Operand rn)
- {
- instruction |= (sz << 22);
-
- if (rd.Type.IsInteger())
- {
- context.Assembler.WriteInstructionAuto(instruction, rd, rn);
- }
- else
- {
- if (rn.Type == OperandType.I64)
- {
- instruction |= Assembler.SfFlag;
- }
-
- context.Assembler.WriteInstruction(instruction, rd, rn);
- }
- }
-
- private static void GenerateScalarFPConvGpr(
- CodeGenContext context,
- uint sz,
- uint instruction,
- Operand rd,
- Operand rn,
- uint fBits)
- {
- Debug.Assert(fBits <= 64);
-
- instruction |= (sz << 22);
- instruction |= (64 - fBits) << 10;
-
- if (rd.Type.IsInteger())
- {
- Debug.Assert(rd.Type != OperandType.I32 || fBits <= 32);
-
- context.Assembler.WriteInstructionAuto(instruction, rd, rn);
- }
- else
- {
- if (rn.Type == OperandType.I64)
- {
- instruction |= Assembler.SfFlag;
- }
- else
- {
- Debug.Assert(fBits <= 32);
- }
-
- context.Assembler.WriteInstruction(instruction, rd, rn);
- }
-
- }
-
- private static void GenerateScalarTernary(
- CodeGenContext context,
- uint sz,
- uint instruction,
- Operand rd,
- Operand rn,
- Operand rm,
- Operand ra)
- {
- instruction |= (sz << 22);
-
- context.Assembler.WriteInstruction(instruction, rd, rn, rm, ra);
- }
-
- private static void GenerateVectorUnary(
- CodeGenContext context,
- uint q,
- uint sz,
- uint instruction,
- Operand rd,
- Operand rn)
- {
- instruction |= (q << 30) | (sz << 22);
-
- context.Assembler.WriteInstruction(instruction, rd, rn);
- }
-
- private static void GenerateVectorUnaryByElem(
- CodeGenContext context,
- uint q,
- uint sz,
- uint instruction,
- uint srcIndex,
- Operand rd,
- Operand rn)
- {
- uint imm5 = (srcIndex << ((int)sz + 1)) | (1u << (int)sz);
-
- instruction |= (q << 30) | (imm5 << 16);
-
- context.Assembler.WriteInstruction(instruction, rd, rn);
- }
-
- private static void GenerateVectorBinary(
- CodeGenContext context,
- uint q,
- uint instruction,
- Operand rd,
- Operand rn,
- Operand rm)
- {
- instruction |= (q << 30);
-
- context.Assembler.WriteInstructionRm16(instruction, rd, rn, rm);
- }
-
- private static void GenerateVectorBinary(
- CodeGenContext context,
- uint q,
- uint sz,
- uint instruction,
- Operand rd,
- Operand rn,
- Operand rm)
- {
- instruction |= (q << 30) | (sz << 22);
-
- context.Assembler.WriteInstructionRm16(instruction, rd, rn, rm);
- }
-
- private static void GenerateVectorBinaryByElem(
- CodeGenContext context,
- uint q,
- uint size,
- uint instruction,
- uint srcIndex,
- Operand rd,
- Operand rn,
- Operand rm)
- {
- instruction |= (q << 30) | (size << 22);
-
- if (size == 2)
- {
- instruction |= ((srcIndex & 1) << 21) | ((srcIndex & 2) << 10);
- }
- else
- {
- instruction |= ((srcIndex & 3) << 20) | ((srcIndex & 4) << 9);
- }
-
- context.Assembler.WriteInstructionRm16(instruction, rd, rn, rm);
- }
-
- private static void GenerateVectorBinaryFPByElem(
- CodeGenContext context,
- uint q,
- uint sz,
- uint instruction,
- uint srcIndex,
- Operand rd,
- Operand rn,
- Operand rm)
- {
- instruction |= (q << 30) | (sz << 22);
-
- if (sz != 0)
- {
- instruction |= (srcIndex & 1) << 11;
- }
- else
- {
- instruction |= ((srcIndex & 1) << 21) | ((srcIndex & 2) << 10);
- }
-
- context.Assembler.WriteInstructionRm16(instruction, rd, rn, rm);
- }
-
- private static void GenerateVectorBinaryShlImm(
- CodeGenContext context,
- uint q,
- uint sz,
- uint instruction,
- Operand rd,
- Operand rn,
- uint shift)
- {
- instruction |= (q << 30);
-
- Debug.Assert(shift >= 0 && shift < (8u << (int)sz));
-
- uint imm = (8u << (int)sz) | (shift & (0x3fu >> (int)(3 - sz)));
-
- instruction |= (imm << 16);
-
- context.Assembler.WriteInstruction(instruction, rd, rn);
- }
-
- private static void GenerateVectorBinaryShrImm(
- CodeGenContext context,
- uint q,
- uint sz,
- uint instruction,
- Operand rd,
- Operand rn,
- uint shift)
- {
- instruction |= (q << 30);
-
- Debug.Assert(shift > 0 && shift <= (8u << (int)sz));
-
- uint imm = (8u << (int)sz) | ((8u << (int)sz) - shift);
-
- instruction |= (imm << 16);
-
- context.Assembler.WriteInstruction(instruction, rd, rn);
- }
-
- private static void GenerateVectorInsertByElem(
- CodeGenContext context,
- uint sz,
- uint instruction,
- uint srcIndex,
- uint dstIndex,
- Operand rd,
- Operand rn)
- {
- uint imm4 = srcIndex << (int)sz;
- uint imm5 = (dstIndex << ((int)sz + 1)) | (1u << (int)sz);
-
- instruction |= imm4 << 11;
- instruction |= imm5 << 16;
-
- context.Assembler.WriteInstruction(instruction, rd, rn);
- }
- }
-} \ No newline at end of file
diff --git a/ARMeilleure/CodeGen/Arm64/HardwareCapabilities.cs b/ARMeilleure/CodeGen/Arm64/HardwareCapabilities.cs
deleted file mode 100644
index 99ff299e..00000000
--- a/ARMeilleure/CodeGen/Arm64/HardwareCapabilities.cs
+++ /dev/null
@@ -1,185 +0,0 @@
-using System;
-using System.Linq;
-using System.Reflection;
-using System.Runtime.CompilerServices;
-using System.Runtime.InteropServices;
-using System.Runtime.Intrinsics.Arm;
-using System.Runtime.Versioning;
-
-namespace ARMeilleure.CodeGen.Arm64
-{
- static partial class HardwareCapabilities
- {
- static HardwareCapabilities()
- {
- if (!ArmBase.Arm64.IsSupported)
- {
- return;
- }
-
- if (OperatingSystem.IsLinux())
- {
- LinuxFeatureInfoHwCap = (LinuxFeatureFlagsHwCap)getauxval(AT_HWCAP);
- LinuxFeatureInfoHwCap2 = (LinuxFeatureFlagsHwCap2)getauxval(AT_HWCAP2);
- }
-
- if (OperatingSystem.IsMacOS())
- {
- for (int i = 0; i < _sysctlNames.Length; i++)
- {
- if (CheckSysctlName(_sysctlNames[i]))
- {
- MacOsFeatureInfo |= (MacOsFeatureFlags)(1 << i);
- }
- }
- }
- }
-
-#region Linux
-
- private const ulong AT_HWCAP = 16;
- private const ulong AT_HWCAP2 = 26;
-
- [LibraryImport("libc", SetLastError = true)]
- private static partial ulong getauxval(ulong type);
-
- [Flags]
- public enum LinuxFeatureFlagsHwCap : ulong
- {
- Fp = 1 << 0,
- Asimd = 1 << 1,
- Evtstrm = 1 << 2,
- Aes = 1 << 3,
- Pmull = 1 << 4,
- Sha1 = 1 << 5,
- Sha2 = 1 << 6,
- Crc32 = 1 << 7,
- Atomics = 1 << 8,
- FpHp = 1 << 9,
- AsimdHp = 1 << 10,
- CpuId = 1 << 11,
- AsimdRdm = 1 << 12,
- Jscvt = 1 << 13,
- Fcma = 1 << 14,
- Lrcpc = 1 << 15,
- DcpOp = 1 << 16,
- Sha3 = 1 << 17,
- Sm3 = 1 << 18,
- Sm4 = 1 << 19,
- AsimdDp = 1 << 20,
- Sha512 = 1 << 21,
- Sve = 1 << 22,
- AsimdFhm = 1 << 23,
- Dit = 1 << 24,
- Uscat = 1 << 25,
- Ilrcpc = 1 << 26,
- FlagM = 1 << 27,
- Ssbs = 1 << 28,
- Sb = 1 << 29,
- Paca = 1 << 30,
- Pacg = 1UL << 31
- }
-
- [Flags]
- public enum LinuxFeatureFlagsHwCap2 : ulong
- {
- Dcpodp = 1 << 0,
- Sve2 = 1 << 1,
- SveAes = 1 << 2,
- SvePmull = 1 << 3,
- SveBitperm = 1 << 4,
- SveSha3 = 1 << 5,
- SveSm4 = 1 << 6,
- FlagM2 = 1 << 7,
- Frint = 1 << 8,
- SveI8mm = 1 << 9,
- SveF32mm = 1 << 10,
- SveF64mm = 1 << 11,
- SveBf16 = 1 << 12,
- I8mm = 1 << 13,
- Bf16 = 1 << 14,
- Dgh = 1 << 15,
- Rng = 1 << 16,
- Bti = 1 << 17,
- Mte = 1 << 18,
- Ecv = 1 << 19,
- Afp = 1 << 20,
- Rpres = 1 << 21,
- Mte3 = 1 << 22,
- Sme = 1 << 23,
- Sme_i16i64 = 1 << 24,
- Sme_f64f64 = 1 << 25,
- Sme_i8i32 = 1 << 26,
- Sme_f16f32 = 1 << 27,
- Sme_b16f32 = 1 << 28,
- Sme_f32f32 = 1 << 29,
- Sme_fa64 = 1 << 30,
- Wfxt = 1UL << 31,
- Ebf16 = 1UL << 32,
- Sve_Ebf16 = 1UL << 33,
- Cssc = 1UL << 34,
- Rprfm = 1UL << 35,
- Sve2p1 = 1UL << 36
- }
-
- public static LinuxFeatureFlagsHwCap LinuxFeatureInfoHwCap { get; } = 0;
- public static LinuxFeatureFlagsHwCap2 LinuxFeatureInfoHwCap2 { get; } = 0;
-
-#endregion
-
-#region macOS
-
- [LibraryImport("libSystem.dylib", SetLastError = true)]
- private static unsafe partial int sysctlbyname([MarshalAs(UnmanagedType.LPStr)] string name, out int oldValue, ref ulong oldSize, IntPtr newValue, ulong newValueSize);
-
- [SupportedOSPlatform("macos")]
- private static bool CheckSysctlName(string name)
- {
- ulong size = sizeof(int);
- if (sysctlbyname(name, out int val, ref size, IntPtr.Zero, 0) == 0 && size == sizeof(int))
- {
- return val != 0;
- }
- return false;
- }
-
- private static string[] _sysctlNames = new string[]
- {
- "hw.optional.floatingpoint",
- "hw.optional.AdvSIMD",
- "hw.optional.arm.FEAT_FP16",
- "hw.optional.arm.FEAT_AES",
- "hw.optional.arm.FEAT_PMULL",
- "hw.optional.arm.FEAT_LSE",
- "hw.optional.armv8_crc32",
- "hw.optional.arm.FEAT_SHA1",
- "hw.optional.arm.FEAT_SHA256"
- };
-
- [Flags]
- public enum MacOsFeatureFlags
- {
- Fp = 1 << 0,
- AdvSimd = 1 << 1,
- Fp16 = 1 << 2,
- Aes = 1 << 3,
- Pmull = 1 << 4,
- Lse = 1 << 5,
- Crc32 = 1 << 6,
- Sha1 = 1 << 7,
- Sha256 = 1 << 8
- }
-
- public static MacOsFeatureFlags MacOsFeatureInfo { get; } = 0;
-
-#endregion
-
- public static bool SupportsAdvSimd => LinuxFeatureInfoHwCap.HasFlag(LinuxFeatureFlagsHwCap.Asimd) || MacOsFeatureInfo.HasFlag(MacOsFeatureFlags.AdvSimd);
- public static bool SupportsAes => LinuxFeatureInfoHwCap.HasFlag(LinuxFeatureFlagsHwCap.Aes) || MacOsFeatureInfo.HasFlag(MacOsFeatureFlags.Aes);
- public static bool SupportsPmull => LinuxFeatureInfoHwCap.HasFlag(LinuxFeatureFlagsHwCap.Pmull) || MacOsFeatureInfo.HasFlag(MacOsFeatureFlags.Pmull);
- public static bool SupportsLse => LinuxFeatureInfoHwCap.HasFlag(LinuxFeatureFlagsHwCap.Atomics) || MacOsFeatureInfo.HasFlag(MacOsFeatureFlags.Lse);
- public static bool SupportsCrc32 => LinuxFeatureInfoHwCap.HasFlag(LinuxFeatureFlagsHwCap.Crc32) || MacOsFeatureInfo.HasFlag(MacOsFeatureFlags.Crc32);
- public static bool SupportsSha1 => LinuxFeatureInfoHwCap.HasFlag(LinuxFeatureFlagsHwCap.Sha1) || MacOsFeatureInfo.HasFlag(MacOsFeatureFlags.Sha1);
- public static bool SupportsSha256 => LinuxFeatureInfoHwCap.HasFlag(LinuxFeatureFlagsHwCap.Sha2) || MacOsFeatureInfo.HasFlag(MacOsFeatureFlags.Sha256);
- }
-}
diff --git a/ARMeilleure/CodeGen/Arm64/IntrinsicInfo.cs b/ARMeilleure/CodeGen/Arm64/IntrinsicInfo.cs
deleted file mode 100644
index 8695db90..00000000
--- a/ARMeilleure/CodeGen/Arm64/IntrinsicInfo.cs
+++ /dev/null
@@ -1,14 +0,0 @@
-namespace ARMeilleure.CodeGen.Arm64
-{
- struct IntrinsicInfo
- {
- public uint Inst { get; }
- public IntrinsicType Type { get; }
-
- public IntrinsicInfo(uint inst, IntrinsicType type)
- {
- Inst = inst;
- Type = type;
- }
- }
-} \ No newline at end of file
diff --git a/ARMeilleure/CodeGen/Arm64/IntrinsicTable.cs b/ARMeilleure/CodeGen/Arm64/IntrinsicTable.cs
deleted file mode 100644
index a309d56d..00000000
--- a/ARMeilleure/CodeGen/Arm64/IntrinsicTable.cs
+++ /dev/null
@@ -1,463 +0,0 @@
-using ARMeilleure.Common;
-using ARMeilleure.IntermediateRepresentation;
-
-namespace ARMeilleure.CodeGen.Arm64
-{
- static class IntrinsicTable
- {
- private static IntrinsicInfo[] _intrinTable;
-
- static IntrinsicTable()
- {
- _intrinTable = new IntrinsicInfo[EnumUtils.GetCount(typeof(Intrinsic))];
-
- Add(Intrinsic.Arm64AbsS, new IntrinsicInfo(0x5e20b800u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64AbsV, new IntrinsicInfo(0x0e20b800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64AddhnV, new IntrinsicInfo(0x0e204000u, IntrinsicType.VectorTernaryRd));
- Add(Intrinsic.Arm64AddpS, new IntrinsicInfo(0x5e31b800u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64AddpV, new IntrinsicInfo(0x0e20bc00u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64AddvV, new IntrinsicInfo(0x0e31b800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64AddS, new IntrinsicInfo(0x5e208400u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64AddV, new IntrinsicInfo(0x0e208400u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64AesdV, new IntrinsicInfo(0x4e285800u, IntrinsicType.Vector128Unary));
- Add(Intrinsic.Arm64AeseV, new IntrinsicInfo(0x4e284800u, IntrinsicType.Vector128Unary));
- Add(Intrinsic.Arm64AesimcV, new IntrinsicInfo(0x4e287800u, IntrinsicType.Vector128Unary));
- Add(Intrinsic.Arm64AesmcV, new IntrinsicInfo(0x4e286800u, IntrinsicType.Vector128Unary));
- Add(Intrinsic.Arm64AndV, new IntrinsicInfo(0x0e201c00u, IntrinsicType.VectorBinaryBitwise));
- Add(Intrinsic.Arm64BicVi, new IntrinsicInfo(0x2f001400u, IntrinsicType.VectorBinaryBitwiseImm));
- Add(Intrinsic.Arm64BicV, new IntrinsicInfo(0x0e601c00u, IntrinsicType.VectorBinaryBitwise));
- Add(Intrinsic.Arm64BifV, new IntrinsicInfo(0x2ee01c00u, IntrinsicType.VectorTernaryRdBitwise));
- Add(Intrinsic.Arm64BitV, new IntrinsicInfo(0x2ea01c00u, IntrinsicType.VectorTernaryRdBitwise));
- Add(Intrinsic.Arm64BslV, new IntrinsicInfo(0x2e601c00u, IntrinsicType.VectorTernaryRdBitwise));
- Add(Intrinsic.Arm64ClsV, new IntrinsicInfo(0x0e204800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64ClzV, new IntrinsicInfo(0x2e204800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64CmeqS, new IntrinsicInfo(0x7e208c00u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64CmeqV, new IntrinsicInfo(0x2e208c00u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64CmeqSz, new IntrinsicInfo(0x5e209800u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64CmeqVz, new IntrinsicInfo(0x0e209800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64CmgeS, new IntrinsicInfo(0x5e203c00u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64CmgeV, new IntrinsicInfo(0x0e203c00u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64CmgeSz, new IntrinsicInfo(0x7e208800u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64CmgeVz, new IntrinsicInfo(0x2e208800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64CmgtS, new IntrinsicInfo(0x5e203400u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64CmgtV, new IntrinsicInfo(0x0e203400u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64CmgtSz, new IntrinsicInfo(0x5e208800u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64CmgtVz, new IntrinsicInfo(0x0e208800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64CmhiS, new IntrinsicInfo(0x7e203400u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64CmhiV, new IntrinsicInfo(0x2e203400u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64CmhsS, new IntrinsicInfo(0x7e203c00u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64CmhsV, new IntrinsicInfo(0x2e203c00u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64CmleSz, new IntrinsicInfo(0x7e209800u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64CmleVz, new IntrinsicInfo(0x2e209800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64CmltSz, new IntrinsicInfo(0x5e20a800u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64CmltVz, new IntrinsicInfo(0x0e20a800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64CmtstS, new IntrinsicInfo(0x5e208c00u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64CmtstV, new IntrinsicInfo(0x0e208c00u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64CntV, new IntrinsicInfo(0x0e205800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64DupSe, new IntrinsicInfo(0x5e000400u, IntrinsicType.ScalarUnaryByElem));
- Add(Intrinsic.Arm64DupVe, new IntrinsicInfo(0x0e000400u, IntrinsicType.VectorUnaryByElem));
- Add(Intrinsic.Arm64DupGp, new IntrinsicInfo(0x0e000c00u, IntrinsicType.VectorUnaryByElem));
- Add(Intrinsic.Arm64EorV, new IntrinsicInfo(0x2e201c00u, IntrinsicType.VectorBinaryBitwise));
- Add(Intrinsic.Arm64ExtV, new IntrinsicInfo(0x2e000000u, IntrinsicType.VectorExt));
- Add(Intrinsic.Arm64FabdS, new IntrinsicInfo(0x7ea0d400u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64FabdV, new IntrinsicInfo(0x2ea0d400u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64FabsV, new IntrinsicInfo(0x0ea0f800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64FabsS, new IntrinsicInfo(0x1e20c000u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64FacgeS, new IntrinsicInfo(0x7e20ec00u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64FacgeV, new IntrinsicInfo(0x2e20ec00u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64FacgtS, new IntrinsicInfo(0x7ea0ec00u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64FacgtV, new IntrinsicInfo(0x2ea0ec00u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64FaddpS, new IntrinsicInfo(0x7e30d800u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64FaddpV, new IntrinsicInfo(0x2e20d400u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64FaddV, new IntrinsicInfo(0x0e20d400u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64FaddS, new IntrinsicInfo(0x1e202800u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64FccmpeS, new IntrinsicInfo(0x1e200410u, IntrinsicType.ScalarFPCompareCond));
- Add(Intrinsic.Arm64FccmpS, new IntrinsicInfo(0x1e200400u, IntrinsicType.ScalarFPCompareCond));
- Add(Intrinsic.Arm64FcmeqS, new IntrinsicInfo(0x5e20e400u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64FcmeqV, new IntrinsicInfo(0x0e20e400u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64FcmeqSz, new IntrinsicInfo(0x5ea0d800u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64FcmeqVz, new IntrinsicInfo(0x0ea0d800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64FcmgeS, new IntrinsicInfo(0x7e20e400u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64FcmgeV, new IntrinsicInfo(0x2e20e400u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64FcmgeSz, new IntrinsicInfo(0x7ea0c800u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64FcmgeVz, new IntrinsicInfo(0x2ea0c800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64FcmgtS, new IntrinsicInfo(0x7ea0e400u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64FcmgtV, new IntrinsicInfo(0x2ea0e400u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64FcmgtSz, new IntrinsicInfo(0x5ea0c800u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64FcmgtVz, new IntrinsicInfo(0x0ea0c800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64FcmleSz, new IntrinsicInfo(0x7ea0d800u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64FcmleVz, new IntrinsicInfo(0x2ea0d800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64FcmltSz, new IntrinsicInfo(0x5ea0e800u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64FcmltVz, new IntrinsicInfo(0x0ea0e800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64FcmpeS, new IntrinsicInfo(0x1e202010u, IntrinsicType.ScalarFPCompare));
- Add(Intrinsic.Arm64FcmpS, new IntrinsicInfo(0x1e202000u, IntrinsicType.ScalarFPCompare));
- Add(Intrinsic.Arm64FcselS, new IntrinsicInfo(0x1e200c00u, IntrinsicType.ScalarFcsel));
- Add(Intrinsic.Arm64FcvtasS, new IntrinsicInfo(0x5e21c800u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64FcvtasV, new IntrinsicInfo(0x0e21c800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64FcvtasGp, new IntrinsicInfo(0x1e240000u, IntrinsicType.ScalarFPConvGpr));
- Add(Intrinsic.Arm64FcvtauS, new IntrinsicInfo(0x7e21c800u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64FcvtauV, new IntrinsicInfo(0x2e21c800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64FcvtauGp, new IntrinsicInfo(0x1e250000u, IntrinsicType.ScalarFPConvGpr));
- Add(Intrinsic.Arm64FcvtlV, new IntrinsicInfo(0x0e217800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64FcvtmsS, new IntrinsicInfo(0x5e21b800u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64FcvtmsV, new IntrinsicInfo(0x0e21b800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64FcvtmsGp, new IntrinsicInfo(0x1e300000u, IntrinsicType.ScalarFPConvGpr));
- Add(Intrinsic.Arm64FcvtmuS, new IntrinsicInfo(0x7e21b800u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64FcvtmuV, new IntrinsicInfo(0x2e21b800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64FcvtmuGp, new IntrinsicInfo(0x1e310000u, IntrinsicType.ScalarFPConvGpr));
- Add(Intrinsic.Arm64FcvtnsS, new IntrinsicInfo(0x5e21a800u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64FcvtnsV, new IntrinsicInfo(0x0e21a800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64FcvtnsGp, new IntrinsicInfo(0x1e200000u, IntrinsicType.ScalarFPConvGpr));
- Add(Intrinsic.Arm64FcvtnuS, new IntrinsicInfo(0x7e21a800u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64FcvtnuV, new IntrinsicInfo(0x2e21a800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64FcvtnuGp, new IntrinsicInfo(0x1e210000u, IntrinsicType.ScalarFPConvGpr));
- Add(Intrinsic.Arm64FcvtnV, new IntrinsicInfo(0x0e216800u, IntrinsicType.VectorBinaryRd));
- Add(Intrinsic.Arm64FcvtpsS, new IntrinsicInfo(0x5ea1a800u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64FcvtpsV, new IntrinsicInfo(0x0ea1a800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64FcvtpsGp, new IntrinsicInfo(0x1e280000u, IntrinsicType.ScalarFPConvGpr));
- Add(Intrinsic.Arm64FcvtpuS, new IntrinsicInfo(0x7ea1a800u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64FcvtpuV, new IntrinsicInfo(0x2ea1a800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64FcvtpuGp, new IntrinsicInfo(0x1e290000u, IntrinsicType.ScalarFPConvGpr));
- Add(Intrinsic.Arm64FcvtxnS, new IntrinsicInfo(0x7e216800u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64FcvtxnV, new IntrinsicInfo(0x2e216800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64FcvtzsSFixed, new IntrinsicInfo(0x5f00fc00u, IntrinsicType.ScalarFPConvFixed));
- Add(Intrinsic.Arm64FcvtzsVFixed, new IntrinsicInfo(0x0f00fc00u, IntrinsicType.VectorFPConvFixed));
- Add(Intrinsic.Arm64FcvtzsS, new IntrinsicInfo(0x5ea1b800u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64FcvtzsV, new IntrinsicInfo(0x0ea1b800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64FcvtzsGpFixed, new IntrinsicInfo(0x1e180000u, IntrinsicType.ScalarFPConvFixedGpr));
- Add(Intrinsic.Arm64FcvtzsGp, new IntrinsicInfo(0x1e380000u, IntrinsicType.ScalarFPConvGpr));
- Add(Intrinsic.Arm64FcvtzuSFixed, new IntrinsicInfo(0x7f00fc00u, IntrinsicType.ScalarFPConvFixed));
- Add(Intrinsic.Arm64FcvtzuVFixed, new IntrinsicInfo(0x2f00fc00u, IntrinsicType.VectorFPConvFixed));
- Add(Intrinsic.Arm64FcvtzuS, new IntrinsicInfo(0x7ea1b800u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64FcvtzuV, new IntrinsicInfo(0x2ea1b800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64FcvtzuGpFixed, new IntrinsicInfo(0x1e190000u, IntrinsicType.ScalarFPConvFixedGpr));
- Add(Intrinsic.Arm64FcvtzuGp, new IntrinsicInfo(0x1e390000u, IntrinsicType.ScalarFPConvGpr));
- Add(Intrinsic.Arm64FcvtS, new IntrinsicInfo(0x1e224000u, IntrinsicType.ScalarFPConv));
- Add(Intrinsic.Arm64FdivV, new IntrinsicInfo(0x2e20fc00u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64FdivS, new IntrinsicInfo(0x1e201800u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64FmaddS, new IntrinsicInfo(0x1f000000u, IntrinsicType.ScalarTernary));
- Add(Intrinsic.Arm64FmaxnmpS, new IntrinsicInfo(0x7e30c800u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64FmaxnmpV, new IntrinsicInfo(0x2e20c400u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64FmaxnmvV, new IntrinsicInfo(0x2e30c800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64FmaxnmV, new IntrinsicInfo(0x0e20c400u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64FmaxnmS, new IntrinsicInfo(0x1e206800u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64FmaxpS, new IntrinsicInfo(0x7e30f800u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64FmaxpV, new IntrinsicInfo(0x2e20f400u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64FmaxvV, new IntrinsicInfo(0x2e30f800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64FmaxV, new IntrinsicInfo(0x0e20f400u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64FmaxS, new IntrinsicInfo(0x1e204800u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64FminnmpS, new IntrinsicInfo(0x7eb0c800u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64FminnmpV, new IntrinsicInfo(0x2ea0c400u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64FminnmvV, new IntrinsicInfo(0x2eb0c800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64FminnmV, new IntrinsicInfo(0x0ea0c400u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64FminnmS, new IntrinsicInfo(0x1e207800u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64FminpS, new IntrinsicInfo(0x7eb0f800u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64FminpV, new IntrinsicInfo(0x2ea0f400u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64FminvV, new IntrinsicInfo(0x2eb0f800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64FminV, new IntrinsicInfo(0x0ea0f400u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64FminS, new IntrinsicInfo(0x1e205800u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64FmlaSe, new IntrinsicInfo(0x5f801000u, IntrinsicType.ScalarTernaryFPRdByElem));
- Add(Intrinsic.Arm64FmlaVe, new IntrinsicInfo(0x0f801000u, IntrinsicType.VectorTernaryFPRdByElem));
- Add(Intrinsic.Arm64FmlaV, new IntrinsicInfo(0x0e20cc00u, IntrinsicType.VectorTernaryRd));
- Add(Intrinsic.Arm64FmlsSe, new IntrinsicInfo(0x5f805000u, IntrinsicType.ScalarTernaryFPRdByElem));
- Add(Intrinsic.Arm64FmlsVe, new IntrinsicInfo(0x0f805000u, IntrinsicType.VectorTernaryFPRdByElem));
- Add(Intrinsic.Arm64FmlsV, new IntrinsicInfo(0x0ea0cc00u, IntrinsicType.VectorTernaryRd));
- Add(Intrinsic.Arm64FmovVi, new IntrinsicInfo(0x0f00f400u, IntrinsicType.VectorFmovi));
- Add(Intrinsic.Arm64FmovS, new IntrinsicInfo(0x1e204000u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64FmovGp, new IntrinsicInfo(0x1e260000u, IntrinsicType.ScalarFPConvGpr));
- Add(Intrinsic.Arm64FmovSi, new IntrinsicInfo(0x1e201000u, IntrinsicType.ScalarFmovi));
- Add(Intrinsic.Arm64FmsubS, new IntrinsicInfo(0x1f008000u, IntrinsicType.ScalarTernary));
- Add(Intrinsic.Arm64FmulxSe, new IntrinsicInfo(0x7f809000u, IntrinsicType.ScalarBinaryFPByElem));
- Add(Intrinsic.Arm64FmulxVe, new IntrinsicInfo(0x2f809000u, IntrinsicType.VectorBinaryFPByElem));
- Add(Intrinsic.Arm64FmulxS, new IntrinsicInfo(0x5e20dc00u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64FmulxV, new IntrinsicInfo(0x0e20dc00u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64FmulSe, new IntrinsicInfo(0x5f809000u, IntrinsicType.ScalarBinaryFPByElem));
- Add(Intrinsic.Arm64FmulVe, new IntrinsicInfo(0x0f809000u, IntrinsicType.VectorBinaryFPByElem));
- Add(Intrinsic.Arm64FmulV, new IntrinsicInfo(0x2e20dc00u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64FmulS, new IntrinsicInfo(0x1e200800u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64FnegV, new IntrinsicInfo(0x2ea0f800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64FnegS, new IntrinsicInfo(0x1e214000u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64FnmaddS, new IntrinsicInfo(0x1f200000u, IntrinsicType.ScalarTernary));
- Add(Intrinsic.Arm64FnmsubS, new IntrinsicInfo(0x1f208000u, IntrinsicType.ScalarTernary));
- Add(Intrinsic.Arm64FnmulS, new IntrinsicInfo(0x1e208800u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64FrecpeS, new IntrinsicInfo(0x5ea1d800u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64FrecpeV, new IntrinsicInfo(0x0ea1d800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64FrecpsS, new IntrinsicInfo(0x5e20fc00u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64FrecpsV, new IntrinsicInfo(0x0e20fc00u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64FrecpxS, new IntrinsicInfo(0x5ea1f800u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64FrintaV, new IntrinsicInfo(0x2e218800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64FrintaS, new IntrinsicInfo(0x1e264000u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64FrintiV, new IntrinsicInfo(0x2ea19800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64FrintiS, new IntrinsicInfo(0x1e27c000u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64FrintmV, new IntrinsicInfo(0x0e219800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64FrintmS, new IntrinsicInfo(0x1e254000u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64FrintnV, new IntrinsicInfo(0x0e218800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64FrintnS, new IntrinsicInfo(0x1e244000u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64FrintpV, new IntrinsicInfo(0x0ea18800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64FrintpS, new IntrinsicInfo(0x1e24c000u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64FrintxV, new IntrinsicInfo(0x2e219800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64FrintxS, new IntrinsicInfo(0x1e274000u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64FrintzV, new IntrinsicInfo(0x0ea19800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64FrintzS, new IntrinsicInfo(0x1e25c000u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64FrsqrteS, new IntrinsicInfo(0x7ea1d800u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64FrsqrteV, new IntrinsicInfo(0x2ea1d800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64FrsqrtsS, new IntrinsicInfo(0x5ea0fc00u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64FrsqrtsV, new IntrinsicInfo(0x0ea0fc00u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64FsqrtV, new IntrinsicInfo(0x2ea1f800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64FsqrtS, new IntrinsicInfo(0x1e21c000u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64FsubV, new IntrinsicInfo(0x0ea0d400u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64FsubS, new IntrinsicInfo(0x1e203800u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64InsVe, new IntrinsicInfo(0x6e000400u, IntrinsicType.VectorInsertByElem));
- Add(Intrinsic.Arm64InsGp, new IntrinsicInfo(0x4e001c00u, IntrinsicType.ScalarUnaryByElem));
- Add(Intrinsic.Arm64Ld1rV, new IntrinsicInfo(0x0d40c000u, IntrinsicType.VectorLdSt));
- Add(Intrinsic.Arm64Ld1Vms, new IntrinsicInfo(0x0c402000u, IntrinsicType.VectorLdSt));
- Add(Intrinsic.Arm64Ld1Vss, new IntrinsicInfo(0x0d400000u, IntrinsicType.VectorLdStSs));
- Add(Intrinsic.Arm64Ld2rV, new IntrinsicInfo(0x0d60c000u, IntrinsicType.VectorLdSt));
- Add(Intrinsic.Arm64Ld2Vms, new IntrinsicInfo(0x0c408000u, IntrinsicType.VectorLdSt));
- Add(Intrinsic.Arm64Ld2Vss, new IntrinsicInfo(0x0d600000u, IntrinsicType.VectorLdStSs));
- Add(Intrinsic.Arm64Ld3rV, new IntrinsicInfo(0x0d40e000u, IntrinsicType.VectorLdSt));
- Add(Intrinsic.Arm64Ld3Vms, new IntrinsicInfo(0x0c404000u, IntrinsicType.VectorLdSt));
- Add(Intrinsic.Arm64Ld3Vss, new IntrinsicInfo(0x0d402000u, IntrinsicType.VectorLdStSs));
- Add(Intrinsic.Arm64Ld4rV, new IntrinsicInfo(0x0d60e000u, IntrinsicType.VectorLdSt));
- Add(Intrinsic.Arm64Ld4Vms, new IntrinsicInfo(0x0c400000u, IntrinsicType.VectorLdSt));
- Add(Intrinsic.Arm64Ld4Vss, new IntrinsicInfo(0x0d602000u, IntrinsicType.VectorLdStSs));
- Add(Intrinsic.Arm64MlaVe, new IntrinsicInfo(0x2f000000u, IntrinsicType.VectorTernaryRdByElem));
- Add(Intrinsic.Arm64MlaV, new IntrinsicInfo(0x0e209400u, IntrinsicType.VectorTernaryRd));
- Add(Intrinsic.Arm64MlsVe, new IntrinsicInfo(0x2f004000u, IntrinsicType.VectorTernaryRdByElem));
- Add(Intrinsic.Arm64MlsV, new IntrinsicInfo(0x2e209400u, IntrinsicType.VectorTernaryRd));
- Add(Intrinsic.Arm64MoviV, new IntrinsicInfo(0x0f000400u, IntrinsicType.VectorMovi));
- Add(Intrinsic.Arm64MrsFpcr, new IntrinsicInfo(0xd53b4400u, IntrinsicType.GetRegister));
- Add(Intrinsic.Arm64MsrFpcr, new IntrinsicInfo(0xd51b4400u, IntrinsicType.SetRegister));
- Add(Intrinsic.Arm64MrsFpsr, new IntrinsicInfo(0xd53b4420u, IntrinsicType.GetRegister));
- Add(Intrinsic.Arm64MsrFpsr, new IntrinsicInfo(0xd51b4420u, IntrinsicType.SetRegister));
- Add(Intrinsic.Arm64MulVe, new IntrinsicInfo(0x0f008000u, IntrinsicType.VectorBinaryByElem));
- Add(Intrinsic.Arm64MulV, new IntrinsicInfo(0x0e209c00u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64MvniV, new IntrinsicInfo(0x2f000400u, IntrinsicType.VectorMvni));
- Add(Intrinsic.Arm64NegS, new IntrinsicInfo(0x7e20b800u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64NegV, new IntrinsicInfo(0x2e20b800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64NotV, new IntrinsicInfo(0x2e205800u, IntrinsicType.VectorUnaryBitwise));
- Add(Intrinsic.Arm64OrnV, new IntrinsicInfo(0x0ee01c00u, IntrinsicType.VectorBinaryBitwise));
- Add(Intrinsic.Arm64OrrVi, new IntrinsicInfo(0x0f001400u, IntrinsicType.VectorBinaryBitwiseImm));
- Add(Intrinsic.Arm64OrrV, new IntrinsicInfo(0x0ea01c00u, IntrinsicType.VectorBinaryBitwise));
- Add(Intrinsic.Arm64PmullV, new IntrinsicInfo(0x0e20e000u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64PmulV, new IntrinsicInfo(0x2e209c00u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64RaddhnV, new IntrinsicInfo(0x2e204000u, IntrinsicType.VectorTernaryRd));
- Add(Intrinsic.Arm64RbitV, new IntrinsicInfo(0x2e605800u, IntrinsicType.VectorUnaryBitwise));
- Add(Intrinsic.Arm64Rev16V, new IntrinsicInfo(0x0e201800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64Rev32V, new IntrinsicInfo(0x2e200800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64Rev64V, new IntrinsicInfo(0x0e200800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64RshrnV, new IntrinsicInfo(0x0f008c00u, IntrinsicType.VectorTernaryShrRd));
- Add(Intrinsic.Arm64RsubhnV, new IntrinsicInfo(0x2e206000u, IntrinsicType.VectorTernaryRd));
- Add(Intrinsic.Arm64SabalV, new IntrinsicInfo(0x0e205000u, IntrinsicType.VectorTernaryRd));
- Add(Intrinsic.Arm64SabaV, new IntrinsicInfo(0x0e207c00u, IntrinsicType.VectorTernaryRd));
- Add(Intrinsic.Arm64SabdlV, new IntrinsicInfo(0x0e207000u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64SabdV, new IntrinsicInfo(0x0e207400u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64SadalpV, new IntrinsicInfo(0x0e206800u, IntrinsicType.VectorBinaryRd));
- Add(Intrinsic.Arm64SaddlpV, new IntrinsicInfo(0x0e202800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64SaddlvV, new IntrinsicInfo(0x0e303800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64SaddlV, new IntrinsicInfo(0x0e200000u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64SaddwV, new IntrinsicInfo(0x0e201000u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64ScvtfSFixed, new IntrinsicInfo(0x5f00e400u, IntrinsicType.ScalarFPConvFixed));
- Add(Intrinsic.Arm64ScvtfVFixed, new IntrinsicInfo(0x0f00e400u, IntrinsicType.VectorFPConvFixed));
- Add(Intrinsic.Arm64ScvtfS, new IntrinsicInfo(0x5e21d800u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64ScvtfV, new IntrinsicInfo(0x0e21d800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64ScvtfGpFixed, new IntrinsicInfo(0x1e020000u, IntrinsicType.ScalarFPConvFixedGpr));
- Add(Intrinsic.Arm64ScvtfGp, new IntrinsicInfo(0x1e220000u, IntrinsicType.ScalarFPConvGpr));
- Add(Intrinsic.Arm64Sha1cV, new IntrinsicInfo(0x5e000000u, IntrinsicType.Vector128Binary));
- Add(Intrinsic.Arm64Sha1hV, new IntrinsicInfo(0x5e280800u, IntrinsicType.Vector128Unary));
- Add(Intrinsic.Arm64Sha1mV, new IntrinsicInfo(0x5e002000u, IntrinsicType.Vector128Binary));
- Add(Intrinsic.Arm64Sha1pV, new IntrinsicInfo(0x5e001000u, IntrinsicType.Vector128Binary));
- Add(Intrinsic.Arm64Sha1su0V, new IntrinsicInfo(0x5e003000u, IntrinsicType.Vector128Binary));
- Add(Intrinsic.Arm64Sha1su1V, new IntrinsicInfo(0x5e281800u, IntrinsicType.Vector128Unary));
- Add(Intrinsic.Arm64Sha256h2V, new IntrinsicInfo(0x5e005000u, IntrinsicType.Vector128Binary));
- Add(Intrinsic.Arm64Sha256hV, new IntrinsicInfo(0x5e004000u, IntrinsicType.Vector128Binary));
- Add(Intrinsic.Arm64Sha256su0V, new IntrinsicInfo(0x5e282800u, IntrinsicType.Vector128Unary));
- Add(Intrinsic.Arm64Sha256su1V, new IntrinsicInfo(0x5e006000u, IntrinsicType.Vector128Binary));
- Add(Intrinsic.Arm64ShaddV, new IntrinsicInfo(0x0e200400u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64ShllV, new IntrinsicInfo(0x2e213800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64ShlS, new IntrinsicInfo(0x5f005400u, IntrinsicType.ScalarBinaryShl));
- Add(Intrinsic.Arm64ShlV, new IntrinsicInfo(0x0f005400u, IntrinsicType.VectorBinaryShl));
- Add(Intrinsic.Arm64ShrnV, new IntrinsicInfo(0x0f008400u, IntrinsicType.VectorTernaryShrRd));
- Add(Intrinsic.Arm64ShsubV, new IntrinsicInfo(0x0e202400u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64SliS, new IntrinsicInfo(0x7f005400u, IntrinsicType.ScalarTernaryShlRd));
- Add(Intrinsic.Arm64SliV, new IntrinsicInfo(0x2f005400u, IntrinsicType.VectorTernaryShlRd));
- Add(Intrinsic.Arm64SmaxpV, new IntrinsicInfo(0x0e20a400u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64SmaxvV, new IntrinsicInfo(0x0e30a800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64SmaxV, new IntrinsicInfo(0x0e206400u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64SminpV, new IntrinsicInfo(0x0e20ac00u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64SminvV, new IntrinsicInfo(0x0e31a800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64SminV, new IntrinsicInfo(0x0e206c00u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64SmlalVe, new IntrinsicInfo(0x0f002000u, IntrinsicType.VectorTernaryRdByElem));
- Add(Intrinsic.Arm64SmlalV, new IntrinsicInfo(0x0e208000u, IntrinsicType.VectorTernaryRd));
- Add(Intrinsic.Arm64SmlslVe, new IntrinsicInfo(0x0f006000u, IntrinsicType.VectorTernaryRdByElem));
- Add(Intrinsic.Arm64SmlslV, new IntrinsicInfo(0x0e20a000u, IntrinsicType.VectorTernaryRd));
- Add(Intrinsic.Arm64SmovV, new IntrinsicInfo(0x0e002c00u, IntrinsicType.VectorUnaryByElem));
- Add(Intrinsic.Arm64SmullVe, new IntrinsicInfo(0x0f00a000u, IntrinsicType.VectorBinaryByElem));
- Add(Intrinsic.Arm64SmullV, new IntrinsicInfo(0x0e20c000u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64SqabsS, new IntrinsicInfo(0x5e207800u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64SqabsV, new IntrinsicInfo(0x0e207800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64SqaddS, new IntrinsicInfo(0x5e200c00u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64SqaddV, new IntrinsicInfo(0x0e200c00u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64SqdmlalSe, new IntrinsicInfo(0x5f003000u, IntrinsicType.ScalarBinaryByElem));
- Add(Intrinsic.Arm64SqdmlalVe, new IntrinsicInfo(0x0f003000u, IntrinsicType.VectorBinaryByElem));
- Add(Intrinsic.Arm64SqdmlalS, new IntrinsicInfo(0x5e209000u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64SqdmlalV, new IntrinsicInfo(0x0e209000u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64SqdmlslSe, new IntrinsicInfo(0x5f007000u, IntrinsicType.ScalarBinaryByElem));
- Add(Intrinsic.Arm64SqdmlslVe, new IntrinsicInfo(0x0f007000u, IntrinsicType.VectorBinaryByElem));
- Add(Intrinsic.Arm64SqdmlslS, new IntrinsicInfo(0x5e20b000u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64SqdmlslV, new IntrinsicInfo(0x0e20b000u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64SqdmulhSe, new IntrinsicInfo(0x5f00c000u, IntrinsicType.ScalarBinaryByElem));
- Add(Intrinsic.Arm64SqdmulhVe, new IntrinsicInfo(0x0f00c000u, IntrinsicType.VectorBinaryByElem));
- Add(Intrinsic.Arm64SqdmulhS, new IntrinsicInfo(0x5e20b400u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64SqdmulhV, new IntrinsicInfo(0x0e20b400u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64SqdmullSe, new IntrinsicInfo(0x5f00b000u, IntrinsicType.ScalarBinaryByElem));
- Add(Intrinsic.Arm64SqdmullVe, new IntrinsicInfo(0x0f00b000u, IntrinsicType.VectorBinaryByElem));
- Add(Intrinsic.Arm64SqdmullS, new IntrinsicInfo(0x5e20d000u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64SqdmullV, new IntrinsicInfo(0x0e20d000u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64SqnegS, new IntrinsicInfo(0x7e207800u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64SqnegV, new IntrinsicInfo(0x2e207800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64SqrdmulhSe, new IntrinsicInfo(0x5f00d000u, IntrinsicType.ScalarBinaryByElem));
- Add(Intrinsic.Arm64SqrdmulhVe, new IntrinsicInfo(0x0f00d000u, IntrinsicType.VectorBinaryByElem));
- Add(Intrinsic.Arm64SqrdmulhS, new IntrinsicInfo(0x7e20b400u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64SqrdmulhV, new IntrinsicInfo(0x2e20b400u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64SqrshlS, new IntrinsicInfo(0x5e205c00u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64SqrshlV, new IntrinsicInfo(0x0e205c00u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64SqrshrnS, new IntrinsicInfo(0x5f009c00u, IntrinsicType.ScalarTernaryShrRd));
- Add(Intrinsic.Arm64SqrshrnV, new IntrinsicInfo(0x0f009c00u, IntrinsicType.VectorTernaryShrRd));
- Add(Intrinsic.Arm64SqrshrunS, new IntrinsicInfo(0x7f008c00u, IntrinsicType.ScalarTernaryShrRd));
- Add(Intrinsic.Arm64SqrshrunV, new IntrinsicInfo(0x2f008c00u, IntrinsicType.VectorTernaryShrRd));
- Add(Intrinsic.Arm64SqshluS, new IntrinsicInfo(0x7f006400u, IntrinsicType.ScalarBinaryShl));
- Add(Intrinsic.Arm64SqshluV, new IntrinsicInfo(0x2f006400u, IntrinsicType.VectorBinaryShl));
- Add(Intrinsic.Arm64SqshlSi, new IntrinsicInfo(0x5f007400u, IntrinsicType.ScalarBinaryShl));
- Add(Intrinsic.Arm64SqshlVi, new IntrinsicInfo(0x0f007400u, IntrinsicType.VectorBinaryShl));
- Add(Intrinsic.Arm64SqshlS, new IntrinsicInfo(0x5e204c00u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64SqshlV, new IntrinsicInfo(0x0e204c00u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64SqshrnS, new IntrinsicInfo(0x5f009400u, IntrinsicType.ScalarTernaryShrRd));
- Add(Intrinsic.Arm64SqshrnV, new IntrinsicInfo(0x0f009400u, IntrinsicType.VectorTernaryShrRd));
- Add(Intrinsic.Arm64SqshrunS, new IntrinsicInfo(0x7f008400u, IntrinsicType.ScalarTernaryShrRd));
- Add(Intrinsic.Arm64SqshrunV, new IntrinsicInfo(0x2f008400u, IntrinsicType.VectorTernaryShrRd));
- Add(Intrinsic.Arm64SqsubS, new IntrinsicInfo(0x5e202c00u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64SqsubV, new IntrinsicInfo(0x0e202c00u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64SqxtnS, new IntrinsicInfo(0x5e214800u, IntrinsicType.ScalarBinaryRd));
- Add(Intrinsic.Arm64SqxtnV, new IntrinsicInfo(0x0e214800u, IntrinsicType.VectorBinaryRd));
- Add(Intrinsic.Arm64SqxtunS, new IntrinsicInfo(0x7e212800u, IntrinsicType.ScalarBinaryRd));
- Add(Intrinsic.Arm64SqxtunV, new IntrinsicInfo(0x2e212800u, IntrinsicType.VectorBinaryRd));
- Add(Intrinsic.Arm64SrhaddV, new IntrinsicInfo(0x0e201400u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64SriS, new IntrinsicInfo(0x7f004400u, IntrinsicType.ScalarTernaryShrRd));
- Add(Intrinsic.Arm64SriV, new IntrinsicInfo(0x2f004400u, IntrinsicType.VectorTernaryShrRd));
- Add(Intrinsic.Arm64SrshlS, new IntrinsicInfo(0x5e205400u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64SrshlV, new IntrinsicInfo(0x0e205400u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64SrshrS, new IntrinsicInfo(0x5f002400u, IntrinsicType.ScalarBinaryShr));
- Add(Intrinsic.Arm64SrshrV, new IntrinsicInfo(0x0f002400u, IntrinsicType.VectorBinaryShr));
- Add(Intrinsic.Arm64SrsraS, new IntrinsicInfo(0x5f003400u, IntrinsicType.ScalarTernaryShrRd));
- Add(Intrinsic.Arm64SrsraV, new IntrinsicInfo(0x0f003400u, IntrinsicType.VectorTernaryShrRd));
- Add(Intrinsic.Arm64SshllV, new IntrinsicInfo(0x0f00a400u, IntrinsicType.VectorBinaryShl));
- Add(Intrinsic.Arm64SshlS, new IntrinsicInfo(0x5e204400u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64SshlV, new IntrinsicInfo(0x0e204400u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64SshrS, new IntrinsicInfo(0x5f000400u, IntrinsicType.ScalarBinaryShr));
- Add(Intrinsic.Arm64SshrV, new IntrinsicInfo(0x0f000400u, IntrinsicType.VectorBinaryShr));
- Add(Intrinsic.Arm64SsraS, new IntrinsicInfo(0x5f001400u, IntrinsicType.ScalarTernaryShrRd));
- Add(Intrinsic.Arm64SsraV, new IntrinsicInfo(0x0f001400u, IntrinsicType.VectorTernaryShrRd));
- Add(Intrinsic.Arm64SsublV, new IntrinsicInfo(0x0e202000u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64SsubwV, new IntrinsicInfo(0x0e203000u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64St1Vms, new IntrinsicInfo(0x0c002000u, IntrinsicType.VectorLdSt));
- Add(Intrinsic.Arm64St1Vss, new IntrinsicInfo(0x0d000000u, IntrinsicType.VectorLdStSs));
- Add(Intrinsic.Arm64St2Vms, new IntrinsicInfo(0x0c008000u, IntrinsicType.VectorLdSt));
- Add(Intrinsic.Arm64St2Vss, new IntrinsicInfo(0x0d200000u, IntrinsicType.VectorLdStSs));
- Add(Intrinsic.Arm64St3Vms, new IntrinsicInfo(0x0c004000u, IntrinsicType.VectorLdSt));
- Add(Intrinsic.Arm64St3Vss, new IntrinsicInfo(0x0d002000u, IntrinsicType.VectorLdStSs));
- Add(Intrinsic.Arm64St4Vms, new IntrinsicInfo(0x0c000000u, IntrinsicType.VectorLdSt));
- Add(Intrinsic.Arm64St4Vss, new IntrinsicInfo(0x0d202000u, IntrinsicType.VectorLdStSs));
- Add(Intrinsic.Arm64SubhnV, new IntrinsicInfo(0x0e206000u, IntrinsicType.VectorTernaryRd));
- Add(Intrinsic.Arm64SubS, new IntrinsicInfo(0x7e208400u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64SubV, new IntrinsicInfo(0x2e208400u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64SuqaddS, new IntrinsicInfo(0x5e203800u, IntrinsicType.ScalarBinaryRd));
- Add(Intrinsic.Arm64SuqaddV, new IntrinsicInfo(0x0e203800u, IntrinsicType.VectorBinaryRd));
- Add(Intrinsic.Arm64TblV, new IntrinsicInfo(0x0e000000u, IntrinsicType.VectorLookupTable));
- Add(Intrinsic.Arm64TbxV, new IntrinsicInfo(0x0e001000u, IntrinsicType.VectorLookupTable));
- Add(Intrinsic.Arm64Trn1V, new IntrinsicInfo(0x0e002800u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64Trn2V, new IntrinsicInfo(0x0e006800u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64UabalV, new IntrinsicInfo(0x2e205000u, IntrinsicType.VectorTernaryRd));
- Add(Intrinsic.Arm64UabaV, new IntrinsicInfo(0x2e207c00u, IntrinsicType.VectorTernaryRd));
- Add(Intrinsic.Arm64UabdlV, new IntrinsicInfo(0x2e207000u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64UabdV, new IntrinsicInfo(0x2e207400u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64UadalpV, new IntrinsicInfo(0x2e206800u, IntrinsicType.VectorBinaryRd));
- Add(Intrinsic.Arm64UaddlpV, new IntrinsicInfo(0x2e202800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64UaddlvV, new IntrinsicInfo(0x2e303800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64UaddlV, new IntrinsicInfo(0x2e200000u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64UaddwV, new IntrinsicInfo(0x2e201000u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64UcvtfSFixed, new IntrinsicInfo(0x7f00e400u, IntrinsicType.ScalarFPConvFixed));
- Add(Intrinsic.Arm64UcvtfVFixed, new IntrinsicInfo(0x2f00e400u, IntrinsicType.VectorFPConvFixed));
- Add(Intrinsic.Arm64UcvtfS, new IntrinsicInfo(0x7e21d800u, IntrinsicType.ScalarUnary));
- Add(Intrinsic.Arm64UcvtfV, new IntrinsicInfo(0x2e21d800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64UcvtfGpFixed, new IntrinsicInfo(0x1e030000u, IntrinsicType.ScalarFPConvFixedGpr));
- Add(Intrinsic.Arm64UcvtfGp, new IntrinsicInfo(0x1e230000u, IntrinsicType.ScalarFPConvGpr));
- Add(Intrinsic.Arm64UhaddV, new IntrinsicInfo(0x2e200400u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64UhsubV, new IntrinsicInfo(0x2e202400u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64UmaxpV, new IntrinsicInfo(0x2e20a400u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64UmaxvV, new IntrinsicInfo(0x2e30a800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64UmaxV, new IntrinsicInfo(0x2e206400u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64UminpV, new IntrinsicInfo(0x2e20ac00u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64UminvV, new IntrinsicInfo(0x2e31a800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64UminV, new IntrinsicInfo(0x2e206c00u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64UmlalVe, new IntrinsicInfo(0x2f002000u, IntrinsicType.VectorTernaryRdByElem));
- Add(Intrinsic.Arm64UmlalV, new IntrinsicInfo(0x2e208000u, IntrinsicType.VectorTernaryRd));
- Add(Intrinsic.Arm64UmlslVe, new IntrinsicInfo(0x2f006000u, IntrinsicType.VectorTernaryRdByElem));
- Add(Intrinsic.Arm64UmlslV, new IntrinsicInfo(0x2e20a000u, IntrinsicType.VectorTernaryRd));
- Add(Intrinsic.Arm64UmovV, new IntrinsicInfo(0x0e003c00u, IntrinsicType.VectorUnaryByElem));
- Add(Intrinsic.Arm64UmullVe, new IntrinsicInfo(0x2f00a000u, IntrinsicType.VectorBinaryByElem));
- Add(Intrinsic.Arm64UmullV, new IntrinsicInfo(0x2e20c000u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64UqaddS, new IntrinsicInfo(0x7e200c00u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64UqaddV, new IntrinsicInfo(0x2e200c00u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64UqrshlS, new IntrinsicInfo(0x7e205c00u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64UqrshlV, new IntrinsicInfo(0x2e205c00u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64UqrshrnS, new IntrinsicInfo(0x7f009c00u, IntrinsicType.ScalarTernaryShrRd));
- Add(Intrinsic.Arm64UqrshrnV, new IntrinsicInfo(0x2f009c00u, IntrinsicType.VectorTernaryShrRd));
- Add(Intrinsic.Arm64UqshlSi, new IntrinsicInfo(0x7f007400u, IntrinsicType.ScalarBinaryShl));
- Add(Intrinsic.Arm64UqshlVi, new IntrinsicInfo(0x2f007400u, IntrinsicType.VectorBinaryShl));
- Add(Intrinsic.Arm64UqshlS, new IntrinsicInfo(0x7e204c00u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64UqshlV, new IntrinsicInfo(0x2e204c00u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64UqshrnS, new IntrinsicInfo(0x7f009400u, IntrinsicType.ScalarTernaryShrRd));
- Add(Intrinsic.Arm64UqshrnV, new IntrinsicInfo(0x2f009400u, IntrinsicType.VectorTernaryShrRd));
- Add(Intrinsic.Arm64UqsubS, new IntrinsicInfo(0x7e202c00u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64UqsubV, new IntrinsicInfo(0x2e202c00u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64UqxtnS, new IntrinsicInfo(0x7e214800u, IntrinsicType.ScalarBinaryRd));
- Add(Intrinsic.Arm64UqxtnV, new IntrinsicInfo(0x2e214800u, IntrinsicType.VectorBinaryRd));
- Add(Intrinsic.Arm64UrecpeV, new IntrinsicInfo(0x0ea1c800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64UrhaddV, new IntrinsicInfo(0x2e201400u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64UrshlS, new IntrinsicInfo(0x7e205400u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64UrshlV, new IntrinsicInfo(0x2e205400u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64UrshrS, new IntrinsicInfo(0x7f002400u, IntrinsicType.ScalarBinaryShr));
- Add(Intrinsic.Arm64UrshrV, new IntrinsicInfo(0x2f002400u, IntrinsicType.VectorBinaryShr));
- Add(Intrinsic.Arm64UrsqrteV, new IntrinsicInfo(0x2ea1c800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64UrsraS, new IntrinsicInfo(0x7f003400u, IntrinsicType.ScalarTernaryShrRd));
- Add(Intrinsic.Arm64UrsraV, new IntrinsicInfo(0x2f003400u, IntrinsicType.VectorTernaryShrRd));
- Add(Intrinsic.Arm64UshllV, new IntrinsicInfo(0x2f00a400u, IntrinsicType.VectorBinaryShl));
- Add(Intrinsic.Arm64UshlS, new IntrinsicInfo(0x7e204400u, IntrinsicType.ScalarBinary));
- Add(Intrinsic.Arm64UshlV, new IntrinsicInfo(0x2e204400u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64UshrS, new IntrinsicInfo(0x7f000400u, IntrinsicType.ScalarBinaryShr));
- Add(Intrinsic.Arm64UshrV, new IntrinsicInfo(0x2f000400u, IntrinsicType.VectorBinaryShr));
- Add(Intrinsic.Arm64UsqaddS, new IntrinsicInfo(0x7e203800u, IntrinsicType.ScalarBinaryRd));
- Add(Intrinsic.Arm64UsqaddV, new IntrinsicInfo(0x2e203800u, IntrinsicType.VectorBinaryRd));
- Add(Intrinsic.Arm64UsraS, new IntrinsicInfo(0x7f001400u, IntrinsicType.ScalarTernaryShrRd));
- Add(Intrinsic.Arm64UsraV, new IntrinsicInfo(0x2f001400u, IntrinsicType.VectorTernaryShrRd));
- Add(Intrinsic.Arm64UsublV, new IntrinsicInfo(0x2e202000u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64UsubwV, new IntrinsicInfo(0x2e203000u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64Uzp1V, new IntrinsicInfo(0x0e001800u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64Uzp2V, new IntrinsicInfo(0x0e005800u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64XtnV, new IntrinsicInfo(0x0e212800u, IntrinsicType.VectorUnary));
- Add(Intrinsic.Arm64Zip1V, new IntrinsicInfo(0x0e003800u, IntrinsicType.VectorBinary));
- Add(Intrinsic.Arm64Zip2V, new IntrinsicInfo(0x0e007800u, IntrinsicType.VectorBinary));
- }
-
- private static void Add(Intrinsic intrin, IntrinsicInfo info)
- {
- _intrinTable[(int)intrin] = info;
- }
-
- public static IntrinsicInfo GetInfo(Intrinsic intrin)
- {
- return _intrinTable[(int)intrin];
- }
- }
-} \ No newline at end of file
diff --git a/ARMeilleure/CodeGen/Arm64/IntrinsicType.cs b/ARMeilleure/CodeGen/Arm64/IntrinsicType.cs
deleted file mode 100644
index 800eca93..00000000
--- a/ARMeilleure/CodeGen/Arm64/IntrinsicType.cs
+++ /dev/null
@@ -1,59 +0,0 @@
-namespace ARMeilleure.CodeGen.Arm64
-{
- enum IntrinsicType
- {
- ScalarUnary,
- ScalarUnaryByElem,
- ScalarBinary,
- ScalarBinaryByElem,
- ScalarBinaryFPByElem,
- ScalarBinaryRd,
- ScalarBinaryShl,
- ScalarBinaryShr,
- ScalarFcsel,
- ScalarFmovi,
- ScalarFPCompare,
- ScalarFPCompareCond,
- ScalarFPConv,
- ScalarFPConvFixed,
- ScalarFPConvFixedGpr,
- ScalarFPConvGpr,
- ScalarTernary,
- ScalarTernaryFPRdByElem,
- ScalarTernaryShlRd,
- ScalarTernaryShrRd,
-
- VectorUnary,
- VectorUnaryBitwise,
- VectorUnaryByElem,
- VectorBinary,
- VectorBinaryBitwise,
- VectorBinaryBitwiseImm,
- VectorBinaryByElem,
- VectorBinaryFPByElem,
- VectorBinaryRd,
- VectorBinaryShl,
- VectorBinaryShr,
- VectorExt,
- VectorFmovi,
- VectorFPConvFixed,
- VectorInsertByElem,
- VectorLdSt,
- VectorLdStSs,
- VectorLookupTable,
- VectorMovi,
- VectorMvni,
- VectorTernaryFPRdByElem,
- VectorTernaryRd,
- VectorTernaryRdBitwise,
- VectorTernaryRdByElem,
- VectorTernaryShlRd,
- VectorTernaryShrRd,
-
- Vector128Unary,
- Vector128Binary,
-
- GetRegister,
- SetRegister
- }
-} \ No newline at end of file
diff --git a/ARMeilleure/CodeGen/Arm64/PreAllocator.cs b/ARMeilleure/CodeGen/Arm64/PreAllocator.cs
deleted file mode 100644
index 6ea9d239..00000000
--- a/ARMeilleure/CodeGen/Arm64/PreAllocator.cs
+++ /dev/null
@@ -1,892 +0,0 @@
-using ARMeilleure.CodeGen.RegisterAllocators;
-using ARMeilleure.IntermediateRepresentation;
-using ARMeilleure.Translation;
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
-using static ARMeilleure.IntermediateRepresentation.Operation.Factory;
-
-namespace ARMeilleure.CodeGen.Arm64
-{
- static class PreAllocator
- {
- private class ConstantDict
- {
- private readonly Dictionary<(ulong, OperandType), Operand> _constants;
-
- public ConstantDict()
- {
- _constants = new Dictionary<(ulong, OperandType), Operand>();
- }
-
- public void Add(ulong value, OperandType type, Operand local)
- {
- _constants.Add((value, type), local);
- }
-
- public bool TryGetValue(ulong value, OperandType type, out Operand local)
- {
- return _constants.TryGetValue((value, type), out local);
- }
- }
-
- public static void RunPass(CompilerContext cctx, StackAllocator stackAlloc, out int maxCallArgs)
- {
- maxCallArgs = -1;
-
- Span<Operation> buffer = default;
-
- Operand[] preservedArgs = new Operand[CallingConvention.GetArgumentsOnRegsCount()];
-
- for (BasicBlock block = cctx.Cfg.Blocks.First; block != null; block = block.ListNext)
- {
- ConstantDict constants = new ConstantDict();
-
- Operation nextNode;
-
- for (Operation node = block.Operations.First; node != default; node = nextNode)
- {
- nextNode = node.ListNext;
-
- if (node.Instruction == Instruction.Phi)
- {
- continue;
- }
-
- InsertConstantRegCopies(constants, block.Operations, node);
- InsertDestructiveRegCopies(block.Operations, node);
-
- switch (node.Instruction)
- {
- case Instruction.Call:
- // Get the maximum number of arguments used on a call.
- // On windows, when a struct is returned from the call,
- // we also need to pass the pointer where the struct
- // should be written on the first argument.
- int argsCount = node.SourcesCount - 1;
-
- if (node.Destination != default && node.Destination.Type == OperandType.V128)
- {
- argsCount++;
- }
-
- if (maxCallArgs < argsCount)
- {
- maxCallArgs = argsCount;
- }
-
- // Copy values to registers expected by the function
- // being called, as mandated by the ABI.
- InsertCallCopies(constants, block.Operations, node);
- break;
- case Instruction.CompareAndSwap:
- case Instruction.CompareAndSwap16:
- case Instruction.CompareAndSwap8:
- nextNode = GenerateCompareAndSwap(block.Operations, node);
- break;
- case Instruction.LoadArgument:
- nextNode = InsertLoadArgumentCopy(cctx, ref buffer, block.Operations, preservedArgs, node);
- break;
- case Instruction.Return:
- InsertReturnCopy(block.Operations, node);
- break;
- case Instruction.Tailcall:
- InsertTailcallCopies(constants, block.Operations, stackAlloc, node, node);
- break;
- }
- }
- }
- }
-
- private static void InsertConstantRegCopies(ConstantDict constants, IntrusiveList<Operation> nodes, Operation node)
- {
- if (node.SourcesCount == 0 || IsIntrinsicWithConst(node))
- {
- return;
- }
-
- Instruction inst = node.Instruction;
-
- Operand src1 = node.GetSource(0);
- Operand src2;
-
- if (src1.Kind == OperandKind.Constant)
- {
- if (!src1.Type.IsInteger())
- {
- // Handle non-integer types (FP32, FP64 and V128).
- // For instructions without an immediate operand, we do the following:
- // - Insert a copy with the constant value (as integer) to a GPR.
- // - Insert a copy from the GPR to a XMM register.
- // - Replace the constant use with the XMM register.
- src1 = AddFloatConstantCopy(constants, nodes, node, src1);
-
- node.SetSource(0, src1);
- }
- else if (!HasConstSrc1(node, src1.Value))
- {
- // Handle integer types.
- // Most ALU instructions accepts a 32-bits immediate on the second operand.
- // We need to ensure the following:
- // - If the constant is on operand 1, we need to move it.
- // -- But first, we try to swap operand 1 and 2 if the instruction is commutative.
- // -- Doing so may allow us to encode the constant as operand 2 and avoid a copy.
- // - If the constant is on operand 2, we check if the instruction supports it,
- // if not, we also add a copy. 64-bits constants are usually not supported.
- if (IsCommutative(node))
- {
- src2 = node.GetSource(1);
-
- Operand temp = src1;
-
- src1 = src2;
- src2 = temp;
-
- node.SetSource(0, src1);
- node.SetSource(1, src2);
- }
-
- if (src1.Kind == OperandKind.Constant)
- {
- src1 = AddIntConstantCopy(constants, nodes, node, src1);
-
- node.SetSource(0, src1);
- }
- }
- }
-
- if (node.SourcesCount < 2)
- {
- return;
- }
-
- src2 = node.GetSource(1);
-
- if (src2.Kind == OperandKind.Constant)
- {
- if (!src2.Type.IsInteger())
- {
- src2 = AddFloatConstantCopy(constants, nodes, node, src2);
-
- node.SetSource(1, src2);
- }
- else if (!HasConstSrc2(inst, src2))
- {
- src2 = AddIntConstantCopy(constants, nodes, node, src2);
-
- node.SetSource(1, src2);
- }
- }
-
- if (node.SourcesCount < 3 ||
- node.Instruction == Instruction.BranchIf ||
- node.Instruction == Instruction.Compare ||
- node.Instruction == Instruction.VectorInsert ||
- node.Instruction == Instruction.VectorInsert16 ||
- node.Instruction == Instruction.VectorInsert8)
- {
- return;
- }
-
- for (int srcIndex = 2; srcIndex < node.SourcesCount; srcIndex++)
- {
- Operand src = node.GetSource(srcIndex);
-
- if (src.Kind == OperandKind.Constant)
- {
- if (!src.Type.IsInteger())
- {
- src = AddFloatConstantCopy(constants, nodes, node, src);
-
- node.SetSource(srcIndex, src);
- }
- else
- {
- src = AddIntConstantCopy(constants, nodes, node, src);
-
- node.SetSource(srcIndex, src);
- }
- }
- }
- }
-
- private static void InsertDestructiveRegCopies(IntrusiveList<Operation> nodes, Operation node)
- {
- if (node.Destination == default || node.SourcesCount == 0)
- {
- return;
- }
-
- Operand dest = node.Destination;
- Operand src1 = node.GetSource(0);
-
- if (IsSameOperandDestSrc1(node) && src1.Kind == OperandKind.LocalVariable)
- {
- bool useNewLocal = false;
-
- for (int srcIndex = 1; srcIndex < node.SourcesCount; srcIndex++)
- {
- if (node.GetSource(srcIndex) == dest)
- {
- useNewLocal = true;
-
- break;
- }
- }
-
- if (useNewLocal)
- {
- // Dest is being used as some source already, we need to use a new
- // local to store the temporary value, otherwise the value on dest
- // local would be overwritten.
- Operand temp = Local(dest.Type);
-
- nodes.AddBefore(node, Operation(Instruction.Copy, temp, src1));
-
- node.SetSource(0, temp);
-
- nodes.AddAfter(node, Operation(Instruction.Copy, dest, temp));
-
- node.Destination = temp;
- }
- else
- {
- nodes.AddBefore(node, Operation(Instruction.Copy, dest, src1));
-
- node.SetSource(0, dest);
- }
- }
- }
-
- private static void InsertCallCopies(ConstantDict constants, IntrusiveList<Operation> nodes, Operation node)
- {
- Operation operation = node;
-
- Operand dest = operation.Destination;
-
- List<Operand> sources = new List<Operand>
- {
- operation.GetSource(0)
- };
-
- int argsCount = operation.SourcesCount - 1;
-
- int intMax = CallingConvention.GetArgumentsOnRegsCount();
- int vecMax = CallingConvention.GetArgumentsOnRegsCount();
-
- int intCount = 0;
- int vecCount = 0;
-
- int stackOffset = 0;
-
- for (int index = 0; index < argsCount; index++)
- {
- Operand source = operation.GetSource(index + 1);
-
- bool passOnReg;
-
- if (source.Type.IsInteger())
- {
- passOnReg = intCount < intMax;
- }
- else if (source.Type == OperandType.V128)
- {
- passOnReg = intCount + 1 < intMax;
- }
- else
- {
- passOnReg = vecCount < vecMax;
- }
-
- if (source.Type == OperandType.V128 && passOnReg)
- {
- // V128 is a struct, we pass each half on a GPR if possible.
- Operand argReg = Gpr(CallingConvention.GetIntArgumentRegister(intCount++), OperandType.I64);
- Operand argReg2 = Gpr(CallingConvention.GetIntArgumentRegister(intCount++), OperandType.I64);
-
- nodes.AddBefore(node, Operation(Instruction.VectorExtract, argReg, source, Const(0)));
- nodes.AddBefore(node, Operation(Instruction.VectorExtract, argReg2, source, Const(1)));
-
- continue;
- }
-
- if (passOnReg)
- {
- Operand argReg = source.Type.IsInteger()
- ? Gpr(CallingConvention.GetIntArgumentRegister(intCount++), source.Type)
- : Xmm(CallingConvention.GetVecArgumentRegister(vecCount++), source.Type);
-
- Operation copyOp = Operation(Instruction.Copy, argReg, source);
-
- InsertConstantRegCopies(constants, nodes, nodes.AddBefore(node, copyOp));
-
- sources.Add(argReg);
- }
- else
- {
- Operand offset = Const(stackOffset);
-
- Operation spillOp = Operation(Instruction.SpillArg, default, offset, source);
-
- InsertConstantRegCopies(constants, nodes, nodes.AddBefore(node, spillOp));
-
- stackOffset += source.Type.GetSizeInBytes();
- }
- }
-
- if (dest != default)
- {
- if (dest.Type == OperandType.V128)
- {
- Operand retLReg = Gpr(CallingConvention.GetIntReturnRegister(), OperandType.I64);
- Operand retHReg = Gpr(CallingConvention.GetIntReturnRegisterHigh(), OperandType.I64);
-
- node = nodes.AddAfter(node, Operation(Instruction.VectorCreateScalar, dest, retLReg));
- nodes.AddAfter(node, Operation(Instruction.VectorInsert, dest, dest, retHReg, Const(1)));
-
- operation.Destination = default;
- }
- else
- {
- Operand retReg = dest.Type.IsInteger()
- ? Gpr(CallingConvention.GetIntReturnRegister(), dest.Type)
- : Xmm(CallingConvention.GetVecReturnRegister(), dest.Type);
-
- Operation copyOp = Operation(Instruction.Copy, dest, retReg);
-
- nodes.AddAfter(node, copyOp);
-
- operation.Destination = retReg;
- }
- }
-
- operation.SetSources(sources.ToArray());
- }
-
- private static void InsertTailcallCopies(
- ConstantDict constants,
- IntrusiveList<Operation> nodes,
- StackAllocator stackAlloc,
- Operation node,
- Operation operation)
- {
- List<Operand> sources = new List<Operand>
- {
- operation.GetSource(0)
- };
-
- int argsCount = operation.SourcesCount - 1;
-
- int intMax = CallingConvention.GetArgumentsOnRegsCount();
- int vecMax = CallingConvention.GetArgumentsOnRegsCount();
-
- int intCount = 0;
- int vecCount = 0;
-
- // Handle arguments passed on registers.
- for (int index = 0; index < argsCount; index++)
- {
- Operand source = operation.GetSource(1 + index);
-
- bool passOnReg;
-
- if (source.Type.IsInteger())
- {
- passOnReg = intCount + 1 < intMax;
- }
- else
- {
- passOnReg = vecCount < vecMax;
- }
-
- if (source.Type == OperandType.V128 && passOnReg)
- {
- // V128 is a struct, we pass each half on a GPR if possible.
- Operand argReg = Gpr(CallingConvention.GetIntArgumentRegister(intCount++), OperandType.I64);
- Operand argReg2 = Gpr(CallingConvention.GetIntArgumentRegister(intCount++), OperandType.I64);
-
- nodes.AddBefore(node, Operation(Instruction.VectorExtract, argReg, source, Const(0)));
- nodes.AddBefore(node, Operation(Instruction.VectorExtract, argReg2, source, Const(1)));
-
- continue;
- }
-
- if (passOnReg)
- {
- Operand argReg = source.Type.IsInteger()
- ? Gpr(CallingConvention.GetIntArgumentRegister(intCount++), source.Type)
- : Xmm(CallingConvention.GetVecArgumentRegister(vecCount++), source.Type);
-
- Operation copyOp = Operation(Instruction.Copy, argReg, source);
-
- InsertConstantRegCopies(constants, nodes, nodes.AddBefore(node, copyOp));
-
- sources.Add(argReg);
- }
- else
- {
- throw new NotImplementedException("Spilling is not currently supported for tail calls. (too many arguments)");
- }
- }
-
- // The target address must be on the return registers, since we
- // don't return anything and it is guaranteed to not be a
- // callee saved register (which would be trashed on the epilogue).
- Operand tcAddress = Gpr(CodeGenCommon.TcAddressRegister, OperandType.I64);
-
- Operation addrCopyOp = Operation(Instruction.Copy, tcAddress, operation.GetSource(0));
-
- nodes.AddBefore(node, addrCopyOp);
-
- sources[0] = tcAddress;
-
- operation.SetSources(sources.ToArray());
- }
-
- private static Operation GenerateCompareAndSwap(IntrusiveList<Operation> nodes, Operation node)
- {
- Operand expected = node.GetSource(1);
-
- if (expected.Type == OperandType.V128)
- {
- Operand dest = node.Destination;
- Operand expectedLow = Local(OperandType.I64);
- Operand expectedHigh = Local(OperandType.I64);
- Operand desiredLow = Local(OperandType.I64);
- Operand desiredHigh = Local(OperandType.I64);
- Operand actualLow = Local(OperandType.I64);
- Operand actualHigh = Local(OperandType.I64);
-
- Operand address = node.GetSource(0);
- Operand desired = node.GetSource(2);
-
- void SplitOperand(Operand source, Operand low, Operand high)
- {
- nodes.AddBefore(node, Operation(Instruction.VectorExtract, low, source, Const(0)));
- nodes.AddBefore(node, Operation(Instruction.VectorExtract, high, source, Const(1)));
- }
-
- SplitOperand(expected, expectedLow, expectedHigh);
- SplitOperand(desired, desiredLow, desiredHigh);
-
- Operation operation = node;
-
- // Update the sources and destinations with split 64-bit halfs of the whole 128-bit values.
- // We also need a additional registers that will be used to store temporary information.
- operation.SetDestinations(new[] { actualLow, actualHigh, Local(OperandType.I64), Local(OperandType.I64) });
- operation.SetSources(new[] { address, expectedLow, expectedHigh, desiredLow, desiredHigh });
-
- // Add some dummy uses of the input operands, as the CAS operation will be a loop,
- // so they can't be used as destination operand.
- for (int i = 0; i < operation.SourcesCount; i++)
- {
- Operand src = operation.GetSource(i);
- node = nodes.AddAfter(node, Operation(Instruction.Copy, src, src));
- }
-
- // Assemble the vector with the 64-bit values at the given memory location.
- node = nodes.AddAfter(node, Operation(Instruction.VectorCreateScalar, dest, actualLow));
- node = nodes.AddAfter(node, Operation(Instruction.VectorInsert, dest, dest, actualHigh, Const(1)));
- }
- else
- {
- // We need a additional register where the store result will be written to.
- node.SetDestinations(new[] { node.Destination, Local(OperandType.I32) });
-
- // Add some dummy uses of the input operands, as the CAS operation will be a loop,
- // so they can't be used as destination operand.
- Operation operation = node;
-
- for (int i = 0; i < operation.SourcesCount; i++)
- {
- Operand src = operation.GetSource(i);
- node = nodes.AddAfter(node, Operation(Instruction.Copy, src, src));
- }
- }
-
- return node.ListNext;
- }
-
- private static void InsertReturnCopy(IntrusiveList<Operation> nodes, Operation node)
- {
- if (node.SourcesCount == 0)
- {
- return;
- }
-
- Operand source = node.GetSource(0);
-
- if (source.Type == OperandType.V128)
- {
- Operand retLReg = Gpr(CallingConvention.GetIntReturnRegister(), OperandType.I64);
- Operand retHReg = Gpr(CallingConvention.GetIntReturnRegisterHigh(), OperandType.I64);
-
- nodes.AddBefore(node, Operation(Instruction.VectorExtract, retLReg, source, Const(0)));
- nodes.AddBefore(node, Operation(Instruction.VectorExtract, retHReg, source, Const(1)));
- }
- else
- {
- Operand retReg = source.Type.IsInteger()
- ? Gpr(CallingConvention.GetIntReturnRegister(), source.Type)
- : Xmm(CallingConvention.GetVecReturnRegister(), source.Type);
-
- Operation retCopyOp = Operation(Instruction.Copy, retReg, source);
-
- nodes.AddBefore(node, retCopyOp);
- }
- }
-
- private static Operation InsertLoadArgumentCopy(
- CompilerContext cctx,
- ref Span<Operation> buffer,
- IntrusiveList<Operation> nodes,
- Operand[] preservedArgs,
- Operation node)
- {
- Operand source = node.GetSource(0);
-
- Debug.Assert(source.Kind == OperandKind.Constant, "Non-constant LoadArgument source kind.");
-
- int index = source.AsInt32();
-
- int intCount = 0;
- int vecCount = 0;
-
- for (int cIndex = 0; cIndex < index; cIndex++)
- {
- OperandType argType = cctx.FuncArgTypes[cIndex];
-
- if (argType.IsInteger())
- {
- intCount++;
- }
- else if (argType == OperandType.V128)
- {
- intCount += 2;
- }
- else
- {
- vecCount++;
- }
- }
-
- bool passOnReg;
-
- if (source.Type.IsInteger())
- {
- passOnReg = intCount < CallingConvention.GetArgumentsOnRegsCount();
- }
- else if (source.Type == OperandType.V128)
- {
- passOnReg = intCount + 1 < CallingConvention.GetArgumentsOnRegsCount();
- }
- else
- {
- passOnReg = vecCount < CallingConvention.GetArgumentsOnRegsCount();
- }
-
- if (passOnReg)
- {
- Operand dest = node.Destination;
-
- if (preservedArgs[index] == default)
- {
- if (dest.Type == OperandType.V128)
- {
- // V128 is a struct, we pass each half on a GPR if possible.
- Operand pArg = Local(OperandType.V128);
-
- Operand argLReg = Gpr(CallingConvention.GetIntArgumentRegister(intCount), OperandType.I64);
- Operand argHReg = Gpr(CallingConvention.GetIntArgumentRegister(intCount + 1), OperandType.I64);
-
- Operation copyL = Operation(Instruction.VectorCreateScalar, pArg, argLReg);
- Operation copyH = Operation(Instruction.VectorInsert, pArg, pArg, argHReg, Const(1));
-
- cctx.Cfg.Entry.Operations.AddFirst(copyH);
- cctx.Cfg.Entry.Operations.AddFirst(copyL);
-
- preservedArgs[index] = pArg;
- }
- else
- {
- Operand pArg = Local(dest.Type);
-
- Operand argReg = dest.Type.IsInteger()
- ? Gpr(CallingConvention.GetIntArgumentRegister(intCount), dest.Type)
- : Xmm(CallingConvention.GetVecArgumentRegister(vecCount), dest.Type);
-
- Operation copyOp = Operation(Instruction.Copy, pArg, argReg);
-
- cctx.Cfg.Entry.Operations.AddFirst(copyOp);
-
- preservedArgs[index] = pArg;
- }
- }
-
- Operation nextNode;
-
- if (dest.AssignmentsCount == 1)
- {
- // Let's propagate the argument if we can to avoid copies.
- PreAllocatorCommon.Propagate(ref buffer, dest, preservedArgs[index]);
- nextNode = node.ListNext;
- }
- else
- {
- Operation argCopyOp = Operation(Instruction.Copy, dest, preservedArgs[index]);
- nextNode = nodes.AddBefore(node, argCopyOp);
- }
-
- Delete(nodes, node);
- return nextNode;
- }
- else
- {
- // TODO: Pass on stack.
- return node;
- }
- }
-
- private static Operand AddFloatConstantCopy(
- ConstantDict constants,
- IntrusiveList<Operation> nodes,
- Operation node,
- Operand source)
- {
- Operand temp = Local(source.Type);
-
- Operand intConst = AddIntConstantCopy(constants, nodes, node, GetIntConst(source));
-
- Operation copyOp = Operation(Instruction.VectorCreateScalar, temp, intConst);
-
- nodes.AddBefore(node, copyOp);
-
- return temp;
- }
-
- private static Operand AddIntConstantCopy(
- ConstantDict constants,
- IntrusiveList<Operation> nodes,
- Operation node,
- Operand source)
- {
- if (constants.TryGetValue(source.Value, source.Type, out Operand temp))
- {
- return temp;
- }
-
- temp = Local(source.Type);
-
- Operation copyOp = Operation(Instruction.Copy, temp, source);
-
- nodes.AddBefore(node, copyOp);
-
- constants.Add(source.Value, source.Type, temp);
-
- return temp;
- }
-
- private static Operand GetIntConst(Operand value)
- {
- if (value.Type == OperandType.FP32)
- {
- return Const(value.AsInt32());
- }
- else if (value.Type == OperandType.FP64)
- {
- return Const(value.AsInt64());
- }
-
- return value;
- }
-
- private static void Delete(IntrusiveList<Operation> nodes, Operation node)
- {
- node.Destination = default;
-
- for (int index = 0; index < node.SourcesCount; index++)
- {
- node.SetSource(index, default);
- }
-
- nodes.Remove(node);
- }
-
- private static Operand Gpr(int register, OperandType type)
- {
- return Register(register, RegisterType.Integer, type);
- }
-
- private static Operand Xmm(int register, OperandType type)
- {
- return Register(register, RegisterType.Vector, type);
- }
-
- private static bool IsSameOperandDestSrc1(Operation operation)
- {
- switch (operation.Instruction)
- {
- case Instruction.Extended:
- return IsSameOperandDestSrc1(operation.Intrinsic);
- case Instruction.VectorInsert:
- case Instruction.VectorInsert16:
- case Instruction.VectorInsert8:
- return true;
- }
-
- return false;
- }
-
- private static bool IsSameOperandDestSrc1(Intrinsic intrinsic)
- {
- IntrinsicInfo info = IntrinsicTable.GetInfo(intrinsic & ~(Intrinsic.Arm64VTypeMask | Intrinsic.Arm64VSizeMask));
-
- return info.Type == IntrinsicType.ScalarBinaryRd ||
- info.Type == IntrinsicType.ScalarTernaryFPRdByElem ||
- info.Type == IntrinsicType.ScalarTernaryShlRd ||
- info.Type == IntrinsicType.ScalarTernaryShrRd ||
- info.Type == IntrinsicType.VectorBinaryRd ||
- info.Type == IntrinsicType.VectorInsertByElem ||
- info.Type == IntrinsicType.VectorTernaryRd ||
- info.Type == IntrinsicType.VectorTernaryRdBitwise ||
- info.Type == IntrinsicType.VectorTernaryFPRdByElem ||
- info.Type == IntrinsicType.VectorTernaryRdByElem ||
- info.Type == IntrinsicType.VectorTernaryShlRd ||
- info.Type == IntrinsicType.VectorTernaryShrRd;
- }
-
- private static bool HasConstSrc1(Operation node, ulong value)
- {
- switch (node.Instruction)
- {
- case Instruction.Add:
- case Instruction.BranchIf:
- case Instruction.Compare:
- case Instruction.Subtract:
- // The immediate encoding of those instructions does not allow Rn to be
- // XZR (it will be SP instead), so we can't allow a Rn constant in this case.
- return value == 0 && NotConstOrConst0(node.GetSource(1));
- case Instruction.BitwiseAnd:
- case Instruction.BitwiseExclusiveOr:
- case Instruction.BitwiseNot:
- case Instruction.BitwiseOr:
- case Instruction.ByteSwap:
- case Instruction.CountLeadingZeros:
- case Instruction.Multiply:
- case Instruction.Negate:
- case Instruction.RotateRight:
- case Instruction.ShiftLeft:
- case Instruction.ShiftRightSI:
- case Instruction.ShiftRightUI:
- return value == 0;
- case Instruction.Copy:
- case Instruction.LoadArgument:
- case Instruction.Spill:
- case Instruction.SpillArg:
- return true;
- case Instruction.Extended:
- return value == 0;
- }
-
- return false;
- }
-
- private static bool NotConstOrConst0(Operand operand)
- {
- return operand.Kind != OperandKind.Constant || operand.Value == 0;
- }
-
- private static bool HasConstSrc2(Instruction inst, Operand operand)
- {
- ulong value = operand.Value;
-
- switch (inst)
- {
- case Instruction.Add:
- case Instruction.BranchIf:
- case Instruction.Compare:
- case Instruction.Subtract:
- return ConstFitsOnUImm12Sh(value);
- case Instruction.BitwiseAnd:
- case Instruction.BitwiseExclusiveOr:
- case Instruction.BitwiseOr:
- return value == 0 || CodeGenCommon.TryEncodeBitMask(operand, out _, out _, out _);
- case Instruction.Multiply:
- case Instruction.Store:
- case Instruction.Store16:
- case Instruction.Store8:
- return value == 0;
- case Instruction.RotateRight:
- case Instruction.ShiftLeft:
- case Instruction.ShiftRightSI:
- case Instruction.ShiftRightUI:
- case Instruction.VectorExtract:
- case Instruction.VectorExtract16:
- case Instruction.VectorExtract8:
- return true;
- case Instruction.Extended:
- // TODO: Check if actual intrinsic is supposed to have consts here?
- // Right now we only hit this case for fixed-point int <-> FP conversion instructions.
- return true;
- }
-
- return false;
- }
-
- private static bool IsCommutative(Operation operation)
- {
- switch (operation.Instruction)
- {
- case Instruction.Add:
- case Instruction.BitwiseAnd:
- case Instruction.BitwiseExclusiveOr:
- case Instruction.BitwiseOr:
- case Instruction.Multiply:
- return true;
-
- case Instruction.BranchIf:
- case Instruction.Compare:
- {
- Operand comp = operation.GetSource(2);
-
- Debug.Assert(comp.Kind == OperandKind.Constant);
-
- var compType = (Comparison)comp.AsInt32();
-
- return compType == Comparison.Equal || compType == Comparison.NotEqual;
- }
- }
-
- return false;
- }
-
- private static bool ConstFitsOnUImm12Sh(ulong value)
- {
- return (value & ~0xfffUL) == 0 || (value & ~0xfff000UL) == 0;
- }
-
- private static bool IsIntrinsicWithConst(Operation operation)
- {
- bool isIntrinsic = IsIntrinsic(operation.Instruction);
-
- if (isIntrinsic)
- {
- Intrinsic intrinsic = operation.Intrinsic;
- IntrinsicInfo info = IntrinsicTable.GetInfo(intrinsic & ~(Intrinsic.Arm64VTypeMask | Intrinsic.Arm64VSizeMask));
-
- // Those have integer inputs that don't support consts.
- return info.Type != IntrinsicType.ScalarFPConvGpr &&
- info.Type != IntrinsicType.ScalarFPConvFixedGpr &&
- info.Type != IntrinsicType.SetRegister;
- }
-
- return false;
- }
-
- private static bool IsIntrinsic(Instruction inst)
- {
- return inst == Instruction.Extended;
- }
- }
-}