From b5c215111de665ef8d18b38405ac55e17996e30e Mon Sep 17 00:00:00 2001 From: LDj3SNuD <35856442+LDj3SNuD@users.noreply.github.com> Date: Thu, 17 Dec 2020 20:32:09 +0100 Subject: PPTC Follow-up. (#1712) * Added support for offline invalidation, via PPTC, of low cq translations replaced by high cq translations; both on a single run and between runs. Added invalidation of .cache files in the event of reuse on a different user operating system. Added .info and .cache files invalidation in case of a failed stream decompression. Nits. * InternalVersion = 1712; * Nits. * Address comment. * Get rid of BinaryFormatter. Nits. * Move Ptc.LoadTranslations(). Nits. * Nits. * Fixed corner cases (in case backup copies have to be used). Added save logs. * Not core fixes. * Complement to the previous commit. Added load logs. Removed BinaryFormatter leftovers. * Add LoadTranslations log. * Nits. * Removed the search and management of LowCq overlapping functions. * Final increment of .info and .cache flags. * Nit. * GetIndirectFunctionAddress(): Validate that writing actually takes place in dynamic table memory range (and not elsewhere). * Fix Ptc.UpdateInfo() due to rebase. * Nit for retrigger Checks. * Nit for retrigger Checks. --- ARMeilleure/Translation/PTC/PtcJumpTable.cs | 281 +++++++++++++++++++++++----- 1 file changed, 237 insertions(+), 44 deletions(-) (limited to 'ARMeilleure/Translation/PTC/PtcJumpTable.cs') diff --git a/ARMeilleure/Translation/PTC/PtcJumpTable.cs b/ARMeilleure/Translation/PTC/PtcJumpTable.cs index e4af68d6..d52d0874 100644 --- a/ARMeilleure/Translation/PTC/PtcJumpTable.cs +++ b/ARMeilleure/Translation/PTC/PtcJumpTable.cs @@ -2,15 +2,15 @@ using ARMeilleure.Translation.Cache; using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Diagnostics; +using System.IO; using System.Runtime.InteropServices; namespace ARMeilleure.Translation.PTC { - [Serializable] class PtcJumpTable { - [Serializable] - private struct TableEntry + public struct TableEntry { public int EntryIndex; public long GuestAddress; @@ -24,82 +24,270 @@ namespace ARMeilleure.Translation.PTC } } - private enum DirectHostAddress + public enum DirectHostAddress { - CallStub, - TailCallStub, - Host + CallStub = 0, + TailCallStub = 1, + Host = 2 } - private enum IndirectHostAddress + public enum IndirectHostAddress { - CallStub, - TailCallStub + CallStub = 0, + TailCallStub = 1 } private readonly List> _jumpTable; private readonly List> _dynamicTable; - private readonly List _targets; - private readonly Dictionary> _dependants; - private readonly Dictionary> _owners; - - public List Targets => _targets; - public Dictionary> Dependants => _dependants; - public Dictionary> Owners => _owners; + public List Targets { get; } + public Dictionary> Dependants { get; } + public Dictionary> Owners { get; } public PtcJumpTable() { _jumpTable = new List>(); _dynamicTable = new List>(); - _targets = new List(); - _dependants = new Dictionary>(); - _owners = new Dictionary>(); + Targets = new List(); + Dependants = new Dictionary>(); + Owners = new Dictionary>(); + } + + public PtcJumpTable( + List> jumpTable, List> dynamicTable, + List targets, Dictionary> dependants, Dictionary> owners) + { + _jumpTable = jumpTable; + _dynamicTable = dynamicTable; + + Targets = targets; + Dependants = dependants; + Owners = owners; + } + + public static PtcJumpTable Deserialize(MemoryStream stream) + { + using (BinaryReader reader = new BinaryReader(stream, EncodingCache.UTF8NoBOM, true)) + { + var jumpTable = new List>(); + + int jumpTableCount = reader.ReadInt32(); + + for (int i = 0; i < jumpTableCount; i++) + { + int entryIndex = reader.ReadInt32(); + long guestAddress = reader.ReadInt64(); + DirectHostAddress hostAddress = (DirectHostAddress)reader.ReadInt32(); + + jumpTable.Add(new TableEntry(entryIndex, guestAddress, hostAddress)); + } + + var dynamicTable = new List>(); + + int dynamicTableCount = reader.ReadInt32(); + + for (int i = 0; i < dynamicTableCount; i++) + { + int entryIndex = reader.ReadInt32(); + long guestAddress = reader.ReadInt64(); + IndirectHostAddress hostAddress = (IndirectHostAddress)reader.ReadInt32(); + + dynamicTable.Add(new TableEntry(entryIndex, guestAddress, hostAddress)); + } + + var targets = new List(); + + int targetsCount = reader.ReadInt32(); + + for (int i = 0; i < targetsCount; i++) + { + ulong address = reader.ReadUInt64(); + + targets.Add(address); + } + + var dependants = new Dictionary>(); + + int dependantsCount = reader.ReadInt32(); + + for (int i = 0; i < dependantsCount; i++) + { + ulong address = reader.ReadUInt64(); + + var entries = new List(); + + int entriesCount = reader.ReadInt32(); + + for (int j = 0; j < entriesCount; j++) + { + int entry = reader.ReadInt32(); + + entries.Add(entry); + } + + dependants.Add(address, entries); + } + + var owners = new Dictionary>(); + + int ownersCount = reader.ReadInt32(); + + for (int i = 0; i < ownersCount; i++) + { + ulong address = reader.ReadUInt64(); + + var entries = new List(); + + int entriesCount = reader.ReadInt32(); + + for (int j = 0; j < entriesCount; j++) + { + int entry = reader.ReadInt32(); + + entries.Add(entry); + } + + owners.Add(address, entries); + } + + return new PtcJumpTable(jumpTable, dynamicTable, targets, dependants, owners); + } + } + + public static void Serialize(MemoryStream stream, PtcJumpTable ptcJumpTable) + { + using (BinaryWriter writer = new BinaryWriter(stream, EncodingCache.UTF8NoBOM, true)) + { + writer.Write((int)ptcJumpTable._jumpTable.Count); + + foreach (var tableEntry in ptcJumpTable._jumpTable) + { + writer.Write((int)tableEntry.EntryIndex); + writer.Write((long)tableEntry.GuestAddress); + writer.Write((int)tableEntry.HostAddress); + } + + writer.Write((int)ptcJumpTable._dynamicTable.Count); + + foreach (var tableEntry in ptcJumpTable._dynamicTable) + { + writer.Write((int)tableEntry.EntryIndex); + writer.Write((long)tableEntry.GuestAddress); + writer.Write((int)tableEntry.HostAddress); + } + + writer.Write((int)ptcJumpTable.Targets.Count); + + foreach (ulong address in ptcJumpTable.Targets) + { + writer.Write((ulong)address); + } + + writer.Write((int)ptcJumpTable.Dependants.Count); + + foreach (var kv in ptcJumpTable.Dependants) + { + writer.Write((ulong)kv.Key); // address + + writer.Write((int)kv.Value.Count); + + foreach (int entry in kv.Value) + { + writer.Write((int)entry); + } + } + + writer.Write((int)ptcJumpTable.Owners.Count); + + foreach (var kv in ptcJumpTable.Owners) + { + writer.Write((ulong)kv.Key); // address + + writer.Write((int)kv.Value.Count); + + foreach (int entry in kv.Value) + { + writer.Write((int)entry); + } + } + } } public void Initialize(JumpTable jumpTable) { - _targets.Clear(); + Targets.Clear(); foreach (ulong guestAddress in jumpTable.Targets.Keys) { - _targets.Add(guestAddress); + Targets.Add(guestAddress); } - _dependants.Clear(); + Dependants.Clear(); - foreach (var item in jumpTable.Dependants) + foreach (var kv in jumpTable.Dependants) { - _dependants.Add(item.Key, new List(item.Value)); + Dependants.Add(kv.Key, new List(kv.Value)); } - _owners.Clear(); + Owners.Clear(); - foreach (var item in jumpTable.Owners) + foreach (var kv in jumpTable.Owners) { - _owners.Add(item.Key, new List(item.Value)); + Owners.Add(kv.Key, new List(kv.Value)); } } + // For future use. + public void Clean(ulong guestAddress) + { + if (Owners.TryGetValue(guestAddress, out List entries)) + { + foreach (int entry in entries) + { + if ((entry & JumpTable.DynamicEntryTag) == 0) + { + int removed = _jumpTable.RemoveAll(tableEntry => tableEntry.EntryIndex == entry); + + Debug.Assert(removed == 1); + } + else + { + if (JumpTable.DynamicTableElems > 1) + { + throw new NotSupportedException(); + } + + int removed = _dynamicTable.RemoveAll(tableEntry => tableEntry.EntryIndex == (entry & ~JumpTable.DynamicEntryTag)); + + Debug.Assert(removed == 1); + } + } + } + + Targets.Remove(guestAddress); + Dependants.Remove(guestAddress); + Owners.Remove(guestAddress); + } + public void Clear() { _jumpTable.Clear(); _dynamicTable.Clear(); - _targets.Clear(); - _dependants.Clear(); - _owners.Clear(); + Targets.Clear(); + Dependants.Clear(); + Owners.Clear(); } public void WriteJumpTable(JumpTable jumpTable, ConcurrentDictionary funcs) { - // Writes internal state to jump table in-memory, after PTC was loaded. + // Writes internal state to jump table in-memory, after PtcJumpTable was deserialized. - foreach (var item in _jumpTable) + foreach (var tableEntry in _jumpTable) { - long guestAddress = item.GuestAddress; - DirectHostAddress directHostAddress = item.HostAddress; + long guestAddress = tableEntry.GuestAddress; + DirectHostAddress directHostAddress = tableEntry.HostAddress; long hostAddress; @@ -119,7 +307,12 @@ namespace ARMeilleure.Translation.PTC } else { - throw new KeyNotFoundException($"({nameof(guestAddress)} = 0x{(ulong)guestAddress:X16})"); + if (!PtcProfiler.ProfiledFuncs.TryGetValue((ulong)guestAddress, out var value) || !value.highCq) + { + throw new KeyNotFoundException($"({nameof(guestAddress)} = 0x{(ulong)guestAddress:X16})"); + } + + hostAddress = 0L; } } else @@ -127,7 +320,7 @@ namespace ARMeilleure.Translation.PTC throw new InvalidOperationException(nameof(directHostAddress)); } - int entry = item.EntryIndex; + int entry = tableEntry.EntryIndex; jumpTable.Table.SetEntry(entry); jumpTable.ExpandIfNeededJumpTable(entry); @@ -141,17 +334,17 @@ namespace ARMeilleure.Translation.PTC public void WriteDynamicTable(JumpTable jumpTable) { - // Writes internal state to jump table in-memory, after PTC was loaded. + // Writes internal state to jump table in-memory, after PtcJumpTable was deserialized. if (JumpTable.DynamicTableElems > 1) { throw new NotSupportedException(); } - foreach (var item in _dynamicTable) + foreach (var tableEntry in _dynamicTable) { - long guestAddress = item.GuestAddress; - IndirectHostAddress indirectHostAddress = item.HostAddress; + long guestAddress = tableEntry.GuestAddress; + IndirectHostAddress indirectHostAddress = tableEntry.HostAddress; long hostAddress; @@ -168,7 +361,7 @@ namespace ARMeilleure.Translation.PTC throw new InvalidOperationException(nameof(indirectHostAddress)); } - int entry = item.EntryIndex; + int entry = tableEntry.EntryIndex; jumpTable.DynTable.SetEntry(entry); jumpTable.ExpandIfNeededDynamicTable(entry); @@ -182,7 +375,7 @@ namespace ARMeilleure.Translation.PTC public void ReadJumpTable(JumpTable jumpTable) { - // Reads in-memory jump table state and store internally for PTC serialization. + // Reads in-memory jump table state and store internally for PtcJumpTable serialization. _jumpTable.Clear(); @@ -216,7 +409,7 @@ namespace ARMeilleure.Translation.PTC public void ReadDynamicTable(JumpTable jumpTable) { - // Reads in-memory jump table state and store internally for PTC serialization. + // Reads in-memory jump table state and store internally for PtcJumpTable serialization. if (JumpTable.DynamicTableElems > 1) { -- cgit v1.2.3