PPTC Follow-up. (#1712)

* Added support for offline invalidation, via PPTC, of low cq translations replaced by high cq translations; both on a single run and between runs.

Added invalidation of .cache files in the event of reuse on a different user operating system.

Added .info and .cache files invalidation in case of a failed stream decompression.

Nits.

* InternalVersion = 1712;

* Nits.

* Address comment.

* Get rid of BinaryFormatter.

Nits.

* Move Ptc.LoadTranslations().

Nits.

* Nits.

* Fixed corner cases (in case backup copies have to be used). Added save logs.

* Not core fixes.

* Complement to the previous commit. Added load logs. Removed BinaryFormatter leftovers.

* Add LoadTranslations log.

* Nits.

* Removed the search and management of LowCq overlapping functions.

* Final increment of .info and .cache flags.

* Nit.

* GetIndirectFunctionAddress(): Validate that writing actually takes place in dynamic table memory range (and not elsewhere).

* Fix Ptc.UpdateInfo() due to rebase.

* Nit for retrigger Checks.

* Nit for retrigger Checks.

1 files changed, 138 insertions, 21 deletions

diff --git a/ARMeilleure/Translation/PTC/PtcProfiler.cs b/ARMeilleure/Translation/PTC/PtcProfiler.cs
index dbb3ed9d..bc9814ec 100644
--- a/ARMeilleure/Translation/PTC/PtcProfiler.cs
+++ b/ARMeilleure/Translation/PTC/PtcProfiler.cs
@@ -1,10 +1,11 @@
 using ARMeilleure.State;
+using Ryujinx.Common.Logging;
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
 using System.IO.Compression;
-using System.Runtime.Serialization.Formatters.Binary;
 using System.Security.Cryptography;
 using System.Threading;
 
@@ -12,12 +13,14 @@ namespace ARMeilleure.Translation.PTC
 {
     public static class PtcProfiler
     {
+        private const string HeaderMagic = "Phd";
+
+        private const uint InternalVersion = 1713; //! Not to be incremented manually for each change to the ARMeilleure project.
+
         private const int SaveInterval = 30; // Seconds.
 
         private const CompressionLevel SaveCompressionLevel = CompressionLevel.Fastest;
 
-        private static readonly BinaryFormatter _binaryFormatter;
-
         private static readonly System.Timers.Timer _timer;
 
         private static readonly ManualResetEvent _waitEvent;
@@ -26,17 +29,15 @@ namespace ARMeilleure.Translation.PTC
 
         private static bool _disposed;
 
-        internal static Dictionary<ulong, (ExecutionMode mode, bool highCq)> ProfiledFuncs { get; private set; } //! Not to be modified.
+        internal static Dictionary<ulong, (ExecutionMode mode, bool highCq)> ProfiledFuncs { get; private set; }
 
         internal static bool Enabled { get; private set; }
 
         public static ulong StaticCodeStart { internal get; set; }
-        public static int   StaticCodeSize  { internal get; set; }
+        public static ulong StaticCodeSize  { internal get; set; }
 
         static PtcProfiler()
         {
-            _binaryFormatter = new BinaryFormatter();
-
             _timer = new System.Timers.Timer((double)SaveInterval * 1000d);
             _timer.Elapsed += PreSave;
 
@@ -55,11 +56,11 @@ namespace ARMeilleure.Translation.PTC
         {
             if (IsAddressInStaticCodeRange(address))
             {
+                Debug.Assert(!highCq);
+
                 lock (_lock)
                 {
-                    Debug.Assert(!highCq && !ProfiledFuncs.ContainsKey(address));
-
-                    ProfiledFuncs.TryAdd(address, (mode, highCq));
+                    ProfiledFuncs.TryAdd(address, (mode, highCq: false));
                 }
             }
         }
@@ -68,18 +69,35 @@ namespace ARMeilleure.Translation.PTC
         {
             if (IsAddressInStaticCodeRange(address))
             {
+                Debug.Assert(highCq);
+
                 lock (_lock)
                 {
-                    Debug.Assert(highCq && ProfiledFuncs.ContainsKey(address));
+                    Debug.Assert(ProfiledFuncs.ContainsKey(address));
 
-                    ProfiledFuncs[address] = (mode, highCq);
+                    ProfiledFuncs[address] = (mode, highCq: true);
                 }
             }
         }
 
         internal static bool IsAddressInStaticCodeRange(ulong address)
         {
-            return address >= StaticCodeStart && address < StaticCodeStart + (ulong)StaticCodeSize;
+            return address >= StaticCodeStart && address < StaticCodeStart + StaticCodeSize;
+        }
+
+        internal static Dictionary<ulong, (ExecutionMode mode, bool highCq)> GetProfiledFuncsToTranslate(ConcurrentDictionary<ulong, TranslatedFunction> funcs)
+        {
+            var profiledFuncsToTranslate = new Dictionary<ulong, (ExecutionMode mode, bool highCq)>(ProfiledFuncs);
+
+            foreach (ulong address in profiledFuncsToTranslate.Keys)
+            {
+                if (funcs.ContainsKey(address))
+                {
+                    profiledFuncsToTranslate.Remove(address);
+                }
+            }
+
+            return profiledFuncsToTranslate;
         }
 
         internal static void ClearEntries()
@@ -97,21 +115,21 @@ namespace ARMeilleure.Translation.PTC
 
             if (fileInfoActual.Exists && fileInfoActual.Length != 0L)
             {
-                if (!Load(fileNameActual))
+                if (!Load(fileNameActual, false))
                 {
                     if (fileInfoBackup.Exists && fileInfoBackup.Length != 0L)
                     {
-                        Load(fileNameBackup);
+                        Load(fileNameBackup, true);
                     }
                 }
             }
             else if (fileInfoBackup.Exists && fileInfoBackup.Length != 0L)
             {
-                Load(fileNameBackup);
+                Load(fileNameBackup, true);
             }
         }
 
-        private static bool Load(string fileName)
+        private static bool Load(string fileName, bool isBackup)
         {
             using (FileStream compressedStream = new FileStream(fileName, FileMode.Open))
             using (DeflateStream deflateStream = new DeflateStream(compressedStream, CompressionMode.Decompress, true))
@@ -147,9 +165,25 @@ namespace ARMeilleure.Translation.PTC
 
                 stream.Seek((long)hashSize, SeekOrigin.Begin);
 
+                Header header = ReadHeader(stream);
+
+                if (header.Magic != HeaderMagic)
+                {
+                    InvalidateCompressedStream(compressedStream);
+
+                    return false;
+                }
+
+                if (header.InfoFileVersion != InternalVersion)
+                {
+                    InvalidateCompressedStream(compressedStream);
+
+                    return false;
+                }
+
                 try
                 {
-                    ProfiledFuncs = (Dictionary<ulong, (ExecutionMode, bool)>)_binaryFormatter.Deserialize(stream);
+                    ProfiledFuncs = Deserialize(stream);
                 }
                 catch
                 {
@@ -159,9 +193,13 @@ namespace ARMeilleure.Translation.PTC
 
                     return false;
                 }
-
-                return true;
             }
+
+            long fileSize = new FileInfo(fileName).Length;
+
+            Logger.Info?.Print(LogClass.Ptc, $"{(isBackup ? "Loaded Backup Profiling Info" : "Loaded Profiling Info")} (size: {fileSize} bytes, profiled functions: {ProfiledFuncs.Count}).");
+
+            return true;
         }
 
         private static bool CompareHash(ReadOnlySpan<byte> currentHash, ReadOnlySpan<byte> expectedHash)
@@ -169,6 +207,42 @@ namespace ARMeilleure.Translation.PTC
             return currentHash.SequenceEqual(expectedHash);
         }
 
+        private static Header ReadHeader(MemoryStream stream)
+        {
+            using (BinaryReader headerReader = new BinaryReader(stream, EncodingCache.UTF8NoBOM, true))
+            {
+                Header header = new Header();
+
+                header.Magic = headerReader.ReadString();
+
+                header.InfoFileVersion = headerReader.ReadUInt32();
+
+                return header;
+            }
+        }
+
+        private static Dictionary<ulong, (ExecutionMode, bool)> Deserialize(MemoryStream stream)
+        {
+            using (BinaryReader reader = new BinaryReader(stream, EncodingCache.UTF8NoBOM, true))
+            {
+                var profiledFuncs = new Dictionary<ulong, (ExecutionMode, bool)>();
+
+                int profiledFuncsCount = reader.ReadInt32();
+
+                for (int i = 0; i < profiledFuncsCount; i++)
+                {
+                    ulong address = reader.ReadUInt64();
+
+                    ExecutionMode mode = (ExecutionMode)reader.ReadInt32();
+                    bool highCq = reader.ReadBoolean();
+
+                    profiledFuncs.Add(address, (mode, highCq));
+                }
+
+                return profiledFuncs;
+            }
+        }
+
         private static void InvalidateCompressedStream(FileStream compressedStream)
         {
             compressedStream.SetLength(0L);
@@ -195,6 +269,8 @@ namespace ARMeilleure.Translation.PTC
 
         private static void Save(string fileName)
         {
+            int profiledFuncsCount;
+
             using (MemoryStream stream = new MemoryStream())
             using (MD5 md5 = MD5.Create())
             {
@@ -202,9 +278,13 @@ namespace ARMeilleure.Translation.PTC
 
                 stream.Seek((long)hashSize, SeekOrigin.Begin);
 
+                WriteHeader(stream);
+
                 lock (_lock)
                 {
-                    _binaryFormatter.Serialize(stream, ProfiledFuncs);
+                    Serialize(stream, ProfiledFuncs);
+
+                    profiledFuncsCount = ProfiledFuncs.Count;
                 }
 
                 stream.Seek((long)hashSize, SeekOrigin.Begin);
@@ -231,6 +311,43 @@ namespace ARMeilleure.Translation.PTC
                     }
                 }
             }
+
+            long fileSize = new FileInfo(fileName).Length;
+
+            Logger.Info?.Print(LogClass.Ptc, $"Saved Profiling Info (size: {fileSize} bytes, profiled functions: {profiledFuncsCount}).");
+        }
+
+        private static void WriteHeader(MemoryStream stream)
+        {
+            using (BinaryWriter headerWriter = new BinaryWriter(stream, EncodingCache.UTF8NoBOM, true))
+            {
+                headerWriter.Write((string)HeaderMagic); // Header.Magic
+
+                headerWriter.Write((uint)InternalVersion); // Header.InfoFileVersion
+            }
+        }
+
+        private static void Serialize(MemoryStream stream, Dictionary<ulong, (ExecutionMode mode, bool highCq)> profiledFuncs)
+        {
+            using (BinaryWriter writer = new BinaryWriter(stream, EncodingCache.UTF8NoBOM, true))
+            {
+                writer.Write((int)profiledFuncs.Count);
+
+                foreach (var kv in profiledFuncs)
+                {
+                    writer.Write((ulong)kv.Key); // address
+
+                    writer.Write((int)kv.Value.mode);
+                    writer.Write((bool)kv.Value.highCq);
+                }
+            }
+        }
+
+        private struct Header
+        {
+            public string Magic;
+
+            public uint InfoFileVersion;
         }
 
         internal static void Start()