From f6d88558b1780df25088042771a75ab174f0a06c Mon Sep 17 00:00:00 2001 From: Mary Date: Tue, 1 Dec 2020 22:48:31 +0100 Subject: salieri: Fix missing guest GPU accessor missing on hashes (#1759) This adds the guest GPU accessor to hashes computation. As this change all the hashes from the cache, I added some migration logic. This is required for #1755. --- Ryujinx.Graphics.Gpu/Shader/Cache/CacheHelper.cs | 482 +++++++++++++++++++++++ 1 file changed, 482 insertions(+) create mode 100644 Ryujinx.Graphics.Gpu/Shader/Cache/CacheHelper.cs (limited to 'Ryujinx.Graphics.Gpu/Shader/Cache/CacheHelper.cs') diff --git a/Ryujinx.Graphics.Gpu/Shader/Cache/CacheHelper.cs b/Ryujinx.Graphics.Gpu/Shader/Cache/CacheHelper.cs new file mode 100644 index 00000000..1d492214 --- /dev/null +++ b/Ryujinx.Graphics.Gpu/Shader/Cache/CacheHelper.cs @@ -0,0 +1,482 @@ +using Ryujinx.Common; +using Ryujinx.Common.Configuration; +using Ryujinx.Common.Logging; +using Ryujinx.Graphics.GAL; +using Ryujinx.Graphics.Gpu.Memory; +using Ryujinx.Graphics.Gpu.Shader.Cache.Definition; +using Ryujinx.Graphics.Shader; +using Ryujinx.Graphics.Shader.Translation; +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace Ryujinx.Graphics.Gpu.Shader.Cache +{ + /// + /// Helper to manipulate the disk shader cache. + /// + static class CacheHelper + { + /// + /// Try to read the manifest header from a given file path. + /// + /// The path to the manifest file + /// The manifest header read + /// Return true if the manifest header was read + public static bool TryReadManifestHeader(string manifestPath, out CacheManifestHeader header) + { + header = default; + + if (File.Exists(manifestPath)) + { + Memory rawManifest = File.ReadAllBytes(manifestPath); + + if (MemoryMarshal.TryRead(rawManifest.Span, out header)) + { + return true; + } + } + + return false; + } + + /// + /// Try to read the manifest from a given file path. + /// + /// The path to the manifest file + /// The graphics api used by the cache + /// The hash type of the cache + /// The manifest header read + /// The entries read from the cache manifest + /// Return true if the manifest was read + public static bool TryReadManifestFile(string manifestPath, CacheGraphicsApi graphicsApi, CacheHashType hashType, out CacheManifestHeader header, out HashSet entries) + { + header = default; + entries = new HashSet(); + + if (File.Exists(manifestPath)) + { + Memory rawManifest = File.ReadAllBytes(manifestPath); + + if (MemoryMarshal.TryRead(rawManifest.Span, out header)) + { + Memory hashTableRaw = rawManifest.Slice(Unsafe.SizeOf()); + + bool isValid = header.IsValid(graphicsApi, hashType, hashTableRaw.Span); + + if (isValid) + { + ReadOnlySpan hashTable = MemoryMarshal.Cast(hashTableRaw.Span); + + foreach (Hash128 hash in hashTable) + { + entries.Add(hash); + } + } + + return isValid; + } + } + + return false; + } + + /// + /// Compute a cache manifest from runtime data. + /// + /// The version of the cache + /// The graphics api used by the cache + /// The hash type of the cache + /// The entries in the cache + /// The cache manifest from runtime data + public static byte[] ComputeManifest(ulong version, CacheGraphicsApi graphicsApi, CacheHashType hashType, HashSet entries) + { + if (hashType != CacheHashType.XxHash128) + { + throw new NotImplementedException($"{hashType}"); + } + + CacheManifestHeader manifestHeader = new CacheManifestHeader(version, graphicsApi, hashType); + + byte[] data = new byte[Unsafe.SizeOf() + entries.Count * Unsafe.SizeOf()]; + + // CacheManifestHeader has the same size as a Hash128. + Span dataSpan = MemoryMarshal.Cast(data.AsSpan()).Slice(1); + + int i = 0; + + foreach (Hash128 hash in entries) + { + dataSpan[i++] = hash; + } + + manifestHeader.UpdateChecksum(data.AsSpan().Slice(Unsafe.SizeOf())); + + MemoryMarshal.Write(data, ref manifestHeader); + + return data; + } + + /// + /// Get the base directory of the shader cache for a given title id. + /// + /// The title id of the target application + /// The base directory of the shader cache for a given title id + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static string GetBaseCacheDirectory(string titleId) => Path.Combine(AppDataManager.GamesDirPath, titleId, "cache", "shader"); + + /// + /// Get the temp path to the cache data directory. + /// + /// The cache directory + /// The temp path to the cache data directory + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static string GetCacheTempDataPath(string cacheDirectory) => Path.Combine(cacheDirectory, "temp"); + + /// + /// The path to the cache archive file. + /// + /// The cache directory + /// The path to the cache archive file + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static string GetArchivePath(string cacheDirectory) => Path.Combine(cacheDirectory, "cache.zip"); + + /// + /// The path to the cache manifest file. + /// + /// The cache directory + /// The path to the cache manifest file + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static string GetManifestPath(string cacheDirectory) => Path.Combine(cacheDirectory, "cache.info"); + + /// + /// Create a new temp path to the given cached file via its hash. + /// + /// The cache directory + /// The hash of the cached data + /// New path to the given cached file + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static string GenCacheTempFilePath(string cacheDirectory, Hash128 key) => Path.Combine(GetCacheTempDataPath(cacheDirectory), key.ToString()); + + /// + /// Generate the path to the cache directory. + /// + /// The base of the cache directory + /// The graphics api in use + /// The name of the shader provider in use + /// The name of the cache + /// The path to the cache directory + public static string GenerateCachePath(string baseCacheDirectory, CacheGraphicsApi graphicsApi, string shaderProvider, string cacheName) + { + string graphicsApiName = graphicsApi switch + { + CacheGraphicsApi.OpenGL => "opengl", + CacheGraphicsApi.OpenGLES => "opengles", + CacheGraphicsApi.Vulkan => "vulkan", + CacheGraphicsApi.DirectX => "directx", + CacheGraphicsApi.Metal => "metal", + CacheGraphicsApi.Guest => "guest", + _ => throw new NotImplementedException(graphicsApi.ToString()), + }; + + return Path.Combine(baseCacheDirectory, graphicsApiName, shaderProvider, cacheName); + } + + /// + /// Read a cached file with the given hash that is present in the archive. + /// + /// The archive in use + /// The given hash + /// The cached file if present or null + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static byte[] ReadFromArchive(ZipArchive archive, Hash128 entry) + { + if (archive != null) + { + ZipArchiveEntry archiveEntry = archive.GetEntry($"{entry}"); + + if (archiveEntry != null) + { + try + { + byte[] result = new byte[archiveEntry.Length]; + + using (Stream archiveStream = archiveEntry.Open()) + { + archiveStream.Read(result); + + return result; + } + } + catch (Exception e) + { + Logger.Error?.Print(LogClass.Gpu, $"Cannot load cache file {entry} from archive"); + Logger.Error?.Print(LogClass.Gpu, e.ToString()); + } + } + } + + return null; + } + + /// + /// Read a cached file with the given hash that is not present in the archive. + /// + /// The cache directory + /// The given hash + /// The cached file if present or null + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static byte[] ReadFromFile(string cacheDirectory, Hash128 entry) + { + string cacheTempFilePath = GenCacheTempFilePath(cacheDirectory, entry); + + try + { + return File.ReadAllBytes(cacheTempFilePath); + } + catch (Exception e) + { + Logger.Error?.Print(LogClass.Gpu, $"Cannot load cache file at {cacheTempFilePath}"); + Logger.Error?.Print(LogClass.Gpu, e.ToString()); + } + + return null; + } + + /// + /// Compute the guest program code for usage while dumping to disk or hash. + /// + /// The guest shader entries to use + /// The transform feedback descriptors + /// Used to determine if the guest program code is generated for hashing + /// The guest program code for usage while dumping to disk or hash + private static byte[] ComputeGuestProgramCode(ReadOnlySpan cachedShaderEntries, TransformFeedbackDescriptor[] tfd, bool forHashCompute = false) + { + using (MemoryStream stream = new MemoryStream()) + { + BinaryWriter writer = new BinaryWriter(stream); + + foreach (GuestShaderCacheEntry cachedShaderEntry in cachedShaderEntries) + { + if (cachedShaderEntry != null) + { + // Code (and Code A if present) + stream.Write(cachedShaderEntry.Code); + + if (forHashCompute) + { + // Guest GPU accessor header (only write this for hashes, already present in the header for dumps) + writer.WriteStruct(cachedShaderEntry.Header.GpuAccessorHeader); + } + + // Texture descriptors + foreach (GuestTextureDescriptor textureDescriptor in cachedShaderEntry.TextureDescriptors.Values) + { + writer.WriteStruct(textureDescriptor); + } + } + } + + // Transformation feedback + if (tfd != null) + { + foreach (TransformFeedbackDescriptor transform in tfd) + { + writer.WriteStruct(new GuestShaderCacheTransformFeedbackHeader(transform.BufferIndex, transform.Stride, transform.VaryingLocations.Length)); + writer.Write(transform.VaryingLocations); + } + } + + return stream.ToArray(); + } + } + + /// + /// Compute a guest hash from shader entries. + /// + /// The guest shader entries to use + /// The optional transform feedback descriptors + /// A guest hash from shader entries + public static Hash128 ComputeGuestHashFromCache(ReadOnlySpan cachedShaderEntries, TransformFeedbackDescriptor[] tfd = null) + { + return XXHash128.ComputeHash(ComputeGuestProgramCode(cachedShaderEntries, tfd, true)); + } + + /// + /// Read transform feedback descriptors from guest. + /// + /// The raw guest transform feedback descriptors + /// The guest shader program header + /// The transform feedback descriptors read from guest + public static TransformFeedbackDescriptor[] ReadTransformationFeedbackInformations(ref ReadOnlySpan data, GuestShaderCacheHeader header) + { + if (header.TransformFeedbackCount != 0) + { + TransformFeedbackDescriptor[] result = new TransformFeedbackDescriptor[header.TransformFeedbackCount]; + + for (int i = 0; i < result.Length; i++) + { + GuestShaderCacheTransformFeedbackHeader feedbackHeader = MemoryMarshal.Read(data); + + result[i] = new TransformFeedbackDescriptor(feedbackHeader.BufferIndex, feedbackHeader.Stride, data.Slice(Unsafe.SizeOf(), feedbackHeader.VaryingLocationsLength).ToArray()); + + data = data.Slice(Unsafe.SizeOf() + feedbackHeader.VaryingLocationsLength); + } + + return result; + } + + return null; + } + + /// + /// Create a new instance of from an gpu accessor. + /// + /// The gpu accessor + /// A new instance of + public static GuestGpuAccessorHeader CreateGuestGpuAccessorCache(IGpuAccessor gpuAccessor) + { + return new GuestGpuAccessorHeader + { + ComputeLocalSizeX = gpuAccessor.QueryComputeLocalSizeX(), + ComputeLocalSizeY = gpuAccessor.QueryComputeLocalSizeY(), + ComputeLocalSizeZ = gpuAccessor.QueryComputeLocalSizeZ(), + ComputeLocalMemorySize = gpuAccessor.QueryComputeLocalMemorySize(), + ComputeSharedMemorySize = gpuAccessor.QueryComputeSharedMemorySize(), + PrimitiveTopology = gpuAccessor.QueryPrimitiveTopology(), + }; + } + + /// + /// Create guest shader cache entries from the runtime contexts. + /// + /// The GPU memory manager in use + /// The runtime contexts + /// Guest shader cahe entries from the runtime contexts + public static GuestShaderCacheEntry[] CreateShaderCacheEntries(MemoryManager memoryManager, ReadOnlySpan shaderContexts) + { + GuestShaderCacheEntry[] entries = new GuestShaderCacheEntry[shaderContexts.Length]; + + for (int i = 0; i < shaderContexts.Length; i++) + { + TranslatorContext context = shaderContexts[i]; + + if (context == null) + { + continue; + } + + int sizeA = context.AddressA == 0 ? 0 : context.SizeA; + + byte[] code = new byte[context.Size + sizeA]; + + memoryManager.GetSpan(context.Address, context.Size).CopyTo(code); + + if (context.AddressA != 0) + { + memoryManager.GetSpan(context.AddressA, context.SizeA).CopyTo(code.AsSpan().Slice(context.Size, context.SizeA)); + } + + GuestGpuAccessorHeader gpuAccessorHeader = CreateGuestGpuAccessorCache(context.GpuAccessor); + + if (context.GpuAccessor is GpuAccessor) + { + gpuAccessorHeader.TextureDescriptorCount = context.TextureHandlesForCache.Count; + } + + GuestShaderCacheEntryHeader header = new GuestShaderCacheEntryHeader(context.Stage, context.Size, sizeA, gpuAccessorHeader); + + GuestShaderCacheEntry entry = new GuestShaderCacheEntry(header, code); + + if (context.GpuAccessor is GpuAccessor gpuAccessor) + { + foreach (int textureHandle in context.TextureHandlesForCache) + { + GuestTextureDescriptor textureDescriptor = ((Image.TextureDescriptor)gpuAccessor.GetTextureDescriptor(textureHandle)).ToCache(); + + textureDescriptor.Handle = (uint)textureHandle; + + entry.TextureDescriptors.Add(textureHandle, textureDescriptor); + } + } + + entries[i] = entry; + } + + return entries; + } + + /// + /// Create a guest shader program. + /// + /// The entries composing the guest program dump + /// The transform feedback descriptors in use + /// The resulting guest shader program + public static byte[] CreateGuestProgramDump(GuestShaderCacheEntry[] shaderCacheEntries, TransformFeedbackDescriptor[] tfd = null) + { + using (MemoryStream resultStream = new MemoryStream()) + { + BinaryWriter resultStreamWriter = new BinaryWriter(resultStream); + + byte transformFeedbackCount = 0; + + if (tfd != null) + { + transformFeedbackCount = (byte)tfd.Length; + } + + // Header + resultStreamWriter.WriteStruct(new GuestShaderCacheHeader((byte)shaderCacheEntries.Length, transformFeedbackCount)); + + // Write all entries header + foreach (GuestShaderCacheEntry entry in shaderCacheEntries) + { + if (entry == null) + { + resultStreamWriter.WriteStruct(new GuestShaderCacheEntryHeader()); + } + else + { + resultStreamWriter.WriteStruct(entry.Header); + } + } + + // Finally, write all program code and all transform feedback information. + resultStreamWriter.Write(ComputeGuestProgramCode(shaderCacheEntries, tfd)); + + return resultStream.ToArray(); + } + } + + /// + /// Save temporary files not in archive. + /// + /// The base of the cache directory + /// The archive to use + /// The entries in the cache + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void EnsureArchiveUpToDate(string baseCacheDirectory, ZipArchive archive, HashSet entries) + { + foreach (Hash128 hash in entries) + { + string cacheTempFilePath = GenCacheTempFilePath(baseCacheDirectory, hash); + + if (File.Exists(cacheTempFilePath)) + { + string cacheHash = $"{hash}"; + + ZipArchiveEntry entry = archive.GetEntry(cacheHash); + + entry?.Delete(); + + archive.CreateEntryFromFile(cacheTempFilePath, cacheHash); + + File.Delete(cacheTempFilePath); + } + } + } + } +} -- cgit v1.2.3