diff options
Diffstat (limited to 'src/Ryujinx.Graphics.Gpu/Image/TextureBindingsArrayCache.cs')
| -rw-r--r-- | src/Ryujinx.Graphics.Gpu/Image/TextureBindingsArrayCache.cs | 714 |
1 files changed, 714 insertions, 0 deletions
diff --git a/src/Ryujinx.Graphics.Gpu/Image/TextureBindingsArrayCache.cs b/src/Ryujinx.Graphics.Gpu/Image/TextureBindingsArrayCache.cs new file mode 100644 index 00000000..70ea1f6b --- /dev/null +++ b/src/Ryujinx.Graphics.Gpu/Image/TextureBindingsArrayCache.cs @@ -0,0 +1,714 @@ +using Ryujinx.Graphics.GAL; +using Ryujinx.Graphics.Gpu.Engine.Types; +using Ryujinx.Graphics.Gpu.Memory; +using Ryujinx.Graphics.Shader; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; + +namespace Ryujinx.Graphics.Gpu.Image +{ + /// <summary> + /// Texture bindings array cache. + /// </summary> + class TextureBindingsArrayCache + { + /// <summary> + /// Minimum timestamp delta until texture array can be removed from the cache. + /// </summary> + private const int MinDeltaForRemoval = 20000; + + private readonly GpuContext _context; + private readonly GpuChannel _channel; + private readonly bool _isCompute; + + /// <summary> + /// Array cache entry key. + /// </summary> + private readonly struct CacheEntryKey : IEquatable<CacheEntryKey> + { + /// <summary> + /// Whether the entry is for an image. + /// </summary> + public readonly bool IsImage; + + /// <summary> + /// Texture or image target type. + /// </summary> + public readonly Target Target; + + /// <summary> + /// Word offset of the first handle on the constant buffer. + /// </summary> + public readonly int HandleIndex; + + /// <summary> + /// Number of entries of the array. + /// </summary> + public readonly int ArrayLength; + + private readonly TexturePool _texturePool; + private readonly SamplerPool _samplerPool; + + private readonly BufferBounds _textureBufferBounds; + + /// <summary> + /// Creates a new array cache entry. + /// </summary> + /// <param name="isImage">Whether the entry is for an image</param> + /// <param name="bindingInfo">Binding information for the array</param> + /// <param name="texturePool">Texture pool where the array textures are located</param> + /// <param name="samplerPool">Sampler pool where the array samplers are located</param> + /// <param name="textureBufferBounds">Constant buffer bounds with the texture handles</param> + public CacheEntryKey( + bool isImage, + TextureBindingInfo bindingInfo, + TexturePool texturePool, + SamplerPool samplerPool, + ref BufferBounds textureBufferBounds) + { + IsImage = isImage; + Target = bindingInfo.Target; + HandleIndex = bindingInfo.Handle; + ArrayLength = bindingInfo.ArrayLength; + + _texturePool = texturePool; + _samplerPool = samplerPool; + + _textureBufferBounds = textureBufferBounds; + } + + /// <summary> + /// Checks if the texture and sampler pools matches the cached pools. + /// </summary> + /// <param name="texturePool">Texture pool instance</param> + /// <param name="samplerPool">Sampler pool instance</param> + /// <returns>True if the pools match, false otherwise</returns> + private bool MatchesPools(TexturePool texturePool, SamplerPool samplerPool) + { + return _texturePool == texturePool && _samplerPool == samplerPool; + } + + /// <summary> + /// Checks if the cached constant buffer address and size matches. + /// </summary> + /// <param name="textureBufferBounds">New buffer address and size</param> + /// <returns>True if the address and size matches, false otherwise</returns> + private bool MatchesBufferBounds(BufferBounds textureBufferBounds) + { + return _textureBufferBounds.Equals(textureBufferBounds); + } + + public bool Equals(CacheEntryKey other) + { + return IsImage == other.IsImage && + Target == other.Target && + HandleIndex == other.HandleIndex && + ArrayLength == other.ArrayLength && + MatchesPools(other._texturePool, other._samplerPool) && + MatchesBufferBounds(other._textureBufferBounds); + } + + public override bool Equals(object obj) + { + return obj is CacheEntryKey other && Equals(other); + } + + public override int GetHashCode() + { + return _textureBufferBounds.Range.GetHashCode(); + } + } + + /// <summary> + /// Array cache entry. + /// </summary> + private class CacheEntry + { + /// <summary> + /// Key for this entry on the cache. + /// </summary> + public readonly CacheEntryKey Key; + + /// <summary> + /// Linked list node used on the texture bindings array cache. + /// </summary> + public LinkedListNode<CacheEntry> CacheNode; + + /// <summary> + /// Timestamp set on the last use of the array by the cache. + /// </summary> + public int CacheTimestamp; + + /// <summary> + /// All cached textures, along with their invalidated sequence number as value. + /// </summary> + public readonly Dictionary<Texture, int> Textures; + + /// <summary> + /// All pool texture IDs along with their textures. + /// </summary> + public readonly Dictionary<int, Texture> TextureIds; + + /// <summary> + /// All pool sampler IDs along with their samplers. + /// </summary> + public readonly Dictionary<int, Sampler> SamplerIds; + + /// <summary> + /// Backend texture array if the entry is for a texture, otherwise null. + /// </summary> + public readonly ITextureArray TextureArray; + + /// <summary> + /// Backend image array if the entry is for an image, otherwise null. + /// </summary> + public readonly IImageArray ImageArray; + + private readonly TexturePool _texturePool; + private readonly SamplerPool _samplerPool; + + private int _texturePoolSequence; + private int _samplerPoolSequence; + + private int[] _cachedTextureBuffer; + private int[] _cachedSamplerBuffer; + + private int _lastSequenceNumber; + + /// <summary> + /// Creates a new array cache entry. + /// </summary> + /// <param name="key">Key for this entry on the cache</param> + /// <param name="texturePool">Texture pool where the array textures are located</param> + /// <param name="samplerPool">Sampler pool where the array samplers are located</param> + private CacheEntry(ref CacheEntryKey key, TexturePool texturePool, SamplerPool samplerPool) + { + Key = key; + Textures = new Dictionary<Texture, int>(); + TextureIds = new Dictionary<int, Texture>(); + SamplerIds = new Dictionary<int, Sampler>(); + + _texturePool = texturePool; + _samplerPool = samplerPool; + + _lastSequenceNumber = -1; + } + + /// <summary> + /// Creates a new array cache entry. + /// </summary> + /// <param name="key">Key for this entry on the cache</param> + /// <param name="array">Backend texture array</param> + /// <param name="texturePool">Texture pool where the array textures are located</param> + /// <param name="samplerPool">Sampler pool where the array samplers are located</param> + public CacheEntry(ref CacheEntryKey key, ITextureArray array, TexturePool texturePool, SamplerPool samplerPool) : this(ref key, texturePool, samplerPool) + { + TextureArray = array; + } + + /// <summary> + /// Creates a new array cache entry. + /// </summary> + /// <param name="key">Key for this entry on the cache</param> + /// <param name="array">Backend image array</param> + /// <param name="texturePool">Texture pool where the array textures are located</param> + /// <param name="samplerPool">Sampler pool where the array samplers are located</param> + public CacheEntry(ref CacheEntryKey key, IImageArray array, TexturePool texturePool, SamplerPool samplerPool) : this(ref key, texturePool, samplerPool) + { + ImageArray = array; + } + + /// <summary> + /// Synchronizes memory for all textures in the array. + /// </summary> + /// <param name="isStore">Indicates if the texture may be modified by the access</param> + public void SynchronizeMemory(bool isStore) + { + foreach (Texture texture in Textures.Keys) + { + texture.SynchronizeMemory(); + + if (isStore) + { + texture.SignalModified(); + } + } + } + + /// <summary> + /// Clears all cached texture instances. + /// </summary> + public void Reset() + { + Textures.Clear(); + TextureIds.Clear(); + SamplerIds.Clear(); + } + + /// <summary> + /// Updates the cached constant buffer data. + /// </summary> + /// <param name="cachedTextureBuffer">Constant buffer data with the texture handles (and sampler handles, if they are combined)</param> + /// <param name="cachedSamplerBuffer">Constant buffer data with the sampler handles</param> + /// <param name="separateSamplerBuffer">Whether <paramref name="cachedTextureBuffer"/> and <paramref name="cachedSamplerBuffer"/> comes from different buffers</param> + public void UpdateData(ReadOnlySpan<int> cachedTextureBuffer, ReadOnlySpan<int> cachedSamplerBuffer, bool separateSamplerBuffer) + { + _cachedTextureBuffer = cachedTextureBuffer.ToArray(); + _cachedSamplerBuffer = separateSamplerBuffer ? cachedSamplerBuffer.ToArray() : _cachedTextureBuffer; + } + + /// <summary> + /// Checks if any texture has been deleted since the last call to this method. + /// </summary> + /// <returns>True if one or more textures have been deleted, false otherwise</returns> + public bool ValidateTextures() + { + foreach ((Texture texture, int invalidatedSequence) in Textures) + { + if (texture.InvalidatedSequence != invalidatedSequence) + { + return false; + } + } + + return true; + } + + /// <summary> + /// Checks if the cached texture or sampler pool has been modified since the last call to this method. + /// </summary> + /// <returns>True if any used entries of the pools might have been modified, false otherwise</returns> + public bool PoolsModified() + { + bool texturePoolModified = _texturePool.WasModified(ref _texturePoolSequence); + bool samplerPoolModified = _samplerPool.WasModified(ref _samplerPoolSequence); + + // If both pools were not modified since the last check, we have nothing else to check. + if (!texturePoolModified && !samplerPoolModified) + { + return false; + } + + // If the pools were modified, let's check if any of the entries we care about changed. + + // Check if any of our cached textures changed on the pool. + foreach ((int textureId, Texture texture) in TextureIds) + { + if (_texturePool.GetCachedItem(textureId) != texture) + { + return true; + } + } + + // Check if any of our cached samplers changed on the pool. + foreach ((int samplerId, Sampler sampler) in SamplerIds) + { + if (_samplerPool.GetCachedItem(samplerId) != sampler) + { + return true; + } + } + + return false; + } + + /// <summary> + /// Checks if the sequence number matches the one used on the last call to this method. + /// </summary> + /// <param name="currentSequenceNumber">Current sequence number</param> + /// <returns>True if the sequence numbers match, false otherwise</returns> + public bool MatchesSequenceNumber(int currentSequenceNumber) + { + if (_lastSequenceNumber == currentSequenceNumber) + { + return true; + } + + _lastSequenceNumber = currentSequenceNumber; + + return false; + } + + /// <summary> + /// Checks if the buffer data matches the cached data. + /// </summary> + /// <param name="cachedTextureBuffer">New texture buffer data</param> + /// <param name="cachedSamplerBuffer">New sampler buffer data</param> + /// <param name="separateSamplerBuffer">Whether <paramref name="cachedTextureBuffer"/> and <paramref name="cachedSamplerBuffer"/> comes from different buffers</param> + /// <param name="samplerWordOffset">Word offset of the sampler constant buffer handle that is used</param> + /// <returns>True if the data matches, false otherwise</returns> + public bool MatchesBufferData( + ReadOnlySpan<int> cachedTextureBuffer, + ReadOnlySpan<int> cachedSamplerBuffer, + bool separateSamplerBuffer, + int samplerWordOffset) + { + if (_cachedTextureBuffer != null && cachedTextureBuffer.Length > _cachedTextureBuffer.Length) + { + cachedTextureBuffer = cachedTextureBuffer[.._cachedTextureBuffer.Length]; + } + + if (!_cachedTextureBuffer.AsSpan().SequenceEqual(cachedTextureBuffer)) + { + return false; + } + + if (separateSamplerBuffer) + { + if (_cachedSamplerBuffer == null || + _cachedSamplerBuffer.Length <= samplerWordOffset || + cachedSamplerBuffer.Length <= samplerWordOffset) + { + return false; + } + + int oldValue = _cachedSamplerBuffer[samplerWordOffset]; + int newValue = cachedSamplerBuffer[samplerWordOffset]; + + return oldValue == newValue; + } + + return true; + } + } + + private readonly Dictionary<CacheEntryKey, CacheEntry> _cache; + private readonly LinkedList<CacheEntry> _lruCache; + + private int _currentTimestamp; + + /// <summary> + /// Creates a new instance of the texture bindings array cache. + /// </summary> + /// <param name="context">GPU context</param> + /// <param name="channel">GPU channel</param> + /// <param name="isCompute">Whether the bindings will be used for compute or graphics pipelines</param> + public TextureBindingsArrayCache(GpuContext context, GpuChannel channel, bool isCompute) + { + _context = context; + _channel = channel; + _isCompute = isCompute; + _cache = new Dictionary<CacheEntryKey, CacheEntry>(); + _lruCache = new LinkedList<CacheEntry>(); + } + + /// <summary> + /// Updates a texture array bindings and textures. + /// </summary> + /// <param name="texturePool">Texture pool</param> + /// <param name="samplerPool">Sampler pool</param> + /// <param name="stage">Shader stage where the array is used</param> + /// <param name="stageIndex">Shader stage index where the array is used</param> + /// <param name="textureBufferIndex">Texture constant buffer index</param> + /// <param name="samplerIndex">Sampler handles source</param> + /// <param name="bindingInfo">Array binding information</param> + public void UpdateTextureArray( + TexturePool texturePool, + SamplerPool samplerPool, + ShaderStage stage, + int stageIndex, + int textureBufferIndex, + SamplerIndex samplerIndex, + TextureBindingInfo bindingInfo) + { + Update(texturePool, samplerPool, stage, stageIndex, textureBufferIndex, isImage: false, samplerIndex, bindingInfo); + } + + /// <summary> + /// Updates a image array bindings and textures. + /// </summary> + /// <param name="texturePool">Texture pool</param> + /// <param name="stage">Shader stage where the array is used</param> + /// <param name="stageIndex">Shader stage index where the array is used</param> + /// <param name="textureBufferIndex">Texture constant buffer index</param> + /// <param name="bindingInfo">Array binding information</param> + public void UpdateImageArray(TexturePool texturePool, ShaderStage stage, int stageIndex, int textureBufferIndex, TextureBindingInfo bindingInfo) + { + Update(texturePool, null, stage, stageIndex, textureBufferIndex, isImage: true, SamplerIndex.ViaHeaderIndex, bindingInfo); + } + + /// <summary> + /// Updates a texture or image array bindings and textures. + /// </summary> + /// <param name="texturePool">Texture pool</param> + /// <param name="samplerPool">Sampler pool</param> + /// <param name="stage">Shader stage where the array is used</param> + /// <param name="stageIndex">Shader stage index where the array is used</param> + /// <param name="textureBufferIndex">Texture constant buffer index</param> + /// <param name="isImage">Whether the array is a image or texture array</param> + /// <param name="samplerIndex">Sampler handles source</param> + /// <param name="bindingInfo">Array binding information</param> + private void Update( + TexturePool texturePool, + SamplerPool samplerPool, + ShaderStage stage, + int stageIndex, + int textureBufferIndex, + bool isImage, + SamplerIndex samplerIndex, + TextureBindingInfo bindingInfo) + { + (textureBufferIndex, int samplerBufferIndex) = TextureHandle.UnpackSlots(bindingInfo.CbufSlot, textureBufferIndex); + + bool separateSamplerBuffer = textureBufferIndex != samplerBufferIndex; + + ref BufferBounds textureBufferBounds = ref _channel.BufferManager.GetUniformBufferBounds(_isCompute, stageIndex, textureBufferIndex); + ref BufferBounds samplerBufferBounds = ref _channel.BufferManager.GetUniformBufferBounds(_isCompute, stageIndex, samplerBufferIndex); + + CacheEntry entry = GetOrAddEntry( + texturePool, + samplerPool, + bindingInfo, + isImage, + ref textureBufferBounds, + out bool isNewEntry); + + bool poolsModified = entry.PoolsModified(); + bool isStore = bindingInfo.Flags.HasFlag(TextureUsageFlags.ImageStore); + + ReadOnlySpan<int> cachedTextureBuffer; + ReadOnlySpan<int> cachedSamplerBuffer; + + if (!poolsModified && !isNewEntry && entry.ValidateTextures()) + { + if (entry.MatchesSequenceNumber(_context.SequenceNumber)) + { + entry.SynchronizeMemory(isStore); + + if (isImage) + { + _context.Renderer.Pipeline.SetImageArray(stage, bindingInfo.Binding, entry.ImageArray); + } + else + { + _context.Renderer.Pipeline.SetTextureArray(stage, bindingInfo.Binding, entry.TextureArray); + } + + return; + } + + cachedTextureBuffer = MemoryMarshal.Cast<byte, int>(_channel.MemoryManager.Physical.GetSpan(textureBufferBounds.Range)); + + if (separateSamplerBuffer) + { + cachedSamplerBuffer = MemoryMarshal.Cast<byte, int>(_channel.MemoryManager.Physical.GetSpan(samplerBufferBounds.Range)); + } + else + { + cachedSamplerBuffer = cachedTextureBuffer; + } + + (_, int samplerWordOffset, _) = TextureHandle.UnpackOffsets(bindingInfo.Handle); + + if (entry.MatchesBufferData(cachedTextureBuffer, cachedSamplerBuffer, separateSamplerBuffer, samplerWordOffset)) + { + entry.SynchronizeMemory(isStore); + + if (isImage) + { + _context.Renderer.Pipeline.SetImageArray(stage, bindingInfo.Binding, entry.ImageArray); + } + else + { + _context.Renderer.Pipeline.SetTextureArray(stage, bindingInfo.Binding, entry.TextureArray); + } + + return; + } + } + else + { + cachedTextureBuffer = MemoryMarshal.Cast<byte, int>(_channel.MemoryManager.Physical.GetSpan(textureBufferBounds.Range)); + + if (separateSamplerBuffer) + { + cachedSamplerBuffer = MemoryMarshal.Cast<byte, int>(_channel.MemoryManager.Physical.GetSpan(samplerBufferBounds.Range)); + } + else + { + cachedSamplerBuffer = cachedTextureBuffer; + } + } + + if (!isNewEntry) + { + entry.Reset(); + } + + entry.UpdateData(cachedTextureBuffer, cachedSamplerBuffer, separateSamplerBuffer); + + Format[] formats = isImage ? new Format[bindingInfo.ArrayLength] : null; + ISampler[] samplers = isImage ? null : new ISampler[bindingInfo.ArrayLength]; + ITexture[] textures = new ITexture[bindingInfo.ArrayLength]; + + for (int index = 0; index < bindingInfo.ArrayLength; index++) + { + int handleIndex = bindingInfo.Handle + index * (Constants.TextureHandleSizeInBytes / sizeof(int)); + int packedId = TextureHandle.ReadPackedId(handleIndex, cachedTextureBuffer, cachedSamplerBuffer); + int textureId = TextureHandle.UnpackTextureId(packedId); + int samplerId; + + if (samplerIndex == SamplerIndex.ViaHeaderIndex) + { + samplerId = textureId; + } + else + { + samplerId = TextureHandle.UnpackSamplerId(packedId); + } + + ref readonly TextureDescriptor descriptor = ref texturePool.GetForBinding(textureId, out Texture texture); + + if (texture != null) + { + entry.Textures[texture] = texture.InvalidatedSequence; + + if (isStore) + { + texture.SignalModified(); + } + } + + Sampler sampler = samplerPool?.Get(samplerId); + + entry.TextureIds[textureId] = texture; + entry.SamplerIds[samplerId] = sampler; + + ITexture hostTexture = texture?.GetTargetTexture(bindingInfo.Target); + ISampler hostSampler = sampler?.GetHostSampler(texture); + + Format format = bindingInfo.Format; + + if (hostTexture != null && texture.Target == Target.TextureBuffer) + { + // Ensure that the buffer texture is using the correct buffer as storage. + // Buffers are frequently re-created to accommodate larger data, so we need to re-bind + // to ensure we're not using a old buffer that was already deleted. + if (isImage) + { + if (format == 0 && texture != null) + { + format = texture.Format; + } + + _channel.BufferManager.SetBufferTextureStorage(entry.ImageArray, hostTexture, texture.Range, bindingInfo, index, format); + } + else + { + _channel.BufferManager.SetBufferTextureStorage(entry.TextureArray, hostTexture, texture.Range, bindingInfo, index, format); + } + } + else if (isImage) + { + if (format == 0 && texture != null) + { + format = texture.Format; + } + + formats[index] = format; + textures[index] = hostTexture; + } + else + { + samplers[index] = hostSampler; + textures[index] = hostTexture; + } + } + + if (isImage) + { + entry.ImageArray.SetFormats(0, formats); + entry.ImageArray.SetImages(0, textures); + + _context.Renderer.Pipeline.SetImageArray(stage, bindingInfo.Binding, entry.ImageArray); + } + else + { + entry.TextureArray.SetSamplers(0, samplers); + entry.TextureArray.SetTextures(0, textures); + + _context.Renderer.Pipeline.SetTextureArray(stage, bindingInfo.Binding, entry.TextureArray); + } + } + + /// <summary> + /// Gets a cached texture entry, or creates a new one if not found. + /// </summary> + /// <param name="texturePool">Texture pool</param> + /// <param name="samplerPool">Sampler pool</param> + /// <param name="bindingInfo">Array binding information</param> + /// <param name="isImage">Whether the array is a image or texture array</param> + /// <param name="textureBufferBounds">Constant buffer bounds with the texture handles</param> + /// <param name="isNew">Whether a new entry was created, or an existing one was returned</param> + /// <returns>Cache entry</returns> + private CacheEntry GetOrAddEntry( + TexturePool texturePool, + SamplerPool samplerPool, + TextureBindingInfo bindingInfo, + bool isImage, + ref BufferBounds textureBufferBounds, + out bool isNew) + { + CacheEntryKey key = new CacheEntryKey( + isImage, + bindingInfo, + texturePool, + samplerPool, + ref textureBufferBounds); + + isNew = !_cache.TryGetValue(key, out CacheEntry entry); + + if (isNew) + { + int arrayLength = bindingInfo.ArrayLength; + + if (isImage) + { + IImageArray array = _context.Renderer.CreateImageArray(arrayLength, bindingInfo.Target == Target.TextureBuffer); + + _cache.Add(key, entry = new CacheEntry(ref key, array, texturePool, samplerPool)); + } + else + { + ITextureArray array = _context.Renderer.CreateTextureArray(arrayLength, bindingInfo.Target == Target.TextureBuffer); + + _cache.Add(key, entry = new CacheEntry(ref key, array, texturePool, samplerPool)); + } + } + + if (entry.CacheNode != null) + { + _lruCache.Remove(entry.CacheNode); + _lruCache.AddLast(entry.CacheNode); + } + else + { + entry.CacheNode = _lruCache.AddLast(entry); + } + + entry.CacheTimestamp = ++_currentTimestamp; + + RemoveLeastUsedEntries(); + + return entry; + } + + /// <summary> + /// Remove entries from the cache that have not been used for some time. + /// </summary> + private void RemoveLeastUsedEntries() + { + LinkedListNode<CacheEntry> nextNode = _lruCache.First; + + while (nextNode != null && _currentTimestamp - nextNode.Value.CacheTimestamp >= MinDeltaForRemoval) + { + LinkedListNode<CacheEntry> toRemove = nextNode; + nextNode = nextNode.Next; + _cache.Remove(toRemove.Value.Key); + _lruCache.Remove(toRemove); + } + } + } +} |