From cee712105850ac3385cd0091a923438167433f9f Mon Sep 17 00:00:00 2001 From: TSR Berry <20988865+TSRBerry@users.noreply.github.com> Date: Sat, 8 Apr 2023 01:22:00 +0200 Subject: Move solution and projects to src --- src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs | 713 +++++++++++++++++++++++ 1 file changed, 713 insertions(+) create mode 100644 src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs (limited to 'src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs') diff --git a/src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs b/src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs new file mode 100644 index 00000000..b0f7e799 --- /dev/null +++ b/src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs @@ -0,0 +1,713 @@ +using Ryujinx.Memory; +using Ryujinx.Memory.Range; +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace Ryujinx.Graphics.Gpu.Memory +{ + /// + /// GPU memory manager. + /// + public class MemoryManager : IWritableBlock + { + private const int PtLvl0Bits = 14; + private const int PtLvl1Bits = 14; + public const int PtPageBits = 12; + + private const ulong PtLvl0Size = 1UL << PtLvl0Bits; + private const ulong PtLvl1Size = 1UL << PtLvl1Bits; + public const ulong PageSize = 1UL << PtPageBits; + + private const ulong PtLvl0Mask = PtLvl0Size - 1; + private const ulong PtLvl1Mask = PtLvl1Size - 1; + public const ulong PageMask = PageSize - 1; + + private const int PtLvl0Bit = PtPageBits + PtLvl1Bits; + private const int PtLvl1Bit = PtPageBits; + private const int AddressSpaceBits = PtPageBits + PtLvl1Bits + PtLvl0Bits; + + public const ulong PteUnmapped = ulong.MaxValue; + + private readonly ulong[][] _pageTable; + + public event EventHandler MemoryUnmapped; + + /// + /// Physical memory where the virtual memory is mapped into. + /// + internal PhysicalMemory Physical { get; } + + /// + /// Cache of GPU counters. + /// + internal CounterCache CounterCache { get; } + + /// + /// Creates a new instance of the GPU memory manager. + /// + /// Physical memory that this memory manager will map into + internal MemoryManager(PhysicalMemory physicalMemory) + { + Physical = physicalMemory; + CounterCache = new CounterCache(); + _pageTable = new ulong[PtLvl0Size][]; + MemoryUnmapped += Physical.TextureCache.MemoryUnmappedHandler; + MemoryUnmapped += Physical.BufferCache.MemoryUnmappedHandler; + MemoryUnmapped += CounterCache.MemoryUnmappedHandler; + } + + /// + /// Reads data from GPU mapped memory. + /// + /// Type of the data + /// GPU virtual address where the data is located + /// True if read tracking is triggered on the memory region + /// The data at the specified memory location + public T Read(ulong va, bool tracked = false) where T : unmanaged + { + int size = Unsafe.SizeOf(); + + if (IsContiguous(va, size)) + { + ulong address = Translate(va); + + if (tracked) + { + return Physical.ReadTracked(address); + } + else + { + return Physical.Read(address); + } + } + else + { + Span data = new byte[size]; + + ReadImpl(va, data, tracked); + + return MemoryMarshal.Cast(data)[0]; + } + } + + /// + /// Gets a read-only span of data from GPU mapped memory. + /// + /// GPU virtual address where the data is located + /// Size of the data + /// True if read tracking is triggered on the span + /// The span of the data at the specified memory location + public ReadOnlySpan GetSpan(ulong va, int size, bool tracked = false) + { + if (IsContiguous(va, size)) + { + return Physical.GetSpan(Translate(va), size, tracked); + } + else + { + Span data = new byte[size]; + + ReadImpl(va, data, tracked); + + return data; + } + } + + /// + /// Gets a read-only span of data from GPU mapped memory, up to the entire range specified, + /// or the last mapped page if the range is not fully mapped. + /// + /// GPU virtual address where the data is located + /// Size of the data + /// True if read tracking is triggered on the span + /// The span of the data at the specified memory location + public ReadOnlySpan GetSpanMapped(ulong va, int size, bool tracked = false) + { + bool isContiguous = true; + int mappedSize; + + if (ValidateAddress(va) && GetPte(va) != PteUnmapped && Physical.IsMapped(Translate(va))) + { + ulong endVa = va + (ulong)size; + ulong endVaAligned = (endVa + PageMask) & ~PageMask; + ulong currentVa = va & ~PageMask; + + int pages = (int)((endVaAligned - currentVa) / PageSize); + + for (int page = 0; page < pages - 1; page++) + { + ulong nextVa = currentVa + PageSize; + ulong nextPa = Translate(nextVa); + + if (!ValidateAddress(nextVa) || GetPte(nextVa) == PteUnmapped || !Physical.IsMapped(nextPa)) + { + break; + } + + if (Translate(currentVa) + PageSize != nextPa) + { + isContiguous = false; + } + + currentVa += PageSize; + } + + currentVa += PageSize; + + if (currentVa > endVa) + { + currentVa = endVa; + } + + mappedSize = (int)(currentVa - va); + } + else + { + return ReadOnlySpan.Empty; + } + + if (isContiguous) + { + return Physical.GetSpan(Translate(va), mappedSize, tracked); + } + else + { + Span data = new byte[mappedSize]; + + ReadImpl(va, data, tracked); + + return data; + } + } + + /// + /// Reads data from a possibly non-contiguous region of GPU mapped memory. + /// + /// GPU virtual address of the data + /// Span to write the read data into + /// True to enable write tracking on read, false otherwise + private void ReadImpl(ulong va, Span data, bool tracked) + { + if (data.Length == 0) + { + return; + } + + int offset = 0, size; + + if ((va & PageMask) != 0) + { + ulong pa = Translate(va); + + size = Math.Min(data.Length, (int)PageSize - (int)(va & PageMask)); + + Physical.GetSpan(pa, size, tracked).CopyTo(data.Slice(0, size)); + + offset += size; + } + + for (; offset < data.Length; offset += size) + { + ulong pa = Translate(va + (ulong)offset); + + size = Math.Min(data.Length - offset, (int)PageSize); + + Physical.GetSpan(pa, size, tracked).CopyTo(data.Slice(offset, size)); + } + } + + /// + /// Gets a writable region from GPU mapped memory. + /// + /// Start address of the range + /// Size in bytes to be range + /// True if write tracking is triggered on the span + /// A writable region with the data at the specified memory location + public WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false) + { + if (IsContiguous(va, size)) + { + return Physical.GetWritableRegion(Translate(va), size, tracked); + } + else + { + Memory memory = new byte[size]; + + GetSpan(va, size).CopyTo(memory.Span); + + return new WritableRegion(this, va, memory, tracked); + } + } + + /// + /// Writes data to GPU mapped memory. + /// + /// Type of the data + /// GPU virtual address to write the value into + /// The value to be written + public void Write(ulong va, T value) where T : unmanaged + { + Write(va, MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref value, 1))); + } + + /// + /// Writes data to GPU mapped memory. + /// + /// GPU virtual address to write the data into + /// The data to be written + public void Write(ulong va, ReadOnlySpan data) + { + WriteImpl(va, data, Physical.Write); + } + + /// + /// Writes data to GPU mapped memory, destined for a tracked resource. + /// + /// GPU virtual address to write the data into + /// The data to be written + public void WriteTrackedResource(ulong va, ReadOnlySpan data) + { + WriteImpl(va, data, Physical.WriteTrackedResource); + } + + /// + /// Writes data to GPU mapped memory without write tracking. + /// + /// GPU virtual address to write the data into + /// The data to be written + public void WriteUntracked(ulong va, ReadOnlySpan data) + { + WriteImpl(va, data, Physical.WriteUntracked); + } + + private delegate void WriteCallback(ulong address, ReadOnlySpan data); + + /// + /// Writes data to possibly non-contiguous GPU mapped memory. + /// + /// GPU virtual address of the region to write into + /// Data to be written + /// Write callback + private void WriteImpl(ulong va, ReadOnlySpan data, WriteCallback writeCallback) + { + if (IsContiguous(va, data.Length)) + { + writeCallback(Translate(va), data); + } + else + { + int offset = 0, size; + + if ((va & PageMask) != 0) + { + ulong pa = Translate(va); + + size = Math.Min(data.Length, (int)PageSize - (int)(va & PageMask)); + + writeCallback(pa, data.Slice(0, size)); + + offset += size; + } + + for (; offset < data.Length; offset += size) + { + ulong pa = Translate(va + (ulong)offset); + + size = Math.Min(data.Length - offset, (int)PageSize); + + writeCallback(pa, data.Slice(offset, size)); + } + } + } + + /// + /// Writes data to GPU mapped memory, stopping at the first unmapped page at the memory region, if any. + /// + /// GPU virtual address to write the data into + /// The data to be written + public void WriteMapped(ulong va, ReadOnlySpan data) + { + if (IsContiguous(va, data.Length)) + { + Physical.Write(Translate(va), data); + } + else + { + int offset = 0, size; + + if ((va & PageMask) != 0) + { + ulong pa = Translate(va); + + size = Math.Min(data.Length, (int)PageSize - (int)(va & PageMask)); + + if (pa != PteUnmapped && Physical.IsMapped(pa)) + { + Physical.Write(pa, data.Slice(0, size)); + } + + offset += size; + } + + for (; offset < data.Length; offset += size) + { + ulong pa = Translate(va + (ulong)offset); + + size = Math.Min(data.Length - offset, (int)PageSize); + + if (pa != PteUnmapped && Physical.IsMapped(pa)) + { + Physical.Write(pa, data.Slice(offset, size)); + } + } + } + } + + /// + /// Maps a given range of pages to the specified CPU virtual address. + /// + /// + /// All addresses and sizes must be page aligned. + /// + /// CPU virtual address to map into + /// GPU virtual address to be mapped + /// Size in bytes of the mapping + /// Kind of the resource located at the mapping + public void Map(ulong pa, ulong va, ulong size, PteKind kind) + { + lock (_pageTable) + { + MemoryUnmapped?.Invoke(this, new UnmapEventArgs(va, size)); + + for (ulong offset = 0; offset < size; offset += PageSize) + { + SetPte(va + offset, PackPte(pa + offset, kind)); + } + } + } + + /// + /// Unmaps a given range of pages at the specified GPU virtual memory region. + /// + /// GPU virtual address to unmap + /// Size in bytes of the region being unmapped + public void Unmap(ulong va, ulong size) + { + lock (_pageTable) + { + // Event handlers are not expected to be thread safe. + MemoryUnmapped?.Invoke(this, new UnmapEventArgs(va, size)); + + for (ulong offset = 0; offset < size; offset += PageSize) + { + SetPte(va + offset, PteUnmapped); + } + } + } + + /// + /// Checks if a region of GPU mapped memory is contiguous. + /// + /// GPU virtual address of the region + /// Size of the region + /// True if the region is contiguous, false otherwise + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool IsContiguous(ulong va, int size) + { + if (!ValidateAddress(va) || GetPte(va) == PteUnmapped) + { + return false; + } + + ulong endVa = (va + (ulong)size + PageMask) & ~PageMask; + + va &= ~PageMask; + + int pages = (int)((endVa - va) / PageSize); + + for (int page = 0; page < pages - 1; page++) + { + if (!ValidateAddress(va + PageSize) || GetPte(va + PageSize) == PteUnmapped) + { + return false; + } + + if (Translate(va) + PageSize != Translate(va + PageSize)) + { + return false; + } + + va += PageSize; + } + + return true; + } + + /// + /// Gets the physical regions that make up the given virtual address region. + /// + /// Virtual address of the range + /// Size of the range + /// Multi-range with the physical regions + public MultiRange GetPhysicalRegions(ulong va, ulong size) + { + if (IsContiguous(va, (int)size)) + { + return new MultiRange(Translate(va), size); + } + + ulong regionStart = Translate(va); + ulong regionSize = Math.Min(size, PageSize - (va & PageMask)); + + ulong endVa = va + size; + ulong endVaRounded = (endVa + PageMask) & ~PageMask; + + va &= ~PageMask; + + int pages = (int)((endVaRounded - va) / PageSize); + + var regions = new List(); + + for (int page = 0; page < pages - 1; page++) + { + ulong currPa = Translate(va); + ulong newPa = Translate(va + PageSize); + + if ((currPa != PteUnmapped || newPa != PteUnmapped) && currPa + PageSize != newPa) + { + regions.Add(new MemoryRange(regionStart, regionSize)); + regionStart = newPa; + regionSize = 0; + } + + va += PageSize; + regionSize += Math.Min(endVa - va, PageSize); + } + + regions.Add(new MemoryRange(regionStart, regionSize)); + + return new MultiRange(regions.ToArray()); + } + + /// + /// Checks if a given GPU virtual memory range is mapped to the same physical regions + /// as the specified physical memory multi-range. + /// + /// Physical memory multi-range + /// GPU virtual memory address + /// True if the virtual memory region is mapped into the specified physical one, false otherwise + public bool CompareRange(MultiRange range, ulong va) + { + va &= ~PageMask; + + for (int i = 0; i < range.Count; i++) + { + MemoryRange currentRange = range.GetSubRange(i); + + if (currentRange.Address != PteUnmapped) + { + ulong address = currentRange.Address & ~PageMask; + ulong endAddress = (currentRange.EndAddress + PageMask) & ~PageMask; + + while (address < endAddress) + { + if (Translate(va) != address) + { + return false; + } + + va += PageSize; + address += PageSize; + } + } + else + { + ulong endVa = va + (((currentRange.Size) + PageMask) & ~PageMask); + + while (va < endVa) + { + if (Translate(va) != PteUnmapped) + { + return false; + } + + va += PageSize; + } + } + } + + return true; + } + + /// + /// Validates a GPU virtual address. + /// + /// Address to validate + /// True if the address is valid, false otherwise + private static bool ValidateAddress(ulong va) + { + return va < (1UL << AddressSpaceBits); + } + + /// + /// Checks if a given page is mapped. + /// + /// GPU virtual address of the page to check + /// True if the page is mapped, false otherwise + public bool IsMapped(ulong va) + { + return Translate(va) != PteUnmapped; + } + + /// + /// Translates a GPU virtual address to a CPU virtual address. + /// + /// GPU virtual address to be translated + /// CPU virtual address, or if unmapped + public ulong Translate(ulong va) + { + if (!ValidateAddress(va)) + { + return PteUnmapped; + } + + ulong pte = GetPte(va); + + if (pte == PteUnmapped) + { + return PteUnmapped; + } + + return UnpackPaFromPte(pte) + (va & PageMask); + } + + /// + /// Translates a GPU virtual address to a CPU virtual address on the first mapped page of memory + /// on the specified region. + /// If no page is mapped on the specified region, is returned. + /// + /// GPU virtual address to be translated + /// Size of the range to be translated + /// CPU virtual address, or if unmapped + public ulong TranslateFirstMapped(ulong va, ulong size) + { + if (!ValidateAddress(va)) + { + return PteUnmapped; + } + + ulong endVa = va + size; + + ulong pte = GetPte(va); + + for (; va < endVa && pte == PteUnmapped; va += PageSize - (va & PageMask)) + { + pte = GetPte(va); + } + + if (pte == PteUnmapped) + { + return PteUnmapped; + } + + return UnpackPaFromPte(pte) + (va & PageMask); + } + + /// + /// Gets the kind of a given memory page. + /// This might indicate the type of resource that can be allocated on the page, and also texture tiling. + /// + /// GPU virtual address + /// Kind of the memory page + public PteKind GetKind(ulong va) + { + if (!ValidateAddress(va)) + { + return PteKind.Invalid; + } + + ulong pte = GetPte(va); + + if (pte == PteUnmapped) + { + return PteKind.Invalid; + } + + return UnpackKindFromPte(pte); + } + + /// + /// Gets the Page Table entry for a given GPU virtual address. + /// + /// GPU virtual address + /// Page table entry (CPU virtual address) + private ulong GetPte(ulong va) + { + ulong l0 = (va >> PtLvl0Bit) & PtLvl0Mask; + ulong l1 = (va >> PtLvl1Bit) & PtLvl1Mask; + + if (_pageTable[l0] == null) + { + return PteUnmapped; + } + + return _pageTable[l0][l1]; + } + + /// + /// Sets a Page Table entry at a given GPU virtual address. + /// + /// GPU virtual address + /// Page table entry (CPU virtual address) + private void SetPte(ulong va, ulong pte) + { + ulong l0 = (va >> PtLvl0Bit) & PtLvl0Mask; + ulong l1 = (va >> PtLvl1Bit) & PtLvl1Mask; + + if (_pageTable[l0] == null) + { + _pageTable[l0] = new ulong[PtLvl1Size]; + + for (ulong index = 0; index < PtLvl1Size; index++) + { + _pageTable[l0][index] = PteUnmapped; + } + } + + _pageTable[l0][l1] = pte; + } + + /// + /// Creates a page table entry from a physical address and kind. + /// + /// Physical address + /// Kind + /// Page table entry + private static ulong PackPte(ulong pa, PteKind kind) + { + return pa | ((ulong)kind << 56); + } + + /// + /// Unpacks kind from a page table entry. + /// + /// Page table entry + /// Kind + private static PteKind UnpackKindFromPte(ulong pte) + { + return (PteKind)(pte >> 56); + } + + /// + /// Unpacks physical address from a page table entry. + /// + /// Page table entry + /// Physical address + private static ulong UnpackPaFromPte(ulong pte) + { + return pte & 0xffffffffffffffUL; + } + } +} \ No newline at end of file -- cgit v1.2.3