From c4f56c570494a6186792439e3c0e74458cc82b5c Mon Sep 17 00:00:00 2001 From: gdkchan Date: Sun, 17 Jan 2021 15:44:34 -0300 Subject: Support for resources on non-contiguous GPU memory regions (#1905) * Support for resources on non-contiguous GPU memory regions * Implement MultiRange physical addresses, only used with a single range for now * Actually use non-contiguous ranges * GetPhysicalRegions fixes * Documentation and remove Address property from TextureInfo * Finish implementing GetWritableRegion * Fix typo --- Ryujinx.Graphics.Gpu/Memory/GpuRegionHandle.cs | 60 ++++++ Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs | 286 +++++++++++++++++++++---- Ryujinx.Graphics.Gpu/Memory/PhysicalMemory.cs | 99 +++++++++ 3 files changed, 407 insertions(+), 38 deletions(-) create mode 100644 Ryujinx.Graphics.Gpu/Memory/GpuRegionHandle.cs (limited to 'Ryujinx.Graphics.Gpu/Memory') diff --git a/Ryujinx.Graphics.Gpu/Memory/GpuRegionHandle.cs b/Ryujinx.Graphics.Gpu/Memory/GpuRegionHandle.cs new file mode 100644 index 00000000..d2a05495 --- /dev/null +++ b/Ryujinx.Graphics.Gpu/Memory/GpuRegionHandle.cs @@ -0,0 +1,60 @@ +using Ryujinx.Cpu.Tracking; +using Ryujinx.Memory.Tracking; +using System; + +namespace Ryujinx.Graphics.Gpu.Memory +{ + class GpuRegionHandle : IRegionHandle + { + private readonly CpuRegionHandle[] _cpuRegionHandles; + + public bool Dirty + { + get + { + foreach (var regionHandle in _cpuRegionHandles) + { + if (regionHandle.Dirty) + { + return true; + } + } + + return false; + } + } + + public ulong Address => throw new NotSupportedException(); + public ulong Size => throw new NotSupportedException(); + public ulong EndAddress => throw new NotSupportedException(); + + public GpuRegionHandle(CpuRegionHandle[] cpuRegionHandles) + { + _cpuRegionHandles = cpuRegionHandles; + } + + public void Dispose() + { + foreach (var regionHandle in _cpuRegionHandles) + { + regionHandle.Dispose(); + } + } + + public void RegisterAction(RegionSignal action) + { + foreach (var regionHandle in _cpuRegionHandles) + { + regionHandle.RegisterAction(action); + } + } + + public void Reprotect() + { + foreach (var regionHandle in _cpuRegionHandles) + { + regionHandle.Reprotect(); + } + } + } +} diff --git a/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs b/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs index cf49fd93..3da22b22 100644 --- a/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs +++ b/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs @@ -1,5 +1,7 @@ using Ryujinx.Memory; +using Ryujinx.Memory.Range; using System; +using System.Collections.Generic; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; @@ -8,7 +10,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// /// GPU memory manager. /// - public class MemoryManager + public class MemoryManager : IWritableBlock { private const int PtLvl0Bits = 14; private const int PtLvl1Bits = 14; @@ -24,6 +26,7 @@ namespace Ryujinx.Graphics.Gpu.Memory private const int PtLvl0Bit = PtPageBits + PtLvl1Bits; private const int PtLvl1Bit = PtPageBits; + private const int AddressSpaceBits = PtPageBits + PtLvl1Bits + PtLvl0Bits; public const ulong PteUnmapped = 0xffffffff_ffffffff; @@ -46,26 +49,69 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Reads data from GPU mapped memory. /// /// Type of the data - /// GPU virtual address where the data is located + /// GPU virtual address where the data is located /// The data at the specified memory location - public T Read(ulong gpuVa) where T : unmanaged + public T Read(ulong va) where T : unmanaged { - ulong processVa = Translate(gpuVa); - - return MemoryMarshal.Cast(_context.PhysicalMemory.GetSpan(processVa, Unsafe.SizeOf()))[0]; + return MemoryMarshal.Cast(GetSpan(va, Unsafe.SizeOf()))[0]; } /// /// Gets a read-only span of data from GPU mapped memory. /// - /// GPU virtual address where the data is located + /// GPU virtual address where the data is located /// Size of the data /// The span of the data at the specified memory location - public ReadOnlySpan GetSpan(ulong gpuVa, int size) + public ReadOnlySpan GetSpan(ulong va, int size, bool tracked = false) { - ulong processVa = Translate(gpuVa); + if (IsContiguous(va, size)) + { + return _context.PhysicalMemory.GetSpan(Translate(va), size, tracked); + } + else + { + Span data = new byte[size]; - return _context.PhysicalMemory.GetSpan(processVa, size); + ReadImpl(va, data, tracked); + + return data; + } + } + + /// + /// Reads data from a possibly non-contiguous region of GPU mapped memory. + /// + /// GPU virtual address of the data + /// Span to write the read data into + /// True to enable write tracking on read, false otherwise + private void ReadImpl(ulong va, Span data, bool tracked) + { + if (data.Length == 0) + { + return; + } + + int offset = 0, size; + + if ((va & PageMask) != 0) + { + ulong pa = Translate(va); + + size = Math.Min(data.Length, (int)PageSize - (int)(va & PageMask)); + + _context.PhysicalMemory.GetSpan(pa, size, tracked).CopyTo(data.Slice(0, size)); + + offset += size; + } + + for (; offset < data.Length; offset += size) + { + ulong pa = Translate(va + (ulong)offset); + + size = Math.Min(data.Length - offset, (int)PageSize); + + _context.PhysicalMemory.GetSpan(pa, size, tracked).CopyTo(data.Slice(offset, size)); + } } /// @@ -74,36 +120,91 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Start address of the range /// Size in bytes to be range /// A writable region with the data at the specified memory location - public WritableRegion GetWritableRegion(ulong gpuVa, int size) + public WritableRegion GetWritableRegion(ulong va, int size) { - ulong processVa = Translate(gpuVa); + if (IsContiguous(va, size)) + { + return _context.PhysicalMemory.GetWritableRegion(Translate(va), size); + } + else + { + Memory memory = new byte[size]; + + GetSpan(va, size).CopyTo(memory.Span); - return _context.PhysicalMemory.GetWritableRegion(processVa, size); + return new WritableRegion(this, va, memory); + } } /// /// Writes data to GPU mapped memory. /// /// Type of the data - /// GPU virtual address to write the value into + /// GPU virtual address to write the value into /// The value to be written - public void Write(ulong gpuVa, T value) where T : unmanaged + public void Write(ulong va, T value) where T : unmanaged { - ulong processVa = Translate(gpuVa); - - _context.PhysicalMemory.Write(processVa, MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref value, 1))); + Write(va, MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref value, 1))); } /// /// Writes data to GPU mapped memory. /// - /// GPU virtual address to write the data into + /// GPU virtual address to write the data into + /// The data to be written + public void Write(ulong va, ReadOnlySpan data) + { + WriteImpl(va, data, _context.PhysicalMemory.Write); + } + + /// + /// Writes data to GPU mapped memory without write tracking. + /// + /// GPU virtual address to write the data into /// The data to be written - public void Write(ulong gpuVa, ReadOnlySpan data) + public void WriteUntracked(ulong va, ReadOnlySpan data) + { + WriteImpl(va, data, _context.PhysicalMemory.WriteUntracked); + } + + private delegate void WriteCallback(ulong address, ReadOnlySpan data); + + /// + /// Writes data to possibly non-contiguous GPU mapped memory. + /// + /// GPU virtual address of the region to write into + /// Data to be written + /// Write callback + private void WriteImpl(ulong va, ReadOnlySpan data, WriteCallback writeCallback) { - ulong processVa = Translate(gpuVa); + if (IsContiguous(va, data.Length)) + { + writeCallback(Translate(va), data); + } + else + { + int offset = 0, size; + + if ((va & PageMask) != 0) + { + ulong pa = Translate(va); + + size = Math.Min(data.Length, (int)PageSize - (int)(va & PageMask)); + + writeCallback(pa, data.Slice(0, size)); - _context.PhysicalMemory.Write(processVa, data); + offset += size; + } + + for (; offset < data.Length; offset += size) + { + ulong pa = Translate(va + (ulong)offset); + + size = Math.Min(data.Length - offset, (int)PageSize); + + writeCallback(pa, data.Slice(offset, size)); + } + } } /// @@ -147,42 +248,151 @@ namespace Ryujinx.Graphics.Gpu.Memory } } + /// + /// Checks if a region of GPU mapped memory is contiguous. + /// + /// GPU virtual address of the region + /// Size of the region + /// True if the region is contiguous, false otherwise + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool IsContiguous(ulong va, int size) + { + if (!ValidateAddress(va) || GetPte(va) == PteUnmapped) + { + return false; + } + + ulong endVa = (va + (ulong)size + PageMask) & ~PageMask; + + va &= ~PageMask; + + int pages = (int)((endVa - va) / PageSize); + + for (int page = 0; page < pages - 1; page++) + { + if (!ValidateAddress(va + PageSize) || GetPte(va + PageSize) == PteUnmapped) + { + return false; + } + + if (Translate(va) + PageSize != Translate(va + PageSize)) + { + return false; + } + + va += PageSize; + } + + return true; + } + + /// + /// Gets the physical regions that make up the given virtual address region. + /// + /// Virtual address of the range + /// Size of the range + /// Multi-range with the physical regions + /// The memory region specified by and is not fully mapped + public MultiRange GetPhysicalRegions(ulong va, ulong size) + { + if (IsContiguous(va, (int)size)) + { + return new MultiRange(Translate(va), size); + } + + if (!IsMapped(va)) + { + throw new InvalidMemoryRegionException($"The specified GPU virtual address 0x{va:X} is not mapped."); + } + + ulong regionStart = Translate(va); + ulong regionSize = Math.Min(size, PageSize - (va & PageMask)); + + ulong endVa = va + size; + ulong endVaRounded = (endVa + PageMask) & ~PageMask; + + va &= ~PageMask; + + int pages = (int)((endVaRounded - va) / PageSize); + + var regions = new List(); + + for (int page = 0; page < pages - 1; page++) + { + if (!IsMapped(va + PageSize)) + { + throw new InvalidMemoryRegionException($"The specified GPU virtual memory range 0x{va:X}..0x{(va + size):X} is not fully mapped."); + } + + ulong newPa = Translate(va + PageSize); + + if (Translate(va) + PageSize != newPa) + { + regions.Add(new MemoryRange(regionStart, regionSize)); + regionStart = newPa; + regionSize = 0; + } + + va += PageSize; + regionSize += Math.Min(endVa - va, PageSize); + } + + regions.Add(new MemoryRange(regionStart, regionSize)); + + return new MultiRange(regions.ToArray()); + } + + /// + /// Validates a GPU virtual address. + /// + /// Address to validate + /// True if the address is valid, false otherwise + private static bool ValidateAddress(ulong va) + { + return va < (1UL << AddressSpaceBits); + } + /// /// Checks if a given page is mapped. /// - /// GPU virtual address of the page to check + /// GPU virtual address of the page to check /// True if the page is mapped, false otherwise - public bool IsMapped(ulong gpuVa) + public bool IsMapped(ulong va) { - return Translate(gpuVa) != PteUnmapped; + return Translate(va) != PteUnmapped; } /// /// Translates a GPU virtual address to a CPU virtual address. /// - /// GPU virtual address to be translated - /// CPU virtual address - public ulong Translate(ulong gpuVa) + /// GPU virtual address to be translated + /// CPU virtual address, or if unmapped + public ulong Translate(ulong va) { - ulong baseAddress = GetPte(gpuVa); + if (!ValidateAddress(va)) + { + return PteUnmapped; + } + + ulong baseAddress = GetPte(va); if (baseAddress == PteUnmapped) { return PteUnmapped; } - return baseAddress + (gpuVa & PageMask); + return baseAddress + (va & PageMask); } /// /// Gets the Page Table entry for a given GPU virtual address. /// - /// GPU virtual address + /// GPU virtual address /// Page table entry (CPU virtual address) - private ulong GetPte(ulong gpuVa) + private ulong GetPte(ulong va) { - ulong l0 = (gpuVa >> PtLvl0Bit) & PtLvl0Mask; - ulong l1 = (gpuVa >> PtLvl1Bit) & PtLvl1Mask; + ulong l0 = (va >> PtLvl0Bit) & PtLvl0Mask; + ulong l1 = (va >> PtLvl1Bit) & PtLvl1Mask; if (_pageTable[l0] == null) { @@ -195,12 +405,12 @@ namespace Ryujinx.Graphics.Gpu.Memory /// /// Sets a Page Table entry at a given GPU virtual address. /// - /// GPU virtual address + /// GPU virtual address /// Page table entry (CPU virtual address) - private void SetPte(ulong gpuVa, ulong pte) + private void SetPte(ulong va, ulong pte) { - ulong l0 = (gpuVa >> PtLvl0Bit) & PtLvl0Mask; - ulong l1 = (gpuVa >> PtLvl1Bit) & PtLvl1Mask; + ulong l0 = (va >> PtLvl0Bit) & PtLvl0Mask; + ulong l1 = (va >> PtLvl1Bit) & PtLvl1Mask; if (_pageTable[l0] == null) { diff --git a/Ryujinx.Graphics.Gpu/Memory/PhysicalMemory.cs b/Ryujinx.Graphics.Gpu/Memory/PhysicalMemory.cs index de41fb9a..8b2401c7 100644 --- a/Ryujinx.Graphics.Gpu/Memory/PhysicalMemory.cs +++ b/Ryujinx.Graphics.Gpu/Memory/PhysicalMemory.cs @@ -1,6 +1,7 @@ using Ryujinx.Cpu; using Ryujinx.Cpu.Tracking; using Ryujinx.Memory; +using Ryujinx.Memory.Range; using System; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; @@ -38,6 +39,37 @@ namespace Ryujinx.Graphics.Gpu.Memory return _cpuMemory.GetSpan(address, size, tracked); } + /// + /// Gets a span of data from the application process. + /// + /// Ranges of physical memory where the data is located + /// True if read tracking is triggered on the span + /// A read only span of the data at the specified memory location + public ReadOnlySpan GetSpan(MultiRange range, bool tracked = false) + { + if (range.Count == 1) + { + var singleRange = range.GetSubRange(0); + return _cpuMemory.GetSpan(singleRange.Address, (int)singleRange.Size, tracked); + } + else + { + Span data = new byte[range.GetSize()]; + + int offset = 0; + + for (int i = 0; i < range.Count; i++) + { + var currentRange = range.GetSubRange(i); + int size = (int)currentRange.Size; + _cpuMemory.GetSpan(currentRange.Address, size, tracked).CopyTo(data.Slice(offset, size)); + offset += size; + } + + return data; + } + } + /// /// Gets a writable region from the application process. /// @@ -70,6 +102,16 @@ namespace Ryujinx.Graphics.Gpu.Memory _cpuMemory.Write(address, data); } + /// + /// Writes data to the application process. + /// + /// Ranges of physical memory where the data is located + /// Data to be written + public void Write(MultiRange range, ReadOnlySpan data) + { + WriteImpl(range, data, _cpuMemory.Write); + } + /// /// Writes data to the application process, without any tracking. /// @@ -80,6 +122,45 @@ namespace Ryujinx.Graphics.Gpu.Memory _cpuMemory.WriteUntracked(address, data); } + /// + /// Writes data to the application process, without any tracking. + /// + /// Ranges of physical memory where the data is located + /// Data to be written + public void WriteUntracked(MultiRange range, ReadOnlySpan data) + { + WriteImpl(range, data, _cpuMemory.WriteUntracked); + } + + private delegate void WriteCallback(ulong address, ReadOnlySpan data); + + /// + /// Writes data to the application process, using the supplied callback method. + /// + /// Ranges of physical memory where the data is located + /// Data to be written + /// Callback method that will perform the write + private void WriteImpl(MultiRange range, ReadOnlySpan data, WriteCallback writeCallback) + { + if (range.Count == 1) + { + var singleRange = range.GetSubRange(0); + writeCallback(singleRange.Address, data); + } + else + { + int offset = 0; + + for (int i = 0; i < range.Count; i++) + { + var currentRange = range.GetSubRange(i); + int size = (int)currentRange.Size; + writeCallback(currentRange.Address, data.Slice(offset, size)); + offset += size; + } + } + } + /// /// Obtains a memory tracking handle for the given virtual region. This should be disposed when finished with. /// @@ -91,6 +172,24 @@ namespace Ryujinx.Graphics.Gpu.Memory return _cpuMemory.BeginTracking(address, size); } + /// + /// Obtains a memory tracking handle for the given virtual region. This should be disposed when finished with. + /// + /// Ranges of physical memory where the data is located + /// The memory tracking handle + public GpuRegionHandle BeginTracking(MultiRange range) + { + var cpuRegionHandles = new CpuRegionHandle[range.Count]; + + for (int i = 0; i < range.Count; i++) + { + var currentRange = range.GetSubRange(i); + cpuRegionHandles[i] = _cpuMemory.BeginTracking(currentRange.Address, currentRange.Size); + } + + return new GpuRegionHandle(cpuRegionHandles); + } + /// /// Obtains a memory tracking handle for the given virtual region, with a specified granularity. This should be disposed when finished with. /// -- cgit v1.2.3