From 139ea93512aeead8a4aee3910a3de86eb109a838 Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Fri, 5 Nov 2021 15:52:31 +0100 Subject: VideoCore: implement channels on gpu caches. --- src/video_core/memory_manager.cpp | 5 ----- 1 file changed, 5 deletions(-) (limited to 'src/video_core/memory_manager.cpp') diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp index bf9eb735d..a3efd365e 100644 --- a/src/video_core/memory_manager.cpp +++ b/src/video_core/memory_manager.cpp @@ -133,11 +133,6 @@ void MemoryManager::SetPageEntry(GPUVAddr gpu_addr, PageEntry page_entry, std::s // TryLockPage(page_entry, size); auto& current_page = page_table[PageEntryIndex(gpu_addr)]; - if ((!current_page.IsValid() && page_entry.IsValid()) || - current_page.ToAddress() != page_entry.ToAddress()) { - rasterizer->ModifyGPUMemory(gpu_addr, size); - } - current_page = page_entry; } -- cgit v1.2.3 From cbaf3fb433a351f7d9509f17f88d4896ba66afd1 Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Thu, 11 Nov 2021 21:24:40 +0100 Subject: VideoCore: Update MemoryManager --- src/video_core/memory_manager.cpp | 147 ++++++++++++++------------------------ 1 file changed, 55 insertions(+), 92 deletions(-) (limited to 'src/video_core/memory_manager.cpp') diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp index a3efd365e..1e090279f 100644 --- a/src/video_core/memory_manager.cpp +++ b/src/video_core/memory_manager.cpp @@ -16,36 +16,63 @@ namespace Tegra { -MemoryManager::MemoryManager(Core::System& system_) - : system{system_}, page_table(page_table_size) {} +MemoryManager::MemoryManager(Core::System& system_, u64 address_space_bits_, u64 page_bits_) + : system{system_}, address_space_bits{address_space_bits_}, page_bits{page_bits_}, entries{}, + page_table{address_space_bits, address_space_bits + page_bits - 38, page_bits} { + address_space_size = 1ULL << address_space_bits; + allocate_start = address_space_bits > 32 ? 1ULL << 32 : 0; + page_size = 1ULL << page_bits; + page_mask = page_size - 1ULL; + const u64 page_table_bits = address_space_bits - cpu_page_bits; + const u64 page_table_size = 1ULL << page_table_bits; + page_table_mask = page_table_size - 1; + + entries.resize(page_table_size / 32, 0); +} MemoryManager::~MemoryManager() = default; -void MemoryManager::BindRasterizer(VideoCore::RasterizerInterface* rasterizer_) { - rasterizer = rasterizer_; +MemoryManager::EntryType MemoryManager::GetEntry(size_t position) const { + position = position >> page_bits; + const u64 entry_mask = entries[position / 32]; + const size_t sub_index = position % 32; + return static_cast((entry_mask >> (2 * sub_index)) & 0x03ULL); +} + +void MemoryManager::SetEntry(size_t position, MemoryManager::EntryType entry) { + position = position >> page_bits; + const u64 entry_mask = entries[position / 32]; + const size_t sub_index = position % 32; + entries[position / 32] = + (~(3ULL << sub_index * 2) & entry_mask) | (static_cast(entry) << sub_index * 2); } -GPUVAddr MemoryManager::UpdateRange(GPUVAddr gpu_addr, PageEntry page_entry, std::size_t size) { +template +GPUVAddr MemoryManager::PageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] VAddr cpu_addr, + size_t size) { u64 remaining_size{size}; + if constexpr (entry_type == EntryType::Mapped) { + page_table.ReserveRange(gpu_addr, size); + } for (u64 offset{}; offset < size; offset += page_size) { - if (remaining_size < page_size) { - SetPageEntry(gpu_addr + offset, page_entry + offset, remaining_size); - } else { - SetPageEntry(gpu_addr + offset, page_entry + offset); + const GPUVAddr current_gpu_addr = gpu_addr + offset; + SetEntry(current_gpu_addr, entry_type); + if constexpr (entry_type == EntryType::Mapped) { + const VAddr current_cpu_addr = cpu_addr + offset; + const auto index = PageEntryIndex(current_gpu_addr); + page_table[index] = static_cast(current_cpu_addr >> 12ULL); } remaining_size -= page_size; } return gpu_addr; } +void MemoryManager::BindRasterizer(VideoCore::RasterizerInterface* rasterizer_) { + rasterizer = rasterizer_; +} + GPUVAddr MemoryManager::Map(VAddr cpu_addr, GPUVAddr gpu_addr, std::size_t size) { - const auto it = std::ranges::lower_bound(map_ranges, gpu_addr, {}, &MapRange::first); - if (it != map_ranges.end() && it->first == gpu_addr) { - it->second = size; - } else { - map_ranges.insert(it, MapRange{gpu_addr, size}); - } - return UpdateRange(gpu_addr, cpu_addr, size); + return PageTableOp(gpu_addr, cpu_addr, size); } GPUVAddr MemoryManager::MapAllocate(VAddr cpu_addr, std::size_t size, std::size_t align) { @@ -62,13 +89,6 @@ void MemoryManager::Unmap(GPUVAddr gpu_addr, std::size_t size) { if (size == 0) { return; } - const auto it = std::ranges::lower_bound(map_ranges, gpu_addr, {}, &MapRange::first); - if (it != map_ranges.end()) { - ASSERT(it->first == gpu_addr); - map_ranges.erase(it); - } else { - ASSERT_MSG(false, "Unmapping non-existent GPU address=0x{:x}", gpu_addr); - } const auto submapped_ranges = GetSubmappedRange(gpu_addr, size); for (const auto& [map_addr, map_size] : submapped_ranges) { @@ -79,63 +99,23 @@ void MemoryManager::Unmap(GPUVAddr gpu_addr, std::size_t size) { rasterizer->UnmapMemory(*cpu_addr, map_size); } - UpdateRange(gpu_addr, PageEntry::State::Unmapped, size); + PageTableOp(gpu_addr, 0, size); } std::optional MemoryManager::AllocateFixed(GPUVAddr gpu_addr, std::size_t size) { for (u64 offset{}; offset < size; offset += page_size) { - if (!GetPageEntry(gpu_addr + offset).IsUnmapped()) { + if (GetEntry(gpu_addr + offset) != EntryType::Free) { return std::nullopt; } } - return UpdateRange(gpu_addr, PageEntry::State::Allocated, size); + return PageTableOp(gpu_addr, 0, size); } GPUVAddr MemoryManager::Allocate(std::size_t size, std::size_t align) { return *AllocateFixed(*FindFreeRange(size, align), size); } -void MemoryManager::TryLockPage(PageEntry page_entry, std::size_t size) { - if (!page_entry.IsValid()) { - return; - } - - ASSERT(system.CurrentProcess() - ->PageTable() - .LockForDeviceAddressSpace(page_entry.ToAddress(), size) - .IsSuccess()); -} - -void MemoryManager::TryUnlockPage(PageEntry page_entry, std::size_t size) { - if (!page_entry.IsValid()) { - return; - } - - ASSERT(system.CurrentProcess() - ->PageTable() - .UnlockForDeviceAddressSpace(page_entry.ToAddress(), size) - .IsSuccess()); -} - -PageEntry MemoryManager::GetPageEntry(GPUVAddr gpu_addr) const { - return page_table[PageEntryIndex(gpu_addr)]; -} - -void MemoryManager::SetPageEntry(GPUVAddr gpu_addr, PageEntry page_entry, std::size_t size) { - // TODO(bunnei): We should lock/unlock device regions. This currently causes issues due to - // improper tracking, but should be fixed in the future. - - //// Unlock the old page - // TryUnlockPage(page_table[PageEntryIndex(gpu_addr)], size); - - //// Lock the new page - // TryLockPage(page_entry, size); - auto& current_page = page_table[PageEntryIndex(gpu_addr)]; - - current_page = page_entry; -} - std::optional MemoryManager::FindFreeRange(std::size_t size, std::size_t align, bool start_32bit_address) const { if (!align) { @@ -145,9 +125,9 @@ std::optional MemoryManager::FindFreeRange(std::size_t size, std::size } u64 available_size{}; - GPUVAddr gpu_addr{start_32bit_address ? address_space_start_low : address_space_start}; + GPUVAddr gpu_addr{allocate_start}; while (gpu_addr + available_size < address_space_size) { - if (GetPageEntry(gpu_addr + available_size).IsUnmapped()) { + if (GetEntry(gpu_addr + available_size) == EntryType::Free) { available_size += page_size; if (available_size >= size) { @@ -168,15 +148,12 @@ std::optional MemoryManager::FindFreeRange(std::size_t size, std::size } std::optional MemoryManager::GpuToCpuAddress(GPUVAddr gpu_addr) const { - if (gpu_addr == 0) { - return std::nullopt; - } - const auto page_entry{GetPageEntry(gpu_addr)}; - if (!page_entry.IsValid()) { + if (GetEntry(gpu_addr) != EntryType::Mapped) { return std::nullopt; } - return page_entry.ToAddress() + (gpu_addr & page_mask); + const VAddr cpu_addr_base = static_cast(page_table[PageEntryIndex(gpu_addr)]) << 12ULL; + return cpu_addr_base + (gpu_addr & page_mask); } std::optional MemoryManager::GpuToCpuAddress(GPUVAddr addr, std::size_t size) const { @@ -227,10 +204,6 @@ template void MemoryManager::Write(GPUVAddr addr, u32 data); template void MemoryManager::Write(GPUVAddr addr, u64 data); u8* MemoryManager::GetPointer(GPUVAddr gpu_addr) { - if (!GetPageEntry(gpu_addr).IsValid()) { - return {}; - } - const auto address{GpuToCpuAddress(gpu_addr)}; if (!address) { return {}; @@ -240,10 +213,6 @@ u8* MemoryManager::GetPointer(GPUVAddr gpu_addr) { } const u8* MemoryManager::GetPointer(GPUVAddr gpu_addr) const { - if (!GetPageEntry(gpu_addr).IsValid()) { - return {}; - } - const auto address{GpuToCpuAddress(gpu_addr)}; if (!address) { return {}; @@ -252,12 +221,6 @@ const u8* MemoryManager::GetPointer(GPUVAddr gpu_addr) const { return system.Memory().GetPointer(*address); } -size_t MemoryManager::BytesToMapEnd(GPUVAddr gpu_addr) const noexcept { - auto it = std::ranges::upper_bound(map_ranges, gpu_addr, {}, &MapRange::first); - --it; - return it->second - (gpu_addr - it->first); -} - void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std::size_t size, bool is_safe) const { std::size_t remaining_size{size}; @@ -268,7 +231,7 @@ void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std: const std::size_t copy_amount{ std::min(static_cast(page_size) - page_offset, remaining_size)}; const auto page_addr{GpuToCpuAddress(page_index << page_bits)}; - if (page_addr && *page_addr != 0) { + if (page_addr) { const auto src_addr{*page_addr + page_offset}; if (is_safe) { // Flush must happen on the rasterizer interface, such that memory is always @@ -307,7 +270,7 @@ void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffe const std::size_t copy_amount{ std::min(static_cast(page_size) - page_offset, remaining_size)}; const auto page_addr{GpuToCpuAddress(page_index << page_bits)}; - if (page_addr && *page_addr != 0) { + if (page_addr) { const auto dest_addr{*page_addr + page_offset}; if (is_safe) { @@ -392,7 +355,7 @@ bool MemoryManager::IsFullyMappedRange(GPUVAddr gpu_addr, std::size_t size) cons size_t page_index{gpu_addr >> page_bits}; const size_t page_last{(gpu_addr + size + page_size - 1) >> page_bits}; while (page_index < page_last) { - if (!page_table[page_index].IsValid() || page_table[page_index].ToAddress() == 0) { + if (GetEntry(page_index << page_bits) == EntryType::Free) { return false; } ++page_index; @@ -408,7 +371,7 @@ std::vector> MemoryManager::GetSubmappedRange( size_t page_offset{gpu_addr & page_mask}; std::optional> last_segment{}; std::optional old_page_addr{}; - const auto extend_size = [&last_segment, &page_index, &page_offset](std::size_t bytes) { + const auto extend_size = [this, &last_segment, &page_index, &page_offset](std::size_t bytes) { if (!last_segment) { const GPUVAddr new_base_addr = (page_index << page_bits) + page_offset; last_segment = {new_base_addr, bytes}; -- cgit v1.2.3 From 0f4ae3cc52a740bac836f8f8c2ff5d560b8ca46e Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Sat, 13 Nov 2021 03:05:32 +0100 Subject: MemoryManager: Temporary Fix for NVDEC. --- src/video_core/memory_manager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/video_core/memory_manager.cpp') diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp index 1e090279f..9e946d448 100644 --- a/src/video_core/memory_manager.cpp +++ b/src/video_core/memory_manager.cpp @@ -125,7 +125,7 @@ std::optional MemoryManager::FindFreeRange(std::size_t size, std::size } u64 available_size{}; - GPUVAddr gpu_addr{allocate_start}; + GPUVAddr gpu_addr{start_32bit_address ? 0 : allocate_start}; while (gpu_addr + available_size < address_space_size) { if (GetEntry(gpu_addr + available_size) == EntryType::Free) { available_size += page_size; -- cgit v1.2.3 From feb49c822d9cabc5bc7be9eab1f2bf4ba460176a Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Sun, 14 Nov 2021 20:55:52 +0100 Subject: NVDRV: Remake ASGPU --- src/video_core/memory_manager.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'src/video_core/memory_manager.cpp') diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp index 9e946d448..fc68bcc73 100644 --- a/src/video_core/memory_manager.cpp +++ b/src/video_core/memory_manager.cpp @@ -71,18 +71,22 @@ void MemoryManager::BindRasterizer(VideoCore::RasterizerInterface* rasterizer_) rasterizer = rasterizer_; } -GPUVAddr MemoryManager::Map(VAddr cpu_addr, GPUVAddr gpu_addr, std::size_t size) { +GPUVAddr MemoryManager::Map(GPUVAddr gpu_addr, VAddr cpu_addr, std::size_t size) { return PageTableOp(gpu_addr, cpu_addr, size); } +GPUVAddr MemoryManager::MapSparse(GPUVAddr gpu_addr, std::size_t size) { + return PageTableOp(gpu_addr, 0, size); +} + GPUVAddr MemoryManager::MapAllocate(VAddr cpu_addr, std::size_t size, std::size_t align) { - return Map(cpu_addr, *FindFreeRange(size, align), size); + return Map(*FindFreeRange(size, align), cpu_addr, size); } GPUVAddr MemoryManager::MapAllocate32(VAddr cpu_addr, std::size_t size) { const std::optional gpu_addr = FindFreeRange(size, 1, true); ASSERT(gpu_addr); - return Map(cpu_addr, *gpu_addr, size); + return Map(*gpu_addr, cpu_addr, size); } void MemoryManager::Unmap(GPUVAddr gpu_addr, std::size_t size) { -- cgit v1.2.3 From e462191482c6507daed67802c6c1d2c50f10c96e Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Fri, 17 Dec 2021 16:45:06 +0100 Subject: Refactor VideoCore to use AS sepparate from Channel. --- src/video_core/memory_manager.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'src/video_core/memory_manager.cpp') diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp index fc68bcc73..d4c0dca78 100644 --- a/src/video_core/memory_manager.cpp +++ b/src/video_core/memory_manager.cpp @@ -16,9 +16,12 @@ namespace Tegra { +std::atomic MemoryManager::unique_identifier_generator{}; + MemoryManager::MemoryManager(Core::System& system_, u64 address_space_bits_, u64 page_bits_) : system{system_}, address_space_bits{address_space_bits_}, page_bits{page_bits_}, entries{}, - page_table{address_space_bits, address_space_bits + page_bits - 38, page_bits} { + page_table{address_space_bits, address_space_bits + page_bits - 38, page_bits}, + unique_identifier{unique_identifier_generator.fetch_add(1, std::memory_order_acq_rel)} { address_space_size = 1ULL << address_space_bits; allocate_start = address_space_bits > 32 ? 1ULL << 32 : 0; page_size = 1ULL << page_bits; -- cgit v1.2.3 From f350c3d74ea7880fc6d21f7f638b0d4a70a3246b Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Sat, 1 Jan 2022 22:03:37 +0100 Subject: Texture cache: Fix the remaining issues with memory mnagement and unmapping. --- src/video_core/memory_manager.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'src/video_core/memory_manager.cpp') diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp index d4c0dca78..b36067613 100644 --- a/src/video_core/memory_manager.cpp +++ b/src/video_core/memory_manager.cpp @@ -59,10 +59,19 @@ GPUVAddr MemoryManager::PageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] VAddr cp } for (u64 offset{}; offset < size; offset += page_size) { const GPUVAddr current_gpu_addr = gpu_addr + offset; + [[maybe_unused]] const auto current_entry_type = GetEntry(current_gpu_addr); SetEntry(current_gpu_addr, entry_type); + if (current_entry_type != entry_type) { + rasterizer->ModifyGPUMemory(unique_identifier, gpu_addr, page_size); + } if constexpr (entry_type == EntryType::Mapped) { const VAddr current_cpu_addr = cpu_addr + offset; const auto index = PageEntryIndex(current_gpu_addr); + const u32 sub_value = static_cast(current_cpu_addr >> 12ULL); + if (current_entry_type == entry_type && sub_value != page_table[index]) { + rasterizer->InvalidateRegion(static_cast(page_table[index]) << 12ULL, + page_size); + } page_table[index] = static_cast(current_cpu_addr >> 12ULL); } remaining_size -= page_size; @@ -168,7 +177,7 @@ std::optional MemoryManager::GpuToCpuAddress(GPUVAddr addr, std::size_t s const size_t page_last{(addr + size + page_size - 1) >> page_bits}; while (page_index < page_last) { const auto page_addr{GpuToCpuAddress(page_index << page_bits)}; - if (page_addr && *page_addr != 0) { + if (page_addr) { return page_addr; } ++page_index; -- cgit v1.2.3 From 4d60410dd979fb688de7735d2b4b25a557bdeac7 Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Sat, 5 Feb 2022 18:15:26 +0100 Subject: MemoryManager: initial multi paging system implementation. --- src/video_core/memory_manager.cpp | 436 +++++++++++++++++++++++--------------- 1 file changed, 263 insertions(+), 173 deletions(-) (limited to 'src/video_core/memory_manager.cpp') diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp index b36067613..836ece136 100644 --- a/src/video_core/memory_manager.cpp +++ b/src/video_core/memory_manager.cpp @@ -7,6 +7,7 @@ #include "common/assert.h" #include "common/logging/log.h" #include "core/core.h" +#include "core/device_memory.h" #include "core/hle/kernel/k_page_table.h" #include "core/hle/kernel/k_process.h" #include "core/memory.h" @@ -14,40 +15,69 @@ #include "video_core/rasterizer_interface.h" #include "video_core/renderer_base.h" +#pragma optimize("", off) + namespace Tegra { std::atomic MemoryManager::unique_identifier_generator{}; -MemoryManager::MemoryManager(Core::System& system_, u64 address_space_bits_, u64 page_bits_) - : system{system_}, address_space_bits{address_space_bits_}, page_bits{page_bits_}, entries{}, - page_table{address_space_bits, address_space_bits + page_bits - 38, page_bits}, +MemoryManager::MemoryManager(Core::System& system_, u64 address_space_bits_, u64 big_page_bits_, + u64 page_bits_) + : system{system_}, memory{system.Memory()}, device_memory{system.DeviceMemory()}, + address_space_bits{address_space_bits_}, page_bits{page_bits_}, big_page_bits{big_page_bits_}, + entries{}, big_entries{}, page_table{address_space_bits, address_space_bits + page_bits - 38, + page_bits != big_page_bits ? page_bits : 0}, unique_identifier{unique_identifier_generator.fetch_add(1, std::memory_order_acq_rel)} { address_space_size = 1ULL << address_space_bits; - allocate_start = address_space_bits > 32 ? 1ULL << 32 : 0; page_size = 1ULL << page_bits; page_mask = page_size - 1ULL; - const u64 page_table_bits = address_space_bits - cpu_page_bits; + big_page_size = 1ULL << big_page_bits; + big_page_mask = big_page_size - 1ULL; + const u64 page_table_bits = address_space_bits - page_bits; + const u64 big_page_table_bits = address_space_bits - big_page_bits; const u64 page_table_size = 1ULL << page_table_bits; + const u64 big_page_table_size = 1ULL << big_page_table_bits; page_table_mask = page_table_size - 1; + big_page_table_mask = big_page_table_size - 1; + big_entries.resize(big_page_table_size / 32, 0); + big_page_table_cpu.resize(big_page_table_size); + big_page_table_physical.resize(big_page_table_size); entries.resize(page_table_size / 32, 0); } MemoryManager::~MemoryManager() = default; +template MemoryManager::EntryType MemoryManager::GetEntry(size_t position) const { - position = position >> page_bits; - const u64 entry_mask = entries[position / 32]; - const size_t sub_index = position % 32; - return static_cast((entry_mask >> (2 * sub_index)) & 0x03ULL); + if constexpr (is_big_page) { + position = position >> big_page_bits; + const u64 entry_mask = big_entries[position / 32]; + const size_t sub_index = position % 32; + return static_cast((entry_mask >> (2 * sub_index)) & 0x03ULL); + } else { + position = position >> page_bits; + const u64 entry_mask = entries[position / 32]; + const size_t sub_index = position % 32; + return static_cast((entry_mask >> (2 * sub_index)) & 0x03ULL); + } } +template void MemoryManager::SetEntry(size_t position, MemoryManager::EntryType entry) { - position = position >> page_bits; - const u64 entry_mask = entries[position / 32]; - const size_t sub_index = position % 32; - entries[position / 32] = - (~(3ULL << sub_index * 2) & entry_mask) | (static_cast(entry) << sub_index * 2); + if constexpr (is_big_page) { + position = position >> big_page_bits; + const u64 entry_mask = big_entries[position / 32]; + const size_t sub_index = position % 32; + big_entries[position / 32] = + (~(3ULL << sub_index * 2) & entry_mask) | (static_cast(entry) << sub_index * 2); + } else { + position = position >> page_bits; + const u64 entry_mask = entries[position / 32]; + const size_t sub_index = position % 32; + entries[position / 32] = + (~(3ULL << sub_index * 2) & entry_mask) | (static_cast(entry) << sub_index * 2); + } } template @@ -59,48 +89,66 @@ GPUVAddr MemoryManager::PageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] VAddr cp } for (u64 offset{}; offset < size; offset += page_size) { const GPUVAddr current_gpu_addr = gpu_addr + offset; - [[maybe_unused]] const auto current_entry_type = GetEntry(current_gpu_addr); - SetEntry(current_gpu_addr, entry_type); + [[maybe_unused]] const auto current_entry_type = GetEntry(current_gpu_addr); + SetEntry(current_gpu_addr, entry_type); if (current_entry_type != entry_type) { rasterizer->ModifyGPUMemory(unique_identifier, gpu_addr, page_size); } if constexpr (entry_type == EntryType::Mapped) { const VAddr current_cpu_addr = cpu_addr + offset; - const auto index = PageEntryIndex(current_gpu_addr); - const u32 sub_value = static_cast(current_cpu_addr >> 12ULL); - if (current_entry_type == entry_type && sub_value != page_table[index]) { - rasterizer->InvalidateRegion(static_cast(page_table[index]) << 12ULL, - page_size); - } - page_table[index] = static_cast(current_cpu_addr >> 12ULL); + const auto index = PageEntryIndex(current_gpu_addr); + const u32 sub_value = static_cast(current_cpu_addr >> cpu_page_bits); + page_table[index] = sub_value; } remaining_size -= page_size; } return gpu_addr; } +template +GPUVAddr MemoryManager::BigPageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] VAddr cpu_addr, + size_t size) { + u64 remaining_size{size}; + for (u64 offset{}; offset < size; offset += big_page_size) { + const GPUVAddr current_gpu_addr = gpu_addr + offset; + [[maybe_unused]] const auto current_entry_type = GetEntry(current_gpu_addr); + SetEntry(current_gpu_addr, entry_type); + if (current_entry_type != entry_type) { + rasterizer->ModifyGPUMemory(unique_identifier, gpu_addr, big_page_size); + } + if constexpr (entry_type == EntryType::Mapped) { + const VAddr current_cpu_addr = cpu_addr + offset; + const auto index = PageEntryIndex(current_gpu_addr); + const u32 sub_value = static_cast(current_cpu_addr >> cpu_page_bits); + big_page_table_cpu[index] = sub_value; + const PAddr phys_address = + device_memory.GetPhysicalAddr(memory.GetPointer(current_cpu_addr)); + big_page_table_physical[index] = static_cast(phys_address); + } + remaining_size -= big_page_size; + } + return gpu_addr; +} + void MemoryManager::BindRasterizer(VideoCore::RasterizerInterface* rasterizer_) { rasterizer = rasterizer_; } -GPUVAddr MemoryManager::Map(GPUVAddr gpu_addr, VAddr cpu_addr, std::size_t size) { +GPUVAddr MemoryManager::Map(GPUVAddr gpu_addr, VAddr cpu_addr, std::size_t size, + bool is_big_pages) { + if (is_big_pages) [[likely]] { + return BigPageTableOp(gpu_addr, cpu_addr, size); + } return PageTableOp(gpu_addr, cpu_addr, size); } -GPUVAddr MemoryManager::MapSparse(GPUVAddr gpu_addr, std::size_t size) { +GPUVAddr MemoryManager::MapSparse(GPUVAddr gpu_addr, std::size_t size, bool is_big_pages) { + if (is_big_pages) [[likely]] { + return BigPageTableOp(gpu_addr, 0, size); + } return PageTableOp(gpu_addr, 0, size); } -GPUVAddr MemoryManager::MapAllocate(VAddr cpu_addr, std::size_t size, std::size_t align) { - return Map(*FindFreeRange(size, align), cpu_addr, size); -} - -GPUVAddr MemoryManager::MapAllocate32(VAddr cpu_addr, std::size_t size) { - const std::optional gpu_addr = FindFreeRange(size, 1, true); - ASSERT(gpu_addr); - return Map(*gpu_addr, cpu_addr, size); -} - void MemoryManager::Unmap(GPUVAddr gpu_addr, std::size_t size) { if (size == 0) { return; @@ -115,61 +163,24 @@ void MemoryManager::Unmap(GPUVAddr gpu_addr, std::size_t size) { rasterizer->UnmapMemory(*cpu_addr, map_size); } + BigPageTableOp(gpu_addr, 0, size); PageTableOp(gpu_addr, 0, size); } -std::optional MemoryManager::AllocateFixed(GPUVAddr gpu_addr, std::size_t size) { - for (u64 offset{}; offset < size; offset += page_size) { - if (GetEntry(gpu_addr + offset) != EntryType::Free) { +std::optional MemoryManager::GpuToCpuAddress(GPUVAddr gpu_addr) const { + if (GetEntry(gpu_addr) != EntryType::Mapped) [[unlikely]] { + if (GetEntry(gpu_addr) != EntryType::Mapped) { return std::nullopt; } - } - return PageTableOp(gpu_addr, 0, size); -} - -GPUVAddr MemoryManager::Allocate(std::size_t size, std::size_t align) { - return *AllocateFixed(*FindFreeRange(size, align), size); -} - -std::optional MemoryManager::FindFreeRange(std::size_t size, std::size_t align, - bool start_32bit_address) const { - if (!align) { - align = page_size; - } else { - align = Common::AlignUp(align, page_size); - } - - u64 available_size{}; - GPUVAddr gpu_addr{start_32bit_address ? 0 : allocate_start}; - while (gpu_addr + available_size < address_space_size) { - if (GetEntry(gpu_addr + available_size) == EntryType::Free) { - available_size += page_size; - - if (available_size >= size) { - return gpu_addr; - } - } else { - gpu_addr += available_size + page_size; - available_size = 0; - - const auto remainder{gpu_addr % align}; - if (remainder) { - gpu_addr = (gpu_addr - remainder) + align; - } - } - } - - return std::nullopt; -} - -std::optional MemoryManager::GpuToCpuAddress(GPUVAddr gpu_addr) const { - if (GetEntry(gpu_addr) != EntryType::Mapped) { - return std::nullopt; + const VAddr cpu_addr_base = static_cast(page_table[PageEntryIndex(gpu_addr)]) + << cpu_page_bits; + return cpu_addr_base + (gpu_addr & page_mask); } - const VAddr cpu_addr_base = static_cast(page_table[PageEntryIndex(gpu_addr)]) << 12ULL; - return cpu_addr_base + (gpu_addr & page_mask); + const VAddr cpu_addr_base = + static_cast(big_page_table_cpu[PageEntryIndex(gpu_addr)]) << cpu_page_bits; + return cpu_addr_base + (gpu_addr & big_page_mask); } std::optional MemoryManager::GpuToCpuAddress(GPUVAddr addr, std::size_t size) const { @@ -225,7 +236,7 @@ u8* MemoryManager::GetPointer(GPUVAddr gpu_addr) { return {}; } - return system.Memory().GetPointer(*address); + return memory.GetPointer(*address); } const u8* MemoryManager::GetPointer(GPUVAddr gpu_addr) const { @@ -234,98 +245,161 @@ const u8* MemoryManager::GetPointer(GPUVAddr gpu_addr) const { return {}; } - return system.Memory().GetPointer(*address); + return memory.GetPointer(*address); } -void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std::size_t size, - bool is_safe) const { +#pragma inline_recursion(on) + +template +inline void MemoryManager::MemoryOperation(GPUVAddr gpu_src_addr, std::size_t size, + FuncMapped&& func_mapped, FuncReserved&& func_reserved, + FuncUnmapped&& func_unmapped) const { + u64 used_page_size; + u64 used_page_mask; + u64 used_page_bits; + if constexpr (is_big_pages) { + used_page_size = big_page_size; + used_page_mask = big_page_mask; + used_page_bits = big_page_bits; + } else { + used_page_size = page_size; + used_page_mask = page_mask; + used_page_bits = page_bits; + } std::size_t remaining_size{size}; - std::size_t page_index{gpu_src_addr >> page_bits}; - std::size_t page_offset{gpu_src_addr & page_mask}; + std::size_t page_index{gpu_src_addr >> used_page_bits}; + std::size_t page_offset{gpu_src_addr & used_page_mask}; + GPUVAddr current_address = gpu_src_addr; while (remaining_size > 0) { const std::size_t copy_amount{ - std::min(static_cast(page_size) - page_offset, remaining_size)}; - const auto page_addr{GpuToCpuAddress(page_index << page_bits)}; - if (page_addr) { - const auto src_addr{*page_addr + page_offset}; - if (is_safe) { - // Flush must happen on the rasterizer interface, such that memory is always - // synchronous when it is read (even when in asynchronous GPU mode). - // Fixes Dead Cells title menu. - rasterizer->FlushRegion(src_addr, copy_amount); - } - system.Memory().ReadBlockUnsafe(src_addr, dest_buffer, copy_amount); - } else { - std::memset(dest_buffer, 0, copy_amount); + std::min(static_cast(used_page_size) - page_offset, remaining_size)}; + auto entry = GetEntry(current_address); + if (entry == EntryType::Mapped) [[likely]] { + func_mapped(page_index, page_offset, copy_amount); + } else if (entry == EntryType::Reserved) { + func_reserved(page_index, page_offset, copy_amount); + } else [[unlikely]] { + func_unmapped(page_index, page_offset, copy_amount); } - page_index++; page_offset = 0; - dest_buffer = static_cast(dest_buffer) + copy_amount; remaining_size -= copy_amount; + current_address += copy_amount; } } +template +void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, + std::size_t size) const { + auto set_to_zero = [&]([[maybe_unused]] std::size_t page_index, + [[maybe_unused]] std::size_t offset, std::size_t copy_amount) { + std::memset(dest_buffer, 0, copy_amount); + dest_buffer = static_cast(dest_buffer) + copy_amount; + }; + auto mapped_normal = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) { + const VAddr cpu_addr_base = + (static_cast(page_table[page_index]) << cpu_page_bits) + offset; + if constexpr (is_safe) { + rasterizer->FlushRegion(cpu_addr_base, copy_amount); + } + memory.ReadBlockUnsafe(cpu_addr_base, dest_buffer, copy_amount); + dest_buffer = static_cast(dest_buffer) + copy_amount; + }; + auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) { + const VAddr cpu_addr_base = + (static_cast(big_page_table_cpu[page_index]) << cpu_page_bits) + offset; + if constexpr (is_safe) { + rasterizer->FlushRegion(cpu_addr_base, copy_amount); + } + memory.ReadBlockUnsafe(cpu_addr_base, dest_buffer, copy_amount); + // u8* physical = device_memory.GetPointer(big_page_table_physical[page_index] + offset); + // std::memcpy(dest_buffer, physical, copy_amount); + dest_buffer = static_cast(dest_buffer) + copy_amount; + }; + auto read_short_pages = [&](std::size_t page_index, std::size_t offset, + std::size_t copy_amount) { + GPUVAddr base = (page_index << big_page_bits) + offset; + MemoryOperation(base, copy_amount, mapped_normal, set_to_zero, set_to_zero); + }; + MemoryOperation(gpu_src_addr, size, mapped_big, set_to_zero, read_short_pages); +} + void MemoryManager::ReadBlock(GPUVAddr gpu_src_addr, void* dest_buffer, std::size_t size) const { - ReadBlockImpl(gpu_src_addr, dest_buffer, size, true); + ReadBlockImpl(gpu_src_addr, dest_buffer, size); } void MemoryManager::ReadBlockUnsafe(GPUVAddr gpu_src_addr, void* dest_buffer, const std::size_t size) const { - ReadBlockImpl(gpu_src_addr, dest_buffer, size, false); + ReadBlockImpl(gpu_src_addr, dest_buffer, size); } -void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size, - bool is_safe) { - std::size_t remaining_size{size}; - std::size_t page_index{gpu_dest_addr >> page_bits}; - std::size_t page_offset{gpu_dest_addr & page_mask}; - - while (remaining_size > 0) { - const std::size_t copy_amount{ - std::min(static_cast(page_size) - page_offset, remaining_size)}; - const auto page_addr{GpuToCpuAddress(page_index << page_bits)}; - if (page_addr) { - const auto dest_addr{*page_addr + page_offset}; - - if (is_safe) { - // Invalidate must happen on the rasterizer interface, such that memory is always - // synchronous when it is written (even when in asynchronous GPU mode). - rasterizer->InvalidateRegion(dest_addr, copy_amount); - } - system.Memory().WriteBlockUnsafe(dest_addr, src_buffer, copy_amount); +template +void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffer, + std::size_t size) { + auto just_advance = [&]([[maybe_unused]] std::size_t page_index, + [[maybe_unused]] std::size_t offset, std::size_t copy_amount) { + src_buffer = static_cast(src_buffer) + copy_amount; + }; + auto mapped_normal = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) { + const VAddr cpu_addr_base = + (static_cast(page_table[page_index]) << cpu_page_bits) + offset; + if constexpr (is_safe) { + rasterizer->InvalidateRegion(cpu_addr_base, copy_amount); } - - page_index++; - page_offset = 0; + memory.WriteBlockUnsafe(cpu_addr_base, src_buffer, copy_amount); src_buffer = static_cast(src_buffer) + copy_amount; - remaining_size -= copy_amount; - } + }; + auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) { + const VAddr cpu_addr_base = + (static_cast(big_page_table_cpu[page_index]) << cpu_page_bits) + offset; + if constexpr (is_safe) { + rasterizer->InvalidateRegion(cpu_addr_base, copy_amount); + } + memory.WriteBlockUnsafe(cpu_addr_base, src_buffer, copy_amount); + /*u8* physical = + device_memory.GetPointer(big_page_table_physical[page_index] << cpu_page_bits) + offset; + std::memcpy(physical, src_buffer, copy_amount);*/ + src_buffer = static_cast(src_buffer) + copy_amount; + }; + auto write_short_pages = [&](std::size_t page_index, std::size_t offset, + std::size_t copy_amount) { + GPUVAddr base = (page_index << big_page_bits) + offset; + MemoryOperation(base, copy_amount, mapped_normal, just_advance, just_advance); + }; + MemoryOperation(gpu_dest_addr, size, mapped_big, just_advance, write_short_pages); } void MemoryManager::WriteBlock(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size) { - WriteBlockImpl(gpu_dest_addr, src_buffer, size, true); + WriteBlockImpl(gpu_dest_addr, src_buffer, size); } void MemoryManager::WriteBlockUnsafe(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size) { - WriteBlockImpl(gpu_dest_addr, src_buffer, size, false); + WriteBlockImpl(gpu_dest_addr, src_buffer, size); } void MemoryManager::FlushRegion(GPUVAddr gpu_addr, size_t size) const { - size_t remaining_size{size}; - size_t page_index{gpu_addr >> page_bits}; - size_t page_offset{gpu_addr & page_mask}; - while (remaining_size > 0) { - const size_t num_bytes{std::min(page_size - page_offset, remaining_size)}; - if (const auto page_addr{GpuToCpuAddress(page_index << page_bits)}; page_addr) { - rasterizer->FlushRegion(*page_addr + page_offset, num_bytes); - } - ++page_index; - page_offset = 0; - remaining_size -= num_bytes; - } + auto do_nothing = [&]([[maybe_unused]] std::size_t page_index, + [[maybe_unused]] std::size_t offset, + [[maybe_unused]] std::size_t copy_amount) {}; + + auto mapped_normal = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) { + const VAddr cpu_addr_base = + (static_cast(page_table[page_index]) << cpu_page_bits) + offset; + rasterizer->FlushRegion(cpu_addr_base, copy_amount); + }; + auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) { + const VAddr cpu_addr_base = + (static_cast(big_page_table_cpu[page_index]) << cpu_page_bits) + offset; + rasterizer->FlushRegion(cpu_addr_base, copy_amount); + }; + auto flush_short_pages = [&](std::size_t page_index, std::size_t offset, + std::size_t copy_amount) { + GPUVAddr base = (page_index << big_page_bits) + offset; + MemoryOperation(base, copy_amount, mapped_normal, do_nothing, do_nothing); + }; + MemoryOperation(gpu_addr, size, mapped_big, do_nothing, flush_short_pages); } void MemoryManager::CopyBlock(GPUVAddr gpu_dest_addr, GPUVAddr gpu_src_addr, std::size_t size) { @@ -348,7 +422,7 @@ bool MemoryManager::IsGranularRange(GPUVAddr gpu_addr, std::size_t size) const { } bool MemoryManager::IsContinousRange(GPUVAddr gpu_addr, std::size_t size) const { - size_t page_index{gpu_addr >> page_bits}; + size_t page_index{gpu_addr >> big_page_bits}; const size_t page_last{(gpu_addr + size + page_size - 1) >> page_bits}; std::optional old_page_addr{}; while (page_index != page_last) { @@ -371,7 +445,7 @@ bool MemoryManager::IsFullyMappedRange(GPUVAddr gpu_addr, std::size_t size) cons size_t page_index{gpu_addr >> page_bits}; const size_t page_last{(gpu_addr + size + page_size - 1) >> page_bits}; while (page_index < page_last) { - if (GetEntry(page_index << page_bits) == EntryType::Free) { + if (GetEntry(page_index << page_bits) == EntryType::Free) { return false; } ++page_index; @@ -379,47 +453,63 @@ bool MemoryManager::IsFullyMappedRange(GPUVAddr gpu_addr, std::size_t size) cons return true; } +#pragma inline_recursion(on) + std::vector> MemoryManager::GetSubmappedRange( GPUVAddr gpu_addr, std::size_t size) const { std::vector> result{}; - size_t page_index{gpu_addr >> page_bits}; - size_t remaining_size{size}; - size_t page_offset{gpu_addr & page_mask}; std::optional> last_segment{}; std::optional old_page_addr{}; - const auto extend_size = [this, &last_segment, &page_index, &page_offset](std::size_t bytes) { - if (!last_segment) { - const GPUVAddr new_base_addr = (page_index << page_bits) + page_offset; - last_segment = {new_base_addr, bytes}; - } else { - last_segment->second += bytes; - } - }; - const auto split = [&last_segment, &result] { + const auto split = [&last_segment, &result]([[maybe_unused]] std::size_t page_index, + [[maybe_unused]] std::size_t offset, + [[maybe_unused]] std::size_t copy_amount) { if (last_segment) { result.push_back(*last_segment); last_segment = std::nullopt; } }; - while (remaining_size > 0) { - const size_t num_bytes{std::min(page_size - page_offset, remaining_size)}; - const auto page_addr{GpuToCpuAddress(page_index << page_bits)}; - if (!page_addr || *page_addr == 0) { - split(); - } else if (old_page_addr) { - if (*old_page_addr + page_size != *page_addr) { - split(); + const auto extend_size_big = [this, &split, &old_page_addr, + &last_segment](std::size_t page_index, std::size_t offset, + std::size_t copy_amount) { + const VAddr cpu_addr_base = + (static_cast(big_page_table_cpu[page_index]) << cpu_page_bits) + offset; + if (old_page_addr) { + if (*old_page_addr != cpu_addr_base) { + split(0, 0, 0); } - extend_size(num_bytes); + } + old_page_addr = {cpu_addr_base + copy_amount}; + if (!last_segment) { + const GPUVAddr new_base_addr = (page_index << big_page_bits) + offset; + last_segment = {new_base_addr, copy_amount}; } else { - extend_size(num_bytes); + last_segment->second += copy_amount; } - ++page_index; - page_offset = 0; - remaining_size -= num_bytes; - old_page_addr = page_addr; - } - split(); + }; + const auto extend_size_short = [this, &split, &old_page_addr, + &last_segment](std::size_t page_index, std::size_t offset, + std::size_t copy_amount) { + const VAddr cpu_addr_base = + (static_cast(page_table[page_index]) << cpu_page_bits) + offset; + if (old_page_addr) { + if (*old_page_addr != cpu_addr_base) { + split(0, 0, 0); + } + } + old_page_addr = {cpu_addr_base + copy_amount}; + if (!last_segment) { + const GPUVAddr new_base_addr = (page_index << page_bits) + offset; + last_segment = {new_base_addr, copy_amount}; + } else { + last_segment->second += copy_amount; + } + }; + auto do_short_pages = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) { + GPUVAddr base = (page_index << big_page_bits) + offset; + MemoryOperation(base, copy_amount, extend_size_short, split, split); + }; + MemoryOperation(gpu_addr, size, extend_size_big, split, do_short_pages); + split(0, 0, 0); return result; } -- cgit v1.2.3 From 359f22b808f54de9f19e40a8f93604ee7e4bac80 Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Sun, 6 Feb 2022 18:51:07 +0100 Subject: MemoryManager: Finish up the initial implementation. --- src/video_core/memory_manager.cpp | 177 ++++++++++++++++++++++++++++---------- 1 file changed, 131 insertions(+), 46 deletions(-) (limited to 'src/video_core/memory_manager.cpp') diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp index 836ece136..e1a8b5391 100644 --- a/src/video_core/memory_manager.cpp +++ b/src/video_core/memory_manager.cpp @@ -15,8 +15,6 @@ #include "video_core/rasterizer_interface.h" #include "video_core/renderer_base.h" -#pragma optimize("", off) - namespace Tegra { std::atomic MemoryManager::unique_identifier_generator{}; @@ -42,7 +40,7 @@ MemoryManager::MemoryManager(Core::System& system_, u64 address_space_bits_, u64 big_entries.resize(big_page_table_size / 32, 0); big_page_table_cpu.resize(big_page_table_size); - big_page_table_physical.resize(big_page_table_size); + big_page_continous.resize(big_page_table_size / continous_bits, 0); entries.resize(page_table_size / 32, 0); } @@ -80,6 +78,19 @@ void MemoryManager::SetEntry(size_t position, MemoryManager::EntryType entry) { } } +inline bool MemoryManager::IsBigPageContinous(size_t big_page_index) const { + const u64 entry_mask = big_page_continous[big_page_index / continous_bits]; + const size_t sub_index = big_page_index % continous_bits; + return ((entry_mask >> sub_index) & 0x1ULL) != 0; +} + +inline void MemoryManager::SetBigPageContinous(size_t big_page_index, bool value) { + const u64 continous_mask = big_page_continous[big_page_index / continous_bits]; + const size_t sub_index = big_page_index % continous_bits; + big_page_continous[big_page_index / continous_bits] = + (~(1ULL << sub_index) & continous_mask) | (value ? 1ULL << sub_index : 0); +} + template GPUVAddr MemoryManager::PageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] VAddr cpu_addr, size_t size) { @@ -121,9 +132,19 @@ GPUVAddr MemoryManager::BigPageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] VAddr const auto index = PageEntryIndex(current_gpu_addr); const u32 sub_value = static_cast(current_cpu_addr >> cpu_page_bits); big_page_table_cpu[index] = sub_value; - const PAddr phys_address = - device_memory.GetPhysicalAddr(memory.GetPointer(current_cpu_addr)); - big_page_table_physical[index] = static_cast(phys_address); + const bool is_continous = ([&] { + uintptr_t base_ptr{ + reinterpret_cast(memory.GetPointer(current_cpu_addr))}; + for (VAddr start_cpu = current_cpu_addr + page_size; + start_cpu < current_cpu_addr + big_page_size; start_cpu += page_size) { + base_ptr += page_size; + if (base_ptr != reinterpret_cast(memory.GetPointer(start_cpu))) { + return false; + } + } + return true; + })(); + SetBigPageContinous(index, is_continous); } remaining_size -= big_page_size; } @@ -248,12 +269,17 @@ const u8* MemoryManager::GetPointer(GPUVAddr gpu_addr) const { return memory.GetPointer(*address); } +#ifdef _MSC_VER // no need for gcc / clang but msvc's compiler is more conservative with inlining. #pragma inline_recursion(on) +#endif template inline void MemoryManager::MemoryOperation(GPUVAddr gpu_src_addr, std::size_t size, FuncMapped&& func_mapped, FuncReserved&& func_reserved, FuncUnmapped&& func_unmapped) const { + static constexpr bool BOOL_BREAK_MAPPED = std::is_same_v; + static constexpr bool BOOL_BREAK_RESERVED = std::is_same_v; + static constexpr bool BOOL_BREAK_UNMAPPED = std::is_same_v; u64 used_page_size; u64 used_page_mask; u64 used_page_bits; @@ -276,11 +302,31 @@ inline void MemoryManager::MemoryOperation(GPUVAddr gpu_src_addr, std::size_t si std::min(static_cast(used_page_size) - page_offset, remaining_size)}; auto entry = GetEntry(current_address); if (entry == EntryType::Mapped) [[likely]] { - func_mapped(page_index, page_offset, copy_amount); + if constexpr (BOOL_BREAK_MAPPED) { + if (func_mapped(page_index, page_offset, copy_amount)) { + return; + } + } else { + func_mapped(page_index, page_offset, copy_amount); + } + } else if (entry == EntryType::Reserved) { - func_reserved(page_index, page_offset, copy_amount); + if constexpr (BOOL_BREAK_RESERVED) { + if (func_reserved(page_index, page_offset, copy_amount)) { + return; + } + } else { + func_reserved(page_index, page_offset, copy_amount); + } + } else [[unlikely]] { - func_unmapped(page_index, page_offset, copy_amount); + if constexpr (BOOL_BREAK_UNMAPPED) { + if (func_unmapped(page_index, page_offset, copy_amount)) { + return; + } + } else { + func_unmapped(page_index, page_offset, copy_amount); + } } page_index++; page_offset = 0; @@ -303,7 +349,8 @@ void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, if constexpr (is_safe) { rasterizer->FlushRegion(cpu_addr_base, copy_amount); } - memory.ReadBlockUnsafe(cpu_addr_base, dest_buffer, copy_amount); + u8* physical = memory.GetPointer(cpu_addr_base); + std::memcpy(dest_buffer, physical, copy_amount); dest_buffer = static_cast(dest_buffer) + copy_amount; }; auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) { @@ -312,9 +359,12 @@ void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, if constexpr (is_safe) { rasterizer->FlushRegion(cpu_addr_base, copy_amount); } - memory.ReadBlockUnsafe(cpu_addr_base, dest_buffer, copy_amount); - // u8* physical = device_memory.GetPointer(big_page_table_physical[page_index] + offset); - // std::memcpy(dest_buffer, physical, copy_amount); + if (!IsBigPageContinous(page_index)) { + memory.ReadBlockUnsafe(cpu_addr_base, dest_buffer, copy_amount); + } else { + u8* physical = memory.GetPointer(cpu_addr_base); + std::memcpy(dest_buffer, physical, copy_amount); + } dest_buffer = static_cast(dest_buffer) + copy_amount; }; auto read_short_pages = [&](std::size_t page_index, std::size_t offset, @@ -347,7 +397,8 @@ void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffe if constexpr (is_safe) { rasterizer->InvalidateRegion(cpu_addr_base, copy_amount); } - memory.WriteBlockUnsafe(cpu_addr_base, src_buffer, copy_amount); + u8* physical = memory.GetPointer(cpu_addr_base); + std::memcpy(physical, src_buffer, copy_amount); src_buffer = static_cast(src_buffer) + copy_amount; }; auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) { @@ -356,10 +407,12 @@ void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffe if constexpr (is_safe) { rasterizer->InvalidateRegion(cpu_addr_base, copy_amount); } - memory.WriteBlockUnsafe(cpu_addr_base, src_buffer, copy_amount); - /*u8* physical = - device_memory.GetPointer(big_page_table_physical[page_index] << cpu_page_bits) + offset; - std::memcpy(physical, src_buffer, copy_amount);*/ + if (!IsBigPageContinous(page_index)) { + memory.WriteBlockUnsafe(cpu_addr_base, src_buffer, copy_amount); + } else { + u8* physical = memory.GetPointer(cpu_addr_base); + std::memcpy(physical, src_buffer, copy_amount); + } src_buffer = static_cast(src_buffer) + copy_amount; }; auto write_short_pages = [&](std::size_t page_index, std::size_t offset, @@ -413,48 +466,80 @@ void MemoryManager::CopyBlock(GPUVAddr gpu_dest_addr, GPUVAddr gpu_src_addr, std } bool MemoryManager::IsGranularRange(GPUVAddr gpu_addr, std::size_t size) const { - const auto cpu_addr{GpuToCpuAddress(gpu_addr)}; - if (!cpu_addr) { + if (GetEntry(gpu_addr) == EntryType::Mapped) [[likely]] { + size_t page_index = gpu_addr >> big_page_bits; + if (IsBigPageContinous(page_index)) [[likely]] { + const std::size_t page{(page_index & big_page_mask) + size}; + return page <= big_page_size; + } + const std::size_t page{(gpu_addr & Core::Memory::YUZU_PAGEMASK) + size}; + return page <= Core::Memory::YUZU_PAGESIZE; + } + if (GetEntry(gpu_addr) != EntryType::Mapped) { return false; } - const std::size_t page{(*cpu_addr & Core::Memory::YUZU_PAGEMASK) + size}; + const std::size_t page{(gpu_addr & Core::Memory::YUZU_PAGEMASK) + size}; return page <= Core::Memory::YUZU_PAGESIZE; } bool MemoryManager::IsContinousRange(GPUVAddr gpu_addr, std::size_t size) const { - size_t page_index{gpu_addr >> big_page_bits}; - const size_t page_last{(gpu_addr + size + page_size - 1) >> page_bits}; std::optional old_page_addr{}; - while (page_index != page_last) { - const auto page_addr{GpuToCpuAddress(page_index << page_bits)}; - if (!page_addr || *page_addr == 0) { - return false; + bool result{true}; + auto fail = [&]([[maybe_unused]] std::size_t page_index, [[maybe_unused]] std::size_t offset, + std::size_t copy_amount) { + result = false; + return true; + }; + auto short_check = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) { + const VAddr cpu_addr_base = + (static_cast(page_table[page_index]) << cpu_page_bits) + offset; + if (old_page_addr && *old_page_addr != cpu_addr_base) { + result = false; + return true; } - if (old_page_addr) { - if (*old_page_addr + page_size != *page_addr) { - return false; - } + old_page_addr = {cpu_addr_base + copy_amount}; + return false; + }; + auto big_check = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) { + const VAddr cpu_addr_base = + (static_cast(big_page_table_cpu[page_index]) << cpu_page_bits) + offset; + if (old_page_addr && *old_page_addr != cpu_addr_base) { + result = false; + return true; } - old_page_addr = page_addr; - ++page_index; - } - return true; + old_page_addr = {cpu_addr_base + copy_amount}; + return false; + }; + auto check_short_pages = [&](std::size_t page_index, std::size_t offset, + std::size_t copy_amount) { + GPUVAddr base = (page_index << big_page_bits) + offset; + MemoryOperation(base, copy_amount, short_check, fail, fail); + return !result; + }; + MemoryOperation(gpu_addr, size, big_check, fail, check_short_pages); + return result; } bool MemoryManager::IsFullyMappedRange(GPUVAddr gpu_addr, std::size_t size) const { - size_t page_index{gpu_addr >> page_bits}; - const size_t page_last{(gpu_addr + size + page_size - 1) >> page_bits}; - while (page_index < page_last) { - if (GetEntry(page_index << page_bits) == EntryType::Free) { - return false; - } - ++page_index; - } - return true; + std::optional old_page_addr{}; + bool result{true}; + auto fail = [&]([[maybe_unused]] std::size_t page_index, [[maybe_unused]] std::size_t offset, + [[maybe_unused]] std::size_t copy_amount) { + result = false; + return true; + }; + auto pass = [&]([[maybe_unused]] std::size_t page_index, [[maybe_unused]] std::size_t offset, + [[maybe_unused]] std::size_t copy_amount) { return false; }; + auto check_short_pages = [&](std::size_t page_index, std::size_t offset, + std::size_t copy_amount) { + GPUVAddr base = (page_index << big_page_bits) + offset; + MemoryOperation(base, copy_amount, pass, pass, fail); + return !result; + }; + MemoryOperation(gpu_addr, size, pass, fail, check_short_pages); + return result; } -#pragma inline_recursion(on) - std::vector> MemoryManager::GetSubmappedRange( GPUVAddr gpu_addr, std::size_t size) const { std::vector> result{}; -- cgit v1.2.3 From 5a568b1655f0d721891083da19e2da2614796389 Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Sat, 19 Feb 2022 14:18:02 +0100 Subject: MemoryManager: Fix errors popping out. --- src/video_core/memory_manager.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'src/video_core/memory_manager.cpp') diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp index e1a8b5391..0f97db272 100644 --- a/src/video_core/memory_manager.cpp +++ b/src/video_core/memory_manager.cpp @@ -134,11 +134,15 @@ GPUVAddr MemoryManager::BigPageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] VAddr big_page_table_cpu[index] = sub_value; const bool is_continous = ([&] { uintptr_t base_ptr{ - reinterpret_cast(memory.GetPointer(current_cpu_addr))}; + reinterpret_cast(memory.GetPointerSilent(current_cpu_addr))}; + if (base_ptr == 0) { + return false; + } for (VAddr start_cpu = current_cpu_addr + page_size; start_cpu < current_cpu_addr + big_page_size; start_cpu += page_size) { base_ptr += page_size; - if (base_ptr != reinterpret_cast(memory.GetPointer(start_cpu))) { + auto next_ptr = reinterpret_cast(memory.GetPointerSilent(start_cpu)); + if (next_ptr == 0 || base_ptr != next_ptr) { return false; } } @@ -359,7 +363,7 @@ void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, if constexpr (is_safe) { rasterizer->FlushRegion(cpu_addr_base, copy_amount); } - if (!IsBigPageContinous(page_index)) { + if (!IsBigPageContinous(page_index)) [[unlikely]] { memory.ReadBlockUnsafe(cpu_addr_base, dest_buffer, copy_amount); } else { u8* physical = memory.GetPointer(cpu_addr_base); @@ -407,7 +411,7 @@ void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffe if constexpr (is_safe) { rasterizer->InvalidateRegion(cpu_addr_base, copy_amount); } - if (!IsBigPageContinous(page_index)) { + if (!IsBigPageContinous(page_index)) [[unlikely]] { memory.WriteBlockUnsafe(cpu_addr_base, src_buffer, copy_amount); } else { u8* physical = memory.GetPointer(cpu_addr_base); -- cgit v1.2.3 From 7cfa28a6664fa79b5b12341a93e995d74cb0deef Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Sat, 2 Apr 2022 22:38:58 +0200 Subject: Memory Manager: ensure safety of GPU to CPU address. --- src/video_core/memory_manager.cpp | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src/video_core/memory_manager.cpp') diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp index 0f97db272..4e52ce0fd 100644 --- a/src/video_core/memory_manager.cpp +++ b/src/video_core/memory_manager.cpp @@ -193,6 +193,9 @@ void MemoryManager::Unmap(GPUVAddr gpu_addr, std::size_t size) { } std::optional MemoryManager::GpuToCpuAddress(GPUVAddr gpu_addr) const { + if (gpu_addr >= address_space_size) [[unlikely]] { + return std::nullopt; + } if (GetEntry(gpu_addr) != EntryType::Mapped) [[unlikely]] { if (GetEntry(gpu_addr) != EntryType::Mapped) { return std::nullopt; -- cgit v1.2.3 From f5fd6b5c8674fcf64a3e70809ee0a34d3a95beb6 Mon Sep 17 00:00:00 2001 From: bunnei Date: Sun, 14 Aug 2022 02:36:36 -0700 Subject: DMA & InlineToMemory Engines Rework. --- src/video_core/memory_manager.cpp | 91 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) (limited to 'src/video_core/memory_manager.cpp') diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp index 4e52ce0fd..4a692448e 100644 --- a/src/video_core/memory_manager.cpp +++ b/src/video_core/memory_manager.cpp @@ -462,6 +462,97 @@ void MemoryManager::FlushRegion(GPUVAddr gpu_addr, size_t size) const { MemoryOperation(gpu_addr, size, mapped_big, do_nothing, flush_short_pages); } +bool MemoryManager::IsMemoryDirty(GPUVAddr gpu_addr, size_t size) const { + bool result = false; + auto do_nothing = [&]([[maybe_unused]] std::size_t page_index, + [[maybe_unused]] std::size_t offset, + [[maybe_unused]] std::size_t copy_amount) { return false; }; + + auto mapped_normal = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) { + const VAddr cpu_addr_base = + (static_cast(page_table[page_index]) << cpu_page_bits) + offset; + result |= rasterizer->MustFlushRegion(cpu_addr_base, copy_amount); + return result; + }; + auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) { + const VAddr cpu_addr_base = + (static_cast(big_page_table_cpu[page_index]) << cpu_page_bits) + offset; + result |= rasterizer->MustFlushRegion(cpu_addr_base, copy_amount); + return result; + }; + auto check_short_pages = [&](std::size_t page_index, std::size_t offset, + std::size_t copy_amount) { + GPUVAddr base = (page_index << big_page_bits) + offset; + MemoryOperation(base, copy_amount, mapped_normal, do_nothing, do_nothing); + return result; + }; + MemoryOperation(gpu_addr, size, mapped_big, do_nothing, check_short_pages); + return result; +} + +size_t MemoryManager::MaxContinousRange(GPUVAddr gpu_addr, size_t size) const { + std::optional old_page_addr{}; + size_t range_so_far = 0; + bool result{false}; + auto fail = [&]([[maybe_unused]] std::size_t page_index, [[maybe_unused]] std::size_t offset, + std::size_t copy_amount) { + result = true; + return true; + }; + auto short_check = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) { + const VAddr cpu_addr_base = + (static_cast(page_table[page_index]) << cpu_page_bits) + offset; + if (old_page_addr && *old_page_addr != cpu_addr_base) { + result = true; + return true; + } + range_so_far += copy_amount; + old_page_addr = {cpu_addr_base + copy_amount}; + return false; + }; + auto big_check = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) { + const VAddr cpu_addr_base = + (static_cast(big_page_table_cpu[page_index]) << cpu_page_bits) + offset; + if (old_page_addr && *old_page_addr != cpu_addr_base) { + return true; + } + range_so_far += copy_amount; + old_page_addr = {cpu_addr_base + copy_amount}; + return false; + }; + auto check_short_pages = [&](std::size_t page_index, std::size_t offset, + std::size_t copy_amount) { + GPUVAddr base = (page_index << big_page_bits) + offset; + MemoryOperation(base, copy_amount, short_check, fail, fail); + return result; + }; + MemoryOperation(gpu_addr, size, big_check, fail, check_short_pages); + return range_so_far; +} + +void MemoryManager::InvalidateRegion(GPUVAddr gpu_addr, size_t size) const { + auto do_nothing = [&]([[maybe_unused]] std::size_t page_index, + [[maybe_unused]] std::size_t offset, + [[maybe_unused]] std::size_t copy_amount) {}; + + auto mapped_normal = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) { + const VAddr cpu_addr_base = + (static_cast(page_table[page_index]) << cpu_page_bits) + offset; + rasterizer->InvalidateRegion(cpu_addr_base, copy_amount); + }; + auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) { + const VAddr cpu_addr_base = + (static_cast(big_page_table_cpu[page_index]) << cpu_page_bits) + offset; + rasterizer->InvalidateRegion(cpu_addr_base, copy_amount); + }; + auto invalidate_short_pages = [&](std::size_t page_index, std::size_t offset, + std::size_t copy_amount) { + GPUVAddr base = (page_index << big_page_bits) + offset; + MemoryOperation(base, copy_amount, mapped_normal, do_nothing, do_nothing); + }; + MemoryOperation(gpu_addr, size, mapped_big, do_nothing, invalidate_short_pages); +} + void MemoryManager::CopyBlock(GPUVAddr gpu_dest_addr, GPUVAddr gpu_src_addr, std::size_t size) { std::vector tmp_buffer(size); ReadBlock(gpu_src_addr, tmp_buffer.data(), size); -- cgit v1.2.3 From 770e19f51a0767c8fc2b204d9d50fa504a3c4e44 Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Wed, 13 Apr 2022 16:20:34 +0200 Subject: Buffer Cache: Deduce vertex array limit from memory layout when limit is the highest possible. --- src/video_core/memory_manager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/video_core/memory_manager.cpp') diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp index 4a692448e..3cb2d9224 100644 --- a/src/video_core/memory_manager.cpp +++ b/src/video_core/memory_manager.cpp @@ -193,7 +193,7 @@ void MemoryManager::Unmap(GPUVAddr gpu_addr, std::size_t size) { } std::optional MemoryManager::GpuToCpuAddress(GPUVAddr gpu_addr) const { - if (gpu_addr >= address_space_size) [[unlikely]] { + if (!IsWithinGPUAddressRange(gpu_addr)) [[unlikely]] { return std::nullopt; } if (GetEntry(gpu_addr) != EntryType::Mapped) [[unlikely]] { -- cgit v1.2.3 From fe24c65153349d3a759a2eef02ec703851a96847 Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Thu, 16 Jun 2022 02:12:21 +0200 Subject: General: Fix clang format. --- src/video_core/memory_manager.cpp | 1 - 1 file changed, 1 deletion(-) (limited to 'src/video_core/memory_manager.cpp') diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp index 3cb2d9224..cca401c74 100644 --- a/src/video_core/memory_manager.cpp +++ b/src/video_core/memory_manager.cpp @@ -619,7 +619,6 @@ bool MemoryManager::IsContinousRange(GPUVAddr gpu_addr, std::size_t size) const } bool MemoryManager::IsFullyMappedRange(GPUVAddr gpu_addr, std::size_t size) const { - std::optional old_page_addr{}; bool result{true}; auto fail = [&]([[maybe_unused]] std::size_t page_index, [[maybe_unused]] std::size_t offset, [[maybe_unused]] std::size_t copy_amount) { -- cgit v1.2.3