From ecb3342145780d811017a3a3c8f14f3e0725db75 Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Sun, 16 Jan 2022 04:43:06 +0100 Subject: Garbage Collection: Redesign the algorithm to do a better use of memory. --- src/video_core/renderer_vulkan/vk_staging_buffer_pool.cpp | 2 +- src/video_core/renderer_vulkan/vk_texture_cache.cpp | 9 +++++++++ src/video_core/renderer_vulkan/vk_texture_cache.h | 4 ++++ 3 files changed, 14 insertions(+), 1 deletion(-) (limited to 'src/video_core/renderer_vulkan') diff --git a/src/video_core/renderer_vulkan/vk_staging_buffer_pool.cpp b/src/video_core/renderer_vulkan/vk_staging_buffer_pool.cpp index 5d5329abf..64a58304b 100644 --- a/src/video_core/renderer_vulkan/vk_staging_buffer_pool.cpp +++ b/src/video_core/renderer_vulkan/vk_staging_buffer_pool.cpp @@ -118,7 +118,7 @@ StagingBufferPool::StagingBufferPool(const Device& device_, MemoryAllocator& mem .image = nullptr, .buffer = *stream_buffer, }; - const auto memory_properties = device.GetPhysical().GetMemoryProperties(); + const auto memory_properties = device.GetPhysical().GetMemoryProperties().memoryProperties; VkMemoryAllocateInfo stream_memory_info{ .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, .pNext = make_dedicated ? &dedicated_info : nullptr, diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp index 83a23b66a..884f501ef 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp @@ -1189,6 +1189,14 @@ u64 TextureCacheRuntime::GetDeviceLocalMemory() const { return device.GetDeviceLocalMemory(); } +u64 TextureCacheRuntime::GetDeviceMemoryUsage() const { + return device.GetDeviceMemoryUsage(); +} + +bool TextureCacheRuntime::CanReportMemoryUsage() const { + return device.CanReportMemoryUsage(); +} + void TextureCacheRuntime::TickFrame() {} Image::Image(TextureCacheRuntime& runtime_, const ImageInfo& info_, GPUVAddr gpu_addr_, @@ -1203,6 +1211,7 @@ Image::Image(TextureCacheRuntime& runtime_, const ImageInfo& info_, GPUVAddr gpu } else { flags |= VideoCommon::ImageFlagBits::Converted; } + flags |= VideoCommon::ImageFlagBits::GCProtected; } if (runtime->device.HasDebuggingToolAttached()) { original_image.SetObjectNameEXT(VideoCommon::Name(*this).c_str()); diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.h b/src/video_core/renderer_vulkan/vk_texture_cache.h index c81130dd2..cb15b4a1c 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.h +++ b/src/video_core/renderer_vulkan/vk_texture_cache.h @@ -55,6 +55,10 @@ public: u64 GetDeviceLocalMemory() const; + u64 GetDeviceMemoryUsage() const; + + bool CanReportMemoryUsage() const; + void BlitImage(Framebuffer* dst_framebuffer, ImageView& dst, ImageView& src, const Region2D& dst_region, const Region2D& src_region, Tegra::Engines::Fermi2D::Filter filter, -- cgit v1.2.3 From 5e982a781201a12c4cee6af2908e4732b4c8d945 Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Sun, 16 Jan 2022 05:05:34 +0100 Subject: Buffer Cache: Tune to the levels of the new GC. --- src/video_core/buffer_cache/buffer_cache.h | 34 +++++++++++++++++++--- src/video_core/renderer_opengl/gl_buffer_cache.cpp | 18 ++++++++++++ src/video_core/renderer_opengl/gl_buffer_cache.h | 12 ++++++++ src/video_core/renderer_vulkan/vk_buffer_cache.cpp | 12 ++++++++ src/video_core/renderer_vulkan/vk_buffer_cache.h | 6 ++++ src/video_core/texture_cache/texture_cache_base.h | 2 -- 6 files changed, 78 insertions(+), 6 deletions(-) (limited to 'src/video_core/renderer_vulkan') diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 200d792dd..644c6e57e 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -76,8 +76,9 @@ class BufferCache { static constexpr BufferId NULL_BUFFER_ID{0}; - static constexpr u64 EXPECTED_MEMORY = 512_MiB; - static constexpr u64 CRITICAL_MEMORY = 1_GiB; + static constexpr s64 DEFAULT_EXPECTED_MEMORY = 512_MiB; + static constexpr s64 DEFAULT_CRITICAL_MEMORY = 1_GiB; + static constexpr s64 TARGET_THRESHOLD = 4_GiB; using Maxwell = Tegra::Engines::Maxwell3D::Regs; @@ -436,6 +437,8 @@ private: Common::LeastRecentlyUsedCache lru_cache; u64 frame_tick = 0; u64 total_used_memory = 0; + u64 minimum_memory = 0; + u64 critical_memory = 0; std::array> PAGE_BITS)> page_table; }; @@ -451,11 +454,30 @@ BufferCache

::BufferCache(VideoCore::RasterizerInterface& rasterizer_, // Ensure the first slot is used for the null buffer void(slot_buffers.insert(runtime, NullBufferParams{})); common_ranges.clear(); + + if (!runtime.CanReportMemoryUsage()) { + minimum_memory = DEFAULT_EXPECTED_MEMORY; + critical_memory = DEFAULT_CRITICAL_MEMORY; + return; + } + + const s64 device_memory = static_cast(runtime.GetDeviceLocalMemory()); + const s64 min_spacing_expected = device_memory - 1_GiB - 512_MiB; + const s64 min_spacing_critical = device_memory - 1_GiB; + const s64 mem_tresshold = std::min(device_memory, TARGET_THRESHOLD); + const s64 min_vacancy_expected = (6 * mem_tresshold) / 10; + const s64 min_vacancy_critical = (3 * mem_tresshold) / 10; + minimum_memory = static_cast( + std::max(std::min(device_memory - min_vacancy_expected, min_spacing_expected), + DEFAULT_EXPECTED_MEMORY)); + critical_memory = static_cast( + std::max(std::min(device_memory - min_vacancy_critical, min_spacing_critical), + DEFAULT_CRITICAL_MEMORY)); } template void BufferCache

::RunGarbageCollector() { - const bool aggressive_gc = total_used_memory >= CRITICAL_MEMORY; + const bool aggressive_gc = total_used_memory >= critical_memory; const u64 ticks_to_destroy = aggressive_gc ? 60 : 120; int num_iterations = aggressive_gc ? 64 : 32; const auto clean_up = [this, &num_iterations](BufferId buffer_id) { @@ -486,7 +508,11 @@ void BufferCache

::TickFrame() { const bool skip_preferred = hits * 256 < shots * 251; uniform_buffer_skip_cache_size = skip_preferred ? DEFAULT_SKIP_CACHE_SIZE : 0; - if (total_used_memory >= EXPECTED_MEMORY) { + // If we can obtain the memory info, use it instead of the estimate. + if (runtime.CanReportMemoryUsage()) { + total_used_memory = runtime.GetDeviceMemoryUsage(); + } + if (total_used_memory >= minimum_memory) { RunGarbageCollector(); } ++frame_tick; diff --git a/src/video_core/renderer_opengl/gl_buffer_cache.cpp b/src/video_core/renderer_opengl/gl_buffer_cache.cpp index d4dd10bb6..45db72e6f 100644 --- a/src/video_core/renderer_opengl/gl_buffer_cache.cpp +++ b/src/video_core/renderer_opengl/gl_buffer_cache.cpp @@ -135,6 +135,24 @@ BufferCacheRuntime::BufferCacheRuntime(const Device& device_) buffer.Create(); glNamedBufferData(buffer.handle, 0x10'000, nullptr, GL_STREAM_COPY); } + + device_access_memory = []() -> u64 { + if (GLAD_GL_NVX_gpu_memory_info) { + GLint cur_avail_mem_kb = 0; + glGetIntegerv(GL_GPU_MEMORY_INFO_TOTAL_AVAILABLE_MEMORY_NVX, &cur_avail_mem_kb); + return static_cast(cur_avail_mem_kb) * 1_KiB; + } + return 2_GiB; // Return minimum requirements + }(); +} + +u64 BufferCacheRuntime::GetDeviceMemoryUsage() const { + if (GLAD_GL_NVX_gpu_memory_info) { + GLint cur_avail_mem_kb = 0; + glGetIntegerv(GL_GPU_MEMORY_INFO_CURRENT_AVAILABLE_VIDMEM_NVX, &cur_avail_mem_kb); + return static_cast(cur_avail_mem_kb) * 1_KiB; + } + return 2_GiB; } void BufferCacheRuntime::CopyBuffer(Buffer& dst_buffer, Buffer& src_buffer, diff --git a/src/video_core/renderer_opengl/gl_buffer_cache.h b/src/video_core/renderer_opengl/gl_buffer_cache.h index 7287731b6..86a265fee 100644 --- a/src/video_core/renderer_opengl/gl_buffer_cache.h +++ b/src/video_core/renderer_opengl/gl_buffer_cache.h @@ -151,6 +151,16 @@ public: use_storage_buffers = use_storage_buffers_; } + u64 GetDeviceLocalMemory() const { + return device_access_memory; + } + + u64 GetDeviceMemoryUsage() const; + + bool CanReportMemoryUsage() const { + return GLAD_GL_NVX_gpu_memory_info; + } + private: static constexpr std::array PABO_LUT{ GL_VERTEX_PROGRAM_PARAMETER_BUFFER_NV, GL_TESS_CONTROL_PROGRAM_PARAMETER_BUFFER_NV, @@ -184,6 +194,8 @@ private: std::array copy_compute_uniforms; u32 index_buffer_offset = 0; + + u64 device_access_memory; }; struct BufferCacheParams { diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp index 5ffd93499..def838c34 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp @@ -141,6 +141,18 @@ StagingBufferRef BufferCacheRuntime::DownloadStagingBuffer(size_t size) { return staging_pool.Request(size, MemoryUsage::Download); } +u64 BufferCacheRuntime::GetDeviceLocalMemory() const { + return device.GetDeviceLocalMemory(); +} + +u64 BufferCacheRuntime::GetDeviceMemoryUsage() const { + return device.GetDeviceMemoryUsage(); +} + +bool BufferCacheRuntime::CanReportMemoryUsage() const { + return device.CanReportMemoryUsage(); +} + void BufferCacheRuntime::Finish() { scheduler.Finish(); } diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.h b/src/video_core/renderer_vulkan/vk_buffer_cache.h index 1ee0d8420..d7fdd18ff 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.h +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.h @@ -65,6 +65,12 @@ public: void Finish(); + u64 GetDeviceLocalMemory() const; + + u64 GetDeviceMemoryUsage() const; + + bool CanReportMemoryUsage() const; + [[nodiscard]] StagingBufferRef UploadStagingBuffer(size_t size); [[nodiscard]] StagingBufferRef DownloadStagingBuffer(size_t size); diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h index 5dabc344b..b1324edf3 100644 --- a/src/video_core/texture_cache/texture_cache_base.h +++ b/src/video_core/texture_cache/texture_cache_base.h @@ -60,8 +60,6 @@ class TextureCache { static constexpr bool HAS_DEVICE_MEMORY_INFO = P::HAS_DEVICE_MEMORY_INFO; static constexpr s64 TARGET_THRESHOLD = 4_GiB; - static constexpr s64 MIN_VACANCY_EXPECTED = (6 * TARGET_THRESHOLD) / 10; - static constexpr s64 MIN_VACANCY_CRITICAL = (3 * TARGET_THRESHOLD) / 10; static constexpr s64 DEFAULT_EXPECTED_MEMORY = 1_GiB + 125_MiB; static constexpr s64 DEFAULT_CRITICAL_MEMORY = 1_GiB + 625_MiB; static constexpr size_t GC_EMERGENCY_COUNTS = 2; -- cgit v1.2.3 From 9edbbf2af401f821c0be6a266e65975e3de25fb3 Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Sun, 16 Jan 2022 06:34:43 +0100 Subject: Garbage Collection: Final tuning. --- src/video_core/buffer_cache/buffer_cache.h | 6 ++-- src/video_core/renderer_opengl/gl_buffer_cache.cpp | 2 +- .../renderer_opengl/gl_texture_cache.cpp | 4 +-- .../renderer_vulkan/vk_texture_cache.cpp | 2 +- src/video_core/texture_cache/image_base.h | 10 +++--- src/video_core/texture_cache/texture_cache.h | 36 ++++++++++++++-------- 6 files changed, 36 insertions(+), 24 deletions(-) (limited to 'src/video_core/renderer_vulkan') diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 644c6e57e..21bfb76a4 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -464,9 +464,9 @@ BufferCache

::BufferCache(VideoCore::RasterizerInterface& rasterizer_, const s64 device_memory = static_cast(runtime.GetDeviceLocalMemory()); const s64 min_spacing_expected = device_memory - 1_GiB - 512_MiB; const s64 min_spacing_critical = device_memory - 1_GiB; - const s64 mem_tresshold = std::min(device_memory, TARGET_THRESHOLD); - const s64 min_vacancy_expected = (6 * mem_tresshold) / 10; - const s64 min_vacancy_critical = (3 * mem_tresshold) / 10; + const s64 mem_threshold = std::min(device_memory, TARGET_THRESHOLD); + const s64 min_vacancy_expected = (6 * mem_threshold) / 10; + const s64 min_vacancy_critical = (3 * mem_threshold) / 10; minimum_memory = static_cast( std::max(std::min(device_memory - min_vacancy_expected, min_spacing_expected), DEFAULT_EXPECTED_MEMORY)); diff --git a/src/video_core/renderer_opengl/gl_buffer_cache.cpp b/src/video_core/renderer_opengl/gl_buffer_cache.cpp index 45db72e6f..f73b0af5f 100644 --- a/src/video_core/renderer_opengl/gl_buffer_cache.cpp +++ b/src/video_core/renderer_opengl/gl_buffer_cache.cpp @@ -150,7 +150,7 @@ u64 BufferCacheRuntime::GetDeviceMemoryUsage() const { if (GLAD_GL_NVX_gpu_memory_info) { GLint cur_avail_mem_kb = 0; glGetIntegerv(GL_GPU_MEMORY_INFO_CURRENT_AVAILABLE_VIDMEM_NVX, &cur_avail_mem_kb); - return static_cast(cur_avail_mem_kb) * 1_KiB; + return device_access_memory - static_cast(cur_avail_mem_kb) * 1_KiB; } return 2_GiB; } diff --git a/src/video_core/renderer_opengl/gl_texture_cache.cpp b/src/video_core/renderer_opengl/gl_texture_cache.cpp index 40f52eacb..63586d9d5 100644 --- a/src/video_core/renderer_opengl/gl_texture_cache.cpp +++ b/src/video_core/renderer_opengl/gl_texture_cache.cpp @@ -513,7 +513,7 @@ u64 TextureCacheRuntime::GetDeviceMemoryUsage() const { if (GLAD_GL_NVX_gpu_memory_info) { GLint cur_avail_mem_kb = 0; glGetIntegerv(GL_GPU_MEMORY_INFO_CURRENT_AVAILABLE_VIDMEM_NVX, &cur_avail_mem_kb); - return static_cast(cur_avail_mem_kb) * 1_KiB; + return device_access_memory - static_cast(cur_avail_mem_kb) * 1_KiB; } return 2_GiB; } @@ -695,7 +695,7 @@ Image::Image(TextureCacheRuntime& runtime_, const VideoCommon::ImageInfo& info_, } if (IsConverted(runtime->device, info.format, info.type)) { flags |= ImageFlagBits::Converted; - flags |= ImageFlagBits::GCProtected; + flags |= ImageFlagBits::CostlyLoad; gl_internal_format = IsPixelFormatSRGB(info.format) ? GL_SRGB8_ALPHA8 : GL_RGBA8; gl_format = GL_RGBA; gl_type = GL_UNSIGNED_INT_8_8_8_8_REV; diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp index 884f501ef..f2890d263 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp @@ -1211,7 +1211,7 @@ Image::Image(TextureCacheRuntime& runtime_, const ImageInfo& info_, GPUVAddr gpu } else { flags |= VideoCommon::ImageFlagBits::Converted; } - flags |= VideoCommon::ImageFlagBits::GCProtected; + flags |= VideoCommon::ImageFlagBits::CostlyLoad; } if (runtime->device.HasDebuggingToolAttached()) { original_image.SetObjectNameEXT(VideoCommon::Name(*this).c_str()); diff --git a/src/video_core/texture_cache/image_base.h b/src/video_core/texture_cache/image_base.h index 279f39269..dd0106432 100644 --- a/src/video_core/texture_cache/image_base.h +++ b/src/video_core/texture_cache/image_base.h @@ -29,11 +29,11 @@ enum class ImageFlagBits : u32 { Sparse = 1 << 9, ///< Image has non continous submemory. // Garbage Collection Flags - BadOverlap = 1 << 10, ///< This image overlaps other but doesn't fit, has higher - ///< garbage collection priority - Alias = 1 << 11, ///< This image has aliases and has priority on garbage - ///< collection - GCProtected = 1 << 12, ///< Protected from low-tier GC as they are costy to load back. + BadOverlap = 1 << 10, ///< This image overlaps other but doesn't fit, has higher + ///< garbage collection priority + Alias = 1 << 11, ///< This image has aliases and has priority on garbage + ///< collection + CostlyLoad = 1 << 12, ///< Protected from low-tier GC as it is costly to load back. // Rescaler Rescaled = 1 << 13, diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 7b6bd8697..efc1c4525 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -53,17 +53,16 @@ TextureCache

::TextureCache(Runtime& runtime_, VideoCore::RasterizerInterface& const s64 device_memory = static_cast(runtime.GetDeviceLocalMemory()); const s64 min_spacing_expected = device_memory - 1_GiB - 512_MiB; const s64 min_spacing_critical = device_memory - 1_GiB; - const s64 mem_tresshold = std::min(device_memory, TARGET_THRESHOLD); - const s64 min_vacancy_expected = (6 * mem_tresshold) / 10; - const s64 min_vacancy_critical = (3 * mem_tresshold) / 10; + const s64 mem_threshold = std::min(device_memory, TARGET_THRESHOLD); + const s64 min_vacancy_expected = (6 * mem_threshold) / 10; + const s64 min_vacancy_critical = (3 * mem_threshold) / 10; expected_memory = static_cast( std::max(std::min(device_memory - min_vacancy_expected, min_spacing_expected), DEFAULT_EXPECTED_MEMORY)); critical_memory = static_cast( std::max(std::min(device_memory - min_vacancy_critical, min_spacing_critical), DEFAULT_CRITICAL_MEMORY)); - minimum_memory = static_cast((device_memory - mem_tresshold) / 2); - LOG_CRITICAL(Debug, "Available Memory: {}", device_memory / 1_MiB); + minimum_memory = static_cast((device_memory - mem_threshold) / 2); } else { expected_memory = DEFAULT_EXPECTED_MEMORY + 512_MiB; critical_memory = DEFAULT_CRITICAL_MEMORY + 1_GiB; @@ -73,11 +72,12 @@ TextureCache

::TextureCache(Runtime& runtime_, VideoCore::RasterizerInterface& template void TextureCache

::RunGarbageCollector() { - const bool high_priority_mode = total_used_memory >= expected_memory; - const bool aggressive_mode = total_used_memory >= critical_memory; - const u64 ticks_to_destroy = aggressive_mode ? 10ULL : high_priority_mode ? 25ULL : 100ULL; - size_t num_iterations = aggressive_mode ? 300 : (high_priority_mode ? 50 : 10); - const auto clean_up = [this, &num_iterations, high_priority_mode](ImageId image_id) { + bool high_priority_mode = total_used_memory >= expected_memory; + bool aggressive_mode = total_used_memory >= critical_memory; + const u64 ticks_to_destroy = aggressive_mode ? 10ULL : high_priority_mode ? 25ULL : 50ULL; + size_t num_iterations = aggressive_mode ? 40 : (high_priority_mode ? 20 : 10); + const auto clean_up = [this, &num_iterations, &high_priority_mode, + &aggressive_mode](ImageId image_id) { if (num_iterations == 0) { return true; } @@ -85,7 +85,8 @@ void TextureCache

::RunGarbageCollector() { auto& image = slot_images[image_id]; const bool must_download = image.IsSafeDownload() && False(image.flags & ImageFlagBits::BadOverlap); - if (!high_priority_mode && must_download) { + if (!high_priority_mode && + (must_download || True(image.flags & ImageFlagBits::CostlyLoad))) { return false; } if (must_download) { @@ -100,6 +101,18 @@ void TextureCache

::RunGarbageCollector() { } UnregisterImage(image_id); DeleteImage(image_id, image.scale_tick > frame_tick + 5); + if (total_used_memory < critical_memory) { + if (aggressive_mode) { + // Sink the aggresiveness. + num_iterations >>= 2; + aggressive_mode = false; + return false; + } + if (high_priority_mode && total_used_memory < expected_memory) { + num_iterations >>= 1; + high_priority_mode = false; + } + } return false; }; lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, clean_up); @@ -120,7 +133,6 @@ void TextureCache

::TickFrame() { runtime.TickFrame(); critical_gc = 0; ++frame_tick; - LOG_CRITICAL(Debug, "Current memory: {}", total_used_memory / 1_MiB); } template -- cgit v1.2.3