From 2b58652f0897053d4da04deb586490220ab5a774 Mon Sep 17 00:00:00 2001 From: ReinUsesLisp Date: Sat, 27 Jul 2019 19:40:10 -0300 Subject: maxwell_3d: Slow implementation of passed samples (query 21) Implements GL_SAMPLES_PASSED by waiting immediately for queries. --- src/video_core/engines/maxwell_3d.cpp | 39 +++++++++++++++++++++++------------ src/video_core/engines/maxwell_3d.h | 38 ++++++++++++++++++++++++++++++---- 2 files changed, 60 insertions(+), 17 deletions(-) (limited to 'src/video_core/engines') diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index 0b3e8749b..fe91ff6a0 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -400,6 +400,10 @@ void Maxwell3D::CallMethod(const GPU::MethodCall& method_call) { ProcessQueryCondition(); break; } + case MAXWELL3D_REG_INDEX(counter_reset): { + ProcessCounterReset(); + break; + } case MAXWELL3D_REG_INDEX(sync_info): { ProcessSyncPoint(); break; @@ -544,23 +548,23 @@ void Maxwell3D::ProcessQueryGet() { "Units other than CROP are unimplemented"); switch (regs.query.query_get.operation) { - case Regs::QueryOperation::Release: { - const u64 result = regs.query.query_sequence; - StampQueryResult(result, regs.query.query_get.short_query == 0); + case Regs::QueryOperation::Release: + StampQueryResult(regs.query.query_sequence, regs.query.query_get.short_query == 0); break; - } - case Regs::QueryOperation::Acquire: { - // Todo(Blinkhawk): Under this operation, the GPU waits for the CPU - // to write a value that matches the current payload. + case Regs::QueryOperation::Acquire: + // TODO(Blinkhawk): Under this operation, the GPU waits for the CPU to write a value that + // matches the current payload. UNIMPLEMENTED_MSG("Unimplemented query operation ACQUIRE"); break; - } case Regs::QueryOperation::Counter: { - u64 result{}; + u64 result; switch (regs.query.query_get.select) { case Regs::QuerySelect::Zero: result = 0; break; + case Regs::QuerySelect::SamplesPassed: + result = rasterizer.Query(VideoCore::QueryType::SamplesPassed); + break; default: result = 1; UNIMPLEMENTED_MSG("Unimplemented query select type {}", @@ -569,15 +573,13 @@ void Maxwell3D::ProcessQueryGet() { StampQueryResult(result, regs.query.query_get.short_query == 0); break; } - case Regs::QueryOperation::Trap: { + case Regs::QueryOperation::Trap: UNIMPLEMENTED_MSG("Unimplemented query operation TRAP"); break; - } - default: { + default: UNIMPLEMENTED_MSG("Unknown query operation"); break; } - } } void Maxwell3D::ProcessQueryCondition() { @@ -619,6 +621,17 @@ void Maxwell3D::ProcessQueryCondition() { } } +void Maxwell3D::ProcessCounterReset() { + switch (regs.counter_reset) { + case Regs::CounterReset::SampleCnt: + rasterizer.ResetCounter(VideoCore::QueryType::SamplesPassed); + break; + default: + UNIMPLEMENTED_MSG("counter_reset={}", static_cast(regs.counter_reset)); + break; + } +} + void Maxwell3D::ProcessSyncPoint() { const u32 sync_point = regs.sync_info.sync_point.Value(); const u32 increment = regs.sync_info.increment.Value(); diff --git a/src/video_core/engines/maxwell_3d.h b/src/video_core/engines/maxwell_3d.h index 0a2af54e5..d21f678ed 100644 --- a/src/video_core/engines/maxwell_3d.h +++ b/src/video_core/engines/maxwell_3d.h @@ -409,6 +409,27 @@ public: Linear = 1, }; + enum class CounterReset : u32 { + SampleCnt = 0x01, + Unk02 = 0x02, + Unk03 = 0x03, + Unk04 = 0x04, + EmittedPrimitives = 0x10, // Not tested + Unk11 = 0x11, + Unk12 = 0x12, + Unk13 = 0x13, + Unk15 = 0x15, + Unk16 = 0x16, + Unk17 = 0x17, + Unk18 = 0x18, + Unk1A = 0x1A, + Unk1B = 0x1B, + Unk1C = 0x1C, + Unk1D = 0x1D, + Unk1E = 0x1E, + GeneratedPrimitives = 0x1F, + }; + struct Cull { enum class FrontFace : u32 { ClockWise = 0x0900, @@ -857,7 +878,7 @@ public: BitField<7, 1, u32> c7; } clip_distance_enabled; - INSERT_UNION_PADDING_WORDS(0x1); + u32 samplecnt_enable; float point_size; @@ -865,7 +886,11 @@ public: u32 point_sprite_enable; - INSERT_UNION_PADDING_WORDS(0x5); + INSERT_UNION_PADDING_WORDS(0x3); + + CounterReset counter_reset; + + INSERT_UNION_PADDING_WORDS(0x1); u32 zeta_enable; @@ -1412,12 +1437,15 @@ private: /// Handles a write to the QUERY_GET register. void ProcessQueryGet(); - // Writes the query result accordingly + /// Writes the query result accordingly. void StampQueryResult(u64 payload, bool long_query); - // Handles Conditional Rendering + /// Handles conditional rendering. void ProcessQueryCondition(); + /// Handles counter resets. + void ProcessCounterReset(); + /// Handles writes to syncing register. void ProcessSyncPoint(); @@ -1499,8 +1527,10 @@ ASSERT_REG_POSITION(screen_y_control, 0x4EB); ASSERT_REG_POSITION(vb_element_base, 0x50D); ASSERT_REG_POSITION(vb_base_instance, 0x50E); ASSERT_REG_POSITION(clip_distance_enabled, 0x544); +ASSERT_REG_POSITION(samplecnt_enable, 0x545); ASSERT_REG_POSITION(point_size, 0x546); ASSERT_REG_POSITION(point_sprite_enable, 0x548); +ASSERT_REG_POSITION(counter_reset, 0x54C); ASSERT_REG_POSITION(zeta_enable, 0x54E); ASSERT_REG_POSITION(multisample_control, 0x54F); ASSERT_REG_POSITION(condition, 0x554); -- cgit v1.2.3 From aae8c180cbbf91ba12f53c37e81a97d4b3cc4ccd Mon Sep 17 00:00:00 2001 From: ReinUsesLisp Date: Tue, 26 Nov 2019 18:52:15 -0300 Subject: gl_query_cache: Implement host queries using a deferred cache Instead of waiting immediately for executed commands, defer the query until the guest CPU reads it. This way we get closer to what the guest program is doing. To archive this we have to build a dependency queue, because host APIs (like OpenGL and Vulkan) use ranged queries instead of counters like NVN. Waiting for queries implicitly uses fences and this requires a command being queued, otherwise the driver will lock waiting until a timeout. To fix this when there are no commands queued, we explicitly call glFlush. --- src/video_core/engines/maxwell_3d.cpp | 41 +++-- src/video_core/engines/maxwell_3d.h | 4 + src/video_core/rasterizer_interface.h | 5 +- src/video_core/renderer_opengl/gl_query_cache.cpp | 201 +++++++++++++++++++--- src/video_core/renderer_opengl/gl_query_cache.h | 123 +++++++++++-- src/video_core/renderer_opengl/gl_rasterizer.cpp | 30 ++-- src/video_core/renderer_opengl/gl_rasterizer.h | 10 +- 7 files changed, 328 insertions(+), 86 deletions(-) (limited to 'src/video_core/engines') diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index fe91ff6a0..9add2bc94 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -556,23 +556,13 @@ void Maxwell3D::ProcessQueryGet() { // matches the current payload. UNIMPLEMENTED_MSG("Unimplemented query operation ACQUIRE"); break; - case Regs::QueryOperation::Counter: { - u64 result; - switch (regs.query.query_get.select) { - case Regs::QuerySelect::Zero: - result = 0; - break; - case Regs::QuerySelect::SamplesPassed: - result = rasterizer.Query(VideoCore::QueryType::SamplesPassed); - break; - default: - result = 1; - UNIMPLEMENTED_MSG("Unimplemented query select type {}", - static_cast(regs.query.query_get.select.Value())); + case Regs::QueryOperation::Counter: + if (const std::optional result = GetQueryResult()) { + // If the query returns an empty optional it means it's cached and deferred. + // In this case we have a non-empty result, so we stamp it immediately. + StampQueryResult(*result, regs.query.query_get.short_query == 0); } - StampQueryResult(result, regs.query.query_get.short_query == 0); break; - } case Regs::QueryOperation::Trap: UNIMPLEMENTED_MSG("Unimplemented query operation TRAP"); break; @@ -595,20 +585,20 @@ void Maxwell3D::ProcessQueryCondition() { } case Regs::ConditionMode::ResNonZero: { Regs::QueryCompare cmp; - memory_manager.ReadBlockUnsafe(condition_address, &cmp, sizeof(cmp)); + memory_manager.ReadBlock(condition_address, &cmp, sizeof(cmp)); execute_on = cmp.initial_sequence != 0U && cmp.initial_mode != 0U; break; } case Regs::ConditionMode::Equal: { Regs::QueryCompare cmp; - memory_manager.ReadBlockUnsafe(condition_address, &cmp, sizeof(cmp)); + memory_manager.ReadBlock(condition_address, &cmp, sizeof(cmp)); execute_on = cmp.initial_sequence == cmp.current_sequence && cmp.initial_mode == cmp.current_mode; break; } case Regs::ConditionMode::NotEqual: { Regs::QueryCompare cmp; - memory_manager.ReadBlockUnsafe(condition_address, &cmp, sizeof(cmp)); + memory_manager.ReadBlock(condition_address, &cmp, sizeof(cmp)); execute_on = cmp.initial_sequence != cmp.current_sequence || cmp.initial_mode != cmp.current_mode; break; @@ -674,6 +664,21 @@ void Maxwell3D::DrawArrays() { } } +std::optional Maxwell3D::GetQueryResult() { + switch (regs.query.query_get.select) { + case Regs::QuerySelect::Zero: + return 0; + case Regs::QuerySelect::SamplesPassed: + // Deferred. + rasterizer.Query(regs.query.QueryAddress(), VideoCore::QueryType::SamplesPassed); + return {}; + default: + UNIMPLEMENTED_MSG("Unimplemented query select type {}", + static_cast(regs.query.query_get.select.Value())); + return 1; + } +} + void Maxwell3D::ProcessCBBind(std::size_t stage_index) { // Bind the buffer currently in CB_ADDRESS to the specified index in the desired shader stage. auto& shader = state.shader_stages[stage_index]; diff --git a/src/video_core/engines/maxwell_3d.h b/src/video_core/engines/maxwell_3d.h index d21f678ed..26939be3f 100644 --- a/src/video_core/engines/maxwell_3d.h +++ b/src/video_core/engines/maxwell_3d.h @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -1462,6 +1463,9 @@ private: // Handles a instance drawcall from MME void StepInstance(MMEDrawMode expected_mode, u32 count); + + /// Returns a query's value or an empty object if the value will be deferred through a cache. + std::optional GetQueryResult(); }; #define ASSERT_REG_POSITION(field_name, position) \ diff --git a/src/video_core/rasterizer_interface.h b/src/video_core/rasterizer_interface.h index 2fc627539..a394f2d3e 100644 --- a/src/video_core/rasterizer_interface.h +++ b/src/video_core/rasterizer_interface.h @@ -20,6 +20,7 @@ namespace VideoCore { enum class QueryType { SamplesPassed, }; +constexpr std::size_t NumQueryTypes = 1; enum class LoadCallbackStage { Prepare, @@ -48,8 +49,8 @@ public: /// Resets the counter of a query virtual void ResetCounter(QueryType type) = 0; - /// Returns the value of a GPU query - virtual u64 Query(QueryType type) = 0; + /// Records a GPU query and caches it + virtual void Query(GPUVAddr gpu_addr, QueryType type) = 0; /// Notify rasterizer that all caches should be flushed to Switch memory virtual void FlushAll() = 0; diff --git a/src/video_core/renderer_opengl/gl_query_cache.cpp b/src/video_core/renderer_opengl/gl_query_cache.cpp index 1c7dc999a..8f0e8241d 100644 --- a/src/video_core/renderer_opengl/gl_query_cache.cpp +++ b/src/video_core/renderer_opengl/gl_query_cache.cpp @@ -2,58 +2,203 @@ // Licensed under GPLv2 or any later version // Refer to the license.txt file included. +#include +#include +#include +#include + #include +#include "common/assert.h" +#include "core/core.h" +#include "video_core/engines/maxwell_3d.h" +#include "video_core/memory_manager.h" #include "video_core/renderer_opengl/gl_query_cache.h" +#include "video_core/renderer_opengl/gl_rasterizer.h" namespace OpenGL { -HostCounter::HostCounter(GLenum target) { - query.Create(target); +using VideoCore::QueryType; + +namespace { + +constexpr std::array QueryTargets = {GL_SAMPLES_PASSED}; + +constexpr GLenum GetTarget(QueryType type) { + return QueryTargets[static_cast(type)]; } -HostCounter::~HostCounter() = default; +} // Anonymous namespace + +CounterStream::CounterStream(QueryCache& cache, QueryType type) + : cache{cache}, type{type}, target{GetTarget(type)} {} -void HostCounter::UpdateState(bool enabled) { +CounterStream::~CounterStream() = default; + +void CounterStream::Update(bool enabled, bool any_command_queued) { if (enabled) { - Enable(); - } else { - Disable(); + if (!current) { + current = cache.GetHostCounter(last, type); + } + return; } + + if (current) { + EndQuery(any_command_queued); + } + last = std::exchange(current, nullptr); } -void HostCounter::Reset() { - counter = 0; - Disable(); +void CounterStream::Reset(bool any_command_queued) { + if (current) { + EndQuery(any_command_queued); + } + current = nullptr; + last = nullptr; } -u64 HostCounter::Query() { - if (!is_beginned) { - return counter; +std::shared_ptr CounterStream::GetCurrent(bool any_command_queued) { + if (!current) { + return nullptr; + } + EndQuery(any_command_queued); + last = std::move(current); + current = cache.GetHostCounter(last, type); + return last; +} + +void CounterStream::EndQuery(bool any_command_queued) { + if (!any_command_queued) { + // There are chances a query waited on without commands (glDraw, glClear, glDispatch). Not + // having any of these causes a lock. glFlush is considered a command, so we can safely wait + // for this. Insert to the OpenGL command stream a flush. + glFlush(); + } + glEndQuery(target); +} + +QueryCache::QueryCache(Core::System& system, RasterizerOpenGL& rasterizer) + : RasterizerCache{rasterizer}, system{system}, + rasterizer{rasterizer}, streams{{CounterStream{*this, QueryType::SamplesPassed}}} {} + +QueryCache::~QueryCache() = default; + +void QueryCache::Query(GPUVAddr gpu_addr, QueryType type) { + auto& memory_manager = system.GPU().MemoryManager(); + const auto host_ptr = memory_manager.GetPointer(gpu_addr); + + auto query = TryGet(host_ptr); + if (!query) { + const auto cpu_addr = memory_manager.GpuToCpuAddress(gpu_addr); + ASSERT_OR_EXECUTE(cpu_addr, return;); + + query = std::make_shared(type, *cpu_addr, host_ptr); + Register(query); + } + + query->SetCounter(GetStream(type).GetCurrent(rasterizer.AnyCommandQueued())); + query->MarkAsModified(true, *this); +} + +void QueryCache::UpdateCounters() { + auto& samples_passed = GetStream(QueryType::SamplesPassed); + + const auto& regs = system.GPU().Maxwell3D().regs; + samples_passed.Update(regs.samplecnt_enable, rasterizer.AnyCommandQueued()); +} + +void QueryCache::ResetCounter(QueryType type) { + GetStream(type).Reset(rasterizer.AnyCommandQueued()); +} + +void QueryCache::Reserve(QueryType type, OGLQuery&& query) { + reserved_queries[static_cast(type)].push_back(std::move(query)); +} + +std::shared_ptr QueryCache::GetHostCounter(std::shared_ptr dependency, + QueryType type) { + const auto type_index = static_cast(type); + auto& reserve = reserved_queries[type_index]; + + if (reserve.empty()) { + return std::make_shared(*this, std::move(dependency), type); } - Disable(); - u64 value; - glGetQueryObjectui64v(query.handle, GL_QUERY_RESULT, &value); - Enable(); - counter += value; + auto counter = std::make_shared(*this, std::move(dependency), type, + std::move(reserve.back())); + reserve.pop_back(); return counter; } -void HostCounter::Enable() { - if (is_beginned) { - return; +void QueryCache::FlushObjectInner(const std::shared_ptr& counter_) { + auto& counter = *counter_; + auto& stream = GetStream(counter.GetType()); + + // Waiting for a query while another query of the same target is enabled locks Nvidia's driver. + // To avoid this disable and re-enable keeping the dependency stream. + const bool is_enabled = stream.IsEnabled(); + if (is_enabled) { + stream.Update(false, false); + } + + counter.Flush(); + + if (is_enabled) { + stream.Update(true, false); } - is_beginned = true; - glBeginQuery(GL_SAMPLES_PASSED, query.handle); } -void HostCounter::Disable() { - if (!is_beginned) { - return; +CounterStream& QueryCache::GetStream(QueryType type) { + return streams[static_cast(type)]; +} + +HostCounter::HostCounter(QueryCache& cache, std::shared_ptr dependency, QueryType type) + : cache{cache}, type{type}, dependency{std::move(dependency)} { + const GLenum target = GetTarget(type); + query.Create(target); + glBeginQuery(target, query.handle); +} + +HostCounter::HostCounter(QueryCache& cache, std::shared_ptr dependency, QueryType type, + OGLQuery&& query_) + : cache{cache}, type{type}, dependency{std::move(dependency)}, query{std::move(query_)} { + glBeginQuery(GetTarget(type), query.handle); +} + +HostCounter::~HostCounter() { + cache.Reserve(type, std::move(query)); +} + +u64 HostCounter::Query() { + if (query.handle == 0) { + return result; + } + + glGetQueryObjectui64v(query.handle, GL_QUERY_RESULT, &result); + + if (dependency) { + result += dependency->Query(); } - glEndQuery(GL_SAMPLES_PASSED); - is_beginned = false; + + return result; +} + +CachedQuery::CachedQuery(QueryType type, VAddr cpu_addr, u8* host_ptr) + : RasterizerCacheObject{host_ptr}, type{type}, cpu_addr{cpu_addr}, host_ptr{host_ptr} {} + +CachedQuery::~CachedQuery() = default; + +void CachedQuery::Flush() { + const u64 value = counter->Query(); + std::memcpy(host_ptr, &value, sizeof(value)); +} + +void CachedQuery::SetCounter(std::shared_ptr counter_) { + counter = std::move(counter_); +} + +QueryType CachedQuery::GetType() const { + return type; } } // namespace OpenGL diff --git a/src/video_core/renderer_opengl/gl_query_cache.h b/src/video_core/renderer_opengl/gl_query_cache.h index 52c6546bf..91594b120 100644 --- a/src/video_core/renderer_opengl/gl_query_cache.h +++ b/src/video_core/renderer_opengl/gl_query_cache.h @@ -4,38 +4,131 @@ #pragma once +#include +#include +#include +#include + #include #include "common/common_types.h" +#include "video_core/rasterizer_cache.h" +#include "video_core/rasterizer_interface.h" #include "video_core/renderer_opengl/gl_resource_manager.h" +namespace Core { +class System; +} + namespace OpenGL { -class HostCounter final { +class CachedQuery; +class HostCounter; +class RasterizerOpenGL; +class QueryCache; + +class CounterStream final { public: - explicit HostCounter(GLenum target); - ~HostCounter(); + explicit CounterStream(QueryCache& cache, VideoCore::QueryType type); + ~CounterStream(); + + void Update(bool enabled, bool any_command_queued); + + void Reset(bool any_command_queued); + + std::shared_ptr GetCurrent(bool any_command_queued); + + bool IsEnabled() const { + return current != nullptr; + } + +private: + void EndQuery(bool any_command_queued); + + QueryCache& cache; + + std::shared_ptr current; + std::shared_ptr last; + VideoCore::QueryType type; + GLenum target; +}; + +class QueryCache final : public RasterizerCache> { +public: + explicit QueryCache(Core::System& system, RasterizerOpenGL& rasterizer); + ~QueryCache(); + + void Query(GPUVAddr gpu_addr, VideoCore::QueryType type); + + void UpdateCounters(); + + void ResetCounter(VideoCore::QueryType type); - /// Enables or disables the counter as required. - void UpdateState(bool enabled); + void Reserve(VideoCore::QueryType type, OGLQuery&& query); - /// Resets the counter disabling it if needed. - void Reset(); + std::shared_ptr GetHostCounter(std::shared_ptr dependency, + VideoCore::QueryType type); + +protected: + void FlushObjectInner(const std::shared_ptr& counter) override; + +private: + CounterStream& GetStream(VideoCore::QueryType type); + + Core::System& system; + RasterizerOpenGL& rasterizer; + + std::array streams; + std::array, VideoCore::NumQueryTypes> reserved_queries; +}; + +class HostCounter final { +public: + explicit HostCounter(QueryCache& cache, std::shared_ptr dependency, + VideoCore::QueryType type); + explicit HostCounter(QueryCache& cache, std::shared_ptr dependency, + VideoCore::QueryType type, OGLQuery&& query); + ~HostCounter(); /// Returns the current value of the query. - /// @note It may harm precision of future queries if the counter is not disabled. u64 Query(); private: - /// Enables the counter when disabled. - void Enable(); + QueryCache& cache; + VideoCore::QueryType type; - /// Disables the counter when enabled. - void Disable(); + std::shared_ptr dependency; ///< Counter queued before this one. + OGLQuery query; ///< OpenGL query. + u64 result; ///< Added values of the counter. +}; + +class CachedQuery final : public RasterizerCacheObject { +public: + explicit CachedQuery(VideoCore::QueryType type, VAddr cpu_addr, u8* host_ptr); + ~CachedQuery(); + + /// Writes the counter value to host memory. + void Flush(); + + /// Updates the counter this cached query registered in guest memory will write when requested. + void SetCounter(std::shared_ptr counter); - OGLQuery query; ///< OpenGL query. - u64 counter{}; ///< Added values of the counter. - bool is_beginned{}; ///< True when the OpenGL query is beginned. + /// Returns the query type. + VideoCore::QueryType GetType() const; + + VAddr GetCpuAddr() const override { + return cpu_addr; + } + + std::size_t GetSizeInBytes() const override { + return sizeof(u64); + } + +private: + VideoCore::QueryType type; + VAddr cpu_addr; ///< Guest CPU address. + u8* host_ptr; ///< Writable host pointer. + std::shared_ptr counter; ///< Host counter to query, owns the dependency tree. }; } // namespace OpenGL diff --git a/src/video_core/renderer_opengl/gl_rasterizer.cpp b/src/video_core/renderer_opengl/gl_rasterizer.cpp index 652db705b..827f85884 100644 --- a/src/video_core/renderer_opengl/gl_rasterizer.cpp +++ b/src/video_core/renderer_opengl/gl_rasterizer.cpp @@ -25,6 +25,7 @@ #include "video_core/engines/maxwell_3d.h" #include "video_core/engines/shader_type.h" #include "video_core/memory_manager.h" +#include "video_core/renderer_opengl/gl_query_cache.h" #include "video_core/renderer_opengl/gl_rasterizer.h" #include "video_core/renderer_opengl/gl_shader_cache.h" #include "video_core/renderer_opengl/gl_shader_gen.h" @@ -92,8 +93,8 @@ std::size_t GetConstBufferSize(const Tegra::Engines::ConstBufferInfo& buffer, RasterizerOpenGL::RasterizerOpenGL(Core::System& system, Core::Frontend::EmuWindow& emu_window, ScreenInfo& info) : RasterizerAccelerated{system.Memory()}, texture_cache{system, *this, device}, - shader_cache{*this, system, emu_window, device}, system{system}, screen_info{info}, - buffer_cache{*this, system, device, STREAM_BUFFER_SIZE} { + shader_cache{*this, system, emu_window, device}, query_cache{system, *this}, system{system}, + screen_info{info}, buffer_cache{*this, system, device, STREAM_BUFFER_SIZE} { shader_program_manager = std::make_unique(); state.draw.shader_program = 0; state.Apply(); @@ -548,9 +549,9 @@ void RasterizerOpenGL::Clear() { void RasterizerOpenGL::Draw(bool is_indexed, bool is_instanced) { MICROPROFILE_SCOPE(OpenGL_Drawing); auto& gpu = system.GPU().Maxwell3D(); - const auto& regs = gpu.regs; - samples_passed.UpdateState(regs.samplecnt_enable); + + query_cache.UpdateCounters(); SyncRasterizeEnable(state); SyncColorMask(); @@ -718,24 +719,11 @@ void RasterizerOpenGL::DispatchCompute(GPUVAddr code_addr) { } void RasterizerOpenGL::ResetCounter(VideoCore::QueryType type) { - switch (type) { - case VideoCore::QueryType::SamplesPassed: - samples_passed.Reset(); - break; - default: - UNIMPLEMENTED_MSG("type={}", static_cast(type)); - break; - } + query_cache.ResetCounter(type); } -u64 RasterizerOpenGL::Query(VideoCore::QueryType type) { - switch (type) { - case VideoCore::QueryType::SamplesPassed: - return samples_passed.Query(); - default: - UNIMPLEMENTED_MSG("type={}", static_cast(type)); - return 1; - } +void RasterizerOpenGL::Query(GPUVAddr gpu_addr, VideoCore::QueryType type) { + query_cache.Query(gpu_addr, type); } void RasterizerOpenGL::FlushAll() {} @@ -747,6 +735,7 @@ void RasterizerOpenGL::FlushRegion(CacheAddr addr, u64 size) { } texture_cache.FlushRegion(addr, size); buffer_cache.FlushRegion(addr, size); + query_cache.FlushRegion(addr, size); } void RasterizerOpenGL::InvalidateRegion(CacheAddr addr, u64 size) { @@ -757,6 +746,7 @@ void RasterizerOpenGL::InvalidateRegion(CacheAddr addr, u64 size) { texture_cache.InvalidateRegion(addr, size); shader_cache.InvalidateRegion(addr, size); buffer_cache.InvalidateRegion(addr, size); + query_cache.InvalidateRegion(addr, size); } void RasterizerOpenGL::FlushAndInvalidateRegion(CacheAddr addr, u64 size) { diff --git a/src/video_core/renderer_opengl/gl_rasterizer.h b/src/video_core/renderer_opengl/gl_rasterizer.h index 857a6c073..4fb6811a7 100644 --- a/src/video_core/renderer_opengl/gl_rasterizer.h +++ b/src/video_core/renderer_opengl/gl_rasterizer.h @@ -63,7 +63,7 @@ public: void Clear() override; void DispatchCompute(GPUVAddr code_addr) override; void ResetCounter(VideoCore::QueryType type) override; - u64 Query(VideoCore::QueryType type) override; + void Query(GPUVAddr gpu_addr, VideoCore::QueryType type) override; void FlushAll() override; void FlushRegion(CacheAddr addr, u64 size) override; void InvalidateRegion(CacheAddr addr, u64 size) override; @@ -78,6 +78,11 @@ public: void LoadDiskResources(const std::atomic_bool& stop_loading, const VideoCore::DiskResourceLoadCallback& callback) override; + /// Returns true when there are commands queued to the OpenGL server. + bool AnyCommandQueued() const { + return num_queued_commands > 0; + } + private: /// Configures the color and depth framebuffer states. void ConfigureFramebuffers(); @@ -207,6 +212,7 @@ private: ShaderCacheOpenGL shader_cache; SamplerCacheOpenGL sampler_cache; FramebufferCacheOpenGL framebuffer_cache; + QueryCache query_cache; Core::System& system; ScreenInfo& screen_info; @@ -224,8 +230,6 @@ private: BindBuffersRangePushBuffer bind_ubo_pushbuffer{GL_UNIFORM_BUFFER}; BindBuffersRangePushBuffer bind_ssbo_pushbuffer{GL_SHADER_STORAGE_BUFFER}; - HostCounter samples_passed{GL_SAMPLES_PASSED}; - /// Number of commands queued to the OpenGL driver. Reseted on flush. std::size_t num_queued_commands = 0; }; -- cgit v1.2.3 From 73d2d3342dc8867d32f08f89b2ca36ff071598dc Mon Sep 17 00:00:00 2001 From: ReinUsesLisp Date: Thu, 28 Nov 2019 02:15:34 -0300 Subject: gl_query_cache: Optimize query cache Use a custom cache instead of relying on a ranged cache. --- src/video_core/engines/maxwell_3d.cpp | 11 +- src/video_core/rasterizer_interface.h | 3 +- src/video_core/renderer_opengl/gl_query_cache.cpp | 214 ++++++++++++++++------ src/video_core/renderer_opengl/gl_query_cache.h | 61 ++++-- src/video_core/renderer_opengl/gl_rasterizer.cpp | 5 +- src/video_core/renderer_opengl/gl_rasterizer.h | 2 +- 6 files changed, 217 insertions(+), 79 deletions(-) (limited to 'src/video_core/engines') diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index 9add2bc94..842cdcbcf 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -4,6 +4,7 @@ #include #include +#include #include "common/assert.h" #include "core/core.h" #include "core/core_timing.h" @@ -16,6 +17,8 @@ namespace Tegra::Engines { +using VideoCore::QueryType; + /// First register id that is actually a Macro call. constexpr u32 MacroRegistersStart = 0xE00; @@ -614,10 +617,11 @@ void Maxwell3D::ProcessQueryCondition() { void Maxwell3D::ProcessCounterReset() { switch (regs.counter_reset) { case Regs::CounterReset::SampleCnt: - rasterizer.ResetCounter(VideoCore::QueryType::SamplesPassed); + rasterizer.ResetCounter(QueryType::SamplesPassed); break; default: - UNIMPLEMENTED_MSG("counter_reset={}", static_cast(regs.counter_reset)); + LOG_WARNING(Render_OpenGL, "Unimplemented counter reset={}", + static_cast(regs.counter_reset)); break; } } @@ -670,7 +674,8 @@ std::optional Maxwell3D::GetQueryResult() { return 0; case Regs::QuerySelect::SamplesPassed: // Deferred. - rasterizer.Query(regs.query.QueryAddress(), VideoCore::QueryType::SamplesPassed); + rasterizer.Query(regs.query.QueryAddress(), VideoCore::QueryType::SamplesPassed, + system.GPU().GetTicks()); return {}; default: UNIMPLEMENTED_MSG("Unimplemented query select type {}", diff --git a/src/video_core/rasterizer_interface.h b/src/video_core/rasterizer_interface.h index a394f2d3e..e9f1436f0 100644 --- a/src/video_core/rasterizer_interface.h +++ b/src/video_core/rasterizer_interface.h @@ -6,6 +6,7 @@ #include #include +#include #include "common/common_types.h" #include "video_core/engines/fermi_2d.h" #include "video_core/gpu.h" @@ -50,7 +51,7 @@ public: virtual void ResetCounter(QueryType type) = 0; /// Records a GPU query and caches it - virtual void Query(GPUVAddr gpu_addr, QueryType type) = 0; + virtual void Query(GPUVAddr gpu_addr, QueryType type, std::optional timestamp) = 0; /// Notify rasterizer that all caches should be flushed to Switch memory virtual void FlushAll() = 0; diff --git a/src/video_core/renderer_opengl/gl_query_cache.cpp b/src/video_core/renderer_opengl/gl_query_cache.cpp index 8f0e8241d..74cb73209 100644 --- a/src/video_core/renderer_opengl/gl_query_cache.cpp +++ b/src/video_core/renderer_opengl/gl_query_cache.cpp @@ -2,8 +2,10 @@ // Licensed under GPLv2 or any later version // Refer to the license.txt file included. +#include #include #include +#include #include #include @@ -22,6 +24,13 @@ using VideoCore::QueryType; namespace { +constexpr std::uintptr_t PAGE_SIZE = 4096; +constexpr int PAGE_SHIFT = 12; + +constexpr std::size_t SMALL_QUERY_SIZE = 8; // Query size without timestamp +constexpr std::size_t LARGE_QUERY_SIZE = 16; // Query size with timestamp +constexpr std::ptrdiff_t TIMESTAMP_OFFSET = 8; + constexpr std::array QueryTargets = {GL_SAMPLES_PASSED}; constexpr GLenum GetTarget(QueryType type) { @@ -37,23 +46,19 @@ CounterStream::~CounterStream() = default; void CounterStream::Update(bool enabled, bool any_command_queued) { if (enabled) { - if (!current) { - current = cache.GetHostCounter(last, type); - } - return; - } - - if (current) { - EndQuery(any_command_queued); + Enable(); + } else { + Disable(any_command_queued); } - last = std::exchange(current, nullptr); } void CounterStream::Reset(bool any_command_queued) { if (current) { EndQuery(any_command_queued); + + // Immediately start a new query to avoid disabling its state. + current = cache.GetHostCounter(nullptr, type); } - current = nullptr; last = nullptr; } @@ -67,6 +72,20 @@ std::shared_ptr CounterStream::GetCurrent(bool any_command_queued) return last; } +void CounterStream::Enable() { + if (current) { + return; + } + current = cache.GetHostCounter(last, type); +} + +void CounterStream::Disable(bool any_command_queued) { + if (current) { + EndQuery(any_command_queued); + } + last = std::exchange(current, nullptr); +} + void CounterStream::EndQuery(bool any_command_queued) { if (!any_command_queued) { // There are chances a query waited on without commands (glDraw, glClear, glDispatch). Not @@ -78,26 +97,57 @@ void CounterStream::EndQuery(bool any_command_queued) { } QueryCache::QueryCache(Core::System& system, RasterizerOpenGL& rasterizer) - : RasterizerCache{rasterizer}, system{system}, - rasterizer{rasterizer}, streams{{CounterStream{*this, QueryType::SamplesPassed}}} {} + : system{system}, rasterizer{rasterizer}, streams{{CounterStream{*this, + QueryType::SamplesPassed}}} {} QueryCache::~QueryCache() = default; -void QueryCache::Query(GPUVAddr gpu_addr, QueryType type) { +void QueryCache::InvalidateRegion(CacheAddr addr, std::size_t size) { + const u64 addr_begin = static_cast(addr); + const u64 addr_end = addr_begin + static_cast(size); + const auto in_range = [addr_begin, addr_end](CachedQuery& query) { + const u64 cache_begin = query.GetCacheAddr(); + const u64 cache_end = cache_begin + query.GetSizeInBytes(); + return cache_begin < addr_end && addr_begin < cache_end; + }; + + const u64 page_end = addr_end >> PAGE_SHIFT; + for (u64 page = addr_begin >> PAGE_SHIFT; page <= page_end; ++page) { + const auto& it = cached_queries.find(page); + if (it == std::end(cached_queries)) { + continue; + } + auto& contents = it->second; + for (auto& query : contents) { + if (!in_range(query)) { + continue; + } + rasterizer.UpdatePagesCachedCount(query.GetCpuAddr(), query.GetSizeInBytes(), -1); + Flush(query); + } + contents.erase(std::remove_if(std::begin(contents), std::end(contents), in_range), + std::end(contents)); + } +} + +void QueryCache::FlushRegion(CacheAddr addr, std::size_t size) { + // We can handle flushes in the same way as invalidations. + InvalidateRegion(addr, size); +} + +void QueryCache::Query(GPUVAddr gpu_addr, QueryType type, std::optional timestamp) { auto& memory_manager = system.GPU().MemoryManager(); const auto host_ptr = memory_manager.GetPointer(gpu_addr); - auto query = TryGet(host_ptr); + CachedQuery* query = TryGet(ToCacheAddr(host_ptr)); if (!query) { const auto cpu_addr = memory_manager.GpuToCpuAddress(gpu_addr); ASSERT_OR_EXECUTE(cpu_addr, return;); - query = std::make_shared(type, *cpu_addr, host_ptr); - Register(query); + query = &Register(CachedQuery(type, *cpu_addr, host_ptr)); } - query->SetCounter(GetStream(type).GetCurrent(rasterizer.AnyCommandQueued())); - query->MarkAsModified(true, *this); + query->SetCounter(GetStream(type).GetCurrent(rasterizer.AnyCommandQueued()), timestamp); } void QueryCache::UpdateCounters() { @@ -117,34 +167,54 @@ void QueryCache::Reserve(QueryType type, OGLQuery&& query) { std::shared_ptr QueryCache::GetHostCounter(std::shared_ptr dependency, QueryType type) { - const auto type_index = static_cast(type); - auto& reserve = reserved_queries[type_index]; - + auto& reserve = reserved_queries[static_cast(type)]; + OGLQuery query; if (reserve.empty()) { - return std::make_shared(*this, std::move(dependency), type); + query.Create(GetTarget(type)); + } else { + query = std::move(reserve.back()); + reserve.pop_back(); } - auto counter = std::make_shared(*this, std::move(dependency), type, - std::move(reserve.back())); - reserve.pop_back(); - return counter; + return std::make_shared(*this, std::move(dependency), type, std::move(query)); +} + +CachedQuery& QueryCache::Register(CachedQuery&& cached_query) { + const u64 page = static_cast(cached_query.GetCacheAddr()) >> PAGE_SHIFT; + auto& stored_ref = cached_queries[page].emplace_back(std::move(cached_query)); + rasterizer.UpdatePagesCachedCount(stored_ref.GetCpuAddr(), stored_ref.GetSizeInBytes(), 1); + return stored_ref; +} + +CachedQuery* QueryCache::TryGet(CacheAddr addr) { + const u64 page = static_cast(addr) >> PAGE_SHIFT; + const auto it = cached_queries.find(page); + if (it == std::end(cached_queries)) { + return nullptr; + } + auto& contents = it->second; + const auto found = + std::find_if(std::begin(contents), std::end(contents), + [addr](const auto& query) { return query.GetCacheAddr() == addr; }); + return found != std::end(contents) ? &*found : nullptr; } -void QueryCache::FlushObjectInner(const std::shared_ptr& counter_) { - auto& counter = *counter_; - auto& stream = GetStream(counter.GetType()); +void QueryCache::Flush(CachedQuery& cached_query) { + auto& stream = GetStream(cached_query.GetType()); // Waiting for a query while another query of the same target is enabled locks Nvidia's driver. // To avoid this disable and re-enable keeping the dependency stream. - const bool is_enabled = stream.IsEnabled(); - if (is_enabled) { - stream.Update(false, false); + // But we only have to do this if we have pending waits to be done. + const bool slice_counter = stream.IsEnabled() && cached_query.WaitPending(); + const bool any_command_queued = rasterizer.AnyCommandQueued(); + if (slice_counter) { + stream.Update(false, any_command_queued); } - counter.Flush(); + cached_query.Flush(); - if (is_enabled) { - stream.Update(true, false); + if (slice_counter) { + stream.Update(true, any_command_queued); } } @@ -152,13 +222,6 @@ CounterStream& QueryCache::GetStream(QueryType type) { return streams[static_cast(type)]; } -HostCounter::HostCounter(QueryCache& cache, std::shared_ptr dependency, QueryType type) - : cache{cache}, type{type}, dependency{std::move(dependency)} { - const GLenum target = GetTarget(type); - query.Create(target); - glBeginQuery(target, query.handle); -} - HostCounter::HostCounter(QueryCache& cache, std::shared_ptr dependency, QueryType type, OGLQuery&& query_) : cache{cache}, type{type}, dependency{std::move(dependency)}, query{std::move(query_)} { @@ -170,35 +233,80 @@ HostCounter::~HostCounter() { } u64 HostCounter::Query() { - if (query.handle == 0) { - return result; + if (result) { + return *result; } - glGetQueryObjectui64v(query.handle, GL_QUERY_RESULT, &result); - + u64 value; + glGetQueryObjectui64v(query.handle, GL_QUERY_RESULT, &value); if (dependency) { - result += dependency->Query(); + value += dependency->Query(); } - return result; + return *(result = value); +} + +bool HostCounter::WaitPending() const noexcept { + return result.has_value(); } CachedQuery::CachedQuery(QueryType type, VAddr cpu_addr, u8* host_ptr) - : RasterizerCacheObject{host_ptr}, type{type}, cpu_addr{cpu_addr}, host_ptr{host_ptr} {} + : type{type}, cpu_addr{cpu_addr}, host_ptr{host_ptr} {} + +CachedQuery::CachedQuery(CachedQuery&& rhs) noexcept + : type{rhs.type}, cpu_addr{rhs.cpu_addr}, host_ptr{rhs.host_ptr}, + counter{std::move(rhs.counter)}, timestamp{rhs.timestamp} {} CachedQuery::~CachedQuery() = default; +CachedQuery& CachedQuery::operator=(CachedQuery&& rhs) noexcept { + type = rhs.type; + cpu_addr = rhs.cpu_addr; + host_ptr = rhs.host_ptr; + counter = std::move(rhs.counter); + timestamp = rhs.timestamp; + return *this; +} + void CachedQuery::Flush() { - const u64 value = counter->Query(); - std::memcpy(host_ptr, &value, sizeof(value)); + // When counter is nullptr it means that it's just been reseted. We are supposed to write a zero + // in these cases. + const u64 value = counter ? counter->Query() : 0; + std::memcpy(host_ptr, &value, sizeof(u64)); + + if (timestamp) { + std::memcpy(host_ptr + TIMESTAMP_OFFSET, &*timestamp, sizeof(u64)); + } } -void CachedQuery::SetCounter(std::shared_ptr counter_) { +void CachedQuery::SetCounter(std::shared_ptr counter_, std::optional timestamp_) { + if (counter) { + // If there's an old counter set it means the query is being rewritten by the game. + // To avoid losing the data forever, flush here. + Flush(); + } counter = std::move(counter_); + timestamp = timestamp_; +} + +bool CachedQuery::WaitPending() const noexcept { + return counter && counter->WaitPending(); } -QueryType CachedQuery::GetType() const { +QueryType CachedQuery::GetType() const noexcept { return type; } +VAddr CachedQuery::GetCpuAddr() const noexcept { + return cpu_addr; +} + +CacheAddr CachedQuery::GetCacheAddr() const noexcept { + return ToCacheAddr(host_ptr); +} + +u64 CachedQuery::GetSizeInBytes() const noexcept { + return timestamp ? LARGE_QUERY_SIZE : SMALL_QUERY_SIZE; +} + } // namespace OpenGL diff --git a/src/video_core/renderer_opengl/gl_query_cache.h b/src/video_core/renderer_opengl/gl_query_cache.h index 91594b120..d9f22b44d 100644 --- a/src/video_core/renderer_opengl/gl_query_cache.h +++ b/src/video_core/renderer_opengl/gl_query_cache.h @@ -7,12 +7,12 @@ #include #include #include +#include #include #include #include "common/common_types.h" -#include "video_core/rasterizer_cache.h" #include "video_core/rasterizer_interface.h" #include "video_core/renderer_opengl/gl_resource_manager.h" @@ -43,6 +43,10 @@ public: } private: + void Enable(); + + void Disable(bool any_command_queued); + void EndQuery(bool any_command_queued); QueryCache& cache; @@ -53,12 +57,16 @@ private: GLenum target; }; -class QueryCache final : public RasterizerCache> { +class QueryCache final { public: explicit QueryCache(Core::System& system, RasterizerOpenGL& rasterizer); ~QueryCache(); - void Query(GPUVAddr gpu_addr, VideoCore::QueryType type); + void InvalidateRegion(CacheAddr addr, std::size_t size); + + void FlushRegion(CacheAddr addr, std::size_t size); + + void Query(GPUVAddr gpu_addr, VideoCore::QueryType type, std::optional timestamp); void UpdateCounters(); @@ -69,23 +77,26 @@ public: std::shared_ptr GetHostCounter(std::shared_ptr dependency, VideoCore::QueryType type); -protected: - void FlushObjectInner(const std::shared_ptr& counter) override; - private: + CachedQuery& Register(CachedQuery&& cached_query); + + CachedQuery* TryGet(CacheAddr addr); + + void Flush(CachedQuery& cached_query); + CounterStream& GetStream(VideoCore::QueryType type); Core::System& system; RasterizerOpenGL& rasterizer; + std::unordered_map> cached_queries; + std::array streams; std::array, VideoCore::NumQueryTypes> reserved_queries; }; class HostCounter final { public: - explicit HostCounter(QueryCache& cache, std::shared_ptr dependency, - VideoCore::QueryType type); explicit HostCounter(QueryCache& cache, std::shared_ptr dependency, VideoCore::QueryType type, OGLQuery&& query); ~HostCounter(); @@ -93,42 +104,54 @@ public: /// Returns the current value of the query. u64 Query(); + /// Returns true when querying this counter will potentially wait for OpenGL. + bool WaitPending() const noexcept; + private: QueryCache& cache; VideoCore::QueryType type; std::shared_ptr dependency; ///< Counter queued before this one. OGLQuery query; ///< OpenGL query. - u64 result; ///< Added values of the counter. + std::optional result; ///< Added values of the counter. }; -class CachedQuery final : public RasterizerCacheObject { +class CachedQuery final { public: explicit CachedQuery(VideoCore::QueryType type, VAddr cpu_addr, u8* host_ptr); + CachedQuery(CachedQuery&&) noexcept; + CachedQuery(const CachedQuery&) = delete; ~CachedQuery(); + CachedQuery& operator=(CachedQuery&&) noexcept; + /// Writes the counter value to host memory. void Flush(); /// Updates the counter this cached query registered in guest memory will write when requested. - void SetCounter(std::shared_ptr counter); + void SetCounter(std::shared_ptr counter, std::optional timestamp); + + /// Returns true when a flushing this query will potentially wait for OpenGL. + bool WaitPending() const noexcept; /// Returns the query type. - VideoCore::QueryType GetType() const; + VideoCore::QueryType GetType() const noexcept; - VAddr GetCpuAddr() const override { - return cpu_addr; - } + /// Returns the guest CPU address for this query. + VAddr GetCpuAddr() const noexcept; - std::size_t GetSizeInBytes() const override { - return sizeof(u64); - } + /// Returns the cache address for this query. + CacheAddr GetCacheAddr() const noexcept; + + /// Returns the number of cached bytes. + u64 GetSizeInBytes() const noexcept; private: - VideoCore::QueryType type; + VideoCore::QueryType type; ///< Abstracted query type (e.g. samples passed). VAddr cpu_addr; ///< Guest CPU address. u8* host_ptr; ///< Writable host pointer. std::shared_ptr counter; ///< Host counter to query, owns the dependency tree. + std::optional timestamp; ///< Timestamp to flush to guest memory. }; } // namespace OpenGL diff --git a/src/video_core/renderer_opengl/gl_rasterizer.cpp b/src/video_core/renderer_opengl/gl_rasterizer.cpp index 827f85884..4bdc8db85 100644 --- a/src/video_core/renderer_opengl/gl_rasterizer.cpp +++ b/src/video_core/renderer_opengl/gl_rasterizer.cpp @@ -722,8 +722,9 @@ void RasterizerOpenGL::ResetCounter(VideoCore::QueryType type) { query_cache.ResetCounter(type); } -void RasterizerOpenGL::Query(GPUVAddr gpu_addr, VideoCore::QueryType type) { - query_cache.Query(gpu_addr, type); +void RasterizerOpenGL::Query(GPUVAddr gpu_addr, VideoCore::QueryType type, + std::optional timestamp) { + query_cache.Query(gpu_addr, type, timestamp); } void RasterizerOpenGL::FlushAll() {} diff --git a/src/video_core/renderer_opengl/gl_rasterizer.h b/src/video_core/renderer_opengl/gl_rasterizer.h index 4fb6811a7..c772fd4ba 100644 --- a/src/video_core/renderer_opengl/gl_rasterizer.h +++ b/src/video_core/renderer_opengl/gl_rasterizer.h @@ -63,7 +63,7 @@ public: void Clear() override; void DispatchCompute(GPUVAddr code_addr) override; void ResetCounter(VideoCore::QueryType type) override; - void Query(GPUVAddr gpu_addr, VideoCore::QueryType type) override; + void Query(GPUVAddr gpu_addr, VideoCore::QueryType type, std::optional timestamp) override; void FlushAll() override; void FlushRegion(CacheAddr addr, u64 size) override; void InvalidateRegion(CacheAddr addr, u64 size) override; -- cgit v1.2.3