Implement Counter Queue and Partial Host Conditional Rendering (#1167)

* Implementation of query queue and host conditional rendering

* Resolve some comments.

* Use overloads instead of passing object.

* Wake the consumer threads when incrementing syncpoints.

Also, do a busy loop when awaiting the counter for a blocking flush, rather than potentially sleeping the thread.

* Ensure there's a command between begin and end query.

7 files changed, 182 insertions, 54 deletions

diff --git a/Ryujinx.Graphics.Gpu/Engine/MethodClear.cs b/Ryujinx.Graphics.Gpu/Engine/MethodClear.cs
index a555015d..a9552762 100644
--- a/Ryujinx.Graphics.Gpu/Engine/MethodClear.cs
+++ b/Ryujinx.Graphics.Gpu/Engine/MethodClear.cs
@@ -13,7 +13,9 @@ namespace Ryujinx.Graphics.Gpu.Engine
         /// <param name="argument">Method call argument</param>
         private void Clear(GpuState state, int argument)
         {
-            if (!GetRenderEnable(state))
+            ConditionalRenderEnabled renderEnable = GetRenderEnable(state);
+
+            if (renderEnable == ConditionalRenderEnabled.False)
             {
                 return;
             }
@@ -68,6 +70,11 @@ namespace Ryujinx.Graphics.Gpu.Engine
             }
 
             UpdateRenderTargetState(state, useControl: true);
+
+            if (renderEnable == ConditionalRenderEnabled.Host)
+            {
+                _context.Renderer.Pipeline.EndHostConditionalRendering();
+            }
         }
     }
 }
\ No newline at end of file
diff --git a/Ryujinx.Graphics.Gpu/Engine/MethodConditionalRendering.cs b/Ryujinx.Graphics.Gpu/Engine/MethodConditionalRendering.cs
index 4775de02..c8d47b9f 100644
--- a/Ryujinx.Graphics.Gpu/Engine/MethodConditionalRendering.cs
+++ b/Ryujinx.Graphics.Gpu/Engine/MethodConditionalRendering.cs
@@ -1,26 +1,34 @@
 using Ryujinx.Common.Logging;
+using Ryujinx.Graphics.GAL;
 using Ryujinx.Graphics.Gpu.State;
 
 namespace Ryujinx.Graphics.Gpu.Engine
 {
     partial class Methods
     {
+        enum ConditionalRenderEnabled
+        {
+            False,
+            True,
+            Host
+        }
+
         /// <summary>
         /// Checks if draws and clears should be performed, according
         /// to currently set conditional rendering conditions.
         /// </summary>
         /// <param name="state">GPU state</param>
         /// <returns>True if rendering is enabled, false otherwise</returns>
-        private bool GetRenderEnable(GpuState state)
+        private ConditionalRenderEnabled GetRenderEnable(GpuState state)
         {
             ConditionState condState = state.Get<ConditionState>(MethodOffset.ConditionState);
 
             switch (condState.Condition)
             {
                 case Condition.Always:
-                    return true;
+                    return ConditionalRenderEnabled.True;
                 case Condition.Never:
-                    return false;
+                    return ConditionalRenderEnabled.False;
                 case Condition.ResultNonZero:
                     return CounterNonZero(condState.Address.Pack());
                 case Condition.Equal:
@@ -31,22 +39,32 @@ namespace Ryujinx.Graphics.Gpu.Engine
 
             Logger.PrintWarning(LogClass.Gpu, $"Invalid conditional render condition \"{condState.Condition}\".");
 
-            return true;
+            return ConditionalRenderEnabled.True;
         }
 
         /// <summary>
         /// Checks if the counter value at a given GPU memory address is non-zero.
         /// </summary>
         /// <param name="gpuVa">GPU virtual address of the counter value</param>
-        /// <returns>True if the value is not zero, false otherwise</returns>
-        private bool CounterNonZero(ulong gpuVa)
+        /// <returns>True if the value is not zero, false otherwise. Returns host if handling with host conditional rendering</returns>
+        private ConditionalRenderEnabled CounterNonZero(ulong gpuVa)
         {
-            if (!FindAndFlush(gpuVa))
+            ICounterEvent evt = _counterCache.FindEvent(gpuVa);
+
+            if (evt == null)
             {
-                return false;
+                return ConditionalRenderEnabled.False;
             }
 
-            return _context.MemoryAccessor.ReadUInt64(gpuVa) != 0;
+            if (_context.Renderer.Pipeline.TryHostConditionalRendering(evt, 0L, false))
+            {
+                return ConditionalRenderEnabled.Host;
+            }
+            else
+            {
+                evt.Flush();
+                return (_context.MemoryAccessor.ReadUInt64(gpuVa) != 0) ? ConditionalRenderEnabled.True : ConditionalRenderEnabled.False;
+            }
         }
 
         /// <summary>
@@ -54,29 +72,57 @@ namespace Ryujinx.Graphics.Gpu.Engine
         /// </summary>
         /// <param name="gpuVa">GPU virtual address</param>
         /// <param name="isEqual">True to check if the values are equal, false to check if they are not equal</param>
-        /// <returns>True if the condition is met, false otherwise</returns>
-        private bool CounterCompare(ulong gpuVa, bool isEqual)
+        /// <returns>True if the condition is met, false otherwise. Returns host if handling with host conditional rendering</returns>
+        private ConditionalRenderEnabled CounterCompare(ulong gpuVa, bool isEqual)
         {
-            if (!FindAndFlush(gpuVa) && !FindAndFlush(gpuVa + 16))
+            ICounterEvent evt = FindEvent(gpuVa);
+            ICounterEvent evt2 = FindEvent(gpuVa + 16);
+
+            if (evt == null && evt2 == null)
+            {
+                return ConditionalRenderEnabled.False;
+            }
+
+            bool useHost;
+
+            if (evt != null && evt2 == null)
+            {
+                useHost = _context.Renderer.Pipeline.TryHostConditionalRendering(evt, _context.MemoryAccessor.ReadUInt64(gpuVa + 16), isEqual);
+            }
+            else if (evt == null && evt2 != null)
+            {
+                useHost = _context.Renderer.Pipeline.TryHostConditionalRendering(evt2, _context.MemoryAccessor.ReadUInt64(gpuVa), isEqual);
+            }
+            else
             {
-                return false;
+                useHost = _context.Renderer.Pipeline.TryHostConditionalRendering(evt, evt2, isEqual);
             }
 
-            ulong x = _context.MemoryAccessor.ReadUInt64(gpuVa);
-            ulong y = _context.MemoryAccessor.ReadUInt64(gpuVa + 16);
+            if (useHost)
+            {
+                return ConditionalRenderEnabled.Host;
+            }
+            else
+            {
+                evt?.Flush();
+                evt2?.Flush();
 
-            return isEqual ? x == y : x != y;
+                ulong x = _context.MemoryAccessor.ReadUInt64(gpuVa);
+                ulong y = _context.MemoryAccessor.ReadUInt64(gpuVa + 16);
+
+                return (isEqual ? x == y : x != y) ? ConditionalRenderEnabled.True : ConditionalRenderEnabled.False;
+            }
         }
 
         /// <summary>
         /// Tries to find a counter that is supposed to be written at the specified address,
-        /// flushing if necessary.
+        /// returning the related event.
         /// </summary>
         /// <param name="gpuVa">GPU virtual address where the counter is supposed to be written</param>
-        /// <returns>True if a counter value is found at the specified address, false otherwise</returns>
-        private bool FindAndFlush(ulong gpuVa)
+        /// <returns>The counter event, or null if not present</returns>
+        private ICounterEvent FindEvent(ulong gpuVa)
         {
-            return _counterCache.Contains(gpuVa);
+            return _counterCache.FindEvent(gpuVa);
         }
     }
 }
diff --git a/Ryujinx.Graphics.Gpu/Engine/MethodDraw.cs b/Ryujinx.Graphics.Gpu/Engine/MethodDraw.cs
index 68131f62..d70402e9 100644
--- a/Ryujinx.Graphics.Gpu/Engine/MethodDraw.cs
+++ b/Ryujinx.Graphics.Gpu/Engine/MethodDraw.cs
@@ -35,17 +35,22 @@ namespace Ryujinx.Graphics.Gpu.Engine
         /// <param name="argument">Method call argument</param>
         private void DrawEnd(GpuState state, int argument)
         {
-            bool renderEnable = GetRenderEnable(state);
+            ConditionalRenderEnabled renderEnable = GetRenderEnable(state);
 
-            if (!renderEnable || _instancedDrawPending)
+            if (renderEnable == ConditionalRenderEnabled.False || _instancedDrawPending)
             {
-                if (!renderEnable)
+                if (renderEnable == ConditionalRenderEnabled.False)
                 {
                     PerformDeferredDraws();
                 }
 
                 _drawIndexed = false;
 
+                if (renderEnable == ConditionalRenderEnabled.Host)
+                {
+                    _context.Renderer.Pipeline.EndHostConditionalRendering();
+                }
+
                 return;
             }
 
@@ -72,6 +77,11 @@ namespace Ryujinx.Graphics.Gpu.Engine
 
                 _drawIndexed = false;
 
+                if (renderEnable == ConditionalRenderEnabled.Host)
+                {
+                    _context.Renderer.Pipeline.EndHostConditionalRendering();
+                }
+
                 return;
             }
 
@@ -100,6 +110,11 @@ namespace Ryujinx.Graphics.Gpu.Engine
                     drawState.First,
                     firstInstance);
             }
+
+            if (renderEnable == ConditionalRenderEnabled.Host)
+            {
+                _context.Renderer.Pipeline.EndHostConditionalRendering();
+            }
         }
 
         /// <summary>
diff --git a/Ryujinx.Graphics.Gpu/Engine/MethodIncrementSyncpoint.cs b/Ryujinx.Graphics.Gpu/Engine/MethodIncrementSyncpoint.cs
index 65742ba7..8fcfb900 100644
--- a/Ryujinx.Graphics.Gpu/Engine/MethodIncrementSyncpoint.cs
+++ b/Ryujinx.Graphics.Gpu/Engine/MethodIncrementSyncpoint.cs
@@ -13,6 +13,7 @@ namespace Ryujinx.Graphics.Gpu.Engine
         {
             uint syncpointId = (uint)(argument) & 0xFFFF;
 
+            _context.Renderer.UpdateCounters(); // Poll the query counters, the game may want an updated result.
             _context.Synchronization.IncrementSyncpoint(syncpointId);
         }
     }
diff --git a/Ryujinx.Graphics.Gpu/Engine/MethodReport.cs b/Ryujinx.Graphics.Gpu/Engine/MethodReport.cs
index 997f55ff..e8efddea 100644
--- a/Ryujinx.Graphics.Gpu/Engine/MethodReport.cs
+++ b/Ryujinx.Graphics.Gpu/Engine/MethodReport.cs
@@ -64,23 +64,9 @@ namespace Ryujinx.Graphics.Gpu.Engine
         {
             CounterData counterData = new CounterData();
 
-            ulong counter = 0;
+            var rs = state.Get<ReportState>(MethodOffset.ReportState);
 
-            switch (type)
-            {
-                case ReportCounterType.Zero:
-                    counter = 0;
-                    break;
-                case ReportCounterType.SamplesPassed:
-                    counter = _context.Renderer.GetCounter(CounterType.SamplesPassed);
-                    break;
-                case ReportCounterType.PrimitivesGenerated:
-                    counter = _context.Renderer.GetCounter(CounterType.PrimitivesGenerated);
-                    break;
-                case ReportCounterType.TransformFeedbackPrimitivesWritten:
-                    counter = _context.Renderer.GetCounter(CounterType.TransformFeedbackPrimitivesWritten);
-                    break;
-            }
+            ulong gpuVa = rs.Address.Pack();
 
             ulong ticks = ConvertNanosecondsToTicks((ulong)PerformanceCounter.ElapsedNanoseconds);
 
@@ -91,18 +77,40 @@ namespace Ryujinx.Graphics.Gpu.Engine
                 ticks /= 256;
             }
 
-            counterData.Counter   = counter;
-            counterData.Timestamp = ticks;
+            ICounterEvent counter = null;
+
+            EventHandler<ulong> resultHandler = (object evt, ulong result) =>
+            {
+                counterData.Counter = result;
+                counterData.Timestamp = ticks;
 
-            Span<CounterData> counterDataSpan = MemoryMarshal.CreateSpan(ref counterData, 1);
+                Span<CounterData> counterDataSpan = MemoryMarshal.CreateSpan(ref counterData, 1);
 
-            Span<byte> data = MemoryMarshal.Cast<CounterData, byte>(counterDataSpan);
+                Span<byte> data = MemoryMarshal.Cast<CounterData, byte>(counterDataSpan);
 
-            var rs = state.Get<ReportState>(MethodOffset.ReportState);
+                if (counter?.Invalid != true)
+                {
+                    _context.MemoryAccessor.Write(gpuVa, data);
+                }
+            };
 
-            _context.MemoryAccessor.Write(rs.Address.Pack(), data);
+            switch (type)
+            {
+                case ReportCounterType.Zero:
+                    resultHandler(null, 0);
+                    break;
+                case ReportCounterType.SamplesPassed:
+                    counter = _context.Renderer.ReportCounter(CounterType.SamplesPassed, resultHandler);
+                    break;
+                case ReportCounterType.PrimitivesGenerated:
+                    counter = _context.Renderer.ReportCounter(CounterType.PrimitivesGenerated, resultHandler);
+                    break;
+                case ReportCounterType.TransformFeedbackPrimitivesWritten:
+                    counter = _context.Renderer.ReportCounter(CounterType.TransformFeedbackPrimitivesWritten, resultHandler);
+                    break;
+            }
 
-            _counterCache.AddOrUpdate(rs.Address.Pack());
+            _counterCache.AddOrUpdate(gpuVa, counter);
         }
 
         /// <summary>
diff --git a/Ryujinx.Graphics.Gpu/Memory/CounterCache.cs b/Ryujinx.Graphics.Gpu/Memory/CounterCache.cs
index 3110cdcc..90b9187b 100644
--- a/Ryujinx.Graphics.Gpu/Memory/CounterCache.cs
+++ b/Ryujinx.Graphics.Gpu/Memory/CounterCache.cs
@@ -1,4 +1,5 @@
-using System.Collections.Generic;
+using Ryujinx.Graphics.GAL;
+using System.Collections.Generic;
 
 namespace Ryujinx.Graphics.Gpu.Memory
 {
@@ -10,10 +11,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
         private struct CounterEntry
         {
             public ulong Address { get; }
+            public ICounterEvent Event { get; }
 
-            public CounterEntry(ulong address)
+            public CounterEntry(ulong address, ICounterEvent evt)
             {
                 Address = address;
+                Event = evt;
             }
         }
 
@@ -31,11 +34,11 @@ namespace Ryujinx.Graphics.Gpu.Memory
         /// Adds a new counter to the counter cache, or updates a existing one.
         /// </summary>
         /// <param name="gpuVa">GPU virtual address where the counter will be written in memory</param>
-        public void AddOrUpdate(ulong gpuVa)
+        public void AddOrUpdate(ulong gpuVa, ICounterEvent evt)
         {
             int index = BinarySearch(gpuVa);
 
-            CounterEntry entry = new CounterEntry(gpuVa);
+            CounterEntry entry = new CounterEntry(gpuVa, evt);
 
             if (index < 0)
             {
@@ -76,6 +79,16 @@ namespace Ryujinx.Graphics.Gpu.Memory
                 count++;
             }
 
+            // Notify the removed counter events that their result should no longer be written out.
+            for (int i = 0; i < count; i++)
+            {
+                ICounterEvent evt = _items[index + i].Event;
+                if (evt != null)
+                {
+                    evt.Invalid = true;
+                }
+            }
+
             _items.RemoveRange(index, count);
         }
 
@@ -102,6 +115,44 @@ namespace Ryujinx.Graphics.Gpu.Memory
         }
 
         /// <summary>
+        /// Flush any counter value written to the specified GPU virtual memory address.
+        /// </summary>
+        /// <param name="gpuVa">GPU virtual address</param>
+        /// <returns>True if any counter value was written on the specified address, false otherwise</returns>
+        public bool FindAndFlush(ulong gpuVa)
+        {
+            int index = BinarySearch(gpuVa);
+            if (index > 0)
+            {
+                _items[index].Event?.Flush();
+
+                return true;
+            } 
+            else
+            {
+                return false;
+            }
+        }
+
+        /// <summary>
+        /// Find any counter event that would write to the specified GPU virtual memory address.
+        /// </summary>
+        /// <param name="gpuVa">GPU virtual address</param>
+        /// <returns>The counter event, or null if not present</returns>
+        public ICounterEvent FindEvent(ulong gpuVa)
+        {
+            int index = BinarySearch(gpuVa);
+            if (index > 0)
+            {
+                return _items[index].Event;
+            }
+            else
+            {
+                return null;
+            }
+        }
+
+        /// <summary>
         /// Performs binary search of an address on the list.
         /// </summary>
         /// <param name="address">Address to search</param>
diff --git a/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs b/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs
index f4af86f3..6f9ee6a4 100644
--- a/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs
+++ b/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs
@@ -181,13 +181,13 @@ namespace Ryujinx.Graphics.Gpu.Memory
         {
             lock (_pageTable)
             {
+                // Event handlers are not expected to be thread safe.
+                MemoryUnmapped?.Invoke(this, new UnmapEventArgs(va, size));
+
                 for (ulong offset = 0; offset < size; offset += PageSize)
                 {
                     SetPte(va + offset, PteUnmapped);
                 }
-
-                // Event handlers are not expected to be thread safe.
-                MemoryUnmapped?.Invoke(this, new UnmapEventArgs(va, size));
             }
         }