vk_scheduler: Allow command submission on worker thread

This changes how Scheduler::Flush works. It queues the current command
buffer to be sent to the GPU but does not do it immediately. The Vulkan
worker thread takes care of that. Users will have to use
Scheduler::Flush + Scheduler::WaitWorker to get the previous behavior.

Scheduler::Finish is unchanged.

To avoid waiting on work never queued, Scheduler::Wait sends the current
command buffer if that's what the caller wants to wait.

1 files changed, 64 insertions, 56 deletions

diff --git a/src/video_core/renderer_vulkan/vk_scheduler.cpp b/src/video_core/renderer_vulkan/vk_scheduler.cpp
index 25a4933e5..81cb330d9 100644
--- a/src/video_core/renderer_vulkan/vk_scheduler.cpp
+++ b/src/video_core/renderer_vulkan/vk_scheduler.cpp
@@ -31,7 +31,7 @@ void VKScheduler::CommandChunk::ExecuteAll(vk::CommandBuffer cmdbuf) {
         command->~Command();
         command = next;
     }
-
+    submit = false;
     command_offset = 0;
     first = nullptr;
     last = nullptr;
@@ -42,7 +42,7 @@ VKScheduler::VKScheduler(const Device& device_, StateTracker& state_tracker_)
       master_semaphore{std::make_unique<MasterSemaphore>(device)},
       command_pool{std::make_unique<CommandPool>(*master_semaphore, device)} {
     AcquireNewChunk();
-    AllocateNewContext();
+    AllocateWorkerCommandBuffer();
     worker_thread = std::thread(&VKScheduler::WorkerThread, this);
 }
 
@@ -60,6 +60,7 @@ void VKScheduler::Flush(VkSemaphore semaphore) {
 void VKScheduler::Finish(VkSemaphore semaphore) {
     const u64 presubmit_tick = CurrentTick();
     SubmitExecution(semaphore);
+    WaitWorker();
     Wait(presubmit_tick);
     AllocateNewContext();
 }
@@ -140,75 +141,82 @@ void VKScheduler::WorkerThread() {
         if (quit) {
             continue;
         }
-        auto extracted_chunk = std::move(chunk_queue.Front());
-        chunk_queue.Pop();
-        extracted_chunk->ExecuteAll(current_cmdbuf);
-        chunk_reserve.Push(std::move(extracted_chunk));
+        while (!chunk_queue.Empty()) {
+            auto extracted_chunk = std::move(chunk_queue.Front());
+            chunk_queue.Pop();
+            const bool has_submit = extracted_chunk->HasSubmit();
+            extracted_chunk->ExecuteAll(current_cmdbuf);
+            if (has_submit) {
+                AllocateWorkerCommandBuffer();
+            }
+            chunk_reserve.Push(std::move(extracted_chunk));
+        }
     } while (!quit);
 }
 
+void VKScheduler::AllocateWorkerCommandBuffer() {
+    current_cmdbuf = vk::CommandBuffer(command_pool->Commit(), device.GetDispatchLoader());
+    current_cmdbuf.Begin({
+        .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
+        .pNext = nullptr,
+        .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT,
+        .pInheritanceInfo = nullptr,
+    });
+}
+
 void VKScheduler::SubmitExecution(VkSemaphore semaphore) {
     EndPendingOperations();
     InvalidateState();
-    WaitWorker();
-
-    std::unique_lock lock{mutex};
-
-    current_cmdbuf.End();
-
-    const VkSemaphore timeline_semaphore = master_semaphore->Handle();
-    const u32 num_signal_semaphores = semaphore ? 2U : 1U;
 
     const u64 signal_value = master_semaphore->CurrentTick();
-    const u64 wait_value = signal_value - 1;
-    const VkPipelineStageFlags wait_stage_mask = VK_PIPELINE_STAGE_ALL_COMMANDS_BIT;
-
     master_semaphore->NextTick();
 
-    const std::array signal_values{signal_value, u64(0)};
-    const std::array signal_semaphores{timeline_semaphore, semaphore};
+    Record([semaphore, signal_value, this](vk::CommandBuffer cmdbuf) {
+        cmdbuf.End();
 
-    const VkTimelineSemaphoreSubmitInfoKHR timeline_si{
-        .sType = VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO_KHR,
-        .pNext = nullptr,
-        .waitSemaphoreValueCount = 1,
-        .pWaitSemaphoreValues = &wait_value,
-        .signalSemaphoreValueCount = num_signal_semaphores,
-        .pSignalSemaphoreValues = signal_values.data(),
-    };
-    const VkSubmitInfo submit_info{
-        .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO,
-        .pNext = &timeline_si,
-        .waitSemaphoreCount = 1,
-        .pWaitSemaphores = &timeline_semaphore,
-        .pWaitDstStageMask = &wait_stage_mask,
-        .commandBufferCount = 1,
-        .pCommandBuffers = current_cmdbuf.address(),
-        .signalSemaphoreCount = num_signal_semaphores,
-        .pSignalSemaphores = signal_semaphores.data(),
-    };
-    switch (const VkResult result = device.GetGraphicsQueue().Submit(submit_info)) {
-    case VK_SUCCESS:
-        break;
-    case VK_ERROR_DEVICE_LOST:
-        device.ReportLoss();
-        [[fallthrough]];
-    default:
-        vk::Check(result);
-    }
-}
+        const u32 num_signal_semaphores = semaphore ? 2U : 1U;
 
-void VKScheduler::AllocateNewContext() {
-    std::unique_lock lock{mutex};
+        const u64 wait_value = signal_value - 1;
+        const VkPipelineStageFlags wait_stage_mask = VK_PIPELINE_STAGE_ALL_COMMANDS_BIT;
 
-    current_cmdbuf = vk::CommandBuffer(command_pool->Commit(), device.GetDispatchLoader());
-    current_cmdbuf.Begin({
-        .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
-        .pNext = nullptr,
-        .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT,
-        .pInheritanceInfo = nullptr,
+        const VkSemaphore timeline_semaphore = master_semaphore->Handle();
+        const std::array signal_values{signal_value, u64(0)};
+        const std::array signal_semaphores{timeline_semaphore, semaphore};
+
+        const VkTimelineSemaphoreSubmitInfoKHR timeline_si{
+            .sType = VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO_KHR,
+            .pNext = nullptr,
+            .waitSemaphoreValueCount = 1,
+            .pWaitSemaphoreValues = &wait_value,
+            .signalSemaphoreValueCount = num_signal_semaphores,
+            .pSignalSemaphoreValues = signal_values.data(),
+        };
+        const VkSubmitInfo submit_info{
+            .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO,
+            .pNext = &timeline_si,
+            .waitSemaphoreCount = 1,
+            .pWaitSemaphores = &timeline_semaphore,
+            .pWaitDstStageMask = &wait_stage_mask,
+            .commandBufferCount = 1,
+            .pCommandBuffers = cmdbuf.address(),
+            .signalSemaphoreCount = num_signal_semaphores,
+            .pSignalSemaphores = signal_semaphores.data(),
+        };
+        switch (const VkResult result = device.GetGraphicsQueue().Submit(submit_info)) {
+        case VK_SUCCESS:
+            break;
+        case VK_ERROR_DEVICE_LOST:
+            device.ReportLoss();
+            [[fallthrough]];
+        default:
+            vk::Check(result);
+        }
     });
+    chunk->MarkSubmit();
+    DispatchWork();
+}
 
+void VKScheduler::AllocateNewContext() {
     // Enable counters once again. These are disabled when a command buffer is finished.
     if (query_cache) {
         query_cache->UpdateCounters();