vk_staging_buffer_pool: Add stream buffer for small uploads

This uses a ring buffer similar to OpenGL's stream buffer for small
uploads. This stops us from allocating several small buffers, reducing
memory fragmentation and cache locality.

It uses dedicated allocations when possible.

4 files changed, 27 insertions, 29 deletions

diff --git a/src/video_core/renderer_opengl/gl_texture_cache.cpp b/src/video_core/renderer_opengl/gl_texture_cache.cpp
index 37572ab28..31eb54123 100644
--- a/src/video_core/renderer_opengl/gl_texture_cache.cpp
+++ b/src/video_core/renderer_opengl/gl_texture_cache.cpp
@@ -550,15 +550,14 @@ void TextureCacheRuntime::BlitFramebuffer(Framebuffer* dst, Framebuffer* src,
 }
 
 void TextureCacheRuntime::AccelerateImageUpload(Image& image, const ImageBufferMap& map,
-                                                size_t buffer_offset,
                                                 std::span<const SwizzleParameters> swizzles) {
     switch (image.info.type) {
     case ImageType::e2D:
-        return util_shaders.BlockLinearUpload2D(image, map, buffer_offset, swizzles);
+        return util_shaders.BlockLinearUpload2D(image, map, swizzles);
     case ImageType::e3D:
-        return util_shaders.BlockLinearUpload3D(image, map, buffer_offset, swizzles);
+        return util_shaders.BlockLinearUpload3D(image, map, swizzles);
     case ImageType::Linear:
-        return util_shaders.PitchUpload(image, map, buffer_offset, swizzles);
+        return util_shaders.PitchUpload(image, map, swizzles);
     default:
         UNREACHABLE();
         break;
@@ -710,10 +709,10 @@ Image::Image(TextureCacheRuntime& runtime, const VideoCommon::ImageInfo& info_,
     }
 }
 
-void Image::UploadMemory(const ImageBufferMap& map, size_t buffer_offset,
+void Image::UploadMemory(const ImageBufferMap& map,
                          std::span<const VideoCommon::BufferImageCopy> copies) {
     glBindBuffer(GL_PIXEL_UNPACK_BUFFER, map.buffer);
-    glFlushMappedBufferRange(GL_PIXEL_UNPACK_BUFFER, buffer_offset, unswizzled_size_bytes);
+    glFlushMappedBufferRange(GL_PIXEL_UNPACK_BUFFER, map.offset, unswizzled_size_bytes);
 
     glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
 
@@ -729,19 +728,19 @@ void Image::UploadMemory(const ImageBufferMap& map, size_t buffer_offset,
             current_image_height = copy.buffer_image_height;
             glPixelStorei(GL_UNPACK_IMAGE_HEIGHT, current_image_height);
         }
-        CopyBufferToImage(copy, buffer_offset);
+        CopyBufferToImage(copy, map.offset);
     }
 }
 
-void Image::UploadMemory(const ImageBufferMap& map, size_t buffer_offset,
+void Image::UploadMemory(const ImageBufferMap& map,
                          std::span<const VideoCommon::BufferCopy> copies) {
     for (const VideoCommon::BufferCopy& copy : copies) {
-        glCopyNamedBufferSubData(map.buffer, buffer.handle, copy.src_offset + buffer_offset,
+        glCopyNamedBufferSubData(map.buffer, buffer.handle, copy.src_offset + map.offset,
                                  copy.dst_offset, copy.size);
     }
 }
 
-void Image::DownloadMemory(ImageBufferMap& map, size_t buffer_offset,
+void Image::DownloadMemory(ImageBufferMap& map,
                            std::span<const VideoCommon::BufferImageCopy> copies) {
     glMemoryBarrier(GL_PIXEL_BUFFER_BARRIER_BIT); // TODO: Move this to its own API
 
@@ -760,7 +759,7 @@ void Image::DownloadMemory(ImageBufferMap& map, size_t buffer_offset,
             current_image_height = copy.buffer_image_height;
             glPixelStorei(GL_PACK_IMAGE_HEIGHT, current_image_height);
         }
-        CopyImageToBuffer(copy, buffer_offset);
+        CopyImageToBuffer(copy, map.offset);
     }
 }
 
diff --git a/src/video_core/renderer_opengl/gl_texture_cache.h b/src/video_core/renderer_opengl/gl_texture_cache.h
index 60d08d6d6..874cf54f4 100644
--- a/src/video_core/renderer_opengl/gl_texture_cache.h
+++ b/src/video_core/renderer_opengl/gl_texture_cache.h
@@ -35,6 +35,7 @@ struct ImageBufferMap {
     ~ImageBufferMap();
 
     std::span<u8> mapped_span;
+    size_t offset = 0;
     OGLSync* sync;
     GLuint buffer;
 };
@@ -78,7 +79,7 @@ public:
                          Tegra::Engines::Fermi2D::Filter filter,
                          Tegra::Engines::Fermi2D::Operation operation);
 
-    void AccelerateImageUpload(Image& image, const ImageBufferMap& map, size_t buffer_offset,
+    void AccelerateImageUpload(Image& image, const ImageBufferMap& map,
                                std::span<const VideoCommon::SwizzleParameters> swizzles);
 
     void InsertUploadMemoryBarrier();
@@ -137,14 +138,12 @@ public:
     explicit Image(TextureCacheRuntime&, const VideoCommon::ImageInfo& info, GPUVAddr gpu_addr,
                    VAddr cpu_addr);
 
-    void UploadMemory(const ImageBufferMap& map, size_t buffer_offset,
+    void UploadMemory(const ImageBufferMap& map,
                       std::span<const VideoCommon::BufferImageCopy> copies);
 
-    void UploadMemory(const ImageBufferMap& map, size_t buffer_offset,
-                      std::span<const VideoCommon::BufferCopy> copies);
+    void UploadMemory(const ImageBufferMap& map, std::span<const VideoCommon::BufferCopy> copies);
 
-    void DownloadMemory(ImageBufferMap& map, size_t buffer_offset,
-                        std::span<const VideoCommon::BufferImageCopy> copies);
+    void DownloadMemory(ImageBufferMap& map, std::span<const VideoCommon::BufferImageCopy> copies);
 
     GLuint Handle() const noexcept {
         return texture.handle;
diff --git a/src/video_core/renderer_opengl/util_shaders.cpp b/src/video_core/renderer_opengl/util_shaders.cpp
index aeb36551c..1b58e8617 100644
--- a/src/video_core/renderer_opengl/util_shaders.cpp
+++ b/src/video_core/renderer_opengl/util_shaders.cpp
@@ -63,7 +63,7 @@ UtilShaders::UtilShaders(ProgramManager& program_manager_)
 
 UtilShaders::~UtilShaders() = default;
 
-void UtilShaders::BlockLinearUpload2D(Image& image, const ImageBufferMap& map, size_t buffer_offset,
+void UtilShaders::BlockLinearUpload2D(Image& image, const ImageBufferMap& map,
                                       std::span<const SwizzleParameters> swizzles) {
     static constexpr Extent3D WORKGROUP_SIZE{32, 32, 1};
     static constexpr GLuint BINDING_SWIZZLE_BUFFER = 0;
@@ -71,13 +71,13 @@ void UtilShaders::BlockLinearUpload2D(Image& image, const ImageBufferMap& map, s
     static constexpr GLuint BINDING_OUTPUT_IMAGE = 0;
 
     program_manager.BindHostCompute(block_linear_unswizzle_2d_program.handle);
-    glFlushMappedNamedBufferRange(map.buffer, buffer_offset, image.guest_size_bytes);
+    glFlushMappedNamedBufferRange(map.buffer, map.offset, image.guest_size_bytes);
     glBindBufferBase(GL_SHADER_STORAGE_BUFFER, BINDING_SWIZZLE_BUFFER, swizzle_table_buffer.handle);
 
     const GLenum store_format = StoreFormat(BytesPerBlock(image.info.format));
     for (const SwizzleParameters& swizzle : swizzles) {
         const Extent3D num_tiles = swizzle.num_tiles;
-        const size_t input_offset = swizzle.buffer_offset + buffer_offset;
+        const size_t input_offset = swizzle.buffer_offset + map.offset;
 
         const u32 num_dispatches_x = Common::DivCeil(num_tiles.width, WORKGROUP_SIZE.width);
         const u32 num_dispatches_y = Common::DivCeil(num_tiles.height, WORKGROUP_SIZE.height);
@@ -100,7 +100,7 @@ void UtilShaders::BlockLinearUpload2D(Image& image, const ImageBufferMap& map, s
     program_manager.RestoreGuestCompute();
 }
 
-void UtilShaders::BlockLinearUpload3D(Image& image, const ImageBufferMap& map, size_t buffer_offset,
+void UtilShaders::BlockLinearUpload3D(Image& image, const ImageBufferMap& map,
                                       std::span<const SwizzleParameters> swizzles) {
     static constexpr Extent3D WORKGROUP_SIZE{16, 8, 8};
 
@@ -108,14 +108,14 @@ void UtilShaders::BlockLinearUpload3D(Image& image, const ImageBufferMap& map, s
     static constexpr GLuint BINDING_INPUT_BUFFER = 1;
     static constexpr GLuint BINDING_OUTPUT_IMAGE = 0;
 
-    glFlushMappedNamedBufferRange(map.buffer, buffer_offset, image.guest_size_bytes);
+    glFlushMappedNamedBufferRange(map.buffer, map.offset, image.guest_size_bytes);
     program_manager.BindHostCompute(block_linear_unswizzle_3d_program.handle);
     glBindBufferBase(GL_SHADER_STORAGE_BUFFER, BINDING_SWIZZLE_BUFFER, swizzle_table_buffer.handle);
 
     const GLenum store_format = StoreFormat(BytesPerBlock(image.info.format));
     for (const SwizzleParameters& swizzle : swizzles) {
         const Extent3D num_tiles = swizzle.num_tiles;
-        const size_t input_offset = swizzle.buffer_offset + buffer_offset;
+        const size_t input_offset = swizzle.buffer_offset + map.offset;
 
         const u32 num_dispatches_x = Common::DivCeil(num_tiles.width, WORKGROUP_SIZE.width);
         const u32 num_dispatches_y = Common::DivCeil(num_tiles.height, WORKGROUP_SIZE.height);
@@ -141,7 +141,7 @@ void UtilShaders::BlockLinearUpload3D(Image& image, const ImageBufferMap& map, s
     program_manager.RestoreGuestCompute();
 }
 
-void UtilShaders::PitchUpload(Image& image, const ImageBufferMap& map, size_t buffer_offset,
+void UtilShaders::PitchUpload(Image& image, const ImageBufferMap& map,
                               std::span<const SwizzleParameters> swizzles) {
     static constexpr Extent3D WORKGROUP_SIZE{32, 32, 1};
     static constexpr GLuint BINDING_INPUT_BUFFER = 0;
@@ -159,7 +159,7 @@ void UtilShaders::PitchUpload(Image& image, const ImageBufferMap& map, size_t bu
                          "Non-power of two images are not implemented");
 
     program_manager.BindHostCompute(pitch_unswizzle_program.handle);
-    glFlushMappedNamedBufferRange(map.buffer, buffer_offset, image.guest_size_bytes);
+    glFlushMappedNamedBufferRange(map.buffer, map.offset, image.guest_size_bytes);
     glUniform2ui(LOC_ORIGIN, 0, 0);
     glUniform2i(LOC_DESTINATION, 0, 0);
     glUniform1ui(LOC_BYTES_PER_BLOCK, bytes_per_block);
@@ -167,7 +167,7 @@ void UtilShaders::PitchUpload(Image& image, const ImageBufferMap& map, size_t bu
     glBindImageTexture(BINDING_OUTPUT_IMAGE, image.Handle(), 0, GL_FALSE, 0, GL_WRITE_ONLY, format);
     for (const SwizzleParameters& swizzle : swizzles) {
         const Extent3D num_tiles = swizzle.num_tiles;
-        const size_t input_offset = swizzle.buffer_offset + buffer_offset;
+        const size_t input_offset = swizzle.buffer_offset + map.offset;
 
         const u32 num_dispatches_x = Common::DivCeil(num_tiles.width, WORKGROUP_SIZE.width);
         const u32 num_dispatches_y = Common::DivCeil(num_tiles.height, WORKGROUP_SIZE.height);
diff --git a/src/video_core/renderer_opengl/util_shaders.h b/src/video_core/renderer_opengl/util_shaders.h
index bec026bc3..7b1d16b09 100644
--- a/src/video_core/renderer_opengl/util_shaders.h
+++ b/src/video_core/renderer_opengl/util_shaders.h
@@ -24,13 +24,13 @@ public:
     explicit UtilShaders(ProgramManager& program_manager);
     ~UtilShaders();
 
-    void BlockLinearUpload2D(Image& image, const ImageBufferMap& map, size_t buffer_offset,
+    void BlockLinearUpload2D(Image& image, const ImageBufferMap& map,
                              std::span<const VideoCommon::SwizzleParameters> swizzles);
 
-    void BlockLinearUpload3D(Image& image, const ImageBufferMap& map, size_t buffer_offset,
+    void BlockLinearUpload3D(Image& image, const ImageBufferMap& map,
                              std::span<const VideoCommon::SwizzleParameters> swizzles);
 
-    void PitchUpload(Image& image, const ImageBufferMap& map, size_t buffer_offset,
+    void PitchUpload(Image& image, const ImageBufferMap& map,
                      std::span<const VideoCommon::SwizzleParameters> swizzles);
 
     void CopyBC4(Image& dst_image, Image& src_image,