nvdec: Implement VA-API hardware video acceleration (#6713)

* nvdec: VA-API

* Verify formatting

* Forgot a semicolon for Windows

* Clarify comment about AV_PIX_FMT_NV12

* Fix assert log spam from missing negation

* vic: Remove forgotten debug code

* Address lioncash's review

* Mention VA-API is Intel/AMD

* Address v1993's review

* Hopefully fix CMakeLists style this time

* vic: Improve cache locality

* vic: Fix off-by-one error

* codec: Async

* codec: Forgot the GetValue()

* nvdec: Address ameerj's review

* codec: Fallback to CPU without VA-API support

* cmake: Address lat9nq's review

* cmake: Make VA-API optional

* vaapi: Multiple GPU

* Apply suggestions from code review

Co-authored-by: Ameer J <52414509+ameerj@users.noreply.github.com>

* nvdec: Address ameerj's review

* codec: Use anonymous instead of static

* nvdec: Remove enum and fix memory leak

* nvdec: Address ameerj's review

* codec: Remove preparation for threading

Co-authored-by: Ameer J <52414509+ameerj@users.noreply.github.com>

1 files changed, 35 insertions, 4 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 92c9929cf..de2413843 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -583,8 +583,32 @@ if (YUZU_USE_BUNDLED_FFMPEG)
             "${FFmpeg_PREFIX};${FFmpeg_BUILD_DIR}"
             CACHE PATH "Path to FFmpeg headers" FORCE)
 
+        if (${CMAKE_SYSTEM_NAME} STREQUAL "Linux")
+            Include(FindPkgConfig REQUIRED)
+            pkg_check_modules(LIBVA libva)
+        endif()
+        if(LIBVA_FOUND)
+            pkg_check_modules(LIBDRM libdrm REQUIRED)
+            find_package(X11 REQUIRED)
+            pkg_check_modules(LIBVA-DRM libva-drm REQUIRED)
+            pkg_check_modules(LIBVA-X11 libva-x11 REQUIRED)
+            set(FFmpeg_LIBVA_LIBRARIES
+                ${LIBDRM_LIBRARIES}
+                ${X11_LIBRARIES}
+                ${LIBVA-DRM_LIBRARIES}
+                ${LIBVA-X11_LIBRARIES}
+                ${LIBVA_LIBRARIES})
+            set(FFmpeg_HWACCEL_FLAGS
+                --enable-hwaccel=h264_vaapi
+                --enable-hwaccel=vp9_vaapi
+                --enable-libdrm)
+            message(STATUS "VA-API found")
+        else()
+            set(FFmpeg_HWACCEL_FLAGS --disable-vaapi)
+        endif()
+
         # `configure` parameters builds only exactly what yuzu needs from FFmpeg
-        # `--disable-{vaapi,vdpau}` is needed to avoid linking issues
+        # `--disable-vdpau` is needed to avoid linking issues
         add_custom_command(
             OUTPUT
                 ${FFmpeg_MAKEFILE}
@@ -600,15 +624,16 @@ if (YUZU_USE_BUNDLED_FFMPEG)
                     --disable-network
                     --disable-postproc
                     --disable-swresample
-                    --disable-vaapi
                     --disable-vdpau
                     --enable-decoder=h264
                     --enable-decoder=vp9
                     --cc="${CMAKE_C_COMPILER}"
                     --cxx="${CMAKE_CXX_COMPILER}"
+                    ${FFmpeg_HWACCEL_FLAGS}
             WORKING_DIRECTORY
                 ${FFmpeg_BUILD_DIR}
         )
+        unset(FFmpeg_HWACCEL_FLAGS)
 
         # Workaround for Ubuntu 18.04's older version of make not being able to call make as a child
         # with context of the jobserver. Also helps ninja users.
@@ -618,9 +643,10 @@ if (YUZU_USE_BUNDLED_FFMPEG)
             OUTPUT_VARIABLE
                 SYSTEM_THREADS)
 
+        set(FFmpeg_BUILD_LIBRARIES ${FFmpeg_LIBRARIES})
         add_custom_command(
             OUTPUT
-                ${FFmpeg_LIBRARIES}
+                ${FFmpeg_BUILD_LIBRARIES}
             COMMAND
                 make -j${SYSTEM_THREADS}
             WORKING_DIRECTORY
@@ -630,7 +656,12 @@ if (YUZU_USE_BUNDLED_FFMPEG)
         # ALL makes this custom target build every time
         # but it won't actually build if the DEPENDS parameter is up to date
         add_custom_target(ffmpeg-configure ALL DEPENDS ${FFmpeg_MAKEFILE})
-        add_custom_target(ffmpeg-build ALL DEPENDS ${FFmpeg_LIBRARIES} ffmpeg-configure)
+        add_custom_target(ffmpeg-build ALL DEPENDS ${FFmpeg_BUILD_LIBRARIES} ffmpeg-configure)
+        link_libraries(${FFmpeg_LIBVA_LIBRARIES})
+        set(FFmpeg_LIBRARIES ${FFmpeg_LIBVA_LIBRARIES} ${FFmpeg_BUILD_LIBRARIES}
+            CACHE PATH "Paths to FFmpeg libraries" FORCE)
+        unset(FFmpeg_BUILD_LIBRARIES)
+        unset(FFmpeg_LIBVA_LIBRARIES)
 
         if (FFmpeg_FOUND)
             message(STATUS "Found FFmpeg version ${FFmpeg_VERSION}")