UI: Add Metal surface creation for MoltenVK (#3980)

* Initial implementation of metal surface across UIs

* Fix SDL2 on windows

* Update Ryujinx/Ryujinx.csproj

Co-authored-by: Mary-nyan <thog@protonmail.com>

* Address Feedback

Co-authored-by: Mary-nyan <thog@protonmail.com>

9 files changed, 182 insertions, 10 deletions

diff --git a/Ryujinx.Ava/AppHost.cs b/Ryujinx.Ava/AppHost.cs
index a016ebd5..0cb3bd13 100644
--- a/Ryujinx.Ava/AppHost.cs
+++ b/Ryujinx.Ava/AppHost.cs
@@ -125,7 +125,7 @@ namespace Ryujinx.Ava
             _inputManager = inputManager;
             _accountManager = accountManager;
             _userChannelPersistence = userChannelPersistence;
-            _renderingThread = new Thread(RenderLoop) { Name = "GUI.RenderThread" };
+            _renderingThread = new Thread(RenderLoop, 1 * 1024 * 1024) { Name = "GUI.RenderThread" };
             _hideCursorOnIdle = ConfigurationState.Instance.HideCursorOnIdle;
             _lastCursorMoveTime = Stopwatch.GetTimestamp();
             _glLogLevel = ConfigurationState.Instance.Logger.GraphicsDebugLevel;
diff --git a/Ryujinx.Ava/Helper/MetalHelper.cs b/Ryujinx.Ava/Helper/MetalHelper.cs
new file mode 100644
index 00000000..ae07ce69
--- /dev/null
+++ b/Ryujinx.Ava/Helper/MetalHelper.cs
@@ -0,0 +1,127 @@
+using System;
+using System.Runtime.Versioning;
+using System.Runtime.InteropServices;
+using Avalonia;
+
+namespace Ryujinx.Ava.Ui.Helper
+{
+    public delegate void UpdateBoundsCallbackDelegate(Rect rect);
+
+    [SupportedOSPlatform("macos")]
+    static class MetalHelper
+    {
+        private const string LibObjCImport = "/usr/lib/libobjc.A.dylib";
+
+        private struct Selector
+        {
+            public readonly IntPtr NativePtr;
+
+            public unsafe Selector(string value)
+            {
+                int size = System.Text.Encoding.UTF8.GetMaxByteCount(value.Length);
+                byte* data = stackalloc byte[size];
+
+                fixed (char* pValue = value)
+                {
+                    System.Text.Encoding.UTF8.GetBytes(pValue, value.Length, data, size);
+                }
+
+                NativePtr = sel_registerName(data);
+            }
+
+            public static implicit operator Selector(string value) => new Selector(value);
+        }
+
+        private static unsafe IntPtr GetClass(string value)
+        {
+            int size = System.Text.Encoding.UTF8.GetMaxByteCount(value.Length);
+            byte* data = stackalloc byte[size];
+
+            fixed (char* pValue = value)
+            {
+                System.Text.Encoding.UTF8.GetBytes(pValue, value.Length, data, size);
+            }
+
+            return objc_getClass(data);
+        }
+
+        private struct NSPoint
+        {
+            public double X;
+            public double Y;
+
+            public NSPoint(double x, double y)
+            {
+                X = x;
+                Y = y;
+            }
+        }
+
+        private struct NSRect
+        {
+            public NSPoint Pos;
+            public NSPoint Size;
+
+            public NSRect(double x, double y, double width, double height)
+            {
+                Pos = new NSPoint(x, y);
+                Size = new NSPoint(width, height);
+            }
+        }
+
+        public static IntPtr GetMetalLayer(out IntPtr nsView, out UpdateBoundsCallbackDelegate updateBounds)
+        {
+            // Create a new CAMetalLayer.
+            IntPtr layerClass = GetClass("CAMetalLayer");
+            IntPtr metalLayer = IntPtr_objc_msgSend(layerClass, "alloc");
+            objc_msgSend(metalLayer, "init");
+
+            // Create a child NSView to render into.
+            IntPtr nsViewClass = GetClass("NSView");
+            IntPtr child = IntPtr_objc_msgSend(nsViewClass, "alloc");
+            objc_msgSend(child, "init", new NSRect(0, 0, 0, 0));
+
+            // Make its renderer our metal layer.
+            objc_msgSend(child, "setWantsLayer:", (byte)1);
+            objc_msgSend(child, "setLayer:", metalLayer);
+            objc_msgSend(metalLayer, "setContentsScale:", Program.DesktopScaleFactor);
+
+            // Ensure the scale factor is up to date.
+            updateBounds = (Rect rect) => {
+                objc_msgSend(metalLayer, "setContentsScale:", Program.DesktopScaleFactor);
+            };
+
+            nsView = child;
+            return metalLayer;
+        }
+
+        public static void DestroyMetalLayer(IntPtr nsView, IntPtr metalLayer)
+        {
+            // TODO
+        }
+
+        [DllImport(LibObjCImport)]
+        private static unsafe extern IntPtr sel_registerName(byte* data);
+
+        [DllImport(LibObjCImport)]
+        private static unsafe extern IntPtr objc_getClass(byte* data);
+
+        [DllImport(LibObjCImport)]
+        private static extern void objc_msgSend(IntPtr receiver, Selector selector);
+
+        [DllImport(LibObjCImport)]
+        private static extern void objc_msgSend(IntPtr receiver, Selector selector, byte value);
+
+        [DllImport(LibObjCImport)]
+        private static extern void objc_msgSend(IntPtr receiver, Selector selector, IntPtr value);
+
+        [DllImport(LibObjCImport)]
+        private static extern void objc_msgSend(IntPtr receiver, Selector selector, NSRect point);
+
+        [DllImport(LibObjCImport)]
+        private static extern void objc_msgSend(IntPtr receiver, Selector selector, double value);
+
+        [DllImport(LibObjCImport, EntryPoint = "objc_msgSend")]
+        private static extern IntPtr IntPtr_objc_msgSend(IntPtr receiver, Selector selector);
+    }
+}
\ No newline at end of file
diff --git a/Ryujinx.Ava/Program.cs b/Ryujinx.Ava/Program.cs
index 23b9ef8e..d929331d 100644
--- a/Ryujinx.Ava/Program.cs
+++ b/Ryujinx.Ava/Program.cs
@@ -22,10 +22,11 @@ namespace Ryujinx.Ava
 {
     internal class Program
     {
-        public static double WindowScaleFactor { get; set; }
-        public static string Version           { get; private set; }
-        public static string ConfigurationPath { get; private set; }
-        public static bool   PreviewerDetached { get; private set; }
+        public static double WindowScaleFactor  { get; set; }
+        public static double DesktopScaleFactor { get; set; } = 1.0;
+        public static string Version            { get; private set; }
+        public static string ConfigurationPath  { get; private set; }
+        public static bool   PreviewerDetached {  get; private set; }
 
         [DllImport("user32.dll", SetLastError = true)]
         public static extern int MessageBoxA(IntPtr hWnd, string text, string caption, uint type);
diff --git a/Ryujinx.Ava/Ryujinx.Ava.csproj b/Ryujinx.Ava/Ryujinx.Ava.csproj
index 6d963a40..24bdf22d 100644
--- a/Ryujinx.Ava/Ryujinx.Ava.csproj
+++ b/Ryujinx.Ava/Ryujinx.Ava.csproj
@@ -31,8 +31,10 @@
     <PackageReference Include="XamlNameReferenceGenerator" Version="1.5.1" />
 
     <PackageReference Include="OpenTK.Core" Version="4.7.5" />
-    <PackageReference Include="Ryujinx.Audio.OpenAL.Dependencies" Version="1.21.0.1" Condition="'$(RuntimeIdentifier)' != 'linux-x64' AND '$(RuntimeIdentifier)' != 'osx-x64'" />
-    <PackageReference Include="Ryujinx.Graphics.Nvdec.Dependencies" Version="5.0.1-build10" Condition="'$(RuntimeIdentifier)' != 'linux-x64' AND '$(RuntimeIdentifier)' != 'osx-x64'" />
+    <PackageReference Include="Ryujinx.Audio.OpenAL.Dependencies" Version="1.21.0.1" Condition="'$(RuntimeIdentifier)' != 'linux-x64' AND '$(RuntimeIdentifier)' != 'osx-x64' AND '$(RuntimeIdentifier)' != 'osx-arm64'" />
+    <PackageReference Include="Ryujinx.Graphics.Nvdec.Dependencies" Version="5.0.1-build10" Condition="'$(RuntimeIdentifier)' != 'linux-x64' AND '$(RuntimeIdentifier)' != 'osx-x64' AND '$(RuntimeIdentifier)' != 'osx-arm64'" />
+    <PackageReference Include="Ryujinx.Graphics.Nvdec.Dependencies.osx" Version="5.0.1" Condition="'$(RuntimeIdentifier)' != 'linux-x64' AND '$(RuntimeIdentifier)' != 'win10-x64'" />
+    <PackageReference Include="Ryujinx.Graphics.Vulkan.Dependencies.MoltenVK" Version="1.2.0" Condition="'$(RuntimeIdentifier)' != 'linux-x64' AND '$(RuntimeIdentifier)' != 'win10-x64'" />
     <PackageReference Include="Silk.NET.Vulkan" Version="2.16.0" />
     <PackageReference Include="Silk.NET.Vulkan.Extensions.EXT" Version="2.16.0" />
     <PackageReference Include="Silk.NET.Vulkan.Extensions.KHR" Version="2.16.0" />
diff --git a/Ryujinx.Ava/Ui/Controls/EmbeddedWindow.cs b/Ryujinx.Ava/Ui/Controls/EmbeddedWindow.cs
index 7acbefca..6ef15982 100644
--- a/Ryujinx.Ava/Ui/Controls/EmbeddedWindow.cs
+++ b/Ryujinx.Ava/Ui/Controls/EmbeddedWindow.cs
@@ -2,11 +2,10 @@ using Avalonia;
 using Avalonia.Controls;
 using Avalonia.Input;
 using Avalonia.Platform;
+using Ryujinx.Ava.Ui.Helper;
 using SPB.Graphics;
 using SPB.Platform;
 using SPB.Platform.GLX;
-using SPB.Platform.X11;
-using SPB.Windowing;
 using System;
 using System.Runtime.InteropServices;
 using System.Runtime.Versioning;
@@ -23,6 +22,10 @@ namespace Ryujinx.Ava.Ui.Controls
         protected GLXWindow X11Window { get; set; }
         protected IntPtr WindowHandle { get; set; }
         protected IntPtr X11Display { get; set; }
+        protected IntPtr NsView { get; set; }
+        protected IntPtr MetalLayer { get; set; }
+
+        private UpdateBoundsCallbackDelegate _updateBoundsCallback;
 
         public event EventHandler<IntPtr> WindowCreated;
         public event EventHandler<Size> SizeChanged;
@@ -58,6 +61,7 @@ namespace Ryujinx.Ava.Ui.Controls
         private void StateChanged(Rect rect)
         {
             SizeChanged?.Invoke(this, rect.Size);
+            _updateBoundsCallback?.Invoke(rect);
         }
 
         protected override IPlatformHandle CreateNativeControlCore(IPlatformHandle parent)
@@ -70,6 +74,11 @@ namespace Ryujinx.Ava.Ui.Controls
             {
                 return CreateWin32(parent);
             }
+            else if (OperatingSystem.IsMacOS())
+            {
+                return CreateMacOs(parent);
+            }
+
             return base.CreateNativeControlCore(parent);
         }
 
@@ -85,6 +94,10 @@ namespace Ryujinx.Ava.Ui.Controls
             {
                 DestroyWin32(control);
             }
+            else if (OperatingSystem.IsMacOS())
+            {
+                DestroyMacOS();
+            }
             else
             {
                 base.DestroyNativeControlCore(control);
@@ -187,6 +200,16 @@ namespace Ryujinx.Ava.Ui.Controls
             return DefWindowProc(hWnd, msg, (IntPtr)wParam, (IntPtr)lParam);
         }
 
+        [SupportedOSPlatform("macos")]
+        IPlatformHandle CreateMacOs(IPlatformHandle parent)
+        {
+            MetalLayer = MetalHelper.GetMetalLayer(out IntPtr nsView, out _updateBoundsCallback);
+
+            NsView = nsView;
+
+            return new PlatformHandle(nsView, "NSView");
+        }
+
         void DestroyLinux()
         {
             X11Window?.Dispose();
@@ -198,5 +221,11 @@ namespace Ryujinx.Ava.Ui.Controls
             DestroyWindow(handle.Handle);
             UnregisterClass(_className, GetModuleHandle(null));
         }
+
+        [SupportedOSPlatform("macos")]
+        void DestroyMacOS()
+        {
+            MetalHelper.DestroyMetalLayer(NsView, MetalLayer);
+        }
     }
 }
\ No newline at end of file
diff --git a/Ryujinx.Ava/Ui/Controls/VulkanEmbeddedWindow.cs b/Ryujinx.Ava/Ui/Controls/VulkanEmbeddedWindow.cs
index 236a0a16..b9c5f75f 100644
--- a/Ryujinx.Ava/Ui/Controls/VulkanEmbeddedWindow.cs
+++ b/Ryujinx.Ava/Ui/Controls/VulkanEmbeddedWindow.cs
@@ -3,6 +3,7 @@ using Ryujinx.Ava.Ui.Controls;
 using Silk.NET.Vulkan;
 using SPB.Graphics.Vulkan;
 using SPB.Platform.GLX;
+using SPB.Platform.Metal;
 using SPB.Platform.Win32;
 using SPB.Platform.X11;
 using SPB.Windowing;
@@ -37,6 +38,10 @@ namespace Ryujinx.Ava.Ui
             {
                 _window = new SimpleX11Window(new NativeHandle(X11Display), new NativeHandle(WindowHandle));
             }
+            else if (OperatingSystem.IsMacOS())
+            {
+                _window = new SimpleMetalWindow(new NativeHandle(NsView), new NativeHandle(MetalLayer));
+            }
             else
             {
                 throw new PlatformNotSupportedException();
diff --git a/Ryujinx.Ava/Ui/ViewModels/SettingsViewModel.cs b/Ryujinx.Ava/Ui/ViewModels/SettingsViewModel.cs
index bd4a55e8..c752697b 100644
--- a/Ryujinx.Ava/Ui/ViewModels/SettingsViewModel.cs
+++ b/Ryujinx.Ava/Ui/ViewModels/SettingsViewModel.cs
@@ -108,6 +108,8 @@ namespace Ryujinx.Ava.Ui.ViewModels
             }
         }
 
+        public bool IsOpenGLAvailable => !OperatingSystem.IsMacOS();
+
         public bool DirectoryChanged
         {
             get => _directoryChanged;
diff --git a/Ryujinx.Ava/Ui/Windows/MainWindow.axaml.cs b/Ryujinx.Ava/Ui/Windows/MainWindow.axaml.cs
index a9da8d7d..33c35c7e 100644
--- a/Ryujinx.Ava/Ui/Windows/MainWindow.axaml.cs
+++ b/Ryujinx.Ava/Ui/Windows/MainWindow.axaml.cs
@@ -154,6 +154,12 @@ namespace Ryujinx.Ava.Ui.Windows
             }
         }
 
+        protected override void HandleScalingChanged(double scale)
+        {
+            Program.DesktopScaleFactor = scale;
+            base.HandleScalingChanged(scale);
+        }
+
         public void Application_Opened(object sender, ApplicationOpenedEventArgs args)
         {
             if (args.Application != null)
diff --git a/Ryujinx.Ava/Ui/Windows/SettingsWindow.axaml b/Ryujinx.Ava/Ui/Windows/SettingsWindow.axaml
index c8c9f59a..bd3dd613 100644
--- a/Ryujinx.Ava/Ui/Windows/SettingsWindow.axaml
+++ b/Ryujinx.Ava/Ui/Windows/SettingsWindow.axaml
@@ -540,7 +540,7 @@
                                     <ComboBoxItem IsVisible="{Binding IsVulkanAvailable}">
                                         <TextBlock Text="Vulkan" />
                                     </ComboBoxItem>
-                                    <ComboBoxItem>
+                                    <ComboBoxItem IsEnabled="{Binding IsOpenGLAvailable}">
                                         <TextBlock Text="OpenGL" />
                                     </ComboBoxItem>
                                 </ComboBox>