Better viewport flipping and depth mode detection method (#1556)

* Use a better viewport flipping approach

* New approach to detect depth mode

* nit: Sort method on the OpenGL backend

* Adjust spacing on comment

* Unswap near and far parameters based on ScaleZ

2 files changed, 65 insertions, 70 deletions

diff --git a/Ryujinx.Graphics.Gpu/Engine/Methods.cs b/Ryujinx.Graphics.Gpu/Engine/Methods.cs
index 79ed3c90..618f6440 100644
--- a/Ryujinx.Graphics.Gpu/Engine/Methods.cs
+++ b/Ryujinx.Graphics.Gpu/Engine/Methods.cs
@@ -488,25 +488,12 @@ namespace Ryujinx.Graphics.Gpu.Engine
         /// <param name="state">Current GPU state</param>
         private void UpdateViewportTransform(GpuState state)
         {
-            DepthMode depthMode = state.Get<DepthMode>(MethodOffset.DepthMode);
+            var yControl = state.Get<YControl> (MethodOffset.YControl);
+            var face     = state.Get<FaceState>(MethodOffset.FaceState);
 
-            _context.Renderer.Pipeline.SetDepthMode(depthMode);
+            UpdateFrontFace(yControl, face.FrontFace);
 
-            YControl yControl = state.Get<YControl>(MethodOffset.YControl);
-
-            bool   flipY  = yControl.HasFlag(YControl.NegateY);
-            Origin origin = yControl.HasFlag(YControl.TriangleRastFlip) ? Origin.LowerLeft : Origin.UpperLeft;
-
-            _context.Renderer.Pipeline.SetOrigin(origin);
-
-            // The triangle rast flip flag only affects rasterization, the viewport is not flipped.
-            // Setting the origin mode to upper left on the host, however, not only affects rasterization,
-            // but also flips the viewport.
-            // We negate the effects of flipping the viewport by flipping it again using the viewport swizzle.
-            if (origin == Origin.UpperLeft)
-            {
-                flipY = !flipY;
-            }
+            bool flipY = yControl.HasFlag(YControl.NegateY);
 
             Span<Viewport> viewports = stackalloc Viewport[Constants.TotalViewports];
 
@@ -515,11 +502,42 @@ namespace Ryujinx.Graphics.Gpu.Engine
                 var transform = state.Get<ViewportTransform>(MethodOffset.ViewportTransform, index);
                 var extents   = state.Get<ViewportExtents>  (MethodOffset.ViewportExtents,   index);
 
-                float x = transform.TranslateX - MathF.Abs(transform.ScaleX);
-                float y = transform.TranslateY - MathF.Abs(transform.ScaleY);
+                float scaleX = MathF.Abs(transform.ScaleX);
+                float scaleY = transform.ScaleY;
 
-                float width  = MathF.Abs(transform.ScaleX) * 2;
-                float height = MathF.Abs(transform.ScaleY) * 2;
+                if (flipY)
+                {
+                    scaleY = -scaleY;
+                }
+
+                if (!_context.Capabilities.SupportsViewportSwizzle && transform.UnpackSwizzleY() == ViewportSwizzle.NegativeY)
+                {
+                    scaleY = -scaleY;
+                }
+
+                if (index == 0)
+                {
+                    // Try to guess the depth mode being used on the high level API
+                    // based on current transform.
+                    // It is setup like so by said APIs:
+                    // If depth mode is ZeroToOne:
+                    //  TranslateZ = Near
+                    //  ScaleZ = Far - Near
+                    // If depth mode is MinusOneToOne:
+                    //  TranslateZ = (Near + Far) / 2
+                    //  ScaleZ = (Far - Near) / 2
+                    // DepthNear/Far are sorted such as that Near is always less than Far.
+                    DepthMode depthMode = extents.DepthNear != transform.TranslateZ &&
+                                          extents.DepthFar  != transform.TranslateZ ? DepthMode.MinusOneToOne : DepthMode.ZeroToOne;
+
+                    _context.Renderer.Pipeline.SetDepthMode(depthMode);
+                }
+
+                float x = transform.TranslateX - scaleX;
+                float y = transform.TranslateY - scaleY;
+
+                float width  = scaleX * 2;
+                float height = scaleY * 2;
 
                 float scale = TextureManager.RenderTargetScale;
                 if (scale != 1f)
@@ -537,34 +555,17 @@ namespace Ryujinx.Graphics.Gpu.Engine
                 ViewportSwizzle swizzleZ = transform.UnpackSwizzleZ();
                 ViewportSwizzle swizzleW = transform.UnpackSwizzleW();
 
-                if (transform.ScaleX < 0)
-                {
-                    swizzleX ^= ViewportSwizzle.NegativeFlag;
-                }
-
-                if (flipY)
-                {
-                    swizzleY ^= ViewportSwizzle.NegativeFlag;
-                }
-
-                if (transform.ScaleY < 0)
-                {
-                    swizzleY ^= ViewportSwizzle.NegativeFlag;
-                }
+                float depthNear = extents.DepthNear;
+                float depthFar  = extents.DepthFar;
 
                 if (transform.ScaleZ < 0)
                 {
-                    swizzleZ ^= ViewportSwizzle.NegativeFlag;
+                    float temp = depthNear;
+                    depthNear  = depthFar;
+                    depthFar   = temp;
                 }
 
-                viewports[index] = new Viewport(
-                    region,
-                    swizzleX,
-                    swizzleY,
-                    swizzleZ,
-                    swizzleW,
-                    extents.DepthNear,
-                    extents.DepthFar);
+                viewports[index] = new Viewport(region, swizzleX, swizzleY, swizzleZ, swizzleW, depthNear, depthFar);
             }
 
             _context.Renderer.Pipeline.SetViewports(0, viewports);
@@ -832,11 +833,29 @@ namespace Ryujinx.Graphics.Gpu.Engine
         /// <param name="state">Current GPU state</param>
         private void UpdateFaceState(GpuState state)
         {
-            var face = state.Get<FaceState>(MethodOffset.FaceState);
+            var yControl = state.Get<YControl> (MethodOffset.YControl);
+            var face     = state.Get<FaceState>(MethodOffset.FaceState);
 
             _context.Renderer.Pipeline.SetFaceCulling(face.CullEnable, face.CullFace);
 
-            _context.Renderer.Pipeline.SetFrontFace(face.FrontFace);
+            UpdateFrontFace(yControl, face.FrontFace);
+        }
+
+        /// <summary>
+        /// Updates the front face based on the current front face and the origin.
+        /// </summary>
+        /// <param name="yControl">Y control register value, where the origin is located</param>
+        /// <param name="frontFace">Front face</param>
+        private void UpdateFrontFace(YControl yControl, FrontFace frontFace)
+        {
+            bool isUpperLeftOrigin = !yControl.HasFlag(YControl.TriangleRastFlip);
+
+            if (isUpperLeftOrigin)
+            {
+                frontFace = frontFace == FrontFace.CounterClockwise ? FrontFace.Clockwise : FrontFace.CounterClockwise;
+            }
+
+            _context.Renderer.Pipeline.SetFrontFace(frontFace);
         }
 
         /// <summary>
diff --git a/Ryujinx.Graphics.Gpu/Shader/GpuAccessor.cs b/Ryujinx.Graphics.Gpu/Shader/GpuAccessor.cs
index 03d8d196..fd0a6b0d 100644
--- a/Ryujinx.Graphics.Gpu/Shader/GpuAccessor.cs
+++ b/Ryujinx.Graphics.Gpu/Shader/GpuAccessor.cs
@@ -189,12 +189,6 @@ namespace Ryujinx.Graphics.Gpu.Shader
         public bool QuerySupportsNonConstantTextureOffset() => _context.Capabilities.SupportsNonConstantTextureOffset;
 
         /// <summary>
-        /// Queries host GPU viewport swizzle support.
-        /// </summary>
-        /// <returns>True if the GPU and driver supports viewport swizzle, false otherwise</returns>
-        public bool QuerySupportsViewportSwizzle() => _context.Capabilities.SupportsViewportSwizzle;
-
-        /// <summary>
         /// Queries texture format information, for shaders using image load or store.
         /// </summary>
         /// <remarks>
@@ -257,24 +251,6 @@ namespace Ryujinx.Graphics.Gpu.Shader
             };
         }
 
-        public int QueryViewportSwizzle(int component)
-        {
-            YControl yControl = _state.Get<YControl>(MethodOffset.YControl);
-
-            bool flipY = yControl.HasFlag(YControl.NegateY) ^ !yControl.HasFlag(YControl.TriangleRastFlip);
-
-            ViewportTransform transform = _state.Get<ViewportTransform>(MethodOffset.ViewportTransform, 0);
-
-            return component switch
-            {
-                0 => (int)(transform.UnpackSwizzleX() ^ (transform.ScaleX < 0 ? ViewportSwizzle.NegativeFlag : 0)),
-                1 => (int)(transform.UnpackSwizzleY() ^ (transform.ScaleY < 0 ? ViewportSwizzle.NegativeFlag : 0) ^ (flipY ? ViewportSwizzle.NegativeFlag : 0)),
-                2 => (int)(transform.UnpackSwizzleZ() ^ (transform.ScaleZ < 0 ? ViewportSwizzle.NegativeFlag : 0)),
-                3 => (int)transform.UnpackSwizzleW(),
-                _ => throw new ArgumentOutOfRangeException(nameof(component))
-            };
-        }
-
         /// <summary>
         /// Gets the texture descriptor for a given texture on the pool.
         /// </summary>