Support 3D BC4 and BC5 compressed textures (#1655)

* Support 3D BC4 and BC5 compressed textures

* PR feedback

* Fix some typos

3 files changed, 95 insertions, 24 deletions

diff --git a/Ryujinx.Graphics.Gpu/Image/Texture.cs b/Ryujinx.Graphics.Gpu/Image/Texture.cs
index 01610629..b1d6af9b 100644
--- a/Ryujinx.Graphics.Gpu/Image/Texture.cs
+++ b/Ryujinx.Graphics.Gpu/Image/Texture.cs
@@ -671,6 +671,9 @@ namespace Ryujinx.Graphics.Gpu.Image
                     data);
             }
 
+            // Handle compressed cases not supported by the host:
+            // - ASTC is usually not supported on desktop cards.
+            // - BC4/BC5 is not supported on 3D textures.
             if (!_context.Capabilities.SupportsAstcCompression && Info.FormatInfo.Format.IsAstc())
             {
                 if (!AstcDecoder.TryDecodeToRgba8(
@@ -691,6 +694,14 @@ namespace Ryujinx.Graphics.Gpu.Image
 
                 data = decoded;
             }
+            else if (Info.Target == Target.Texture3D && Info.FormatInfo.Format.IsBc4())
+            {
+                data = BCnDecoder.DecodeBC4(data, Info.Width, Info.Height, _depth, Info.Levels, _layers, Info.FormatInfo.Format == Format.Bc4Snorm);
+            }
+            else if (Info.Target == Target.Texture3D && Info.FormatInfo.Format.IsBc5())
+            {
+                data = BCnDecoder.DecodeBC5(data, Info.Width, Info.Height, _depth, Info.Levels, _layers, Info.FormatInfo.Format == Format.Bc5Snorm);
+            }
 
             return data;
         }
@@ -707,8 +718,7 @@ namespace Ryujinx.Graphics.Gpu.Image
         public void Flush(bool tracked = true)
         {
             IsModified = false;
-
-            if (Info.FormatInfo.Format.IsAstc())
+            if (TextureCompatibility.IsFormatHostIncompatible(Info, _context.Capabilities))
             {
                 return; // Flushing this format is not supported, as it may have been converted to another host format.
             }
@@ -739,10 +749,9 @@ namespace Ryujinx.Graphics.Gpu.Image
             _context.Renderer.BackgroundContextAction(() =>
             {
                 IsModified = false;
-                if (Info.FormatInfo.Format.IsAstc())
+                if (TextureCompatibility.IsFormatHostIncompatible(Info, _context.Capabilities))
                 {
-                    // ASTC textures are not in their original format, so cannot be flushed.
-                    return;
+                    return; // Flushing this format is not supported, as it may have been converted to another host format.
                 }
 
                 ITexture texture = HostTexture;
diff --git a/Ryujinx.Graphics.Gpu/Image/TextureCompatibility.cs b/Ryujinx.Graphics.Gpu/Image/TextureCompatibility.cs
index 22473a1b..b07aeaf1 100644
--- a/Ryujinx.Graphics.Gpu/Image/TextureCompatibility.cs
+++ b/Ryujinx.Graphics.Gpu/Image/TextureCompatibility.cs
@@ -27,6 +27,68 @@ namespace Ryujinx.Graphics.Gpu.Image
         }
 
         /// <summary>
+        /// Checks if a format is host incompatible.
+        /// </summary>
+        /// <remarks>
+        /// Host incompatible formats can't be used directly, the texture data needs to be converted
+        /// to a compatible format first.
+        /// </remarks>
+        /// <param name="info">Texture information</param>
+        /// <param name="caps">Host GPU capabilities</param>
+        /// <returns>True if the format is incompatible, false otherwise</returns>
+        public static bool IsFormatHostIncompatible(TextureInfo info, Capabilities caps)
+        {
+            Format originalFormat = info.FormatInfo.Format;
+            return ToHostCompatibleFormat(info, caps).Format != originalFormat;
+        }
+
+        /// <summary>
+        /// Converts a incompatible format to a host compatible format, or return the format directly
+        /// if it is already host compatible.
+        /// </summary>
+        /// <remarks>
+        /// This can be used to convert a incompatible compressed format to the decompressor
+        /// output format.
+        /// </remarks>
+        /// <param name="info">Texture information</param>
+        /// <param name="caps">Host GPU capabilities</param>
+        /// <returns>A host compatible format</returns>
+        public static FormatInfo ToHostCompatibleFormat(TextureInfo info, Capabilities caps)
+        {
+            if (!caps.SupportsAstcCompression)
+            {
+                if (info.FormatInfo.Format.IsAstcUnorm())
+                {
+                    return new FormatInfo(Format.R8G8B8A8Unorm, 1, 1, 4, 4);
+                }
+                else if (info.FormatInfo.Format.IsAstcSrgb())
+                {
+                    return new FormatInfo(Format.R8G8B8A8Srgb, 1, 1, 4, 4);
+                }
+            }
+
+            if (info.Target == Target.Texture3D)
+            {
+                // The host API does not support 3D BC4/BC5 compressed formats.
+                // We assume software decompression will be done for those textures,
+                // and so we adjust the format here to match the decompressor output.
+                switch (info.FormatInfo.Format)
+                {
+                    case Format.Bc4Unorm:
+                        return new FormatInfo(Format.R8Unorm, 1, 1, 1, 1);
+                    case Format.Bc4Snorm:
+                        return new FormatInfo(Format.R8Snorm, 1, 1, 1, 1);
+                    case Format.Bc5Unorm:
+                        return new FormatInfo(Format.R8G8Unorm, 1, 1, 2, 2);
+                    case Format.Bc5Snorm:
+                        return new FormatInfo(Format.R8G8Snorm, 1, 1, 2, 2);
+                }
+            }
+
+            return info.FormatInfo;
+        }
+
+        /// <summary>
         /// Finds the appropriate depth format for a copy texture if the source texture has a depth format.
         /// </summary>
         /// <param name="dstTextureFormat">Destination CopyTexture Format</param>
diff --git a/Ryujinx.Graphics.Gpu/Image/TextureManager.cs b/Ryujinx.Graphics.Gpu/Image/TextureManager.cs
index b0e715ea..7588258f 100644
--- a/Ryujinx.Graphics.Gpu/Image/TextureManager.cs
+++ b/Ryujinx.Graphics.Gpu/Image/TextureManager.cs
@@ -1058,19 +1058,7 @@ namespace Ryujinx.Graphics.Gpu.Image
         /// <returns>The texture creation information</returns>
         public static TextureCreateInfo GetCreateInfo(TextureInfo info, Capabilities caps, float scale)
         {
-            FormatInfo formatInfo = info.FormatInfo;
-
-            if (!caps.SupportsAstcCompression)
-            {
-                if (formatInfo.Format.IsAstcUnorm())
-                {
-                    formatInfo = new FormatInfo(Format.R8G8B8A8Unorm, 1, 1, 4, 4);
-                }
-                else if (formatInfo.Format.IsAstcSrgb())
-                {
-                    formatInfo = new FormatInfo(Format.R8G8B8A8Srgb, 1, 1, 4, 4);
-                }
-            }
+            FormatInfo formatInfo = TextureCompatibility.ToHostCompatibleFormat(info, caps);
 
             if (info.Target == Target.TextureBuffer)
             {
@@ -1079,12 +1067,24 @@ namespace Ryujinx.Graphics.Gpu.Image
                 // The shader will need the appropriate conversion code to compensate.
                 switch (formatInfo.Format)
                 {
-                    case Format.R8Snorm:           formatInfo = new FormatInfo(Format.R8Sint,           1, 1, 1, 1); break;
-                    case Format.R16Snorm:          formatInfo = new FormatInfo(Format.R16Sint,          1, 1, 2, 1); break;
-                    case Format.R8G8Snorm:         formatInfo = new FormatInfo(Format.R8G8Sint,         1, 1, 2, 2); break;
-                    case Format.R16G16Snorm:       formatInfo = new FormatInfo(Format.R16G16Sint,       1, 1, 4, 2); break;
-                    case Format.R8G8B8A8Snorm:     formatInfo = new FormatInfo(Format.R8G8B8A8Sint,     1, 1, 4, 4); break;
-                    case Format.R16G16B16A16Snorm: formatInfo = new FormatInfo(Format.R16G16B16A16Sint, 1, 1, 8, 4); break;
+                    case Format.R8Snorm:
+                        formatInfo = new FormatInfo(Format.R8Sint, 1, 1, 1, 1);
+                        break;
+                    case Format.R16Snorm:
+                        formatInfo = new FormatInfo(Format.R16Sint, 1, 1, 2, 1);
+                        break;
+                    case Format.R8G8Snorm:
+                        formatInfo = new FormatInfo(Format.R8G8Sint, 1, 1, 2, 2);
+                        break;
+                    case Format.R16G16Snorm:
+                        formatInfo = new FormatInfo(Format.R16G16Sint, 1, 1, 4, 2);
+                        break;
+                    case Format.R8G8B8A8Snorm:
+                        formatInfo = new FormatInfo(Format.R8G8B8A8Sint, 1, 1, 4, 4);
+                        break;
+                    case Format.R16G16B16A16Snorm:
+                        formatInfo = new FormatInfo(Format.R16G16B16A16Sint, 1, 1, 8, 4);
+                        break;
                 }
             }