Add support for multi game XCIs (#5638)

* Add default values to ApplicationData directly

* Refactor application loading

It should now be possible to load multi game XCIs.
Included updates won't be detected for now.
Opening a game from the command line currently only opens the first one.

* Only include program NCAs where at least one tuple item is not null

* Get application data by title id and add programIndex check back

* Refactor application loading again and remove duplicate code

* Actually use patch ncas for updates

* Fix number of applications found with multi game xcis

* Don't load bundled updates from multi game xcis

* Change ApplicationData.TitleId type to ulong & Add TitleIdString property

* Use cnmt files and ContentCollection to load programs

* Ava: Add updates and DLCs from gamecarts

* Get the cnmt file from its NCA

* Ava: Identify bundled updates in updater window

* Fix the (hopefully) last few bugs

* Add idOffset parameter to GetNcaByType

* Handle missing file for dlc.json

* Ava: Shorten error message for invalid files

* Gtk: Add additional string for bundled updates in TitleUpdateWindow

* Hopefully fix DLC issues

* Apply formatting

* Finally fix DLC issues

* Adjust property names and fileSize field

* Read the correct update file

* Fix wrong casing for application id strings

* Rename TitleId to ApplicationId

* Address review comments

* Fix formatting issues

* Apply suggestions from code review

Co-authored-by: gdkchan <gab.dark.100@gmail.com>

* Gracefully fail when loading pfs for update and dlc window

* Fix applications with multiple programs

* Fix DLCWindow crash on GTK

* Fix some GUI issues

* Remove IsXci again

---------

Co-authored-by: gdkchan <gab.dark.100@gmail.com>

4 files changed, 185 insertions, 85 deletions

diff --git a/src/Ryujinx.HLE/Loaders/Processes/Extensions/NcaExtensions.cs b/src/Ryujinx.HLE/Loaders/Processes/Extensions/NcaExtensions.cs
index 4568b44d..6863d1a7 100644
--- a/src/Ryujinx.HLE/Loaders/Processes/Extensions/NcaExtensions.cs
+++ b/src/Ryujinx.HLE/Loaders/Processes/Extensions/NcaExtensions.cs
@@ -2,21 +2,31 @@
 using LibHac.Common;
 using LibHac.Fs;
 using LibHac.Fs.Fsa;
+using LibHac.FsSystem;
 using LibHac.Loader;
 using LibHac.Ncm;
 using LibHac.Ns;
+using LibHac.Tools.Fs;
 using LibHac.Tools.FsSystem;
 using LibHac.Tools.FsSystem.NcaUtils;
+using LibHac.Tools.Ncm;
+using Ryujinx.Common.Configuration;
 using Ryujinx.Common.Logging;
+using Ryujinx.Common.Utilities;
+using Ryujinx.HLE.FileSystem;
 using Ryujinx.HLE.HOS;
 using System.IO;
 using System.Linq;
 using ApplicationId = LibHac.Ncm.ApplicationId;
+using ContentType = LibHac.Ncm.ContentType;
+using Path = System.IO.Path;
 
 namespace Ryujinx.HLE.Loaders.Processes.Extensions
 {
-    static class NcaExtensions
+    public static class NcaExtensions
     {
+        private static readonly TitleUpdateMetadataJsonSerializerContext _titleSerializerContext = new(JsonHelper.GetDefaultSerializerOptions());
+
         public static ProcessResult Load(this Nca nca, Switch device, Nca patchNca, Nca controlNca)
         {
             // Extract RomFs and ExeFs from NCA.
@@ -47,7 +57,7 @@ namespace Ryujinx.HLE.Loaders.Processes.Extensions
                 nacpData = controlNca.GetNacp(device);
             }
 
-            /* TODO: Rework this since it's wrong and doesn't work as it takes the DisplayVersion from a "potential" inexistant update.
+            /* TODO: Rework this since it's wrong and doesn't work as it takes the DisplayVersion from a "potential" non-existent update.
 
             // Load program 0 control NCA as we are going to need it for display version.
             (_, Nca updateProgram0ControlNca) = GetGameUpdateData(_device.Configuration.VirtualFileSystem, mainNca.Header.TitleId.ToString("x16"), 0, out _);
@@ -86,6 +96,11 @@ namespace Ryujinx.HLE.Loaders.Processes.Extensions
             return processResult;
         }
 
+        public static ulong GetProgramIdBase(this Nca nca)
+        {
+            return nca.Header.TitleId & ~0x1FFFUL;
+        }
+
         public static int GetProgramIndex(this Nca nca)
         {
             return (int)(nca.Header.TitleId & 0xF);
@@ -96,6 +111,11 @@ namespace Ryujinx.HLE.Loaders.Processes.Extensions
             return nca.Header.ContentType == NcaContentType.Program;
         }
 
+        public static bool IsMain(this Nca nca)
+        {
+            return nca.IsProgram() && !nca.IsPatch();
+        }
+
         public static bool IsPatch(this Nca nca)
         {
             int dataIndex = Nca.GetSectionIndexFromType(NcaSectionType.Data, NcaContentType.Program);
@@ -108,6 +128,56 @@ namespace Ryujinx.HLE.Loaders.Processes.Extensions
             return nca.Header.ContentType == NcaContentType.Control;
         }
 
+        public static (Nca, Nca) GetUpdateData(this Nca mainNca, VirtualFileSystem fileSystem, IntegrityCheckLevel checkLevel, int programIndex, out string updatePath)
+        {
+            updatePath = "(unknown)";
+
+            // Load Update NCAs.
+            Nca updatePatchNca = null;
+            Nca updateControlNca = null;
+
+            // Clear the program index part.
+            ulong titleIdBase = mainNca.GetProgramIdBase();
+
+            // Load update information if exists.
+            string titleUpdateMetadataPath = Path.Combine(AppDataManager.GamesDirPath, mainNca.Header.TitleId.ToString("x16"), "updates.json");
+            if (File.Exists(titleUpdateMetadataPath))
+            {
+                updatePath = JsonHelper.DeserializeFromFile(titleUpdateMetadataPath, _titleSerializerContext.TitleUpdateMetadata).Selected;
+                if (File.Exists(updatePath))
+                {
+                    var updateFile = new FileStream(updatePath, FileMode.Open, FileAccess.Read);
+
+                    IFileSystem updatePartitionFileSystem;
+
+                    if (Path.GetExtension(updatePath).ToLower() == ".xci")
+                    {
+                        updatePartitionFileSystem = new Xci(fileSystem.KeySet, updateFile.AsStorage()).OpenPartition(XciPartitionType.Secure);
+                    }
+                    else
+                    {
+                        PartitionFileSystem pfsTemp = new();
+                        pfsTemp.Initialize(updateFile.AsStorage()).ThrowIfFailure();
+                        updatePartitionFileSystem = pfsTemp;
+                    }
+
+                    foreach ((ulong updateTitleId, ContentCollection content) in updatePartitionFileSystem.GetUpdateData(fileSystem, checkLevel))
+                    {
+                        if ((updateTitleId & ~0x1FFFUL) != titleIdBase)
+                        {
+                            continue;
+                        }
+
+                        updatePatchNca = content.GetNcaByType(fileSystem.KeySet, ContentType.Program, programIndex);
+                        updateControlNca = content.GetNcaByType(fileSystem.KeySet, ContentType.Control, programIndex);
+                        break;
+                    }
+                }
+            }
+
+            return (updatePatchNca, updateControlNca);
+        }
+
         public static IFileSystem GetExeFs(this Nca nca, Switch device, Nca patchNca = null)
         {
             IFileSystem exeFs = null;
@@ -172,5 +242,31 @@ namespace Ryujinx.HLE.Loaders.Processes.Extensions
 
             return nacpData;
         }
+
+        public static Cnmt GetCnmt(this Nca cnmtNca, IntegrityCheckLevel checkLevel, ContentMetaType metaType)
+        {
+            string path = $"/{metaType}_{cnmtNca.Header.TitleId:x16}.cnmt";
+            using var cnmtFile = new UniqueRef<IFile>();
+
+            try
+            {
+                Result result = cnmtNca.OpenFileSystem(0, checkLevel)
+                                       .OpenFile(ref cnmtFile.Ref, path.ToU8Span(), OpenMode.Read);
+
+                if (result.IsSuccess())
+                {
+                    return new Cnmt(cnmtFile.Release().AsStream());
+                }
+            }
+            catch (HorizonResultException ex)
+            {
+                if (!ResultFs.PathNotFound.Includes(ex.ResultValue))
+                {
+                    Logger.Warning?.Print(LogClass.Application, $"Failed get cnmt for '{cnmtNca.Header.TitleId:x16}' from nca: {ex.Message}");
+                }
+            }
+
+            return null;
+        }
     }
 }
diff --git a/src/Ryujinx.HLE/Loaders/Processes/Extensions/PartitionFileSystemExtensions.cs b/src/Ryujinx.HLE/Loaders/Processes/Extensions/PartitionFileSystemExtensions.cs
index 50f7d585..5f45cd45 100644
--- a/src/Ryujinx.HLE/Loaders/Processes/Extensions/PartitionFileSystemExtensions.cs
+++ b/src/Ryujinx.HLE/Loaders/Processes/Extensions/PartitionFileSystemExtensions.cs
@@ -1,26 +1,87 @@
 using LibHac.Common;
+using LibHac.Common.Keys;
 using LibHac.Fs;
 using LibHac.Fs.Fsa;
 using LibHac.FsSystem;
+using LibHac.Ncm;
 using LibHac.Tools.Fs;
 using LibHac.Tools.FsSystem;
 using LibHac.Tools.FsSystem.NcaUtils;
+using LibHac.Tools.Ncm;
 using Ryujinx.Common.Configuration;
 using Ryujinx.Common.Logging;
 using Ryujinx.Common.Utilities;
+using Ryujinx.HLE.FileSystem;
 using System;
 using System.Collections.Generic;
-using System.Globalization;
 using System.IO;
+using ContentType = LibHac.Ncm.ContentType;
 
 namespace Ryujinx.HLE.Loaders.Processes.Extensions
 {
     public static class PartitionFileSystemExtensions
     {
         private static readonly DownloadableContentJsonSerializerContext _contentSerializerContext = new(JsonHelper.GetDefaultSerializerOptions());
-        private static readonly TitleUpdateMetadataJsonSerializerContext _titleSerializerContext = new(JsonHelper.GetDefaultSerializerOptions());
 
-        internal static (bool, ProcessResult) TryLoad<TMetaData, TFormat, THeader, TEntry>(this PartitionFileSystemCore<TMetaData, TFormat, THeader, TEntry> partitionFileSystem, Switch device, string path, out string errorMessage)
+        public static Dictionary<ulong, ContentCollection> GetApplicationData(this IFileSystem partitionFileSystem,
+            VirtualFileSystem fileSystem, IntegrityCheckLevel checkLevel)
+        {
+            fileSystem.ImportTickets(partitionFileSystem);
+
+            var programs = new Dictionary<ulong, ContentCollection>();
+
+            foreach (DirectoryEntryEx fileEntry in partitionFileSystem.EnumerateEntries("/", "*.cnmt.nca"))
+            {
+                Cnmt cnmt = partitionFileSystem.GetNca(fileSystem.KeySet, fileEntry.FullPath).GetCnmt(checkLevel, ContentMetaType.Application);
+
+                if (cnmt == null)
+                {
+                    continue;
+                }
+
+                ContentCollection content = new(partitionFileSystem, cnmt);
+
+                if (content.Type != ContentMetaType.Application)
+                {
+                    continue;
+                }
+
+                programs.TryAdd(content.ApplicationId, content);
+            }
+
+            return programs;
+        }
+
+        public static Dictionary<ulong, ContentCollection> GetUpdateData(this IFileSystem partitionFileSystem,
+            VirtualFileSystem fileSystem, IntegrityCheckLevel checkLevel)
+        {
+            fileSystem.ImportTickets(partitionFileSystem);
+
+            var programs = new Dictionary<ulong, ContentCollection>();
+
+            foreach (DirectoryEntryEx fileEntry in partitionFileSystem.EnumerateEntries("/", "*.cnmt.nca"))
+            {
+                Cnmt cnmt = partitionFileSystem.GetNca(fileSystem.KeySet, fileEntry.FullPath).GetCnmt(checkLevel, ContentMetaType.Patch);
+
+                if (cnmt == null)
+                {
+                    continue;
+                }
+
+                ContentCollection content = new(partitionFileSystem, cnmt);
+
+                if (content.Type != ContentMetaType.Patch)
+                {
+                    continue;
+                }
+
+                programs.TryAdd(content.ApplicationId, content);
+            }
+
+            return programs;
+        }
+
+        internal static (bool, ProcessResult) TryLoad<TMetaData, TFormat, THeader, TEntry>(this PartitionFileSystemCore<TMetaData, TFormat, THeader, TEntry> partitionFileSystem, Switch device, string path, ulong titleId, out string errorMessage)
             where TMetaData : PartitionFileSystemMetaCore<TFormat, THeader, TEntry>, new()
             where TFormat : IPartitionFileSystemFormat
             where THeader : unmanaged, IPartitionFileSystemHeader
@@ -35,31 +96,22 @@ namespace Ryujinx.HLE.Loaders.Processes.Extensions
 
             try
             {
-                device.Configuration.VirtualFileSystem.ImportTickets(partitionFileSystem);
+                Dictionary<ulong, ContentCollection> applications = partitionFileSystem.GetApplicationData(device.FileSystem, device.System.FsIntegrityCheckLevel);
 
-                // TODO: To support multi-games container, this should use CNMT NCA instead.
-                foreach (DirectoryEntryEx fileEntry in partitionFileSystem.EnumerateEntries("/", "*.nca"))
+                if (titleId == 0)
                 {
-                    Nca nca = partitionFileSystem.GetNca(device, fileEntry.FullPath);
-
-                    if (nca.GetProgramIndex() != device.Configuration.UserChannelPersistence.Index)
-                    {
-                        continue;
-                    }
-
-                    if (nca.IsPatch())
-                    {
-                        patchNca = nca;
-                    }
-                    else if (nca.IsProgram())
+                    foreach ((ulong _, ContentCollection content) in applications)
                     {
-                        mainNca = nca;
-                    }
-                    else if (nca.IsControl())
-                    {
-                        controlNca = nca;
+                        mainNca = content.GetNcaByType(device.FileSystem.KeySet, ContentType.Program, device.Configuration.UserChannelPersistence.Index);
+                        controlNca = content.GetNcaByType(device.FileSystem.KeySet, ContentType.Control, device.Configuration.UserChannelPersistence.Index);
+                        break;
                     }
                 }
+                else if (applications.TryGetValue(titleId, out ContentCollection content))
+                {
+                    mainNca = content.GetNcaByType(device.FileSystem.KeySet, ContentType.Program, device.Configuration.UserChannelPersistence.Index);
+                    controlNca = content.GetNcaByType(device.FileSystem.KeySet, ContentType.Control, device.Configuration.UserChannelPersistence.Index);
+                }
 
                 ProcessLoaderHelper.RegisterProgramMapInfo(device, partitionFileSystem).ThrowIfFailure();
             }
@@ -79,54 +131,7 @@ namespace Ryujinx.HLE.Loaders.Processes.Extensions
                     return (false, ProcessResult.Failed);
                 }
 
-                // Load Update NCAs.
-                Nca updatePatchNca = null;
-                Nca updateControlNca = null;
-
-                if (ulong.TryParse(mainNca.Header.TitleId.ToString("x16"), NumberStyles.HexNumber, CultureInfo.InvariantCulture, out ulong titleIdBase))
-                {
-                    // Clear the program index part.
-                    titleIdBase &= ~0xFUL;
-
-                    // Load update information if exists.
-                    string titleUpdateMetadataPath = System.IO.Path.Combine(AppDataManager.GamesDirPath, titleIdBase.ToString("x16"), "updates.json");
-                    if (File.Exists(titleUpdateMetadataPath))
-                    {
-                        string updatePath = JsonHelper.DeserializeFromFile(titleUpdateMetadataPath, _titleSerializerContext.TitleUpdateMetadata).Selected;
-                        if (File.Exists(updatePath))
-                        {
-                            PartitionFileSystem updatePartitionFileSystem = new();
-                            updatePartitionFileSystem.Initialize(new FileStream(updatePath, FileMode.Open, FileAccess.Read).AsStorage()).ThrowIfFailure();
-
-                            device.Configuration.VirtualFileSystem.ImportTickets(updatePartitionFileSystem);
-
-                            // TODO: This should use CNMT NCA instead.
-                            foreach (DirectoryEntryEx fileEntry in updatePartitionFileSystem.EnumerateEntries("/", "*.nca"))
-                            {
-                                Nca nca = updatePartitionFileSystem.GetNca(device, fileEntry.FullPath);
-
-                                if (nca.GetProgramIndex() != device.Configuration.UserChannelPersistence.Index)
-                                {
-                                    continue;
-                                }
-
-                                if ($"{nca.Header.TitleId.ToString("x16")[..^3]}000" != titleIdBase.ToString("x16"))
-                                {
-                                    break;
-                                }
-
-                                if (nca.IsProgram())
-                                {
-                                    updatePatchNca = nca;
-                                }
-                                else if (nca.IsControl())
-                                {
-                                    updateControlNca = nca;
-                                }
-                            }
-                        }
-                    }
-                }
+                (Nca updatePatchNca, Nca updateControlNca) = mainNca.GetUpdateData(device.FileSystem, device.System.FsIntegrityCheckLevel, device.Configuration.UserChannelPersistence.Index, out string _);
 
                 if (updatePatchNca != null)
                 {
@@ -168,18 +173,18 @@ namespace Ryujinx.HLE.Loaders.Processes.Extensions
                 return (true, mainNca.Load(device, patchNca, controlNca));
             }
 
-            errorMessage = "Unable to load: Could not find Main NCA";
+            errorMessage = $"Unable to load: Could not find Main NCA for title \"{titleId:X16}\"";
 
             return (false, ProcessResult.Failed);
         }
 
-        public static Nca GetNca(this IFileSystem fileSystem, Switch device, string path)
+        public static Nca GetNca(this IFileSystem fileSystem, KeySet keySet, string path)
         {
             using var ncaFile = new UniqueRef<IFile>();
 
             fileSystem.OpenFile(ref ncaFile.Ref, path.ToU8Span(), OpenMode.Read).ThrowIfFailure();
 
-            return new Nca(device.Configuration.VirtualFileSystem.KeySet, ncaFile.Release().AsStorage());
+            return new Nca(keySet, ncaFile.Release().AsStorage());
         }
     }
 }
diff --git a/src/Ryujinx.HLE/Loaders/Processes/ProcessLoader.cs b/src/Ryujinx.HLE/Loaders/Processes/ProcessLoader.cs
index 220b868d..6b4a64be 100644
--- a/src/Ryujinx.HLE/Loaders/Processes/ProcessLoader.cs
+++ b/src/Ryujinx.HLE/Loaders/Processes/ProcessLoader.cs
@@ -32,7 +32,7 @@ namespace Ryujinx.HLE.Loaders.Processes
             _processesByPid = new ConcurrentDictionary<ulong, ProcessResult>();
         }
 
-        public bool LoadXci(string path)
+        public bool LoadXci(string path, ulong titleId)
         {
             FileStream stream = new(path, FileMode.Open, FileAccess.Read);
             Xci xci = new(_device.Configuration.VirtualFileSystem.KeySet, stream.AsStorage());
@@ -44,7 +44,7 @@ namespace Ryujinx.HLE.Loaders.Processes
                 return false;
             }
 
-            (bool success, ProcessResult processResult) = xci.OpenPartition(XciPartitionType.Secure).TryLoad(_device, path, out string errorMessage);
+            (bool success, ProcessResult processResult) = xci.OpenPartition(XciPartitionType.Secure).TryLoad(_device, path, titleId, out string errorMessage);
 
             if (!success)
             {
@@ -66,13 +66,13 @@ namespace Ryujinx.HLE.Loaders.Processes
             return false;
         }
 
-        public bool LoadNsp(string path)
+        public bool LoadNsp(string path, ulong titleId)
         {
             FileStream file = new(path, FileMode.Open, FileAccess.Read);
             PartitionFileSystem partitionFileSystem = new();
             partitionFileSystem.Initialize(file.AsStorage()).ThrowIfFailure();
 
-            (bool success, ProcessResult processResult) = partitionFileSystem.TryLoad(_device, path, out string errorMessage);
+            (bool success, ProcessResult processResult) = partitionFileSystem.TryLoad(_device, path, titleId, out string errorMessage);
 
             if (processResult.ProcessId == 0)
             {
diff --git a/src/Ryujinx.HLE/Loaders/Processes/ProcessLoaderHelper.cs b/src/Ryujinx.HLE/Loaders/Processes/ProcessLoaderHelper.cs
index c229b174..110bb092 100644
--- a/src/Ryujinx.HLE/Loaders/Processes/ProcessLoaderHelper.cs
+++ b/src/Ryujinx.HLE/Loaders/Processes/ProcessLoaderHelper.cs
@@ -42,15 +42,14 @@ namespace Ryujinx.HLE.Loaders.Processes
 
             foreach (DirectoryEntryEx fileEntry in partitionFileSystem.EnumerateEntries("/", "*.nca"))
             {
-                Nca nca = partitionFileSystem.GetNca(device, fileEntry.FullPath);
+                Nca nca = partitionFileSystem.GetNca(device.FileSystem.KeySet, fileEntry.FullPath);
 
-                if (!nca.IsProgram() && nca.IsPatch())
+                if (!nca.IsProgram())
                 {
                     continue;
                 }
 
-                ulong currentProgramId = nca.Header.TitleId;
-                ulong currentMainProgramId = currentProgramId & ~0xFFFul;
+                ulong currentMainProgramId = nca.GetProgramIdBase();
 
                 if (applicationId == 0 && currentMainProgramId != 0)
                 {
@@ -67,7 +66,7 @@ namespace Ryujinx.HLE.Loaders.Processes
                     break;
                 }
 
-                hasIndex[(int)(currentProgramId & 0xF)] = true;
+                hasIndex[nca.GetProgramIndex()] = true;
             }
 
             if (programCount == 0)