Merge pull request #1005 from DarkLordZach/registered-fmt

file_sys: Add support for registration format

src/core/file_sys/bis_factory.cpp

@@ -0,0 +1,31 @@
+// Copyright 2018 yuzu emulator team
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#include "core/file_sys/bis_factory.h"
+
+namespace FileSys {
+
+static VirtualDir GetOrCreateDirectory(const VirtualDir& dir, std::string_view path) {
+    const auto res = dir->GetDirectoryRelative(path);
+    if (res == nullptr)
+        return dir->CreateDirectoryRelative(path);
+    return res;
+}
+
+BISFactory::BISFactory(VirtualDir nand_root_)
+    : nand_root(std::move(nand_root_)),
+      sysnand_cache(std::make_shared<RegisteredCache>(
+          GetOrCreateDirectory(nand_root, "/system/Contents/registered"))),
+      usrnand_cache(std::make_shared<RegisteredCache>(
+          GetOrCreateDirectory(nand_root, "/user/Contents/registered"))) {}
+
+std::shared_ptr<RegisteredCache> BISFactory::GetSystemNANDContents() const {
+    return sysnand_cache;
+}
+
+std::shared_ptr<RegisteredCache> BISFactory::GetUserNANDContents() const {
+    return usrnand_cache;
+}
+
+} // namespace FileSys

src/core/file_sys/bis_factory.h

@@ -0,0 +1,30 @@
+// Copyright 2018 yuzu emulator team
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#pragma once
+
+#include <memory>
+#include "core/loader/loader.h"
+#include "registered_cache.h"
+
+namespace FileSys {
+
+/// File system interface to the Built-In Storage
+/// This is currently missing accessors to BIS partitions, but seemed like a good place for the NAND
+/// registered caches.
+class BISFactory {
+public:
+    explicit BISFactory(VirtualDir nand_root);
+
+    std::shared_ptr<RegisteredCache> GetSystemNANDContents() const;
+    std::shared_ptr<RegisteredCache> GetUserNANDContents() const;
+
+private:
+    VirtualDir nand_root;
+
+    std::shared_ptr<RegisteredCache> sysnand_cache;
+    std::shared_ptr<RegisteredCache> usrnand_cache;
+};
+
+} // namespace FileSys

src/core/file_sys/card_image.cpp

@@ -96,6 +96,10 @@ VirtualDir XCI::GetLogoPartition() const {
     return GetPartition(XCIPartition::Logo);
 }
 
+const std::vector<std::shared_ptr<NCA>>& XCI::GetNCAs() const {
+    return ncas;
+}
+
 std::shared_ptr<NCA> XCI::GetNCAByType(NCAContentType type) const {
     const auto iter =
         std::find_if(ncas.begin(), ncas.end(),

src/core/file_sys/card_image.h

@@ -68,6 +68,7 @@ public:
     VirtualDir GetUpdatePartition() const;
     VirtualDir GetLogoPartition() const;
 
+    const std::vector<std::shared_ptr<NCA>>& GetNCAs() const;
     std::shared_ptr<NCA> GetNCAByType(NCAContentType type) const;
     VirtualFile GetNCAFileByType(NCAContentType type) const;
 

src/core/file_sys/control_metadata.cpp

@@ -16,7 +16,7 @@ std::string LanguageEntry::GetDeveloperName() const {
     return Common::StringFromFixedZeroTerminatedBuffer(developer_name.data(), 0x100);
 }
 
-NACP::NACP(VirtualFile file_) : file(std::move(file_)), raw(std::make_unique<RawNACP>()) {
+NACP::NACP(VirtualFile file) : raw(std::make_unique<RawNACP>()) {
     file->ReadObject(raw.get());
 }
 

src/core/file_sys/control_metadata.h

@@ -81,7 +81,6 @@ public:
     std::string GetVersionString() const;
 
 private:
-    VirtualFile file;
     std::unique_ptr<RawNACP> raw;
 };
 

src/core/file_sys/nca_metadata.cpp

@@ -0,0 +1,131 @@
+// Copyright 2018 yuzu emulator team
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#include <cstring>
+#include "common/common_funcs.h"
+#include "common/logging/log.h"
+#include "common/swap.h"
+#include "content_archive.h"
+#include "core/file_sys/nca_metadata.h"
+
+namespace FileSys {
+
+bool operator>=(TitleType lhs, TitleType rhs) {
+    return static_cast<size_t>(lhs) >= static_cast<size_t>(rhs);
+}
+
+bool operator<=(TitleType lhs, TitleType rhs) {
+    return static_cast<size_t>(lhs) <= static_cast<size_t>(rhs);
+}
+
+CNMT::CNMT(VirtualFile file) {
+    if (file->ReadObject(&header) != sizeof(CNMTHeader))
+        return;
+
+    // If type is {Application, Update, AOC} has opt-header.
+    if (header.type >= TitleType::Application && header.type <= TitleType::AOC) {
+        if (file->ReadObject(&opt_header, sizeof(CNMTHeader)) != sizeof(OptionalHeader)) {
+            LOG_WARNING(Loader, "Failed to read optional header.");
+        }
+    }
+
+    for (u16 i = 0; i < header.number_content_entries; ++i) {
+        auto& next = content_records.emplace_back(ContentRecord{});
+        if (file->ReadObject(&next, sizeof(CNMTHeader) + i * sizeof(ContentRecord) +
+                                        header.table_offset) != sizeof(ContentRecord)) {
+            content_records.erase(content_records.end() - 1);
+        }
+    }
+
+    for (u16 i = 0; i < header.number_meta_entries; ++i) {
+        auto& next = meta_records.emplace_back(MetaRecord{});
+        if (file->ReadObject(&next, sizeof(CNMTHeader) + i * sizeof(MetaRecord) +
+                                        header.table_offset) != sizeof(MetaRecord)) {
+            meta_records.erase(meta_records.end() - 1);
+        }
+    }
+}
+
+CNMT::CNMT(CNMTHeader header, OptionalHeader opt_header, std::vector<ContentRecord> content_records,
+           std::vector<MetaRecord> meta_records)
+    : header(std::move(header)), opt_header(std::move(opt_header)),
+      content_records(std::move(content_records)), meta_records(std::move(meta_records)) {}
+
+u64 CNMT::GetTitleID() const {
+    return header.title_id;
+}
+
+u32 CNMT::GetTitleVersion() const {
+    return header.title_version;
+}
+
+TitleType CNMT::GetType() const {
+    return header.type;
+}
+
+const std::vector<ContentRecord>& CNMT::GetContentRecords() const {
+    return content_records;
+}
+
+const std::vector<MetaRecord>& CNMT::GetMetaRecords() const {
+    return meta_records;
+}
+
+bool CNMT::UnionRecords(const CNMT& other) {
+    bool change = false;
+    for (const auto& rec : other.content_records) {
+        const auto iter = std::find_if(content_records.begin(), content_records.end(),
+                                       [&rec](const ContentRecord& r) {
+                                           return r.nca_id == rec.nca_id && r.type == rec.type;
+                                       });
+        if (iter == content_records.end()) {
+            content_records.emplace_back(rec);
+            ++header.number_content_entries;
+            change = true;
+        }
+    }
+    for (const auto& rec : other.meta_records) {
+        const auto iter =
+            std::find_if(meta_records.begin(), meta_records.end(), [&rec](const MetaRecord& r) {
+                return r.title_id == rec.title_id && r.title_version == rec.title_version &&
+                       r.type == rec.type;
+            });
+        if (iter == meta_records.end()) {
+            meta_records.emplace_back(rec);
+            ++header.number_meta_entries;
+            change = true;
+        }
+    }
+    return change;
+}
+
+std::vector<u8> CNMT::Serialize() const {
+    const bool has_opt_header =
+        header.type >= TitleType::Application && header.type <= TitleType::AOC;
+    const auto dead_zone = header.table_offset + sizeof(CNMTHeader);
+    std::vector<u8> out(
+        std::max(sizeof(CNMTHeader) + (has_opt_header ? sizeof(OptionalHeader) : 0), dead_zone) +
+        content_records.size() * sizeof(ContentRecord) + meta_records.size() * sizeof(MetaRecord));
+    memcpy(out.data(), &header, sizeof(CNMTHeader));
+
+    // Optional Header
+    if (has_opt_header) {
+        memcpy(out.data() + sizeof(CNMTHeader), &opt_header, sizeof(OptionalHeader));
+    }
+
+    auto offset = header.table_offset;
+
+    for (const auto& rec : content_records) {
+        memcpy(out.data() + offset + sizeof(CNMTHeader), &rec, sizeof(ContentRecord));
+        offset += sizeof(ContentRecord);
+    }
+
+    for (const auto& rec : meta_records) {
+        memcpy(out.data() + offset + sizeof(CNMTHeader), &rec, sizeof(MetaRecord));
+        offset += sizeof(MetaRecord);
+    }
+
+    return out;
+}
+} // namespace FileSys

src/core/file_sys/nca_metadata.h

@@ -0,0 +1,111 @@
+// Copyright 2018 yuzu emulator team
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#pragma once
+
+#include <cstring>
+#include <memory>
+#include <vector>
+#include "common/common_types.h"
+#include "common/swap.h"
+#include "core/file_sys/vfs.h"
+
+namespace FileSys {
+class CNMT;
+
+struct CNMTHeader;
+struct OptionalHeader;
+
+enum class TitleType : u8 {
+    SystemProgram = 0x01,
+    SystemDataArchive = 0x02,
+    SystemUpdate = 0x03,
+    FirmwarePackageA = 0x04,
+    FirmwarePackageB = 0x05,
+    Application = 0x80,
+    Update = 0x81,
+    AOC = 0x82,
+    DeltaTitle = 0x83,
+};
+
+bool operator>=(TitleType lhs, TitleType rhs);
+bool operator<=(TitleType lhs, TitleType rhs);
+
+enum class ContentRecordType : u8 {
+    Meta = 0,
+    Program = 1,
+    Data = 2,
+    Control = 3,
+    Manual = 4,
+    Legal = 5,
+    Patch = 6,
+};
+
+struct ContentRecord {
+    std::array<u8, 0x20> hash;
+    std::array<u8, 0x10> nca_id;
+    std::array<u8, 0x6> size;
+    ContentRecordType type;
+    INSERT_PADDING_BYTES(1);
+};
+static_assert(sizeof(ContentRecord) == 0x38, "ContentRecord has incorrect size.");
+
+constexpr ContentRecord EMPTY_META_CONTENT_RECORD{{}, {}, {}, ContentRecordType::Meta, {}};
+
+struct MetaRecord {
+    u64_le title_id;
+    u32_le title_version;
+    TitleType type;
+    u8 install_byte;
+    INSERT_PADDING_BYTES(2);
+};
+static_assert(sizeof(MetaRecord) == 0x10, "MetaRecord has incorrect size.");
+
+struct OptionalHeader {
+    u64_le title_id;
+    u64_le minimum_version;
+};
+static_assert(sizeof(OptionalHeader) == 0x10, "OptionalHeader has incorrect size.");
+
+struct CNMTHeader {
+    u64_le title_id;
+    u32_le title_version;
+    TitleType type;
+    INSERT_PADDING_BYTES(1);
+    u16_le table_offset;
+    u16_le number_content_entries;
+    u16_le number_meta_entries;
+    INSERT_PADDING_BYTES(12);
+};
+static_assert(sizeof(CNMTHeader) == 0x20, "CNMTHeader has incorrect size.");
+
+// A class representing the format used by NCA metadata files, typically named {}.cnmt.nca or
+// meta0.ncd. These describe which NCA's belong with which titles in the registered cache.
+class CNMT {
+public:
+    explicit CNMT(VirtualFile file);
+    CNMT(CNMTHeader header, OptionalHeader opt_header, std::vector<ContentRecord> content_records,
+         std::vector<MetaRecord> meta_records);
+
+    u64 GetTitleID() const;
+    u32 GetTitleVersion() const;
+    TitleType GetType() const;
+
+    const std::vector<ContentRecord>& GetContentRecords() const;
+    const std::vector<MetaRecord>& GetMetaRecords() const;
+
+    bool UnionRecords(const CNMT& other);
+    std::vector<u8> Serialize() const;
+
+private:
+    CNMTHeader header;
+    OptionalHeader opt_header;
+    std::vector<ContentRecord> content_records;
+    std::vector<MetaRecord> meta_records;
+
+    // TODO(DarkLordZach): According to switchbrew, for Patch-type there is additional data
+    // after the table. This is not documented, unfortunately.
+};
+
+} // namespace FileSys

src/core/file_sys/registered_cache.cpp

@@ -0,0 +1,476 @@
+// Copyright 2018 yuzu emulator team
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#include <regex>
+#include <mbedtls/sha256.h>
+#include "common/assert.h"
+#include "common/hex_util.h"
+#include "common/logging/log.h"
+#include "core/crypto/encryption_layer.h"
+#include "core/file_sys/card_image.h"
+#include "core/file_sys/nca_metadata.h"
+#include "core/file_sys/registered_cache.h"
+#include "core/file_sys/vfs_concat.h"
+
+namespace FileSys {
+std::string RegisteredCacheEntry::DebugInfo() const {
+    return fmt::format("title_id={:016X}, content_type={:02X}", title_id, static_cast<u8>(type));
+}
+
+bool operator<(const RegisteredCacheEntry& lhs, const RegisteredCacheEntry& rhs) {
+    return (lhs.title_id < rhs.title_id) || (lhs.title_id == rhs.title_id && lhs.type < rhs.type);
+}
+
+static bool FollowsTwoDigitDirFormat(std::string_view name) {
+    static const std::regex two_digit_regex("000000[0-9A-F]{2}", std::regex_constants::ECMAScript |
+                                                                     std::regex_constants::icase);
+    return std::regex_match(name.begin(), name.end(), two_digit_regex);
+}
+
+static bool FollowsNcaIdFormat(std::string_view name) {
+    static const std::regex nca_id_regex("[0-9A-F]{32}\\.nca", std::regex_constants::ECMAScript |
+                                                                   std::regex_constants::icase);
+    return name.size() == 36 && std::regex_match(name.begin(), name.end(), nca_id_regex);
+}
+
+static std::string GetRelativePathFromNcaID(const std::array<u8, 16>& nca_id, bool second_hex_upper,
+                                            bool within_two_digit) {
+    if (!within_two_digit)
+        return fmt::format("/{}.nca", HexArrayToString(nca_id, second_hex_upper));
+
+    Core::Crypto::SHA256Hash hash{};
+    mbedtls_sha256(nca_id.data(), nca_id.size(), hash.data(), 0);
+    return fmt::format("/000000{:02X}/{}.nca", hash[0], HexArrayToString(nca_id, second_hex_upper));
+}
+
+static std::string GetCNMTName(TitleType type, u64 title_id) {
+    constexpr std::array<const char*, 9> TITLE_TYPE_NAMES{
+        "SystemProgram",
+        "SystemData",
+        "SystemUpdate",
+        "BootImagePackage",
+        "BootImagePackageSafe",
+        "Application",
+        "Patch",
+        "AddOnContent",
+        "" ///< Currently unknown 'DeltaTitle'
+    };
+
+    auto index = static_cast<size_t>(type);
+    // If the index is after the jump in TitleType, subtract it out.
+    if (index >= static_cast<size_t>(TitleType::Application)) {
+        index -= static_cast<size_t>(TitleType::Application) -
+                 static_cast<size_t>(TitleType::FirmwarePackageB);
+    }
+    return fmt::format("{}_{:016x}.cnmt", TITLE_TYPE_NAMES[index], title_id);
+}
+
+static ContentRecordType GetCRTypeFromNCAType(NCAContentType type) {
+    switch (type) {
+    case NCAContentType::Program:
+        // TODO(DarkLordZach): Differentiate between Program and Patch
+        return ContentRecordType::Program;
+    case NCAContentType::Meta:
+        return ContentRecordType::Meta;
+    case NCAContentType::Control:
+        return ContentRecordType::Control;
+    case NCAContentType::Data:
+        return ContentRecordType::Data;
+    case NCAContentType::Manual:
+        // TODO(DarkLordZach): Peek at NCA contents to differentiate Manual and Legal.
+        return ContentRecordType::Manual;
+    default:
+        UNREACHABLE();
+    }
+}
+
+VirtualFile RegisteredCache::OpenFileOrDirectoryConcat(const VirtualDir& dir,
+                                                       std::string_view path) const {
+    if (dir->GetFileRelative(path) != nullptr)
+        return dir->GetFileRelative(path);
+    if (dir->GetDirectoryRelative(path) != nullptr) {
+        const auto nca_dir = dir->GetDirectoryRelative(path);
+        VirtualFile file = nullptr;
+
+        const auto files = nca_dir->GetFiles();
+        if (files.size() == 1 && files[0]->GetName() == "00") {
+            file = files[0];
+        } else {
+            std::vector<VirtualFile> concat;
+            // Since the files are a two-digit hex number, max is FF.
+            for (size_t i = 0; i < 0x100; ++i) {
+                auto next = nca_dir->GetFile(fmt::format("{:02X}", i));
+                if (next != nullptr) {
+                    concat.push_back(std::move(next));
+                } else {
+                    next = nca_dir->GetFile(fmt::format("{:02x}", i));
+                    if (next != nullptr)
+                        concat.push_back(std::move(next));
+                    else
+                        break;
+                }
+            }
+
+            if (concat.empty())
+                return nullptr;
+
+            file = FileSys::ConcatenateFiles(concat);
+        }
+
+        return file;
+    }
+    return nullptr;
+}
+
+VirtualFile RegisteredCache::GetFileAtID(NcaID id) const {
+    VirtualFile file;
+    // Try all four modes of file storage:
+    // (bit 1 = uppercase/lower, bit 0 = within a two-digit dir)
+    // 00: /000000**/{:032X}.nca
+    // 01: /{:032X}.nca
+    // 10: /000000**/{:032x}.nca
+    // 11: /{:032x}.nca
+    for (u8 i = 0; i < 4; ++i) {
+        const auto path = GetRelativePathFromNcaID(id, (i & 0b10) == 0, (i & 0b01) == 0);
+        file = OpenFileOrDirectoryConcat(dir, path);
+        if (file != nullptr)
+            return file;
+    }
+    return file;
+}
+
+static boost::optional<NcaID> CheckMapForContentRecord(
+    const boost::container::flat_map<u64, CNMT>& map, u64 title_id, ContentRecordType type) {
+    if (map.find(title_id) == map.end())
+        return boost::none;
+
+    const auto& cnmt = map.at(title_id);
+
+    const auto iter = std::find_if(cnmt.GetContentRecords().begin(), cnmt.GetContentRecords().end(),
+                                   [type](const ContentRecord& rec) { return rec.type == type; });
+    if (iter == cnmt.GetContentRecords().end())
+        return boost::none;
+
+    return boost::make_optional(iter->nca_id);
+}
+
+boost::optional<NcaID> RegisteredCache::GetNcaIDFromMetadata(u64 title_id,
+                                                             ContentRecordType type) const {
+    if (type == ContentRecordType::Meta && meta_id.find(title_id) != meta_id.end())
+        return meta_id.at(title_id);
+
+    const auto res1 = CheckMapForContentRecord(yuzu_meta, title_id, type);
+    if (res1 != boost::none)
+        return res1;
+    return CheckMapForContentRecord(meta, title_id, type);
+}
+
+std::vector<NcaID> RegisteredCache::AccumulateFiles() const {
+    std::vector<NcaID> ids;
+    for (const auto& d2_dir : dir->GetSubdirectories()) {
+        if (FollowsNcaIdFormat(d2_dir->GetName())) {
+            ids.push_back(HexStringToArray<0x10, true>(d2_dir->GetName().substr(0, 0x20)));
+            continue;
+        }
+
+        if (!FollowsTwoDigitDirFormat(d2_dir->GetName()))
+            continue;
+
+        for (const auto& nca_dir : d2_dir->GetSubdirectories()) {
+            if (!FollowsNcaIdFormat(nca_dir->GetName()))
+                continue;
+
+            ids.push_back(HexStringToArray<0x10, true>(nca_dir->GetName().substr(0, 0x20)));
+        }
+
+        for (const auto& nca_file : d2_dir->GetFiles()) {
+            if (!FollowsNcaIdFormat(nca_file->GetName()))
+                continue;
+
+            ids.push_back(HexStringToArray<0x10, true>(nca_file->GetName().substr(0, 0x20)));
+        }
+    }
+
+    for (const auto& d2_file : dir->GetFiles()) {
+        if (FollowsNcaIdFormat(d2_file->GetName()))
+            ids.push_back(HexStringToArray<0x10, true>(d2_file->GetName().substr(0, 0x20)));
+    }
+    return ids;
+}
+
+void RegisteredCache::ProcessFiles(const std::vector<NcaID>& ids) {
+    for (const auto& id : ids) {
+        const auto file = GetFileAtID(id);
+
+        if (file == nullptr)
+            continue;
+        const auto nca = std::make_shared<NCA>(parser(file, id));
+        if (nca->GetStatus() != Loader::ResultStatus::Success ||
+            nca->GetType() != NCAContentType::Meta) {
+            continue;
+        }
+
+        const auto section0 = nca->GetSubdirectories()[0];
+
+        for (const auto& file : section0->GetFiles()) {
+            if (file->GetExtension() != "cnmt")
+                continue;
+
+            meta.insert_or_assign(nca->GetTitleId(), CNMT(file));
+            meta_id.insert_or_assign(nca->GetTitleId(), id);
+            break;
+        }
+    }
+}
+
+void RegisteredCache::AccumulateYuzuMeta() {
+    const auto dir = this->dir->GetSubdirectory("yuzu_meta");
+    if (dir == nullptr)
+        return;
+
+    for (const auto& file : dir->GetFiles()) {
+        if (file->GetExtension() != "cnmt")
+            continue;
+
+        CNMT cnmt(file);
+        yuzu_meta.insert_or_assign(cnmt.GetTitleID(), std::move(cnmt));
+    }
+}
+
+void RegisteredCache::Refresh() {
+    if (dir == nullptr)
+        return;
+    const auto ids = AccumulateFiles();
+    ProcessFiles(ids);
+    AccumulateYuzuMeta();
+}
+
+RegisteredCache::RegisteredCache(VirtualDir dir_, RegisteredCacheParsingFunction parsing_function)
+    : dir(std::move(dir_)), parser(std::move(parsing_function)) {
+    Refresh();
+}
+
+bool RegisteredCache::HasEntry(u64 title_id, ContentRecordType type) const {
+    return GetEntryRaw(title_id, type) != nullptr;
+}
+
+bool RegisteredCache::HasEntry(RegisteredCacheEntry entry) const {
+    return GetEntryRaw(entry) != nullptr;
+}
+
+VirtualFile RegisteredCache::GetEntryRaw(u64 title_id, ContentRecordType type) const {
+    const auto id = GetNcaIDFromMetadata(title_id, type);
+    if (id == boost::none)
+        return nullptr;
+
+    return parser(GetFileAtID(id.get()), id.get());
+}
+
+VirtualFile RegisteredCache::GetEntryRaw(RegisteredCacheEntry entry) const {
+    return GetEntryRaw(entry.title_id, entry.type);
+}
+
+std::shared_ptr<NCA> RegisteredCache::GetEntry(u64 title_id, ContentRecordType type) const {
+    const auto raw = GetEntryRaw(title_id, type);
+    if (raw == nullptr)
+        return nullptr;
+    return std::make_shared<NCA>(raw);
+}
+
+std::shared_ptr<NCA> RegisteredCache::GetEntry(RegisteredCacheEntry entry) const {
+    return GetEntry(entry.title_id, entry.type);
+}
+
+template <typename T>
+void RegisteredCache::IterateAllMetadata(
+    std::vector<T>& out, std::function<T(const CNMT&, const ContentRecord&)> proc,
+    std::function<bool(const CNMT&, const ContentRecord&)> filter) const {
+    for (const auto& kv : meta) {
+        const auto& cnmt = kv.second;
+        if (filter(cnmt, EMPTY_META_CONTENT_RECORD))
+            out.push_back(proc(cnmt, EMPTY_META_CONTENT_RECORD));
+        for (const auto& rec : cnmt.GetContentRecords()) {
+            if (GetFileAtID(rec.nca_id) != nullptr && filter(cnmt, rec)) {
+                out.push_back(proc(cnmt, rec));
+            }
+        }
+    }
+    for (const auto& kv : yuzu_meta) {
+        const auto& cnmt = kv.second;
+        for (const auto& rec : cnmt.GetContentRecords()) {
+            if (GetFileAtID(rec.nca_id) != nullptr && filter(cnmt, rec)) {
+                out.push_back(proc(cnmt, rec));
+            }
+        }
+    }
+}
+
+std::vector<RegisteredCacheEntry> RegisteredCache::ListEntries() const {
+    std::vector<RegisteredCacheEntry> out;
+    IterateAllMetadata<RegisteredCacheEntry>(
+        out,
+        [](const CNMT& c, const ContentRecord& r) {
+            return RegisteredCacheEntry{c.GetTitleID(), r.type};
+        },
+        [](const CNMT& c, const ContentRecord& r) { return true; });
+    return out;
+}
+
+std::vector<RegisteredCacheEntry> RegisteredCache::ListEntriesFilter(
+    boost::optional<TitleType> title_type, boost::optional<ContentRecordType> record_type,
+    boost::optional<u64> title_id) const {
+    std::vector<RegisteredCacheEntry> out;
+    IterateAllMetadata<RegisteredCacheEntry>(
+        out,
+        [](const CNMT& c, const ContentRecord& r) {
+            return RegisteredCacheEntry{c.GetTitleID(), r.type};
+        },
+        [&title_type, &record_type, &title_id](const CNMT& c, const ContentRecord& r) {
+            if (title_type != boost::none && title_type.get() != c.GetType())
+                return false;
+            if (record_type != boost::none && record_type.get() != r.type)
+                return false;
+            if (title_id != boost::none && title_id.get() != c.GetTitleID())
+                return false;
+            return true;
+        });
+    return out;
+}
+
+static std::shared_ptr<NCA> GetNCAFromXCIForID(std::shared_ptr<XCI> xci, const NcaID& id) {
+    const auto filename = fmt::format("{}.nca", HexArrayToString(id, false));
+    const auto iter =
+        std::find_if(xci->GetNCAs().begin(), xci->GetNCAs().end(),
+                     [&filename](std::shared_ptr<NCA> nca) { return nca->GetName() == filename; });
+    return iter == xci->GetNCAs().end() ? nullptr : *iter;
+}
+
+InstallResult RegisteredCache::InstallEntry(std::shared_ptr<XCI> xci, bool overwrite_if_exists,
+                                            const VfsCopyFunction& copy) {
+    const auto& ncas = xci->GetNCAs();
+    const auto& meta_iter = std::find_if(ncas.begin(), ncas.end(), [](std::shared_ptr<NCA> nca) {
+        return nca->GetType() == NCAContentType::Meta;
+    });
+
+    if (meta_iter == ncas.end()) {
+        LOG_ERROR(Loader, "The XCI you are attempting to install does not have a metadata NCA and "
+                          "is therefore malformed. Double check your encryption keys.");
+        return InstallResult::ErrorMetaFailed;
+    }
+
+    // Install Metadata File
+    const auto meta_id_raw = (*meta_iter)->GetName().substr(0, 32);
+    const auto meta_id = HexStringToArray<16>(meta_id_raw);
+
+    const auto res = RawInstallNCA(*meta_iter, copy, overwrite_if_exists, meta_id);
+    if (res != InstallResult::Success)
+        return res;
+
+    // Install all the other NCAs
+    const auto section0 = (*meta_iter)->GetSubdirectories()[0];
+    const auto cnmt_file = section0->GetFiles()[0];
+    const CNMT cnmt(cnmt_file);
+    for (const auto& record : cnmt.GetContentRecords()) {
+        const auto nca = GetNCAFromXCIForID(xci, record.nca_id);
+        if (nca == nullptr)
+            return InstallResult::ErrorCopyFailed;
+        const auto res2 = RawInstallNCA(nca, copy, overwrite_if_exists, record.nca_id);
+        if (res2 != InstallResult::Success)
+            return res2;
+    }
+
+    Refresh();
+    return InstallResult::Success;
+}
+
+InstallResult RegisteredCache::InstallEntry(std::shared_ptr<NCA> nca, TitleType type,
+                                            bool overwrite_if_exists, const VfsCopyFunction& copy) {
+    CNMTHeader header{
+        nca->GetTitleId(), ///< Title ID
+        0,                 ///< Ignore/Default title version
+        type,              ///< Type
+        {},                ///< Padding
+        0x10,              ///< Default table offset
+        1,                 ///< 1 Content Entry
+        0,                 ///< No Meta Entries
+        {},                ///< Padding
+    };
+    OptionalHeader opt_header{0, 0};
+    ContentRecord c_rec{{}, {}, {}, GetCRTypeFromNCAType(nca->GetType()), {}};
+    const auto& data = nca->GetBaseFile()->ReadBytes(0x100000);
+    mbedtls_sha256(data.data(), data.size(), c_rec.hash.data(), 0);
+    memcpy(&c_rec.nca_id, &c_rec.hash, 16);
+    const CNMT new_cnmt(header, opt_header, {c_rec}, {});
+    if (!RawInstallYuzuMeta(new_cnmt))
+        return InstallResult::ErrorMetaFailed;
+    return RawInstallNCA(nca, copy, overwrite_if_exists, c_rec.nca_id);
+}
+
+InstallResult RegisteredCache::RawInstallNCA(std::shared_ptr<NCA> nca, const VfsCopyFunction& copy,
+                                             bool overwrite_if_exists,
+                                             boost::optional<NcaID> override_id) {
+    const auto in = nca->GetBaseFile();
+    Core::Crypto::SHA256Hash hash{};
+
+    // Calculate NcaID
+    // NOTE: Because computing the SHA256 of an entire NCA is quite expensive (especially if the
+    // game is massive), we're going to cheat and only hash the first MB of the NCA.
+    // Also, for XCIs the NcaID matters, so if the override id isn't none, use that.
+    NcaID id{};
+    if (override_id == boost::none) {
+        const auto& data = in->ReadBytes(0x100000);
+        mbedtls_sha256(data.data(), data.size(), hash.data(), 0);
+        memcpy(id.data(), hash.data(), 16);
+    } else {
+        id = override_id.get();
+    }
+
+    std::string path = GetRelativePathFromNcaID(id, false, true);
+
+    if (GetFileAtID(id) != nullptr && !overwrite_if_exists) {
+        LOG_WARNING(Loader, "Attempting to overwrite existing NCA. Skipping...");
+        return InstallResult::ErrorAlreadyExists;
+    }
+
+    if (GetFileAtID(id) != nullptr) {
+        LOG_WARNING(Loader, "Overwriting existing NCA...");
+        VirtualDir c_dir;
+        { c_dir = dir->GetFileRelative(path)->GetContainingDirectory(); }
+        c_dir->DeleteFile(FileUtil::GetFilename(path));
+    }
+
+    auto out = dir->CreateFileRelative(path);
+    if (out == nullptr)
+        return InstallResult::ErrorCopyFailed;
+    return copy(in, out) ? InstallResult::Success : InstallResult::ErrorCopyFailed;
+}
+
+bool RegisteredCache::RawInstallYuzuMeta(const CNMT& cnmt) {
+    // Reasoning behind this method can be found in the comment for InstallEntry, NCA overload.
+    const auto dir = this->dir->CreateDirectoryRelative("yuzu_meta");
+    const auto filename = GetCNMTName(cnmt.GetType(), cnmt.GetTitleID());
+    if (dir->GetFile(filename) == nullptr) {
+        auto out = dir->CreateFile(filename);
+        const auto buffer = cnmt.Serialize();
+        out->Resize(buffer.size());
+        out->WriteBytes(buffer);
+    } else {
+        auto out = dir->GetFile(filename);
+        CNMT old_cnmt(out);
+        // Returns true on change
+        if (old_cnmt.UnionRecords(cnmt)) {
+            out->Resize(0);
+            const auto buffer = old_cnmt.Serialize();
+            out->Resize(buffer.size());
+            out->WriteBytes(buffer);
+        }
+    }
+    Refresh();
+    return std::find_if(yuzu_meta.begin(), yuzu_meta.end(),
+                        [&cnmt](const std::pair<u64, CNMT>& kv) {
+                            return kv.second.GetType() == cnmt.GetType() &&
+                                   kv.second.GetTitleID() == cnmt.GetTitleID();
+                        }) != yuzu_meta.end();
+}
+} // namespace FileSys

src/core/file_sys/registered_cache.h

@@ -0,0 +1,124 @@
+// Copyright 2018 yuzu emulator team
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#pragma once
+
+#include <array>
+#include <functional>
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+#include <boost/container/flat_map.hpp>
+#include "common/common_funcs.h"
+#include "common/common_types.h"
+#include "content_archive.h"
+#include "core/file_sys/nca_metadata.h"
+#include "core/file_sys/vfs.h"
+
+namespace FileSys {
+class XCI;
+class CNMT;
+
+using NcaID = std::array<u8, 0x10>;
+using RegisteredCacheParsingFunction = std::function<VirtualFile(const VirtualFile&, const NcaID&)>;
+using VfsCopyFunction = std::function<bool(VirtualFile, VirtualFile)>;
+
+enum class InstallResult {
+    Success,
+    ErrorAlreadyExists,
+    ErrorCopyFailed,
+    ErrorMetaFailed,
+};
+
+struct RegisteredCacheEntry {
+    u64 title_id;
+    ContentRecordType type;
+
+    std::string DebugInfo() const;
+};
+
+// boost flat_map requires operator< for O(log(n)) lookups.
+bool operator<(const RegisteredCacheEntry& lhs, const RegisteredCacheEntry& rhs);
+
+/*
+ * A class that catalogues NCAs in the registered directory structure.
+ * Nintendo's registered format follows this structure:
+ *
+ * Root
+ *   | 000000XX <- XX is the ____ two digits of the NcaID
+ *       | <hash>.nca <- hash is the NcaID (first half of SHA256 over entire file) (folder)
+ *         | 00
+ *         | 01 <- Actual content split along 4GB boundaries. (optional)
+ *
+ * (This impl also supports substituting the nca dir for an nca file, as that's more convenient when
+ * 4GB splitting can be ignored.)
+ */
+class RegisteredCache {
+public:
+    // Parsing function defines the conversion from raw file to NCA. If there are other steps
+    // besides creating the NCA from the file (e.g. NAX0 on SD Card), that should go in a custom
+    // parsing function.
+    explicit RegisteredCache(VirtualDir dir,
+                             RegisteredCacheParsingFunction parsing_function =
+                                 [](const VirtualFile& file, const NcaID& id) { return file; });
+
+    void Refresh();
+
+    bool HasEntry(u64 title_id, ContentRecordType type) const;
+    bool HasEntry(RegisteredCacheEntry entry) const;
+
+    VirtualFile GetEntryRaw(u64 title_id, ContentRecordType type) const;
+    VirtualFile GetEntryRaw(RegisteredCacheEntry entry) const;
+
+    std::shared_ptr<NCA> GetEntry(u64 title_id, ContentRecordType type) const;
+    std::shared_ptr<NCA> GetEntry(RegisteredCacheEntry entry) const;
+
+    std::vector<RegisteredCacheEntry> ListEntries() const;
+    // If a parameter is not boost::none, it will be filtered for from all entries.
+    std::vector<RegisteredCacheEntry> ListEntriesFilter(
+        boost::optional<TitleType> title_type = boost::none,
+        boost::optional<ContentRecordType> record_type = boost::none,
+        boost::optional<u64> title_id = boost::none) const;
+
+    // Raw copies all the ncas from the xci to the csache. Does some quick checks to make sure there
+    // is a meta NCA and all of them are accessible.
+    InstallResult InstallEntry(std::shared_ptr<XCI> xci, bool overwrite_if_exists = false,
+                               const VfsCopyFunction& copy = &VfsRawCopy);
+
+    // Due to the fact that we must use Meta-type NCAs to determine the existance of files, this
+    // poses quite a challenge. Instead of creating a new meta NCA for this file, yuzu will create a
+    // dir inside the NAND called 'yuzu_meta' and store the raw CNMT there.
+    // TODO(DarkLordZach): Author real meta-type NCAs and install those.
+    InstallResult InstallEntry(std::shared_ptr<NCA> nca, TitleType type,
+                               bool overwrite_if_exists = false,
+                               const VfsCopyFunction& copy = &VfsRawCopy);
+
+private:
+    template <typename T>
+    void IterateAllMetadata(std::vector<T>& out,
+                            std::function<T(const CNMT&, const ContentRecord&)> proc,
+                            std::function<bool(const CNMT&, const ContentRecord&)> filter) const;
+    std::vector<NcaID> AccumulateFiles() const;
+    void ProcessFiles(const std::vector<NcaID>& ids);
+    void AccumulateYuzuMeta();
+    boost::optional<NcaID> GetNcaIDFromMetadata(u64 title_id, ContentRecordType type) const;
+    VirtualFile GetFileAtID(NcaID id) const;
+    VirtualFile OpenFileOrDirectoryConcat(const VirtualDir& dir, std::string_view path) const;
+    InstallResult RawInstallNCA(std::shared_ptr<NCA> nca, const VfsCopyFunction& copy,
+                                bool overwrite_if_exists,
+                                boost::optional<NcaID> override_id = boost::none);
+    bool RawInstallYuzuMeta(const CNMT& cnmt);
+
+    VirtualDir dir;
+    RegisteredCacheParsingFunction parser;
+    // maps tid -> NcaID of meta
+    boost::container::flat_map<u64, NcaID> meta_id;
+    // maps tid -> meta
+    boost::container::flat_map<u64, CNMT> meta;
+    // maps tid -> meta for CNMT in yuzu_meta
+    boost::container::flat_map<u64, CNMT> yuzu_meta;
+};
+
+} // namespace FileSys

src/core/file_sys/romfs.cpp

@@ -65,7 +65,7 @@ void ProcessFile(VirtualFile file, size_t file_offset, size_t data_offset, u32 t
         auto entry = GetEntry<FileEntry>(file, file_offset + this_file_offset);
 
         parent->AddFile(std::make_shared<OffsetVfsFile>(
-            file, entry.first.size, entry.first.offset + data_offset, entry.second, parent));
+            file, entry.first.size, entry.first.offset + data_offset, entry.second));
 
         if (entry.first.sibling == ROMFS_ENTRY_EMPTY)
             break;
@@ -79,7 +79,7 @@ void ProcessDirectory(VirtualFile file, size_t dir_offset, size_t file_offset, s
     while (true) {
         auto entry = GetEntry<DirectoryEntry>(file, dir_offset + this_dir_offset);
         auto current = std::make_shared<VectorVfsDirectory>(
-            std::vector<VirtualFile>{}, std::vector<VirtualDir>{}, parent, entry.second);
+            std::vector<VirtualFile>{}, std::vector<VirtualDir>{}, entry.second);
 
         if (entry.first.child_file != ROMFS_ENTRY_EMPTY) {
             ProcessFile(file, file_offset, data_offset, entry.first.child_file, current);
@@ -108,9 +108,9 @@ VirtualDir ExtractRomFS(VirtualFile file) {
     const u64 file_offset = header.file_meta.offset;
     const u64 dir_offset = header.directory_meta.offset + 4;
 
-    const auto root =
+    auto root =
         std::make_shared<VectorVfsDirectory>(std::vector<VirtualFile>{}, std::vector<VirtualDir>{},
-                                             file->GetContainingDirectory(), file->GetName());
+                                             file->GetName(), file->GetContainingDirectory());
 
     ProcessDirectory(file, dir_offset, file_offset, header.data_offset, 0, root);
 

src/core/file_sys/vfs_concat.cpp

@@ -0,0 +1,94 @@
+// Copyright 2018 yuzu emulator team
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#include <algorithm>
+#include <utility>
+
+#include "core/file_sys/vfs_concat.h"
+
+namespace FileSys {
+
+VirtualFile ConcatenateFiles(std::vector<VirtualFile> files, std::string name) {
+    if (files.empty())
+        return nullptr;
+    if (files.size() == 1)
+        return files[0];
+
+    return std::shared_ptr<VfsFile>(new ConcatenatedVfsFile(std::move(files), std::move(name)));
+}
+
+ConcatenatedVfsFile::ConcatenatedVfsFile(std::vector<VirtualFile> files_, std::string name)
+    : name(std::move(name)) {
+    size_t next_offset = 0;
+    for (const auto& file : files_) {
+        files[next_offset] = file;
+        next_offset += file->GetSize();
+    }
+}
+
+std::string ConcatenatedVfsFile::GetName() const {
+    if (files.empty())
+        return "";
+    if (!name.empty())
+        return name;
+    return files.begin()->second->GetName();
+}
+
+size_t ConcatenatedVfsFile::GetSize() const {
+    if (files.empty())
+        return 0;
+    return files.rbegin()->first + files.rbegin()->second->GetSize();
+}
+
+bool ConcatenatedVfsFile::Resize(size_t new_size) {
+    return false;
+}
+
+std::shared_ptr<VfsDirectory> ConcatenatedVfsFile::GetContainingDirectory() const {
+    if (files.empty())
+        return nullptr;
+    return files.begin()->second->GetContainingDirectory();
+}
+
+bool ConcatenatedVfsFile::IsWritable() const {
+    return false;
+}
+
+bool ConcatenatedVfsFile::IsReadable() const {
+    return true;
+}
+
+size_t ConcatenatedVfsFile::Read(u8* data, size_t length, size_t offset) const {
+    auto entry = files.end();
+    for (auto iter = files.begin(); iter != files.end(); ++iter) {
+        if (iter->first > offset) {
+            entry = --iter;
+            break;
+        }
+    }
+
+    // Check if the entry should be the last one. The loop above will make it end().
+    if (entry == files.end() && offset < files.rbegin()->first + files.rbegin()->second->GetSize())
+        --entry;
+
+    if (entry == files.end())
+        return 0;
+
+    const auto remaining = entry->second->GetSize() + offset - entry->first;
+    if (length > remaining) {
+        return entry->second->Read(data, remaining, offset - entry->first) +
+               Read(data + remaining, length - remaining, offset + remaining);
+    }
+
+    return entry->second->Read(data, length, offset - entry->first);
+}
+
+size_t ConcatenatedVfsFile::Write(const u8* data, size_t length, size_t offset) {
+    return 0;
+}
+
+bool ConcatenatedVfsFile::Rename(std::string_view name) {
+    return false;
+}
+} // namespace FileSys

src/core/file_sys/vfs_concat.h

@@ -0,0 +1,41 @@
+// Copyright 2018 yuzu emulator team
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#pragma once
+
+#include <memory>
+#include <string_view>
+#include <boost/container/flat_map.hpp>
+#include "core/file_sys/vfs.h"
+
+namespace FileSys {
+
+// Wrapper function to allow for more efficient handling of files.size() == 0, 1 cases.
+VirtualFile ConcatenateFiles(std::vector<VirtualFile> files, std::string name = "");
+
+// Class that wraps multiple vfs files and concatenates them, making reads seamless. Currently
+// read-only.
+class ConcatenatedVfsFile : public VfsFile {
+    friend VirtualFile ConcatenateFiles(std::vector<VirtualFile> files, std::string name);
+
+    ConcatenatedVfsFile(std::vector<VirtualFile> files, std::string name);
+
+public:
+    std::string GetName() const override;
+    size_t GetSize() const override;
+    bool Resize(size_t new_size) override;
+    std::shared_ptr<VfsDirectory> GetContainingDirectory() const override;
+    bool IsWritable() const override;
+    bool IsReadable() const override;
+    size_t Read(u8* data, size_t length, size_t offset) const override;
+    size_t Write(const u8* data, size_t length, size_t offset) override;
+    bool Rename(std::string_view name) override;
+
+private:
+    // Maps starting offset to file -- more efficient.
+    boost::container::flat_map<u64, VirtualFile> files;
+    std::string name;
+};
+
+} // namespace FileSys

src/core/file_sys/vfs_real.cpp

@@ -83,8 +83,12 @@ VirtualFile RealVfsFilesystem::OpenFile(std::string_view path_, Mode perms) {
 
 VirtualFile RealVfsFilesystem::CreateFile(std::string_view path_, Mode perms) {
     const auto path = FileUtil::SanitizePath(path_, FileUtil::DirectorySeparator::PlatformDefault);
-    if (!FileUtil::Exists(path) && !FileUtil::CreateEmptyFile(path))
-        return nullptr;
+    const auto path_fwd = FileUtil::SanitizePath(path, FileUtil::DirectorySeparator::ForwardSlash);
+    if (!FileUtil::Exists(path)) {
+        FileUtil::CreateFullPath(path_fwd);
+        if (!FileUtil::CreateEmptyFile(path))
+            return nullptr;
+    }
     return OpenFile(path, perms);
 }
 
@@ -140,8 +144,12 @@ VirtualDir RealVfsFilesystem::OpenDirectory(std::string_view path_, Mode perms)
 
 VirtualDir RealVfsFilesystem::CreateDirectory(std::string_view path_, Mode perms) {
     const auto path = FileUtil::SanitizePath(path_, FileUtil::DirectorySeparator::PlatformDefault);
-    if (!FileUtil::Exists(path) && !FileUtil::CreateDir(path))
-        return nullptr;
+    const auto path_fwd = FileUtil::SanitizePath(path, FileUtil::DirectorySeparator::ForwardSlash);
+    if (!FileUtil::Exists(path)) {
+        FileUtil::CreateFullPath(path_fwd);
+        if (!FileUtil::CreateDir(path))
+            return nullptr;
+    }
     // Cannot use make_shared as RealVfsDirectory constructor is private
     return std::shared_ptr<RealVfsDirectory>(new RealVfsDirectory(*this, path, perms));
 }
@@ -306,14 +314,14 @@ RealVfsDirectory::RealVfsDirectory(RealVfsFilesystem& base_, const std::string&
 
 std::shared_ptr<VfsFile> RealVfsDirectory::GetFileRelative(std::string_view path) const {
     const auto full_path = FileUtil::SanitizePath(this->path + DIR_SEP + std::string(path));
-    if (!FileUtil::Exists(full_path))
+    if (!FileUtil::Exists(full_path) || FileUtil::IsDirectory(full_path))
         return nullptr;
     return base.OpenFile(full_path, perms);
 }
 
 std::shared_ptr<VfsDirectory> RealVfsDirectory::GetDirectoryRelative(std::string_view path) const {
     const auto full_path = FileUtil::SanitizePath(this->path + DIR_SEP + std::string(path));
-    if (!FileUtil::Exists(full_path))
+    if (!FileUtil::Exists(full_path) || !FileUtil::IsDirectory(full_path))
         return nullptr;
     return base.OpenDirectory(full_path, perms);
 }

src/core/file_sys/vfs_real.h

@@ -5,7 +5,6 @@
 #pragma once
 
 #include <string_view>
-
 #include <boost/container/flat_map.hpp>
 #include "common/file_util.h"
 #include "core/file_sys/mode.h"

src/core/file_sys/vfs_vector.cpp

@@ -8,8 +8,8 @@
 
 namespace FileSys {
 VectorVfsDirectory::VectorVfsDirectory(std::vector<VirtualFile> files_,
-                                       std::vector<VirtualDir> dirs_, VirtualDir parent_,
-                                       std::string name_)
+                                       std::vector<VirtualDir> dirs_, std::string name_,
+                                       VirtualDir parent_)
     : files(std::move(files_)), dirs(std::move(dirs_)), parent(std::move(parent_)),
       name(std::move(name_)) {}
 

src/core/file_sys/vfs_vector.h

@@ -13,8 +13,8 @@ namespace FileSys {
 class VectorVfsDirectory : public VfsDirectory {
 public:
     explicit VectorVfsDirectory(std::vector<VirtualFile> files = {},
-                                std::vector<VirtualDir> dirs = {}, VirtualDir parent = nullptr,
-                                std::string name = "");
+                                std::vector<VirtualDir> dirs = {}, std::string name = "",
+                                VirtualDir parent = nullptr);
 
     std::vector<std::shared_ptr<VfsFile>> GetFiles() const override;
     std::vector<std::shared_ptr<VfsDirectory>> GetSubdirectories() const override;