Add CIA building

Quite a lot of code, yeah.

The built CIA is almost identical to GM9, with the following differences:
1. Paddings are zeroed out
2. Title key is not written (GM9 gets it from support data/ticket db)
3. Ticket content index is slightly different (GM9 likely takes it from the legit ticket, while we are building a fake one)

The 2, 3 points can be fixed probably.

src/core/CMakeLists.txt

@@ -11,6 +11,8 @@ add_library(core STATIC
   key/arithmetic128.h
   key/key.cpp
   key/key.h
+  ncch/cia_builder.cpp
+  ncch/cia_builder.h
   ncch/ncch_container.cpp
   ncch/ncch_container.h
   ncch/seed_db.cpp

src/core/importer.cpp

@@ -12,6 +12,7 @@
 #include "core/importer.h"
 #include "core/inner_fat.h"
 #include "core/key/key.h"
+#include "core/ncch/cia_builder.h"
 #include "core/ncch/ncch_container.h"
 #include "core/ncch/seed_db.h"
 #include "core/ncch/smdh.h"
@@ -53,6 +54,7 @@ bool SDMCImporter::Init() {
     }
 
     decryptor = std::make_unique<SDMCDecryptor>(config.sdmc_path);
+    cia_builder = std::make_unique<CIABuilder>();
 
     FileUtil::SetUserPath(config.user_path);
     return true;
@@ -493,6 +495,69 @@ void SDMCImporter::AbortDumpCXI() {
     dump_cxi_ncch->AbortDecryptToFile();
 }
 
+bool SDMCImporter::BuildCIA(const ContentSpecifier& specifier, const std::string& destination,
+                            const ProgressCallback& callback) {
+
+    if (config.certs_db_path.empty()) {
+        LOG_ERROR(Core, "Missing certs.db");
+        return false;
+    }
+
+    if (specifier.type != ContentType::Application && specifier.type != ContentType::Update &&
+        specifier.type != ContentType::DLC) {
+
+        LOG_ERROR(Core, "Unsupported specifier type {}", static_cast<int>(specifier.type));
+        return false;
+    }
+
+    // Load TMD
+    const auto path = fmt::format("/title/{:08x}/{:08x}/content/", (specifier.id >> 32),
+                                  (specifier.id & 0xFFFFFFFF));
+    TitleMetadata tmd;
+    if (!LoadTMD(config.sdmc_path, path, *decryptor, tmd)) {
+        LOG_ERROR(Core, "Failed to load TMD from {}", path);
+        return false;
+    }
+
+    const auto physical_path = config.sdmc_path + path.substr(1);
+    bool ret = cia_builder->Init(destination, std::move(tmd), config.certs_db_path,
+                                 FileUtil::GetDirectoryTreeSize(physical_path), callback);
+    if (!ret) {
+        return false;
+    }
+
+    ret = FileUtil::ForeachDirectoryEntry(
+        nullptr, physical_path,
+        [this, path](u64* /*num_entries_out*/, const std::string& directory,
+                     const std::string& virtual_name) {
+            if (FileUtil::IsDirectory(directory + virtual_name + "/")) {
+                return true;
+            }
+
+            static const std::regex app_regex{"([0-9a-f]{8})\\.app"};
+
+            std::smatch match;
+            if (!std::regex_match(virtual_name, match, app_regex)) {
+                return true;
+            }
+            ASSERT(match.size() >= 2);
+
+            const u32 id = static_cast<u32>(std::stoul(match[1], nullptr, 16));
+            NCCHContainer ncch(
+                std::make_shared<SDMCFile>(config.sdmc_path, path + virtual_name, "rb"));
+            return cia_builder->AddContent(id, ncch);
+        });
+    if (!ret) {
+        return false;
+    }
+
+    return cia_builder->Finalize();
+}
+
+void SDMCImporter::AbortBuildCIA() {
+    cia_builder->Abort();
+}
+
 void SDMCImporter::ListTitle(std::vector<ContentSpecifier>& out) const {
     const auto ProcessDirectory = [this, &out, &sdmc_path = config.sdmc_path](ContentType type,
                                                                               u64 high_id) {

src/core/importer.h

@@ -12,6 +12,7 @@
 
 namespace Core {
 
+class CIABuilder;
 class SDMCDecryptor;
 
 /**
@@ -130,6 +131,19 @@ public:
      */
     void AbortDumpCXI();
 
+    /**
+     * Builds a CIA from a content.
+     * Blocks, but can be aborted on another thread.
+     * @return true on success, false otherwise
+     */
+    bool BuildCIA(const ContentSpecifier& specifier, const std::string& destination,
+                  const ProgressCallback& callback = [](std::size_t, std::size_t) {});
+
+    /**
+     * Aborts current CIA building
+     */
+    void AbortBuildCIA();
+
     /**
      * Deletes/Cleans up a content. Used for deleting contents that have
      * not been fully imported.
@@ -172,6 +186,7 @@ private:
     bool is_good{};
     Config config;
     std::unique_ptr<SDMCDecryptor> decryptor;
+    std::unique_ptr<CIABuilder> cia_builder;
 
     // The NCCH used to dump CXIs.
     std::unique_ptr<NCCHContainer> dump_cxi_ncch;

src/core/ncch/cia_builder.cpp

@@ -0,0 +1,237 @@
+// Copyright 2020 Pengfei Zhu
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#include <cryptopp/sha.h>
+#include "common/alignment.h"
+#include "core/ncch/cia_builder.h"
+#include "core/ncch/ticket.h"
+#include "core/ncch/title_metadata.h"
+
+namespace Core {
+
+constexpr std::size_t CIA_ALIGNMENT = 0x40;
+
+class HashedFile : public FileUtil::IOFile {
+public:
+    explicit HashedFile(const std::string& filename, const char openmode[], int flags = 0)
+        : FileUtil::IOFile(filename, openmode, flags) {}
+    ~HashedFile() override = default;
+
+    void SetHashEnabled(bool enabled) {
+        hash_enabled = enabled;
+        if (enabled) { // Restart when hash is newly restarted
+            sha.Restart();
+        }
+    }
+
+    void GetHash(u8* out) {
+        sha.Final(out);
+    }
+
+    std::size_t Write(const char* data, std::size_t length) override {
+        const std::size_t length_written = FileUtil::IOFile::Write(data, length);
+        sha.Update(reinterpret_cast<const CryptoPP::byte*>(data), length_written);
+        return length_written;
+    }
+
+private:
+    CryptoPP::SHA256 sha;
+    bool hash_enabled{};
+};
+
+CIABuilder::CIABuilder() = default;
+CIABuilder::~CIABuilder() = default;
+
+bool CIABuilder::Init(const std::string& destination, TitleMetadata tmd_,
+                      const std::string& certs_db_path, std::size_t total_size_,
+                      const ProgressCallback& callback_) {
+
+    file = std::make_shared<HashedFile>(destination, "wb");
+    if (!*file) {
+        LOG_ERROR(Core, "Could not open file {}", destination);
+        file.reset();
+        return false;
+    }
+
+    tmd = std::move(tmd_);
+    // Remove encrypted flag from TMD chunks
+    for (auto& chunk : tmd.tmd_chunks) {
+        chunk.type &= ~0x01;
+    }
+
+    header.header_size = sizeof(header);
+    // Header will be written in Finalize
+
+    // Cert
+    cert_offset = Common::AlignUp(header.header_size, CIA_ALIGNMENT);
+    header.cert_size = CIA_CERT_SIZE;
+    if (!WriteCert(certs_db_path)) {
+        LOG_ERROR(Core, "Could not write cert to file {}", destination);
+        file.reset();
+        return false;
+    }
+
+    // Ticket
+    ticket_offset = Common::AlignUp(cert_offset + header.cert_size, CIA_ALIGNMENT);
+    header.tik_size = sizeof(Ticket);
+
+    Ticket fake_ticket = BuildFakeTicket(tmd.GetTitleID());
+    file->Seek(ticket_offset, SEEK_SET);
+    if (file->WriteBytes(&fake_ticket, sizeof(fake_ticket)) != sizeof(fake_ticket)) {
+        LOG_ERROR(Core, "Could not write ticket to file {}", destination);
+        file.reset();
+        return false;
+    }
+
+    // TMD will be written in Finalize (we need to set content hash, etc)
+    tmd_offset = Common::AlignUp(ticket_offset + header.tik_size, CIA_ALIGNMENT);
+    header.tmd_size = tmd.GetSize();
+
+    content_offset = Common::AlignUp(tmd_offset + header.tmd_size, CIA_ALIGNMENT);
+    header.content_size = 0;
+
+    // Meta will be written in Finalize
+    header.meta_size = 0;
+
+    written = content_offset;
+    total_size = total_size_;
+    callback = callback_;
+
+    callback(written, total_size);
+    return true;
+}
+
+bool CIABuilder::WriteCert(const std::string& certs_db_path) {
+    FileUtil::IOFile certs_db(certs_db_path, "rb");
+    if (!certs_db) {
+        LOG_ERROR(Core, "Could not open {}", certs_db_path);
+        return false;
+    }
+
+    std::array<u8, CIA_CERT_SIZE> cert;
+    // Read CIA cert
+    certs_db.Seek(0x0C10, SEEK_SET);
+    if (certs_db.ReadBytes(cert.data(), 0x1F0) != 0x1F0) {
+        return false;
+    }
+
+    certs_db.Seek(0x3A00, SEEK_SET);
+    if (certs_db.ReadBytes(cert.data() + 0x1F0, 0x210) != 0x210) {
+        return false;
+    }
+
+    certs_db.Seek(0x3F10, SEEK_SET);
+    if (certs_db.ReadBytes(cert.data() + 0x400, 0x300) != 0x300) {
+        return false;
+    }
+
+    certs_db.Seek(0x3C10, SEEK_SET);
+    if (certs_db.ReadBytes(cert.data() + 0x700, 0x300) != 0x300) {
+        return false;
+    }
+
+    // Write CIA cert to file
+    file->Seek(cert_offset, SEEK_SET);
+    if (file->WriteBytes(cert.data(), cert.size()) != cert.size()) {
+        LOG_ERROR(Core, "Could not write cert");
+        return false;
+    }
+    return true;
+}
+
+bool CIABuilder::AddContent(u16 content_id, NCCHContainer& ncch) {
+    file->Seek(written, SEEK_SET); // To enforce alignment
+    file->SetHashEnabled(true);
+
+    {
+        std::lock_guard lock{abort_ncch_mutex};
+        abort_ncch = &ncch;
+    }
+
+    const auto ret = ncch.DecryptToFile(file, [this](std::size_t current, std::size_t total) {
+        callback(written + current, total_size);
+    });
+
+    {
+        std::lock_guard lock{abort_ncch_mutex};
+        abort_ncch = nullptr;
+    }
+
+    if (ret != ResultStatus::Success) {
+        file.reset();
+        return false;
+    }
+    written = Common::AlignUp(file->Tell(), CIA_ALIGNMENT);
+
+    header.content_size = written - content_offset;
+    header.SetContentPresent(content_id);
+
+    auto& tmd_chunk = tmd.GetContentChunkByID(content_id);
+    file->GetHash(tmd_chunk.hash.data());
+    file->SetHashEnabled(false);
+
+    if (tmd_chunk.index != TMDContentIndex::Main) {
+        return true;
+    }
+
+    // Load meta if the content is main
+    header.meta_size = sizeof(meta);
+
+    static_assert(sizeof(ncch.exheader_header.dependency_list) == sizeof(meta.dependencies),
+                  "Dependency list should be of the same size in NCCH and CIA");
+    std::memcpy(meta.dependencies.data(), &ncch.exheader_header.dependency_list,
+                sizeof(meta.dependencies));
+
+    meta.core_version = ncch.exheader_header.arm11_system_local_caps.core_version;
+
+    std::vector<u8> smdh_buffer;
+    if (ncch.LoadSectionExeFS("icon", smdh_buffer) != ResultStatus::Success) {
+        LOG_WARNING(Core, "Failed to load icon in ExeFS");
+        return true;
+    }
+    std::memcpy(meta.icon_data.data(), smdh_buffer.data(),
+                std::min(meta.icon_data.size(), smdh_buffer.size()));
+
+    return true;
+}
+
+bool CIABuilder::Finalize() {
+    // Write header
+    file->Seek(0, SEEK_SET);
+    if (file->WriteBytes(&header, sizeof(header)) != sizeof(header)) {
+        LOG_ERROR(Core, "Failed to write header");
+        file.reset();
+        return false;
+    }
+
+    // Write TMD
+    file->Seek(tmd_offset, SEEK_SET);
+    if (tmd.Save(*file) != ResultStatus::Success) {
+        file.reset();
+        return false;
+    }
+
+    // Write meta
+    if (header.meta_size) {
+        file->Seek(written, SEEK_SET);
+        if (file->WriteBytes(&meta, sizeof(meta)) != sizeof(meta)) {
+            LOG_ERROR(Core, "Failed to write meta");
+            file.reset();
+            return false;
+        }
+    }
+
+    callback(total_size, total_size);
+    file.reset();
+    return true;
+}
+
+void CIABuilder::Abort() {
+    std::lock_guard lock{abort_ncch_mutex};
+    if (abort_ncch) {
+        abort_ncch->AbortDecryptToFile();
+    }
+}
+
+} // namespace Core

src/core/ncch/cia_builder.h

@@ -0,0 +1,112 @@
+// Copyright 2017 Citra Emulator Project / 2020 Pengfei Zhu
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#include <memory>
+#include <mutex>
+#include <string>
+#include "common/file_util.h"
+#include "common/swap.h"
+#include "core/ncch/ncch_container.h"
+#include "core/ncch/title_metadata.h"
+#include "core/quick_decryptor.h"
+
+namespace Core {
+
+constexpr std::size_t CIA_CONTENT_MAX_COUNT = 0x10000;
+constexpr std::size_t CIA_CONTENT_BITS_SIZE = (CIA_CONTENT_MAX_COUNT / 8);
+constexpr std::size_t CIA_HEADER_SIZE = 0x2020;
+constexpr std::size_t CIA_CERT_SIZE = 0xA00;
+constexpr std::size_t CIA_METADATA_SIZE = 0x3AC0;
+
+class HashedFile;
+
+class CIABuilder {
+public:
+    explicit CIABuilder();
+    ~CIABuilder();
+
+    /**
+     * Initializes the building of the CIA.
+     * @return true on success, false otherwise
+     */
+    bool Init(const std::string& destination, TitleMetadata tmd, const std::string& certs_db_path,
+              std::size_t total_size, const ProgressCallback& callback);
+
+    /**
+     * Adds an NCCH content to the CIA.
+     * @return true on success, false otherwise
+     */
+    bool AddContent(u16 content_id, NCCHContainer& ncch);
+
+    /**
+     * Finalizes this CIA and write remaining data.
+     * @return true on success, false otherwise
+     */
+    bool Finalize();
+
+    /**
+     * Aborts the current work. In fact, only usable during AddContent.
+     */
+    void Abort();
+
+private:
+    struct Header {
+        u32_le header_size;
+        u16_le type;
+        u16_le version;
+        u32_le cert_size;
+        u32_le tik_size;
+        u32_le tmd_size;
+        u32_le meta_size;
+        u64_le content_size;
+        std::array<u8, CIA_CONTENT_BITS_SIZE> content_present;
+
+        bool IsContentPresent(u16 index) const {
+            // The content_present is a bit array which defines which content in the TMD
+            // is included in the CIA, so check the bit for this index and add if set.
+            // The bits in the content index are arranged w/ index 0 as the MSB, 7 as the LSB, etc.
+            return (content_present[index >> 3] & (0x80 >> (index & 7)));
+        }
+
+        void SetContentPresent(u16 index) {
+            content_present[index >> 3] |= (0x80 >> (index & 7));
+        }
+    };
+
+    static_assert(sizeof(Header) == CIA_HEADER_SIZE, "CIA Header structure size is wrong");
+
+    struct Metadata {
+        std::array<u64_le, 0x30> dependencies;
+        std::array<u8, 0x180> reserved;
+        u32_le core_version;
+        std::array<u8, 0xfc> reserved_2;
+        std::array<u8, 0x36c0> icon_data;
+    };
+
+    static_assert(sizeof(Metadata) == CIA_METADATA_SIZE, "CIA Metadata structure size is wrong");
+
+    bool WriteCert(const std::string& certs_db_path);
+
+    Header header{};
+    Metadata meta{};
+
+    TitleMetadata tmd;
+
+    std::size_t cert_offset{};
+    std::size_t ticket_offset{};
+    std::size_t tmd_offset{};
+    std::size_t content_offset{};
+    std::size_t metadata_offset{};
+
+    std::shared_ptr<HashedFile> file;
+    std::size_t written{}; // size written (with alignment)
+    std::size_t total_size{};
+    ProgressCallback callback;
+
+    // The NCCH to abort on
+    std::mutex abort_ncch_mutex;
+    NCCHContainer* abort_ncch{};
+};
+
+} // namespace Core

src/core/quick_decryptor.h

@@ -23,10 +23,6 @@ using ProgressCallback = std::function<void(std::size_t, std::size_t)>;
  */
 class QuickDecryptor {
 public:
-    /**
-     * Initializes the decryptor.
-     * @param root_folder Path to the "Nintendo 3DS/<ID0>/<ID1>" folder.
-     */
     explicit QuickDecryptor();
     ~QuickDecryptor();