From 49ddd86b7aeb7fbddb3e93e1d9c39da86488e848 Mon Sep 17 00:00:00 2001 From: zhupengfei Date: Fri, 7 Aug 2020 08:58:09 +0800 Subject: [PATCH] Add CIA building Quite a lot of code, yeah. The built CIA is almost identical to GM9, with the following differences: 1. Paddings are zeroed out 2. Title key is not written (GM9 gets it from support data/ticket db) 3. Ticket content index is slightly different (GM9 likely takes it from the legit ticket, while we are building a fake one) The 2, 3 points can be fixed probably. --- src/core/CMakeLists.txt | 2 + src/core/importer.cpp | 65 +++++++++ src/core/importer.h | 15 +++ src/core/ncch/cia_builder.cpp | 237 +++++++++++++++++++++++++++++++++ src/core/ncch/cia_builder.h | 112 ++++++++++++++++ src/core/quick_decryptor.h | 4 - src/frontend/import_dialog.cpp | 109 ++++++++++----- src/frontend/import_dialog.h | 11 +- 8 files changed, 514 insertions(+), 41 deletions(-) create mode 100644 src/core/ncch/cia_builder.cpp create mode 100644 src/core/ncch/cia_builder.h diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index f8291f1..b3a9db8 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -11,6 +11,8 @@ add_library(core STATIC key/arithmetic128.h key/key.cpp key/key.h + ncch/cia_builder.cpp + ncch/cia_builder.h ncch/ncch_container.cpp ncch/ncch_container.h ncch/seed_db.cpp diff --git a/src/core/importer.cpp b/src/core/importer.cpp index ab9b5cd..3fde228 100644 --- a/src/core/importer.cpp +++ b/src/core/importer.cpp @@ -12,6 +12,7 @@ #include "core/importer.h" #include "core/inner_fat.h" #include "core/key/key.h" +#include "core/ncch/cia_builder.h" #include "core/ncch/ncch_container.h" #include "core/ncch/seed_db.h" #include "core/ncch/smdh.h" @@ -53,6 +54,7 @@ bool SDMCImporter::Init() { } decryptor = std::make_unique(config.sdmc_path); + cia_builder = std::make_unique(); FileUtil::SetUserPath(config.user_path); return true; @@ -493,6 +495,69 @@ void SDMCImporter::AbortDumpCXI() { dump_cxi_ncch->AbortDecryptToFile(); } +bool SDMCImporter::BuildCIA(const ContentSpecifier& specifier, const std::string& destination, + const ProgressCallback& callback) { + + if (config.certs_db_path.empty()) { + LOG_ERROR(Core, "Missing certs.db"); + return false; + } + + if (specifier.type != ContentType::Application && specifier.type != ContentType::Update && + specifier.type != ContentType::DLC) { + + LOG_ERROR(Core, "Unsupported specifier type {}", static_cast(specifier.type)); + return false; + } + + // Load TMD + const auto path = fmt::format("/title/{:08x}/{:08x}/content/", (specifier.id >> 32), + (specifier.id & 0xFFFFFFFF)); + TitleMetadata tmd; + if (!LoadTMD(config.sdmc_path, path, *decryptor, tmd)) { + LOG_ERROR(Core, "Failed to load TMD from {}", path); + return false; + } + + const auto physical_path = config.sdmc_path + path.substr(1); + bool ret = cia_builder->Init(destination, std::move(tmd), config.certs_db_path, + FileUtil::GetDirectoryTreeSize(physical_path), callback); + if (!ret) { + return false; + } + + ret = FileUtil::ForeachDirectoryEntry( + nullptr, physical_path, + [this, path](u64* /*num_entries_out*/, const std::string& directory, + const std::string& virtual_name) { + if (FileUtil::IsDirectory(directory + virtual_name + "/")) { + return true; + } + + static const std::regex app_regex{"([0-9a-f]{8})\\.app"}; + + std::smatch match; + if (!std::regex_match(virtual_name, match, app_regex)) { + return true; + } + ASSERT(match.size() >= 2); + + const u32 id = static_cast(std::stoul(match[1], nullptr, 16)); + NCCHContainer ncch( + std::make_shared(config.sdmc_path, path + virtual_name, "rb")); + return cia_builder->AddContent(id, ncch); + }); + if (!ret) { + return false; + } + + return cia_builder->Finalize(); +} + +void SDMCImporter::AbortBuildCIA() { + cia_builder->Abort(); +} + void SDMCImporter::ListTitle(std::vector& out) const { const auto ProcessDirectory = [this, &out, &sdmc_path = config.sdmc_path](ContentType type, u64 high_id) { diff --git a/src/core/importer.h b/src/core/importer.h index 28f5f2c..e83af7b 100644 --- a/src/core/importer.h +++ b/src/core/importer.h @@ -12,6 +12,7 @@ namespace Core { +class CIABuilder; class SDMCDecryptor; /** @@ -130,6 +131,19 @@ public: */ void AbortDumpCXI(); + /** + * Builds a CIA from a content. + * Blocks, but can be aborted on another thread. + * @return true on success, false otherwise + */ + bool BuildCIA(const ContentSpecifier& specifier, const std::string& destination, + const ProgressCallback& callback = [](std::size_t, std::size_t) {}); + + /** + * Aborts current CIA building + */ + void AbortBuildCIA(); + /** * Deletes/Cleans up a content. Used for deleting contents that have * not been fully imported. @@ -172,6 +186,7 @@ private: bool is_good{}; Config config; std::unique_ptr decryptor; + std::unique_ptr cia_builder; // The NCCH used to dump CXIs. std::unique_ptr dump_cxi_ncch; diff --git a/src/core/ncch/cia_builder.cpp b/src/core/ncch/cia_builder.cpp new file mode 100644 index 0000000..ab20d85 --- /dev/null +++ b/src/core/ncch/cia_builder.cpp @@ -0,0 +1,237 @@ +// Copyright 2020 Pengfei Zhu +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#include +#include "common/alignment.h" +#include "core/ncch/cia_builder.h" +#include "core/ncch/ticket.h" +#include "core/ncch/title_metadata.h" + +namespace Core { + +constexpr std::size_t CIA_ALIGNMENT = 0x40; + +class HashedFile : public FileUtil::IOFile { +public: + explicit HashedFile(const std::string& filename, const char openmode[], int flags = 0) + : FileUtil::IOFile(filename, openmode, flags) {} + ~HashedFile() override = default; + + void SetHashEnabled(bool enabled) { + hash_enabled = enabled; + if (enabled) { // Restart when hash is newly restarted + sha.Restart(); + } + } + + void GetHash(u8* out) { + sha.Final(out); + } + + std::size_t Write(const char* data, std::size_t length) override { + const std::size_t length_written = FileUtil::IOFile::Write(data, length); + sha.Update(reinterpret_cast(data), length_written); + return length_written; + } + +private: + CryptoPP::SHA256 sha; + bool hash_enabled{}; +}; + +CIABuilder::CIABuilder() = default; +CIABuilder::~CIABuilder() = default; + +bool CIABuilder::Init(const std::string& destination, TitleMetadata tmd_, + const std::string& certs_db_path, std::size_t total_size_, + const ProgressCallback& callback_) { + + file = std::make_shared(destination, "wb"); + if (!*file) { + LOG_ERROR(Core, "Could not open file {}", destination); + file.reset(); + return false; + } + + tmd = std::move(tmd_); + // Remove encrypted flag from TMD chunks + for (auto& chunk : tmd.tmd_chunks) { + chunk.type &= ~0x01; + } + + header.header_size = sizeof(header); + // Header will be written in Finalize + + // Cert + cert_offset = Common::AlignUp(header.header_size, CIA_ALIGNMENT); + header.cert_size = CIA_CERT_SIZE; + if (!WriteCert(certs_db_path)) { + LOG_ERROR(Core, "Could not write cert to file {}", destination); + file.reset(); + return false; + } + + // Ticket + ticket_offset = Common::AlignUp(cert_offset + header.cert_size, CIA_ALIGNMENT); + header.tik_size = sizeof(Ticket); + + Ticket fake_ticket = BuildFakeTicket(tmd.GetTitleID()); + file->Seek(ticket_offset, SEEK_SET); + if (file->WriteBytes(&fake_ticket, sizeof(fake_ticket)) != sizeof(fake_ticket)) { + LOG_ERROR(Core, "Could not write ticket to file {}", destination); + file.reset(); + return false; + } + + // TMD will be written in Finalize (we need to set content hash, etc) + tmd_offset = Common::AlignUp(ticket_offset + header.tik_size, CIA_ALIGNMENT); + header.tmd_size = tmd.GetSize(); + + content_offset = Common::AlignUp(tmd_offset + header.tmd_size, CIA_ALIGNMENT); + header.content_size = 0; + + // Meta will be written in Finalize + header.meta_size = 0; + + written = content_offset; + total_size = total_size_; + callback = callback_; + + callback(written, total_size); + return true; +} + +bool CIABuilder::WriteCert(const std::string& certs_db_path) { + FileUtil::IOFile certs_db(certs_db_path, "rb"); + if (!certs_db) { + LOG_ERROR(Core, "Could not open {}", certs_db_path); + return false; + } + + std::array cert; + // Read CIA cert + certs_db.Seek(0x0C10, SEEK_SET); + if (certs_db.ReadBytes(cert.data(), 0x1F0) != 0x1F0) { + return false; + } + + certs_db.Seek(0x3A00, SEEK_SET); + if (certs_db.ReadBytes(cert.data() + 0x1F0, 0x210) != 0x210) { + return false; + } + + certs_db.Seek(0x3F10, SEEK_SET); + if (certs_db.ReadBytes(cert.data() + 0x400, 0x300) != 0x300) { + return false; + } + + certs_db.Seek(0x3C10, SEEK_SET); + if (certs_db.ReadBytes(cert.data() + 0x700, 0x300) != 0x300) { + return false; + } + + // Write CIA cert to file + file->Seek(cert_offset, SEEK_SET); + if (file->WriteBytes(cert.data(), cert.size()) != cert.size()) { + LOG_ERROR(Core, "Could not write cert"); + return false; + } + return true; +} + +bool CIABuilder::AddContent(u16 content_id, NCCHContainer& ncch) { + file->Seek(written, SEEK_SET); // To enforce alignment + file->SetHashEnabled(true); + + { + std::lock_guard lock{abort_ncch_mutex}; + abort_ncch = &ncch; + } + + const auto ret = ncch.DecryptToFile(file, [this](std::size_t current, std::size_t total) { + callback(written + current, total_size); + }); + + { + std::lock_guard lock{abort_ncch_mutex}; + abort_ncch = nullptr; + } + + if (ret != ResultStatus::Success) { + file.reset(); + return false; + } + written = Common::AlignUp(file->Tell(), CIA_ALIGNMENT); + + header.content_size = written - content_offset; + header.SetContentPresent(content_id); + + auto& tmd_chunk = tmd.GetContentChunkByID(content_id); + file->GetHash(tmd_chunk.hash.data()); + file->SetHashEnabled(false); + + if (tmd_chunk.index != TMDContentIndex::Main) { + return true; + } + + // Load meta if the content is main + header.meta_size = sizeof(meta); + + static_assert(sizeof(ncch.exheader_header.dependency_list) == sizeof(meta.dependencies), + "Dependency list should be of the same size in NCCH and CIA"); + std::memcpy(meta.dependencies.data(), &ncch.exheader_header.dependency_list, + sizeof(meta.dependencies)); + + meta.core_version = ncch.exheader_header.arm11_system_local_caps.core_version; + + std::vector smdh_buffer; + if (ncch.LoadSectionExeFS("icon", smdh_buffer) != ResultStatus::Success) { + LOG_WARNING(Core, "Failed to load icon in ExeFS"); + return true; + } + std::memcpy(meta.icon_data.data(), smdh_buffer.data(), + std::min(meta.icon_data.size(), smdh_buffer.size())); + + return true; +} + +bool CIABuilder::Finalize() { + // Write header + file->Seek(0, SEEK_SET); + if (file->WriteBytes(&header, sizeof(header)) != sizeof(header)) { + LOG_ERROR(Core, "Failed to write header"); + file.reset(); + return false; + } + + // Write TMD + file->Seek(tmd_offset, SEEK_SET); + if (tmd.Save(*file) != ResultStatus::Success) { + file.reset(); + return false; + } + + // Write meta + if (header.meta_size) { + file->Seek(written, SEEK_SET); + if (file->WriteBytes(&meta, sizeof(meta)) != sizeof(meta)) { + LOG_ERROR(Core, "Failed to write meta"); + file.reset(); + return false; + } + } + + callback(total_size, total_size); + file.reset(); + return true; +} + +void CIABuilder::Abort() { + std::lock_guard lock{abort_ncch_mutex}; + if (abort_ncch) { + abort_ncch->AbortDecryptToFile(); + } +} + +} // namespace Core diff --git a/src/core/ncch/cia_builder.h b/src/core/ncch/cia_builder.h new file mode 100644 index 0000000..6d654ef --- /dev/null +++ b/src/core/ncch/cia_builder.h @@ -0,0 +1,112 @@ +// Copyright 2017 Citra Emulator Project / 2020 Pengfei Zhu +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#include +#include +#include +#include "common/file_util.h" +#include "common/swap.h" +#include "core/ncch/ncch_container.h" +#include "core/ncch/title_metadata.h" +#include "core/quick_decryptor.h" + +namespace Core { + +constexpr std::size_t CIA_CONTENT_MAX_COUNT = 0x10000; +constexpr std::size_t CIA_CONTENT_BITS_SIZE = (CIA_CONTENT_MAX_COUNT / 8); +constexpr std::size_t CIA_HEADER_SIZE = 0x2020; +constexpr std::size_t CIA_CERT_SIZE = 0xA00; +constexpr std::size_t CIA_METADATA_SIZE = 0x3AC0; + +class HashedFile; + +class CIABuilder { +public: + explicit CIABuilder(); + ~CIABuilder(); + + /** + * Initializes the building of the CIA. + * @return true on success, false otherwise + */ + bool Init(const std::string& destination, TitleMetadata tmd, const std::string& certs_db_path, + std::size_t total_size, const ProgressCallback& callback); + + /** + * Adds an NCCH content to the CIA. + * @return true on success, false otherwise + */ + bool AddContent(u16 content_id, NCCHContainer& ncch); + + /** + * Finalizes this CIA and write remaining data. + * @return true on success, false otherwise + */ + bool Finalize(); + + /** + * Aborts the current work. In fact, only usable during AddContent. + */ + void Abort(); + +private: + struct Header { + u32_le header_size; + u16_le type; + u16_le version; + u32_le cert_size; + u32_le tik_size; + u32_le tmd_size; + u32_le meta_size; + u64_le content_size; + std::array content_present; + + bool IsContentPresent(u16 index) const { + // The content_present is a bit array which defines which content in the TMD + // is included in the CIA, so check the bit for this index and add if set. + // The bits in the content index are arranged w/ index 0 as the MSB, 7 as the LSB, etc. + return (content_present[index >> 3] & (0x80 >> (index & 7))); + } + + void SetContentPresent(u16 index) { + content_present[index >> 3] |= (0x80 >> (index & 7)); + } + }; + + static_assert(sizeof(Header) == CIA_HEADER_SIZE, "CIA Header structure size is wrong"); + + struct Metadata { + std::array dependencies; + std::array reserved; + u32_le core_version; + std::array reserved_2; + std::array icon_data; + }; + + static_assert(sizeof(Metadata) == CIA_METADATA_SIZE, "CIA Metadata structure size is wrong"); + + bool WriteCert(const std::string& certs_db_path); + + Header header{}; + Metadata meta{}; + + TitleMetadata tmd; + + std::size_t cert_offset{}; + std::size_t ticket_offset{}; + std::size_t tmd_offset{}; + std::size_t content_offset{}; + std::size_t metadata_offset{}; + + std::shared_ptr file; + std::size_t written{}; // size written (with alignment) + std::size_t total_size{}; + ProgressCallback callback; + + // The NCCH to abort on + std::mutex abort_ncch_mutex; + NCCHContainer* abort_ncch{}; +}; + +} // namespace Core diff --git a/src/core/quick_decryptor.h b/src/core/quick_decryptor.h index c407db2..de9f670 100644 --- a/src/core/quick_decryptor.h +++ b/src/core/quick_decryptor.h @@ -23,10 +23,6 @@ using ProgressCallback = std::function; */ class QuickDecryptor { public: - /** - * Initializes the decryptor. - * @param root_folder Path to the "Nintendo 3DS//" folder. - */ explicit QuickDecryptor(); ~QuickDecryptor(); diff --git a/src/frontend/import_dialog.cpp b/src/frontend/import_dialog.cpp index 7baa0b3..b981dc4 100644 --- a/src/frontend/import_dialog.cpp +++ b/src/frontend/import_dialog.cpp @@ -85,7 +85,7 @@ QPixmap GetContentIcon(const Core::ContentSpecifier& specifier, bool use_categor ImportDialog::ImportDialog(QWidget* parent, const Core::Config& config) : QDialog(parent), ui(std::make_unique()), user_path(config.user_path), - importer(config) { + has_cert_db(!config.certs_db_path.empty()), importer(config) { qRegisterMetaType("u64"); qRegisterMetaType(); @@ -564,12 +564,18 @@ void ImportDialog::OnContextMenu(const QPoint& point) { QMenu context_menu; if (item->parent()) { // Second level const auto& specifier = SpecifierFromItem(item); - if (specifier.type != Core::ContentType::Application) { - return; + if (specifier.type == Core::ContentType::Application) { + QAction* dump_cxi = context_menu.addAction(tr("Dump CXI file")); + connect(dump_cxi, &QAction::triggered, + [this, specifier] { StartDumpingCXI(specifier); }); + } + if (specifier.type == Core::ContentType::Application || + specifier.type == Core::ContentType::Update || + specifier.type == Core::ContentType::DLC) { + QAction* build_cia = context_menu.addAction(tr("Build CIA (standard)")); + connect(build_cia, &QAction::triggered, + [this, specifier] { StartBuildingCIA(specifier); }); } - - QAction* dump_cxi = context_menu.addAction(tr("Dump CXI file")); - connect(dump_cxi, &QAction::triggered, [this, specifier] { StartDumpingCXI(specifier); }); } else { // Top level if (!title_view) { return; @@ -581,14 +587,53 @@ void ImportDialog::OnContextMenu(const QPoint& point) { QAction* dump_base_cxi = context_menu.addAction(tr("Dump Base CXI file")); connect(dump_base_cxi, &QAction::triggered, [this, specifier] { StartDumpingCXI(specifier); }); + QAction* build_base_cia = context_menu.addAction(tr("Build Base CIA")); + connect(build_base_cia, &QAction::triggered, + [this, specifier] { StartBuildingCIA(specifier); }); break; + } else if (specifier.type == Core::ContentType::Update) { + QAction* build_update_cia = context_menu.addAction(tr("Build Update CIA")); + connect(build_update_cia, &QAction::triggered, + [this, specifier] { StartBuildingCIA(specifier); }); + } else if (specifier.type == Core::ContentType::DLC) { + QAction* build_dlc_cia = context_menu.addAction(tr("Build DLC CIA")); + connect(build_dlc_cia, &QAction::triggered, + [this, specifier] { StartBuildingCIA(specifier); }); } - // TODO: Add updates, etc } } context_menu.exec(ui->main->viewport()->mapToGlobal(point)); } +// Runs the job, opening a dialog to report its progress. +void ImportDialog::RunProgressiveJob(ProgressiveJob* job) { + auto* dialog = new QProgressDialog(tr("Initializing..."), tr("Cancel"), 0, 100, this); + dialog->setWindowModality(Qt::WindowModal); + dialog->setMinimumDuration(0); + dialog->setValue(0); + + connect(job, &ProgressiveJob::ProgressUpdated, this, [dialog](u64 current, u64 total) { + // Try to map total to int range + // This is equal to ceil(total / INT_MAX) + const u64 multiplier = + (total + std::numeric_limits::max() - 1) / std::numeric_limits::max(); + dialog->setMaximum(static_cast(total / multiplier)); + dialog->setValue(static_cast(current / multiplier)); + dialog->setLabelText( + tr("%1 / %2").arg(ReadableByteSize(current)).arg(ReadableByteSize(total))); + }); + connect(job, &ProgressiveJob::ErrorOccured, this, [this, dialog] { + QMessageBox::critical(this, tr("threeSD"), + tr("Operation failed. Please refer to the log.")); + dialog->hide(); + }); + connect(job, &ProgressiveJob::Completed, this, + [dialog] { dialog->setValue(dialog->maximum()); }); + connect(dialog, &QProgressDialog::canceled, this, [job] { job->Cancel(); }); + + job->start(); +} + void ImportDialog::StartDumpingCXI(const Core::ContentSpecifier& specifier) { const QString path = QFileDialog::getSaveFileName(this, tr("Dump CXI file"), last_dump_cxi_path, tr("CTR Executable Image (*.CXI)")); @@ -597,17 +642,6 @@ void ImportDialog::StartDumpingCXI(const Core::ContentSpecifier& specifier) { } last_dump_cxi_path = QFileInfo(path).path(); - // Try to map total_size to int range - // This is equal to ceil(total_size / INT_MAX) - const u64 multiplier = (specifier.maximum_size + std::numeric_limits::max() - 1) / - std::numeric_limits::max(); - - auto* dialog = new QProgressDialog(tr("Initializing..."), tr("Cancel"), 0, - static_cast(specifier.maximum_size / multiplier), this); - dialog->setWindowModality(Qt::WindowModal); - dialog->setMinimumDuration(0); - dialog->setValue(0); - auto* job = new ProgressiveJob( this, [this, specifier, path](const ProgressiveJob::ProgressCallback& callback) { @@ -618,21 +652,26 @@ void ImportDialog::StartDumpingCXI(const Core::ContentSpecifier& specifier) { return true; }, [this] { importer.AbortDumpCXI(); }); - - connect(job, &ProgressiveJob::ProgressUpdated, this, - [specifier, dialog, multiplier](u64 current, u64 total) { - dialog->setValue(static_cast(current / multiplier)); - dialog->setLabelText(tr("%1 / %2") - .arg(ReadableByteSize(current)) - .arg(ReadableByteSize(specifier.maximum_size))); - }); - connect(job, &ProgressiveJob::ErrorOccured, this, [this, dialog] { - QMessageBox::critical(this, tr("threeSD"), tr("Failed to dump CXI!")); - dialog->hide(); - }); - connect(job, &ProgressiveJob::Completed, this, - [dialog] { dialog->setValue(dialog->maximum()); }); - connect(dialog, &QProgressDialog::canceled, this, [job] { job->Cancel(); }); - - job->start(); + RunProgressiveJob(job); +} + +void ImportDialog::StartBuildingCIA(const Core::ContentSpecifier& specifier) { + const QString path = QFileDialog::getSaveFileName(this, tr("Build CIA"), last_build_cia_path, + tr("CTR Importable Archive (*.CIA)")); + if (path.isEmpty()) { + return; + } + last_build_cia_path = QFileInfo(path).path(); + + auto* job = new ProgressiveJob( + this, + [this, specifier, path](const ProgressiveJob::ProgressCallback& callback) { + if (!importer.BuildCIA(specifier, path.toStdString(), callback)) { + FileUtil::Delete(path.toStdString()); + return false; + } + return true; + }, + [this] { importer.AbortBuildCIA(); }); + RunProgressiveJob(job); } diff --git a/src/frontend/import_dialog.h b/src/frontend/import_dialog.h index cdcfcdc..4785101 100644 --- a/src/frontend/import_dialog.h +++ b/src/frontend/import_dialog.h @@ -12,6 +12,7 @@ #include "core/importer.h" #include "core/ncch/ncch_container.h" +class ProgressiveJob; class QTreeWidgetItem; namespace Ui { @@ -41,13 +42,19 @@ private: Core::ContentSpecifier SpecifierFromItem(QTreeWidgetItem* item) const; void OnContextMenu(const QPoint& point); + + void RunProgressiveJob(ProgressiveJob* job); + void StartDumpingCXI(const Core::ContentSpecifier& content); - Core::NCCHContainer dump_cxi_container; // NCCH container used for dumping CXI - QString last_dump_cxi_path; // Used for recording last path in StartDumpingCXI + QString last_dump_cxi_path; // Used for recording last path in StartDumpingCXI + + void StartBuildingCIA(const Core::ContentSpecifier& content); + QString last_build_cia_path; // Used for recording last path in StartBuildingCIA std::unique_ptr ui; std::string user_path; + bool has_cert_db = false; Core::SDMCImporter importer; std::vector contents; u64 total_size = 0;