Add CIA building

Quite a lot of code, yeah.

The built CIA is almost identical to GM9, with the following differences:
1. Paddings are zeroed out
2. Title key is not written (GM9 gets it from support data/ticket db)
3. Ticket content index is slightly different (GM9 likely takes it from the legit ticket, while we are building a fake one)

The 2, 3 points can be fixed probably.
This commit is contained in:
zhupengfei
2020-08-07 08:58:09 +08:00
parent 769162e95c
commit 49ddd86b7a
8 changed files with 514 additions and 41 deletions
+2
View File
@@ -11,6 +11,8 @@ add_library(core STATIC
key/arithmetic128.h
key/key.cpp
key/key.h
ncch/cia_builder.cpp
ncch/cia_builder.h
ncch/ncch_container.cpp
ncch/ncch_container.h
ncch/seed_db.cpp
+65
View File
@@ -12,6 +12,7 @@
#include "core/importer.h"
#include "core/inner_fat.h"
#include "core/key/key.h"
#include "core/ncch/cia_builder.h"
#include "core/ncch/ncch_container.h"
#include "core/ncch/seed_db.h"
#include "core/ncch/smdh.h"
@@ -53,6 +54,7 @@ bool SDMCImporter::Init() {
}
decryptor = std::make_unique<SDMCDecryptor>(config.sdmc_path);
cia_builder = std::make_unique<CIABuilder>();
FileUtil::SetUserPath(config.user_path);
return true;
@@ -493,6 +495,69 @@ void SDMCImporter::AbortDumpCXI() {
dump_cxi_ncch->AbortDecryptToFile();
}
bool SDMCImporter::BuildCIA(const ContentSpecifier& specifier, const std::string& destination,
const ProgressCallback& callback) {
if (config.certs_db_path.empty()) {
LOG_ERROR(Core, "Missing certs.db");
return false;
}
if (specifier.type != ContentType::Application && specifier.type != ContentType::Update &&
specifier.type != ContentType::DLC) {
LOG_ERROR(Core, "Unsupported specifier type {}", static_cast<int>(specifier.type));
return false;
}
// Load TMD
const auto path = fmt::format("/title/{:08x}/{:08x}/content/", (specifier.id >> 32),
(specifier.id & 0xFFFFFFFF));
TitleMetadata tmd;
if (!LoadTMD(config.sdmc_path, path, *decryptor, tmd)) {
LOG_ERROR(Core, "Failed to load TMD from {}", path);
return false;
}
const auto physical_path = config.sdmc_path + path.substr(1);
bool ret = cia_builder->Init(destination, std::move(tmd), config.certs_db_path,
FileUtil::GetDirectoryTreeSize(physical_path), callback);
if (!ret) {
return false;
}
ret = FileUtil::ForeachDirectoryEntry(
nullptr, physical_path,
[this, path](u64* /*num_entries_out*/, const std::string& directory,
const std::string& virtual_name) {
if (FileUtil::IsDirectory(directory + virtual_name + "/")) {
return true;
}
static const std::regex app_regex{"([0-9a-f]{8})\\.app"};
std::smatch match;
if (!std::regex_match(virtual_name, match, app_regex)) {
return true;
}
ASSERT(match.size() >= 2);
const u32 id = static_cast<u32>(std::stoul(match[1], nullptr, 16));
NCCHContainer ncch(
std::make_shared<SDMCFile>(config.sdmc_path, path + virtual_name, "rb"));
return cia_builder->AddContent(id, ncch);
});
if (!ret) {
return false;
}
return cia_builder->Finalize();
}
void SDMCImporter::AbortBuildCIA() {
cia_builder->Abort();
}
void SDMCImporter::ListTitle(std::vector<ContentSpecifier>& out) const {
const auto ProcessDirectory = [this, &out, &sdmc_path = config.sdmc_path](ContentType type,
u64 high_id) {
+15
View File
@@ -12,6 +12,7 @@
namespace Core {
class CIABuilder;
class SDMCDecryptor;
/**
@@ -130,6 +131,19 @@ public:
*/
void AbortDumpCXI();
/**
* Builds a CIA from a content.
* Blocks, but can be aborted on another thread.
* @return true on success, false otherwise
*/
bool BuildCIA(const ContentSpecifier& specifier, const std::string& destination,
const ProgressCallback& callback = [](std::size_t, std::size_t) {});
/**
* Aborts current CIA building
*/
void AbortBuildCIA();
/**
* Deletes/Cleans up a content. Used for deleting contents that have
* not been fully imported.
@@ -172,6 +186,7 @@ private:
bool is_good{};
Config config;
std::unique_ptr<SDMCDecryptor> decryptor;
std::unique_ptr<CIABuilder> cia_builder;
// The NCCH used to dump CXIs.
std::unique_ptr<NCCHContainer> dump_cxi_ncch;
+237
View File
@@ -0,0 +1,237 @@
// Copyright 2020 Pengfei Zhu
// Licensed under GPLv2 or any later version
// Refer to the license.txt file included.
#include <cryptopp/sha.h>
#include "common/alignment.h"
#include "core/ncch/cia_builder.h"
#include "core/ncch/ticket.h"
#include "core/ncch/title_metadata.h"
namespace Core {
constexpr std::size_t CIA_ALIGNMENT = 0x40;
class HashedFile : public FileUtil::IOFile {
public:
explicit HashedFile(const std::string& filename, const char openmode[], int flags = 0)
: FileUtil::IOFile(filename, openmode, flags) {}
~HashedFile() override = default;
void SetHashEnabled(bool enabled) {
hash_enabled = enabled;
if (enabled) { // Restart when hash is newly restarted
sha.Restart();
}
}
void GetHash(u8* out) {
sha.Final(out);
}
std::size_t Write(const char* data, std::size_t length) override {
const std::size_t length_written = FileUtil::IOFile::Write(data, length);
sha.Update(reinterpret_cast<const CryptoPP::byte*>(data), length_written);
return length_written;
}
private:
CryptoPP::SHA256 sha;
bool hash_enabled{};
};
CIABuilder::CIABuilder() = default;
CIABuilder::~CIABuilder() = default;
bool CIABuilder::Init(const std::string& destination, TitleMetadata tmd_,
const std::string& certs_db_path, std::size_t total_size_,
const ProgressCallback& callback_) {
file = std::make_shared<HashedFile>(destination, "wb");
if (!*file) {
LOG_ERROR(Core, "Could not open file {}", destination);
file.reset();
return false;
}
tmd = std::move(tmd_);
// Remove encrypted flag from TMD chunks
for (auto& chunk : tmd.tmd_chunks) {
chunk.type &= ~0x01;
}
header.header_size = sizeof(header);
// Header will be written in Finalize
// Cert
cert_offset = Common::AlignUp(header.header_size, CIA_ALIGNMENT);
header.cert_size = CIA_CERT_SIZE;
if (!WriteCert(certs_db_path)) {
LOG_ERROR(Core, "Could not write cert to file {}", destination);
file.reset();
return false;
}
// Ticket
ticket_offset = Common::AlignUp(cert_offset + header.cert_size, CIA_ALIGNMENT);
header.tik_size = sizeof(Ticket);
Ticket fake_ticket = BuildFakeTicket(tmd.GetTitleID());
file->Seek(ticket_offset, SEEK_SET);
if (file->WriteBytes(&fake_ticket, sizeof(fake_ticket)) != sizeof(fake_ticket)) {
LOG_ERROR(Core, "Could not write ticket to file {}", destination);
file.reset();
return false;
}
// TMD will be written in Finalize (we need to set content hash, etc)
tmd_offset = Common::AlignUp(ticket_offset + header.tik_size, CIA_ALIGNMENT);
header.tmd_size = tmd.GetSize();
content_offset = Common::AlignUp(tmd_offset + header.tmd_size, CIA_ALIGNMENT);
header.content_size = 0;
// Meta will be written in Finalize
header.meta_size = 0;
written = content_offset;
total_size = total_size_;
callback = callback_;
callback(written, total_size);
return true;
}
bool CIABuilder::WriteCert(const std::string& certs_db_path) {
FileUtil::IOFile certs_db(certs_db_path, "rb");
if (!certs_db) {
LOG_ERROR(Core, "Could not open {}", certs_db_path);
return false;
}
std::array<u8, CIA_CERT_SIZE> cert;
// Read CIA cert
certs_db.Seek(0x0C10, SEEK_SET);
if (certs_db.ReadBytes(cert.data(), 0x1F0) != 0x1F0) {
return false;
}
certs_db.Seek(0x3A00, SEEK_SET);
if (certs_db.ReadBytes(cert.data() + 0x1F0, 0x210) != 0x210) {
return false;
}
certs_db.Seek(0x3F10, SEEK_SET);
if (certs_db.ReadBytes(cert.data() + 0x400, 0x300) != 0x300) {
return false;
}
certs_db.Seek(0x3C10, SEEK_SET);
if (certs_db.ReadBytes(cert.data() + 0x700, 0x300) != 0x300) {
return false;
}
// Write CIA cert to file
file->Seek(cert_offset, SEEK_SET);
if (file->WriteBytes(cert.data(), cert.size()) != cert.size()) {
LOG_ERROR(Core, "Could not write cert");
return false;
}
return true;
}
bool CIABuilder::AddContent(u16 content_id, NCCHContainer& ncch) {
file->Seek(written, SEEK_SET); // To enforce alignment
file->SetHashEnabled(true);
{
std::lock_guard lock{abort_ncch_mutex};
abort_ncch = &ncch;
}
const auto ret = ncch.DecryptToFile(file, [this](std::size_t current, std::size_t total) {
callback(written + current, total_size);
});
{
std::lock_guard lock{abort_ncch_mutex};
abort_ncch = nullptr;
}
if (ret != ResultStatus::Success) {
file.reset();
return false;
}
written = Common::AlignUp(file->Tell(), CIA_ALIGNMENT);
header.content_size = written - content_offset;
header.SetContentPresent(content_id);
auto& tmd_chunk = tmd.GetContentChunkByID(content_id);
file->GetHash(tmd_chunk.hash.data());
file->SetHashEnabled(false);
if (tmd_chunk.index != TMDContentIndex::Main) {
return true;
}
// Load meta if the content is main
header.meta_size = sizeof(meta);
static_assert(sizeof(ncch.exheader_header.dependency_list) == sizeof(meta.dependencies),
"Dependency list should be of the same size in NCCH and CIA");
std::memcpy(meta.dependencies.data(), &ncch.exheader_header.dependency_list,
sizeof(meta.dependencies));
meta.core_version = ncch.exheader_header.arm11_system_local_caps.core_version;
std::vector<u8> smdh_buffer;
if (ncch.LoadSectionExeFS("icon", smdh_buffer) != ResultStatus::Success) {
LOG_WARNING(Core, "Failed to load icon in ExeFS");
return true;
}
std::memcpy(meta.icon_data.data(), smdh_buffer.data(),
std::min(meta.icon_data.size(), smdh_buffer.size()));
return true;
}
bool CIABuilder::Finalize() {
// Write header
file->Seek(0, SEEK_SET);
if (file->WriteBytes(&header, sizeof(header)) != sizeof(header)) {
LOG_ERROR(Core, "Failed to write header");
file.reset();
return false;
}
// Write TMD
file->Seek(tmd_offset, SEEK_SET);
if (tmd.Save(*file) != ResultStatus::Success) {
file.reset();
return false;
}
// Write meta
if (header.meta_size) {
file->Seek(written, SEEK_SET);
if (file->WriteBytes(&meta, sizeof(meta)) != sizeof(meta)) {
LOG_ERROR(Core, "Failed to write meta");
file.reset();
return false;
}
}
callback(total_size, total_size);
file.reset();
return true;
}
void CIABuilder::Abort() {
std::lock_guard lock{abort_ncch_mutex};
if (abort_ncch) {
abort_ncch->AbortDecryptToFile();
}
}
} // namespace Core
+112
View File
@@ -0,0 +1,112 @@
// Copyright 2017 Citra Emulator Project / 2020 Pengfei Zhu
// Licensed under GPLv2 or any later version
// Refer to the license.txt file included.
#include <memory>
#include <mutex>
#include <string>
#include "common/file_util.h"
#include "common/swap.h"
#include "core/ncch/ncch_container.h"
#include "core/ncch/title_metadata.h"
#include "core/quick_decryptor.h"
namespace Core {
constexpr std::size_t CIA_CONTENT_MAX_COUNT = 0x10000;
constexpr std::size_t CIA_CONTENT_BITS_SIZE = (CIA_CONTENT_MAX_COUNT / 8);
constexpr std::size_t CIA_HEADER_SIZE = 0x2020;
constexpr std::size_t CIA_CERT_SIZE = 0xA00;
constexpr std::size_t CIA_METADATA_SIZE = 0x3AC0;
class HashedFile;
class CIABuilder {
public:
explicit CIABuilder();
~CIABuilder();
/**
* Initializes the building of the CIA.
* @return true on success, false otherwise
*/
bool Init(const std::string& destination, TitleMetadata tmd, const std::string& certs_db_path,
std::size_t total_size, const ProgressCallback& callback);
/**
* Adds an NCCH content to the CIA.
* @return true on success, false otherwise
*/
bool AddContent(u16 content_id, NCCHContainer& ncch);
/**
* Finalizes this CIA and write remaining data.
* @return true on success, false otherwise
*/
bool Finalize();
/**
* Aborts the current work. In fact, only usable during AddContent.
*/
void Abort();
private:
struct Header {
u32_le header_size;
u16_le type;
u16_le version;
u32_le cert_size;
u32_le tik_size;
u32_le tmd_size;
u32_le meta_size;
u64_le content_size;
std::array<u8, CIA_CONTENT_BITS_SIZE> content_present;
bool IsContentPresent(u16 index) const {
// The content_present is a bit array which defines which content in the TMD
// is included in the CIA, so check the bit for this index and add if set.
// The bits in the content index are arranged w/ index 0 as the MSB, 7 as the LSB, etc.
return (content_present[index >> 3] & (0x80 >> (index & 7)));
}
void SetContentPresent(u16 index) {
content_present[index >> 3] |= (0x80 >> (index & 7));
}
};
static_assert(sizeof(Header) == CIA_HEADER_SIZE, "CIA Header structure size is wrong");
struct Metadata {
std::array<u64_le, 0x30> dependencies;
std::array<u8, 0x180> reserved;
u32_le core_version;
std::array<u8, 0xfc> reserved_2;
std::array<u8, 0x36c0> icon_data;
};
static_assert(sizeof(Metadata) == CIA_METADATA_SIZE, "CIA Metadata structure size is wrong");
bool WriteCert(const std::string& certs_db_path);
Header header{};
Metadata meta{};
TitleMetadata tmd;
std::size_t cert_offset{};
std::size_t ticket_offset{};
std::size_t tmd_offset{};
std::size_t content_offset{};
std::size_t metadata_offset{};
std::shared_ptr<HashedFile> file;
std::size_t written{}; // size written (with alignment)
std::size_t total_size{};
ProgressCallback callback;
// The NCCH to abort on
std::mutex abort_ncch_mutex;
NCCHContainer* abort_ncch{};
};
} // namespace Core
-4
View File
@@ -23,10 +23,6 @@ using ProgressCallback = std::function<void(std::size_t, std::size_t)>;
*/
class QuickDecryptor {
public:
/**
* Initializes the decryptor.
* @param root_folder Path to the "Nintendo 3DS/<ID0>/<ID1>" folder.
*/
explicit QuickDecryptor();
~QuickDecryptor();
+74 -35
View File
@@ -85,7 +85,7 @@ QPixmap GetContentIcon(const Core::ContentSpecifier& specifier, bool use_categor
ImportDialog::ImportDialog(QWidget* parent, const Core::Config& config)
: QDialog(parent), ui(std::make_unique<Ui::ImportDialog>()), user_path(config.user_path),
importer(config) {
has_cert_db(!config.certs_db_path.empty()), importer(config) {
qRegisterMetaType<u64>("u64");
qRegisterMetaType<Core::ContentSpecifier>();
@@ -564,12 +564,18 @@ void ImportDialog::OnContextMenu(const QPoint& point) {
QMenu context_menu;
if (item->parent()) { // Second level
const auto& specifier = SpecifierFromItem(item);
if (specifier.type != Core::ContentType::Application) {
return;
if (specifier.type == Core::ContentType::Application) {
QAction* dump_cxi = context_menu.addAction(tr("Dump CXI file"));
connect(dump_cxi, &QAction::triggered,
[this, specifier] { StartDumpingCXI(specifier); });
}
if (specifier.type == Core::ContentType::Application ||
specifier.type == Core::ContentType::Update ||
specifier.type == Core::ContentType::DLC) {
QAction* build_cia = context_menu.addAction(tr("Build CIA (standard)"));
connect(build_cia, &QAction::triggered,
[this, specifier] { StartBuildingCIA(specifier); });
}
QAction* dump_cxi = context_menu.addAction(tr("Dump CXI file"));
connect(dump_cxi, &QAction::triggered, [this, specifier] { StartDumpingCXI(specifier); });
} else { // Top level
if (!title_view) {
return;
@@ -581,14 +587,53 @@ void ImportDialog::OnContextMenu(const QPoint& point) {
QAction* dump_base_cxi = context_menu.addAction(tr("Dump Base CXI file"));
connect(dump_base_cxi, &QAction::triggered,
[this, specifier] { StartDumpingCXI(specifier); });
QAction* build_base_cia = context_menu.addAction(tr("Build Base CIA"));
connect(build_base_cia, &QAction::triggered,
[this, specifier] { StartBuildingCIA(specifier); });
break;
} else if (specifier.type == Core::ContentType::Update) {
QAction* build_update_cia = context_menu.addAction(tr("Build Update CIA"));
connect(build_update_cia, &QAction::triggered,
[this, specifier] { StartBuildingCIA(specifier); });
} else if (specifier.type == Core::ContentType::DLC) {
QAction* build_dlc_cia = context_menu.addAction(tr("Build DLC CIA"));
connect(build_dlc_cia, &QAction::triggered,
[this, specifier] { StartBuildingCIA(specifier); });
}
// TODO: Add updates, etc
}
}
context_menu.exec(ui->main->viewport()->mapToGlobal(point));
}
// Runs the job, opening a dialog to report its progress.
void ImportDialog::RunProgressiveJob(ProgressiveJob* job) {
auto* dialog = new QProgressDialog(tr("Initializing..."), tr("Cancel"), 0, 100, this);
dialog->setWindowModality(Qt::WindowModal);
dialog->setMinimumDuration(0);
dialog->setValue(0);
connect(job, &ProgressiveJob::ProgressUpdated, this, [dialog](u64 current, u64 total) {
// Try to map total to int range
// This is equal to ceil(total / INT_MAX)
const u64 multiplier =
(total + std::numeric_limits<int>::max() - 1) / std::numeric_limits<int>::max();
dialog->setMaximum(static_cast<int>(total / multiplier));
dialog->setValue(static_cast<int>(current / multiplier));
dialog->setLabelText(
tr("%1 / %2").arg(ReadableByteSize(current)).arg(ReadableByteSize(total)));
});
connect(job, &ProgressiveJob::ErrorOccured, this, [this, dialog] {
QMessageBox::critical(this, tr("threeSD"),
tr("Operation failed. Please refer to the log."));
dialog->hide();
});
connect(job, &ProgressiveJob::Completed, this,
[dialog] { dialog->setValue(dialog->maximum()); });
connect(dialog, &QProgressDialog::canceled, this, [job] { job->Cancel(); });
job->start();
}
void ImportDialog::StartDumpingCXI(const Core::ContentSpecifier& specifier) {
const QString path = QFileDialog::getSaveFileName(this, tr("Dump CXI file"), last_dump_cxi_path,
tr("CTR Executable Image (*.CXI)"));
@@ -597,17 +642,6 @@ void ImportDialog::StartDumpingCXI(const Core::ContentSpecifier& specifier) {
}
last_dump_cxi_path = QFileInfo(path).path();
// Try to map total_size to int range
// This is equal to ceil(total_size / INT_MAX)
const u64 multiplier = (specifier.maximum_size + std::numeric_limits<int>::max() - 1) /
std::numeric_limits<int>::max();
auto* dialog = new QProgressDialog(tr("Initializing..."), tr("Cancel"), 0,
static_cast<int>(specifier.maximum_size / multiplier), this);
dialog->setWindowModality(Qt::WindowModal);
dialog->setMinimumDuration(0);
dialog->setValue(0);
auto* job = new ProgressiveJob(
this,
[this, specifier, path](const ProgressiveJob::ProgressCallback& callback) {
@@ -618,21 +652,26 @@ void ImportDialog::StartDumpingCXI(const Core::ContentSpecifier& specifier) {
return true;
},
[this] { importer.AbortDumpCXI(); });
connect(job, &ProgressiveJob::ProgressUpdated, this,
[specifier, dialog, multiplier](u64 current, u64 total) {
dialog->setValue(static_cast<int>(current / multiplier));
dialog->setLabelText(tr("%1 / %2")
.arg(ReadableByteSize(current))
.arg(ReadableByteSize(specifier.maximum_size)));
});
connect(job, &ProgressiveJob::ErrorOccured, this, [this, dialog] {
QMessageBox::critical(this, tr("threeSD"), tr("Failed to dump CXI!"));
dialog->hide();
});
connect(job, &ProgressiveJob::Completed, this,
[dialog] { dialog->setValue(dialog->maximum()); });
connect(dialog, &QProgressDialog::canceled, this, [job] { job->Cancel(); });
job->start();
RunProgressiveJob(job);
}
void ImportDialog::StartBuildingCIA(const Core::ContentSpecifier& specifier) {
const QString path = QFileDialog::getSaveFileName(this, tr("Build CIA"), last_build_cia_path,
tr("CTR Importable Archive (*.CIA)"));
if (path.isEmpty()) {
return;
}
last_build_cia_path = QFileInfo(path).path();
auto* job = new ProgressiveJob(
this,
[this, specifier, path](const ProgressiveJob::ProgressCallback& callback) {
if (!importer.BuildCIA(specifier, path.toStdString(), callback)) {
FileUtil::Delete(path.toStdString());
return false;
}
return true;
},
[this] { importer.AbortBuildCIA(); });
RunProgressiveJob(job);
}
+9 -2
View File
@@ -12,6 +12,7 @@
#include "core/importer.h"
#include "core/ncch/ncch_container.h"
class ProgressiveJob;
class QTreeWidgetItem;
namespace Ui {
@@ -41,13 +42,19 @@ private:
Core::ContentSpecifier SpecifierFromItem(QTreeWidgetItem* item) const;
void OnContextMenu(const QPoint& point);
void RunProgressiveJob(ProgressiveJob* job);
void StartDumpingCXI(const Core::ContentSpecifier& content);
Core::NCCHContainer dump_cxi_container; // NCCH container used for dumping CXI
QString last_dump_cxi_path; // Used for recording last path in StartDumpingCXI
QString last_dump_cxi_path; // Used for recording last path in StartDumpingCXI
void StartBuildingCIA(const Core::ContentSpecifier& content);
QString last_build_cia_path; // Used for recording last path in StartBuildingCIA
std::unique_ptr<Ui::ImportDialog> ui;
std::string user_path;
bool has_cert_db = false;
Core::SDMCImporter importer;
std::vector<Core::ContentSpecifier> contents;
u64 total_size = 0;