From 67e6b05e8759a9f09f953272f73f8e0c13ccdbb3 Mon Sep 17 00:00:00 2001 From: zhupengfei Date: Fri, 30 Aug 2019 15:33:47 +0800 Subject: [PATCH] lots of important fixes - DPFS container is fixed - SD Savegame is fixed - added slot0x25KeyX load - added regex for titile ID - added aes_keys.txt import - sd savegame listing is fixed (uninitialized won't be listed any more) - error handling is improved (removed asserts and replaced with return values) - UI is now functional - config is now checked in main --- README.md | 41 +++++++- dist/threeSDumper.gm9 | 4 + src/CMakeLists.txt | 29 +++++- src/common/common_paths.h | 1 + src/common/logging/log.h | 3 +- src/core/data_container.cpp | 47 +++++++--- src/core/data_container.h | 7 +- src/core/decryptor.cpp | 4 + src/core/importer.cpp | 79 ++++++++++++++-- src/core/importer.h | 2 + src/core/inner_fat.cpp | 46 ++++++--- src/core/inner_fat.h | 3 +- src/core/key/key.cpp | 26 +++++- src/core/key/key.h | 5 + src/frontend/CMakeLists.txt | 2 + src/frontend/import_dialog.cpp | 166 +++++++++++++++++++++++++++++---- src/frontend/import_dialog.h | 7 +- src/frontend/import_dialog.ui | 4 +- src/frontend/import_job.cpp | 35 +++++++ src/frontend/import_job.h | 33 +++++++ src/frontend/main.cpp | 21 ++++- src/frontend/main.h | 3 +- 22 files changed, 490 insertions(+), 78 deletions(-) create mode 100644 src/frontend/import_job.cpp create mode 100644 src/frontend/import_job.h diff --git a/README.md b/README.md index ed366e2..7031a39 100644 --- a/README.md +++ b/README.md @@ -31,4 +31,43 @@ You will need to run a GodMode9 script. If you are unsure about the script's saf Make sure the SD card is properly recognized and shows up as a disk. -1. + + +... to do ... + +## Contributing + +This project has a lot of code in common with Citra (most of the `common` module are copy-pasta from there), and follows the same styles and practices. Please refer to [this](https://github.com/citra-emu/citra/wiki/Contributing) for a detailed code style guide. + +Note that this project enables quite a lot of warnings, and all warnings are treated as fatal errors by default. + +Some Qt features are disabled to avoid mistakes, these includes: + +```cmake +# Use QStringBuilder for string concatenation to reduce +# the overall number of temporary strings created. +-DQT_USE_QSTRINGBUILDER + +# Disable implicit type narrowing in signal/slot connec() calls. +-DQT_NO_NARROWING_CONVERSIONS_IN_CONNECT + +# Disable unsafe overloads of QProcess' start() function. +-DQT_NO_PROCESS_COMBINED_ARGUMENT_START + +# Disable implicit QString->QUrl conversions to enforce use of proper resolving functions. +-DQT_NO_URL_CAST_FROM_STRING + +# Disable automatic conversions from 8-bit strings (char *) to unicode QStrings +-DQT_NO_CAST_FROM_ASCII +``` + +## TODO + +1. Clear all the `TODO`s in the code +1. Wireless transfer (probably FTP?) +1. System archives +1. UI improvements + * Better error messages + * Beautiful icons + * Better progress indicator +1. Bug fixes diff --git a/dist/threeSDumper.gm9 b/dist/threeSDumper.gm9 index 4213f1f..09de7cb 100644 --- a/dist/threeSDumper.gm9 +++ b/dist/threeSDumper.gm9 @@ -28,12 +28,16 @@ if chk $[ONTYPE] "N3DS" end cp -w -n "1:/title/00040138/20000003/content" $[OUT]/firm/new rm $[OUT]/firm/new/cmd + find $[OUT]/firm/new/*.app APP + decrypt $[APP] else if not find $[OUT]/firm/old NULL mkdir $[OUT]/firm/old end cp -w -n "1:/title/00040138/00000003/content" $[OUT]/firm/old rm $[OUT]/firm/old/cmd + find $[OUT]/firm/old/*.app APP + decrypt $[APP] end if chk $[ONTYPE] "N3DS" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 859566c..8808a20 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -5,6 +5,9 @@ include_directories(.) set_property(DIRECTORY APPEND PROPERTY COMPILE_DEFINITIONS $<$:_DEBUG> $<$>:NDEBUG>) +# Ensure that projects build with Unicode support. +add_definitions(-DUNICODE -D_UNICODE) + # Set compilation flags if (MSVC) set(CMAKE_CONFIGURATION_TYPES Debug Release CACHE STRING "" FORCE) @@ -18,10 +21,15 @@ if (MSVC) # Avoid windows.h from including some usually unused libs like winsocks.h, since this might cause some redefinition errors. add_definitions(-DWIN32_LEAN_AND_MEAN) - # Ensure that projects build with Unicode support. - add_definitions(-DUNICODE -D_UNICODE) - - # /W3 - Level 3 warnings + # /W4 - Level 4 warnings + # /w34263 - Non-virtual member function hides base class virtual function + # /w44265 - Class has virtual functions, but destructor is not virtual + # /w34456 - Declaration of 'var' hides previous local declaration + # /w34457 - Declaration of 'var' hides function parameter + # /w34458 - Declaration of 'var' hides class member + # /w34459 - Declaration of 'var' hides global definition + # /w34946 - Reinterpret-cast between related types + # /wd4592 - Symbol will be dynamically initialized (implementation limitation) # /MP - Multi-threaded compilation # /Zi - Output debugging information # /Zo - Enhanced debug info for optimized builds @@ -32,7 +40,15 @@ if (MSVC) # /Zc:inline - Let codegen omit inline functions in object files # /Zc:throwingNew - Let codegen assume `operator new` (without std::nothrow) will never return null add_compile_options( - /W3 + /W4 + /w34263 + /w44265 + /w34456 + /w34457 + /w34458 + /w34459 + /w34946 + /wd4592 /MP /Zi /Zo @@ -58,6 +74,9 @@ else() add_compile_options( -Wall -Wno-attributes + -pedantic + -pedantic-errors + -Wno-missing-braces ) if (WARNINGS_AS_ERRORS) diff --git a/src/common/common_paths.h b/src/common/common_paths.h index 4469f02..7bc7711 100644 --- a/src/common/common_paths.h +++ b/src/common/common_paths.h @@ -38,3 +38,4 @@ #define SECRET_SECTOR "sector0x96.bin" #define MOVABLE_SED "movable.sed" #define SEED_DB "seeddb.bin" +#define AES_KEYS "aes_keys.txt" diff --git a/src/common/logging/log.h b/src/common/logging/log.h index bd42654..e982ed2 100644 --- a/src/common/logging/log.h +++ b/src/common/logging/log.h @@ -36,7 +36,8 @@ void PrintLog(std::FILE* f, const std::string& log_class, const std::string& lev #ifdef _DEBUG #define LOG_TRACE(log_class, ...) \ - PrintLog(stderr, #log_class, "Trace", "[1;30m", __FILE__, __LINE__, __func__, __VA_ARGS__) + (void(0)) // PrintLog(stderr, #log_class, "Trace", "[1;30m", __FILE__, __LINE__, __func__, + // __VA_ARGS__) #else #define LOG_TRACE(log_class, fmt, ...) (void(0)) #endif diff --git a/src/core/data_container.cpp b/src/core/data_container.cpp index 2529e95..a4f07e6 100644 --- a/src/core/data_container.cpp +++ b/src/core/data_container.cpp @@ -25,7 +25,7 @@ u8 DPFSContainer::GetBit(u8 level, u8 selector, u64 index) const { return (data[(descriptor.levels[level].offset + selector * descriptor.levels[level].size) / 4 + index / 32] >> (31 - (index % 32))) & - 1; + static_cast(1); } u8 DPFSContainer::GetByte(u8 level, u8 selector, u64 index) const { @@ -38,9 +38,9 @@ u8 DPFSContainer::GetByte(u8 level, u8 selector, u64 index) const { std::vector DPFSContainer::GetLevel3Data() const { std::vector level3_data(descriptor.levels[2].size); for (std::size_t i = 0; i < level3_data.size(); i++) { - auto level2_bit_index = i / std::pow(2, descriptor.levels[1].block_size); + auto level2_bit_index = i / std::pow(2, descriptor.levels[2].block_size); auto level1_bit_index = - (level2_bit_index / 8) / std::pow(2, descriptor.levels[0].block_size); + (level2_bit_index / 8) / std::pow(2, descriptor.levels[1].block_size); auto level2_selector = GetBit(0, level1_selector, level1_bit_index); auto level3_selector = GetBit(1, level2_selector, level2_bit_index); level3_data[i] = GetByte(2, level3_selector, i); @@ -49,27 +49,38 @@ std::vector DPFSContainer::GetLevel3Data() const { } DataContainer::DataContainer(std::vector data_) : data(std::move(data_)) { - ASSERT_MSG(data.size() >= 0x200, "Data size is too small"); + if (data.size() < 0x200) { + LOG_ERROR(Core, "Data size {:X} is too small", data.size()); + is_good = false; + return; + } u32_le magic; std::memcpy(&magic, data.data() + 0x100, sizeof(u32_le)); if (magic == MakeMagic('D', 'I', 'S', 'A')) { - InitAsDISA(); + is_good = InitAsDISA(); } else if (magic == MakeMagic('D', 'I', 'F', 'F')) { - InitAsDIFF(); + is_good = InitAsDIFF(); } else { - // TODO: Add error handling - UNREACHABLE_MSG("Unknown magic"); + LOG_ERROR(Core, "Unknown magic 0x{:08x}", magic); + is_good = false; } } DataContainer::~DataContainer() = default; -void DataContainer::InitAsDISA() { +bool DataContainer::IsGood() const { + return is_good; +} + +bool DataContainer::InitAsDISA() { DISAHeader header; std::memcpy(&header, data.data() + 0x100, sizeof(header)); - ASSERT_MSG(header.version == 0x40000, "DISA Version is not correct"); + if (header.version != 0x40000) { + LOG_ERROR(Core, "DISA Version {:x} is not correct", header.version); + return false; + } if (header.active_partition_table == 0) { // primary partition_table_offset = header.primary_partition_table_offset; @@ -86,13 +97,18 @@ void DataContainer::InitAsDISA() { partition_descriptors = {header.partition_descriptors[0]}; partitions = {header.partitions[0]}; } + + return true; } -void DataContainer::InitAsDIFF() { +bool DataContainer::InitAsDIFF() { DIFFHeader header; std::memcpy(&header, data.data() + 0x100, sizeof(header)); - ASSERT_MSG(header.version == 0x30000, "DIFF Version is not correct"); + if (header.version != 0x30000) { + LOG_ERROR(Core, "DIFF Version {:x} is not correct", header.version); + return false; + } if (header.active_partition_table == 0) { // primary partition_table_offset = header.primary_partition_table_offset; @@ -103,6 +119,8 @@ void DataContainer::InitAsDIFF() { partition_count = 1; partition_descriptors = {{/* offset */ 0, /* size */ header.partition_table_size}}; partitions = {header.partition_A}; + + return true; } std::vector DataContainer::GetPartitionData(u8 index) const { @@ -132,11 +150,12 @@ std::vector DataContainer::GetPartitionData(u8 index) const { std::memcpy(&dpfs_descriptor, data.data() + partition_descriptor_offset + difi.dpfs.offset, sizeof(dpfs_descriptor)); - std::vector partition_data(partitions[index].size / 4); + std::vector partition_data(partitions[index].size / 4); std::memcpy(partition_data.data(), data.data() + partitions[index].offset, partitions[index].size); - DPFSContainer dpfs_container(dpfs_descriptor, difi.dpfs_level1_selector, partition_data); + DPFSContainer dpfs_container(dpfs_descriptor, difi.dpfs_level1_selector, + std::move(partition_data)); auto ivfc_data = dpfs_container.GetLevel3Data(); std::vector result(ivfc_data.data() + ivfc_descriptor.levels[3].offset, diff --git a/src/core/data_container.h b/src/core/data_container.h index 18d09cf..6184779 100644 --- a/src/core/data_container.h +++ b/src/core/data_container.h @@ -116,13 +116,16 @@ public: /// Unwraps the whole container, returning the data in IVFC Level 4 of all partitions. std::vector> GetIVFCLevel4Data() const; + bool IsGood() const; + private: - void InitAsDISA(); - void InitAsDIFF(); + bool InitAsDISA(); + bool InitAsDIFF(); /// Unwraps the whole container, returning the data in IVFC Level 4 of a partition. std::vector GetPartitionData(u8 index) const; + bool is_good = false; std::vector data; u32 partition_count; u64_le partition_table_offset; diff --git a/src/core/decryptor.cpp b/src/core/decryptor.cpp index ff20a87..45bb953 100644 --- a/src/core/decryptor.cpp +++ b/src/core/decryptor.cpp @@ -54,6 +54,10 @@ bool SDMCDecryptor::DecryptAndWriteFile(const std::string& source, CryptoPP::CTR_Mode::Decryption aes; aes.SetKeyWithIV(key.data(), key.size(), ctr.data()); + if (!FileUtil::CreateFullPath(destination)) { + return false; + } + std::string absolute_source = root_folder + source; try { CryptoPP::FileSource(absolute_source.c_str(), true, diff --git a/src/core/importer.cpp b/src/core/importer.cpp index 70372bb..da5e849 100644 --- a/src/core/importer.cpp +++ b/src/core/importer.cpp @@ -6,6 +6,7 @@ #include "common/assert.h" #include "common/common_paths.h" #include "common/file_util.h" +#include "core/data_container.h" #include "core/decryptor.h" #include "core/importer.h" #include "core/inner_fat.h" @@ -89,7 +90,13 @@ bool SDMCImporter::ImportTitle(u64 id) { bool SDMCImporter::ImportSavegame(u64 id) { const auto path = fmt::format("title/{:08x}/{:08x}/data/", (id >> 32), (id & 0xFFFFFFFF)); - SDSavegame save(decryptor->DecryptFile(fmt::format("/{}00000001.sav", path))); + + DataContainer container(decryptor->DecryptFile(fmt::format("/{}00000001.sav", path))); + if (!container.IsGood()) { + return false; + } + + SDSavegame save(std::move(container.GetIVFCLevel4Data())); if (!save.IsGood()) { return false; } @@ -116,6 +123,9 @@ bool SDMCImporter::ImportSysdata(u64 id) { case 0: { // boot9.bin const auto target_path = FileUtil::GetUserPath(FileUtil::UserPath::SysDataDir) + BOOTROM9; LOG_INFO(Core, "Copying {} from {} to {}", BOOTROM9, config.bootrom_path, target_path); + if (!FileUtil::CreateFullPath(target_path)) { + return false; + } return FileUtil::Copy(config.bootrom_path, target_path); } case 1: { // safe mode firm @@ -129,22 +139,31 @@ bool SDMCImporter::ImportSysdata(u64 id) { real_path = config.safe_mode_firm_path + "old/"; } return FileUtil::ForeachDirectoryEntry( - nullptr, config.safe_mode_firm_path, + nullptr, real_path, [is_new_3ds](u64* /*num_entries_out*/, const std::string& directory, const std::string& virtual_name) { if (FileUtil::IsDirectory(directory + virtual_name)) { return true; } - return FileUtil::Copy( - directory + virtual_name, + + const auto target_path = fmt::format("{}00000000000000000000000000000000/title/00040138/{}/content/{}", FileUtil::GetUserPath(FileUtil::UserPath::NANDDir), - (is_new_3ds ? "20000003" : "00000003"), virtual_name)); + (is_new_3ds ? "20000003" : "00000003"), virtual_name); + + if (!FileUtil::CreateFullPath(target_path)) { + return false; + } + + return FileUtil::Copy(directory + virtual_name, target_path); }); } case 2: { // seed db const auto target_path = FileUtil::GetUserPath(FileUtil::UserPath::SysDataDir) + SEED_DB; LOG_INFO(Core, "Copying {} from {} to {}", SEED_DB, config.seed_db_path, target_path); + if (!FileUtil::CreateFullPath(target_path)) { + return false; + } return FileUtil::Copy(config.seed_db_path, target_path); } case 3: { // secret sector @@ -152,8 +171,23 @@ bool SDMCImporter::ImportSysdata(u64 id) { FileUtil::GetUserPath(FileUtil::UserPath::SysDataDir) + SECRET_SECTOR; LOG_INFO(Core, "Copying {} from {} to {}", SECRET_SECTOR, config.secret_sector_path, target_path); + if (!FileUtil::CreateFullPath(target_path)) { + return false; + } return FileUtil::Copy(config.secret_sector_path, target_path); } + case 4: { // aes_keys.txt + const auto target_path = FileUtil::GetUserPath(FileUtil::UserPath::SysDataDir) + AES_KEYS; + if (!FileUtil::CreateFullPath(target_path)) { + return false; + } + FileUtil::IOFile file(target_path, "w"); + if (!file) { + return false; + } + file.WriteString("slot0x25KeyX=" + Key::KeyToString(Key::GetKeyX(0x25)) + "\n"); + return true; + } default: UNREACHABLE_MSG("Unexpected sysdata id {}", id); } @@ -167,17 +201,25 @@ std::vector SDMCImporter::ListContent() const { return content_list; } +// Regex for half Title IDs +static const std::regex title_regex{"[0-9a-f]{8}"}; + void SDMCImporter::ListTitle(std::vector& out) const { - const auto ProcessDirectory = [&out, &sdmc_path = config.sdmc_path](ContentType type, - u64 high_id) { + const auto ProcessDirectory = [& decryptor = this->decryptor, &out, + &sdmc_path = config.sdmc_path](ContentType type, u64 high_id) { FileUtil::ForeachDirectoryEntry( nullptr, fmt::format("{}title/{:08x}/", sdmc_path, high_id), - [type, high_id, &out](u64* /*num_entries_out*/, const std::string& directory, - const std::string& virtual_name) { + [&decryptor, type, high_id, &out](u64* /*num_entries_out*/, + const std::string& directory, + const std::string& virtual_name) { if (!FileUtil::IsDirectory(directory + virtual_name + "/")) { return true; } + if (!std::regex_match(virtual_name, title_regex)) { + return true; + } + const u64 id = (high_id << 32) + std::stoull(virtual_name, nullptr, 16); const auto citra_path = fmt::format( "{}Nintendo " @@ -194,6 +236,15 @@ void SDMCImporter::ListTitle(std::vector& out) const { return true; } if (FileUtil::Exists(directory + virtual_name + "/data/")) { + // Savegames can be uninitialized. + // TODO: Is there a better way of checking this other than performing the + // decryption? (Very costy) + DataContainer container(decryptor->DecryptFile( + fmt::format("/title/{:08x}/{}/data/00000001.sav", high_id, virtual_name))); + if (!container.IsGood()) { + return true; + } + out.push_back( {ContentType::Savegame, id, FileUtil::Exists(citra_path + "data/"), FileUtil::GetDirectoryTreeSize(directory + virtual_name + "/data/")}); @@ -216,6 +267,10 @@ void SDMCImporter::ListExtdata(std::vector& out) const { return true; } + if (!std::regex_match(virtual_name, title_regex)) { + return true; + } + const u64 id = std::stoull(virtual_name, nullptr, 16); const auto citra_path = fmt::format("{}Nintendo " @@ -240,6 +295,12 @@ void SDMCImporter::ListSysdata(std::vector& out) const { CHECK_CONTENT(0, config.bootrom_path, sysdata_path + BOOTROM9, BOOTROM9); CHECK_CONTENT(2, config.seed_db_path, sysdata_path + SEED_DB, SEED_DB); CHECK_CONTENT(3, config.secret_sector_path, sysdata_path + SECRET_SECTOR, SECRET_SECTOR); + if (!config.bootrom_path.empty()) { + // 47 bytes = "slot0x26KeyX=<32>\r\n" is only for Windows, + // but it's maximum_size so probably okay + out.push_back( + {ContentType::Sysdata, 4, FileUtil::Exists(sysdata_path + AES_KEYS), 47, AES_KEYS}); + } } #undef CHECK_CONTENT diff --git a/src/core/importer.h b/src/core/importer.h index 2aa4fce..f48c2b5 100644 --- a/src/core/importer.h +++ b/src/core/importer.h @@ -55,6 +55,8 @@ struct Config { std::string safe_mode_firm_path; ///< Path to safe mode firm (A folder) (Sysdata 1) std::string seed_db_path; ///< Path to seeddb.bin (Sysdata 2) std::string secret_sector_path; ///< Path to secret sector (New3DS only) (Sysdata 3) + + // Sysdata 4 is aes_keys.db (slot0x25KeyX) }; class SDMCImporter { diff --git a/src/core/inner_fat.cpp b/src/core/inner_fat.cpp index 4b6839a..7f656dd 100644 --- a/src/core/inner_fat.cpp +++ b/src/core/inner_fat.cpp @@ -74,11 +74,17 @@ bool InnerFAT::WriteMetadata(const std::string& path) const { return true; } -SDSavegame::SDSavegame(std::vector data_) : duplicate_data(true), data(std::move(data_)) {} - -SDSavegame::SDSavegame(std::vector partitionA_, std::vector partitionB_) - : duplicate_data(false), partitionA(std::move(partitionA_)), - partitionB(std::move(partitionB_)) { +SDSavegame::SDSavegame(std::vector> partitions) { + if (partitions.size() == 1) { + duplicate_data = true; + data = std::move(partitions[0]); + } else if (partitions.size() == 2) { + duplicate_data = false; + partitionA = std::move(partitions[0]); + partitionB = std::move(partitions[1]); + } else { + UNREACHABLE(); + } is_good = Init(); } @@ -151,9 +157,9 @@ bool SDSavegame::ExtractFile(const std::string& path, std::size_t index) const { std::array name_data = {}; // Append a null terminator std::memcpy(name_data.data(), entry.name.data(), entry.name.size()); - std::string name{name_data.data()}; + std::string name = name_data.data(); FileUtil::IOFile file(path + name, "wb"); - if (!file.IsOpen()) { + if (!file) { LOG_ERROR(Core, "Could not open file {}", path + name); return false; } @@ -163,6 +169,7 @@ bool SDSavegame::ExtractFile(const std::string& path, std::size_t index) const { return true; } + u64 file_size = entry.file_size; while (true) { // Entry index is block index + 1 auto block_data = fat[block + 1]; @@ -172,14 +179,16 @@ bool SDSavegame::ExtractFile(const std::string& path, std::size_t index) const { last_block = fat[block + 2].v.index - 1; } - std::size_t size = fs_info.data_region_block_size * (last_block - block + 1); - if (file.WriteBytes(data_region.data() + fs_info.data_region_block_size * block, size) != - size) { + const std::size_t size = fs_info.data_region_block_size * (last_block - block + 1); + const std::size_t to_write = std::min(file_size, size); + if (file.WriteBytes(data_region.data() + fs_info.data_region_block_size * block, + to_write) != to_write) { LOG_ERROR(Core, "Write data failed (file: {})", path + name); return false; } + file_size -= to_write; - if (block_data.v.index == 0) // last node + if (block_data.v.index == 0 || file_size == 0) // last node break; block = block_data.v.index - 1; @@ -209,7 +218,7 @@ ArchiveFormatInfo SDSavegame::GetFormatInfo() const { // Tests on a physical 3DS shows that the `total_size` field seems to always be 0 // when requested with the UserSaveData archive, and 134328448 when requested with // the SaveData archive. More investigation is required to tell whether this is a fixed value. - ArchiveFormatInfo format_info = {/* total_size */ 134328448, + ArchiveFormatInfo format_info = {/* total_size */ 0x40000, /* number_directories */ fs_info.maximum_directory_count, /* number_files */ fs_info.maximum_file_count, /* duplicate_data */ duplicate_data}; @@ -236,7 +245,10 @@ bool SDExtdata::Init() { LOG_ERROR(Core, "Failed to load or decrypt VSXE"); return false; } - DataContainer vsxe_container(vsxe_raw); + DataContainer vsxe_container(std::move(vsxe_raw)); + if (!vsxe_container.IsGood()) { + return false; + } auto vsxe = vsxe_container.GetIVFCLevel4Data()[0]; // Read header @@ -298,7 +310,7 @@ bool SDExtdata::ExtractFile(const std::string& path, std::size_t index) const { std::array name_data = {}; // Append a null terminator std::memcpy(name_data.data(), entry.name.data(), entry.name.size()); - std::string name{name_data.data()}; + std::string name = name_data.data(); FileUtil::IOFile file(path + name, "wb"); if (!file) { LOG_ERROR(Core, "Could not open file {}", path + name); @@ -317,7 +329,11 @@ bool SDExtdata::ExtractFile(const std::string& path, std::size_t index) const { return true; } - DataContainer container(container_data); + DataContainer container(std::move(container_data)); + if (!container.IsGood()) { + return false; + } + auto data = container.GetIVFCLevel4Data()[0]; if (file.WriteBytes(data.data(), data.size()) != data.size()) { LOG_ERROR(Core, "Write data failed (file: {})", path + name); diff --git a/src/core/inner_fat.h b/src/core/inner_fat.h index d5a198f..b06fc73 100644 --- a/src/core/inner_fat.h +++ b/src/core/inner_fat.h @@ -156,8 +156,7 @@ protected: class SDSavegame : public InnerFAT { public: - explicit SDSavegame(std::vector data); - explicit SDSavegame(std::vector partitionA, std::vector partitionB); + explicit SDSavegame(std::vector> partitions); ~SDSavegame() override; bool Extract(std::string path) const override; diff --git a/src/core/key/key.cpp b/src/core/key/key.cpp index 7d3b52a..0e389b3 100644 --- a/src/core/key/key.cpp +++ b/src/core/key/key.cpp @@ -82,14 +82,15 @@ struct KeySlot { std::array key_slots; std::array, 6> common_key_y_slots; -std::string KeyToString(AESKey& key) { +} // namespace + +std::string KeyToString(const AESKey& key) { std::string s; for (auto pos : key) { s += fmt::format("{:02X}", pos); } return s; } -} // namespace void LoadBootromKeys(const std::string& path) { constexpr std::array keys = { @@ -160,6 +161,18 @@ void LoadBootromKeys(const std::string& path) { break; } } + + // HACK: "Dump" 0x25 KeyX + // TODO: Is this legal? + constexpr std::array offsets{{0x138A, 0xCAB, 0xD07, 0x3004, 0x2C, 0x49, 0xE6, 0x146E, + 0x1126, 0xD0, 0x85C, 0x47, 0x70A, 0x112C, 0x808, 0x89}}; + + for (std::size_t i = 0; i < offsets.size(); ++i) { + file.Seek(offsets[i], SEEK_SET); + file.ReadBytes(&new_key[i], 1); + } + LOG_DEBUG(Key, "Loaded Slot0x25 KeyX: {}", KeyToString(new_key)); + SetKeyX(0x25, new_key); } void LoadMovableSedKeys(const std::string& path) { @@ -174,7 +187,7 @@ void LoadMovableSedKeys(const std::string& path) { return; } - constexpr std::size_t KEY_SECTION_START = 0x118; + constexpr std::size_t KEY_SECTION_START = 0x110; file.Seek(KEY_SECTION_START, SEEK_SET); // Jump to the key section AESKey key; @@ -184,7 +197,8 @@ void LoadMovableSedKeys(const std::string& path) { return; } - SetKeyY(0x26, key); + LOG_DEBUG(Key, "Loaded Slot0x34KeyY: {}", KeyToString(key)); + SetKeyY(0x34, key); } void ClearKeys() { @@ -212,6 +226,10 @@ AESKey GetNormalKey(std::size_t slot_id) { return key_slots.at(slot_id).normal.value_or(AESKey{}); } +AESKey GetKeyX(std::size_t slot_id) { + return key_slots.at(slot_id).x.value_or(AESKey{}); +} + void SelectCommonKeyIndex(u8 index) { key_slots[KeySlotID::TicketCommonKey].SetKeyY(common_key_y_slots.at(index)); } diff --git a/src/core/key/key.h b/src/core/key/key.h index 0fa5c2d..8dc7db7 100644 --- a/src/core/key/key.h +++ b/src/core/key/key.h @@ -56,6 +56,8 @@ constexpr std::size_t AES_BLOCK_SIZE = 16; using AESKey = std::array; +std::string KeyToString(const AESKey& key); + void LoadBootromKeys(const std::string& path); void LoadMovableSedKeys(const std::string& path); void ClearKeys(); @@ -67,6 +69,9 @@ void SetNormalKey(std::size_t slot_id, const AESKey& key); bool IsNormalKeyAvailable(std::size_t slot_id); AESKey GetNormalKey(std::size_t slot_id); +// For importing aes_keys.txt +AESKey GetKeyX(std::size_t slot_id); + void SelectCommonKeyIndex(u8 index); } // namespace Core::Key diff --git a/src/frontend/CMakeLists.txt b/src/frontend/CMakeLists.txt index 6300da7..06d0af6 100644 --- a/src/frontend/CMakeLists.txt +++ b/src/frontend/CMakeLists.txt @@ -10,6 +10,8 @@ add_executable(threeSD import_dialog.cpp import_dialog.h import_dialog.ui + import_job.cpp + import_job.h main.cpp main.h main.ui diff --git a/src/frontend/import_dialog.cpp b/src/frontend/import_dialog.cpp index 1165cb6..0daf039 100644 --- a/src/frontend/import_dialog.cpp +++ b/src/frontend/import_dialog.cpp @@ -4,12 +4,16 @@ #include #include +#include #include +#include #include #include +#include #include "common/logging/log.h" #include "common/scope_exit.h" #include "frontend/import_dialog.h" +#include "frontend/import_job.h" #include "ui_import_dialog.h" QString ReadableByteSize(qulonglong size) { @@ -25,10 +29,30 @@ QString ReadableByteSize(qulonglong size) { .arg(QObject::tr(units[digit_groups], "ImportDialog")); } +static const std::map ContentTypeMap{ + {Core::ContentType::Application, QT_TR_NOOP("Application")}, + {Core::ContentType::Update, QT_TR_NOOP("Update")}, + {Core::ContentType::DLC, QT_TR_NOOP("DLC")}, + {Core::ContentType::Savegame, QT_TR_NOOP("Save Data")}, + {Core::ContentType::Extdata, QT_TR_NOOP("Extra Data")}, + {Core::ContentType::Sysdata, QT_TR_NOOP("System Data")}, +}; + +QString GetContentName(const Core::ContentSpecifier& specifier) { + return specifier.name.empty() + ? QStringLiteral("0x%1 (%2)") + .arg(specifier.id, 16, 16, QLatin1Char('0')) + .arg(QObject::tr(ContentTypeMap.at(specifier.type), "ImportDialog")) + : QString::fromStdString(specifier.name); +} + ImportDialog::ImportDialog(QWidget* parent, const Core::Config& config) : QDialog(parent), ui(std::make_unique()), user_path(config.user_path), importer(config) { + qRegisterMetaType("u64"); + qRegisterMetaType(); + ui->setupUi(this); if (!importer.IsGood()) { QMessageBox::critical( @@ -37,29 +61,57 @@ ImportDialog::ImportDialog(QWidget* parent, const Core::Config& config) reject(); } - PopulateContent(); + ui->buttonBox->button(QDialogButtonBox::StandardButton::Reset)->setText(tr("Refresh")); + connect(ui->buttonBox, &QDialogButtonBox::clicked, [this](QAbstractButton* button) { + if (button == ui->buttonBox->button(QDialogButtonBox::StandardButton::Ok)) { + StartImporting(); + } else if (button == ui->buttonBox->button(QDialogButtonBox::StandardButton::Cancel)) { + reject(); + } else { + RelistContent(); + } + }); + + RelistContent(); UpdateSizeDisplay(); + + // Set up column widths + ui->main->setColumnWidth(0, width() / 8); + ui->main->setColumnWidth(1, width() / 2); + ui->main->setColumnWidth(2, width() / 6); + ui->main->setColumnWidth(3, width() / 10); } ImportDialog::~ImportDialog() = default; -void ImportDialog::PopulateContent() { +void ImportDialog::RelistContent() { + auto* dialog = new QProgressDialog(tr("Loading Contents..."), tr("Cancel"), 0, 0, this); + dialog->setWindowModality(Qt::WindowModal); + dialog->setCancelButton(nullptr); + dialog->setMinimumDuration(0); + dialog->setValue(0); + + using FutureWatcher = QFutureWatcher>; + auto* future_watcher = new FutureWatcher(this); + connect(future_watcher, &FutureWatcher::finished, this, [this, dialog] { + dialog->hide(); + RepopulateContent(); + }); + + auto future = + QtConcurrent::run([& importer = this->importer] { return importer.ListContent(); }); + future_watcher->setFuture(future); +} + +void ImportDialog::RepopulateContent() { + total_size = 0; contents = importer.ListContent(); ui->main->clear(); ui->main->setSortingEnabled(false); - const std::map content_type_map{ - {Core::ContentType::Application, QStringLiteral("Application")}, - {Core::ContentType::Update, QStringLiteral("Update")}, - {Core::ContentType::DLC, QStringLiteral("DLC (Add-on Content)")}, - {Core::ContentType::Savegame, QStringLiteral("Save Data")}, - {Core::ContentType::Extdata, QStringLiteral("Extra Data")}, - {Core::ContentType::Sysdata, QStringLiteral("System Data")}, - }; - - for (const auto& [type, name] : content_type_map) { + for (const auto& [type, name] : ContentTypeMap) { auto* checkBox = new QCheckBox(); - checkBox->setText(name); + checkBox->setText(tr(name)); checkBox->setStyleSheet(QStringLiteral("margin-left:7px")); checkBox->setTristate(true); checkBox->setProperty("previousState", static_cast(Qt::Unchecked)); @@ -96,15 +148,16 @@ void ImportDialog::PopulateContent() { ui->main->setItemWidget(item, 0, checkBox); } - for (const auto& content : contents) { + for (std::size_t i = 0; i < contents.size(); ++i) { + const auto& content = contents[i]; + auto* checkBox = new QCheckBox(); checkBox->setStyleSheet(QStringLiteral("margin-left:7px")); + // HACK: The checkbox is used to record ID. Is there a better way? + checkBox->setProperty("id", i); auto* item = new QTreeWidgetItem{ - {QString{}, - content.name.empty() ? QStringLiteral("0x%1").arg(content.id, 16, 16, QLatin1Char('0')) - : QString::fromStdString(content.name), - ReadableByteSize(content.maximum_size), + {QString{}, GetContentName(content), ReadableByteSize(content.maximum_size), content.already_exists ? QStringLiteral("Yes") : QStringLiteral("No")}}; ui->main->invisibleRootItem()->child(static_cast(content.type))->addChild(item); @@ -123,6 +176,10 @@ void ImportDialog::PopulateContent() { UpdateItemCheckState(item->parent()); } }); + + if (!content.already_exists) { + checkBox->setChecked(true); + } } ui->main->setSortingEnabled(true); @@ -172,3 +229,76 @@ void ImportDialog::UpdateItemCheckState(QTreeWidgetItem* item) { } program_trigger = false; } + +std::vector ImportDialog::GetSelectedContentList() { + std::vector to_import; + for (int i = 0; i < ui->main->invisibleRootItem()->childCount(); ++i) { + const auto* item = ui->main->invisibleRootItem()->child(i); + for (int j = 0; j < item->childCount(); ++j) { + const auto* checkBox = static_cast(ui->main->itemWidget(item->child(j), 0)); + if (checkBox->isChecked()) { + to_import.emplace_back(contents[checkBox->property("id").toInt()]); + } + } + } + + return to_import; +} + +void ImportDialog::StartImporting() { + UpdateSizeDisplay(); + if (!ui->buttonBox->button(QDialogButtonBox::StandardButton::Ok)->isEnabled()) { + // Space is no longer enough + QMessageBox::warning(this, tr("Not Enough Space"), + tr("Your disk does not have enough space to hold imported data.")); + return; + } + + const auto& to_import = GetSelectedContentList(); + const std::size_t total_count = to_import.size(); + + // Try to map total_size to int range + // This is equal to ceil(total_size / INT_MAX) + const u64 multiplier = + (total_size + std::numeric_limits::max() - 1) / std::numeric_limits::max(); + + auto* dialog = new QProgressDialog(tr("Initializing..."), tr("Cancel"), 0, + static_cast(total_size / multiplier), this); + dialog->setWindowModality(Qt::WindowModal); + dialog->setValue(0); + + auto* job = new ImportJob(this, importer, std::move(to_import)); + + connect(job, &ImportJob::ProgressUpdated, this, + [dialog, multiplier, total_count](u64 size_imported, u64 count, + Core::ContentSpecifier next_content) { + dialog->setValue(static_cast(size_imported / multiplier)); + dialog->setLabelText(tr("Importing %1 (%2/%3)...") + .arg(GetContentName(next_content)) + .arg(count) + .arg(total_count)); + }); + connect(job, &ImportJob::ErrorOccured, this, + [this, dialog](Core::ContentSpecifier current_content) { + QMessageBox::critical( + this, tr("Error"), + tr("Failed to import content %1!").arg(GetContentName(current_content))); + dialog->hide(); + }); + connect(job, &ImportJob::Completed, this, [this, dialog] { + dialog->setValue(dialog->maximum()); + RepopulateContent(); + }); + connect(dialog, &QProgressDialog::canceled, this, [this, dialog, job] { + // Add yet-another-ProgressDialog to indicate cancel progress + auto* cancel_dialog = new QProgressDialog(tr("Canceling..."), tr("Cancel"), 0, 0, this); + cancel_dialog->setWindowModality(Qt::WindowModal); + cancel_dialog->setCancelButton(nullptr); + cancel_dialog->setMinimumDuration(0); + cancel_dialog->setValue(0); + connect(job, &ImportJob::Completed, cancel_dialog, &QProgressDialog::hide); + job->Cancel(); + }); + + job->start(); +} diff --git a/src/frontend/import_dialog.h b/src/frontend/import_dialog.h index 04f6394..9282736 100644 --- a/src/frontend/import_dialog.h +++ b/src/frontend/import_dialog.h @@ -17,16 +17,19 @@ class ImportDialog; } class ImportDialog : public QDialog { - Q_OBJECT; + Q_OBJECT public: explicit ImportDialog(QWidget* parent, const Core::Config& config); ~ImportDialog() override; private: - void PopulateContent(); + void RelistContent(); + void RepopulateContent(); void UpdateSizeDisplay(); void UpdateItemCheckState(QTreeWidgetItem* item); + std::vector GetSelectedContentList(); + void StartImporting(); std::unique_ptr ui; diff --git a/src/frontend/import_dialog.ui b/src/frontend/import_dialog.ui index 0e06ee6..a2e7068 100644 --- a/src/frontend/import_dialog.ui +++ b/src/frontend/import_dialog.ui @@ -33,7 +33,7 @@ - Imported + Exists @@ -66,7 +66,7 @@ - QDialogButtonBox::Ok|QDialogButtonBox::Cancel + QDialogButtonBox::Ok|QDialogButtonBox::Cancel|QDialogButtonBox::Reset diff --git a/src/frontend/import_job.cpp b/src/frontend/import_job.cpp new file mode 100644 index 0000000..05cee32 --- /dev/null +++ b/src/frontend/import_job.cpp @@ -0,0 +1,35 @@ +// Copyright 2019 threeSD Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#include "frontend/import_job.h" + +#include "common/assert.h" + +ImportJob::ImportJob(QObject* parent, Core::SDMCImporter& importer_, + std::vector contents_) + : QThread(parent), importer(importer_), contents(std::move(contents_)) {} + +ImportJob::~ImportJob() = default; + +void ImportJob::run() { + u64 size_imported = 0, count = 0; + for (const auto& content : contents) { + emit ProgressUpdated(size_imported, count + 1, content); + if (!importer.ImportContent(content)) { + emit ErrorOccured(content); + return; + } + count++; + size_imported += content.maximum_size; + + if (cancelled) { + break; + } + } + emit Completed(); +} + +void ImportJob::Cancel() { + cancelled.store(true); +} diff --git a/src/frontend/import_job.h b/src/frontend/import_job.h new file mode 100644 index 0000000..aac1c03 --- /dev/null +++ b/src/frontend/import_job.h @@ -0,0 +1,33 @@ +// Copyright 2019 threeSD Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#pragma once + +#include +#include +#include "core/importer.h" + +class ImportJob : public QThread { + Q_OBJECT + +public: + explicit ImportJob(QObject* parent, Core::SDMCImporter& importer, + std::vector contents); + ~ImportJob() override; + + void run() override; + void Cancel(); + +signals: + void ProgressUpdated(u64 size_imported, u64 count, Core::ContentSpecifier next_content); + void Completed(); + void ErrorOccured(Core::ContentSpecifier current_content); + +private: + std::atomic_bool cancelled{false}; + Core::SDMCImporter& importer; + std::vector contents; +}; + +Q_DECLARE_METATYPE(Core::ContentSpecifier) diff --git a/src/frontend/main.cpp b/src/frontend/main.cpp index 9fa835e..8f827f2 100644 --- a/src/frontend/main.cpp +++ b/src/frontend/main.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include "common/file_util.h" @@ -16,6 +17,11 @@ #include "common/common_paths.h" #endif +bool IsConfigGood(const Core::Config& config) { + return !config.sdmc_path.empty() && !config.user_path.empty() && + !config.movable_sed_path.empty() && !config.bootrom_path.empty(); +} + MainDialog::MainDialog(QWidget* parent) : QDialog(parent), ui(std::make_unique()) { ui->setupUi(this); @@ -36,8 +42,7 @@ MainDialog::MainDialog(QWidget* parent) : QDialog(parent), ui(std::make_uniquebuttonBox->button(QDialogButtonBox::StandardButton::Reset)) { LoadPresetConfig(); } else if (button == ui->buttonBox->button(QDialogButtonBox::StandardButton::Ok)) { - ImportDialog dialog(this, GetCurrentConfig()); - dialog.exec(); + LaunchImportDialog(); } }); @@ -145,6 +150,18 @@ Core::Config MainDialog::GetCurrentConfig() { } } +void MainDialog::LaunchImportDialog() { + const auto& config = GetCurrentConfig(); + if (!IsConfigGood(config)) { + QMessageBox::critical(this, tr("Incomplete Config"), + tr("Your config is missing some of the required fields.")); + return; + } + + ImportDialog dialog(this, config); + dialog.exec(); +} + int main(int argc, char* argv[]) { // Init settings params QCoreApplication::setOrganizationName(QStringLiteral("zhaowenlan1779")); diff --git a/src/frontend/main.h b/src/frontend/main.h index d33231b..9a9dab2 100644 --- a/src/frontend/main.h +++ b/src/frontend/main.h @@ -13,7 +13,7 @@ class MainDialog; } class MainDialog : public QDialog { - Q_OBJECT; + Q_OBJECT public: explicit MainDialog(QWidget* parent = nullptr); @@ -24,6 +24,7 @@ private: void ShowAdvanced(); void HideAdvanced(); Core::Config GetCurrentConfig(); + void LaunchImportDialog(); std::vector preset_config_list; std::unique_ptr ui;