misc changes

- Add dumper script
- Fill importer code
- other fixes

dist/threeSDumper.gm9

@@ -0,0 +1,50 @@
+# Copyright 2019 threeSD Project
+# Licensed under GPLv2 or any later version
+# Refer to the license.txt file included.
+
+# GM9 Script for dumping necessary files automatically.
+
+set PREVIEW_MODE "threeSD Dumper\nby zhaowenlan1779"
+set OUT "0:/threeSD"
+if not find $[OUT] NULL
+    mkdir $[OUT]
+end
+
+if not ask "Execute threeSD Dumper?"
+    goto Exit
+end
+
+set PREVIEW_MODE "threeSD Dumper\nby zhaowenlan1779\n \nWorking..."
+
+cp -w -n "1:/private/movable.sed" $[OUT]/movable.sed
+cp -w -n "M:/boot9.bin" $[OUT]/boot9.bin
+
+if not find $[OUT]/firm NULL
+    mkdir $[OUT]/firm
+end
+if chk $[ONTYPE] "N3DS"
+    if not find $[OUT]/firm/new NULL
+        mkdir $[OUT]/firm/new
+    end
+    cp -w -n "1:/title/00040138/20000003/content" $[OUT]/firm/new
+    rm $[OUT]/firm/new/cmd
+else
+    if not find $[OUT]/firm/old NULL
+        mkdir $[OUT]/firm/old
+    end
+    cp -w -n "1:/title/00040138/00000003/content" $[OUT]/firm/old
+    rm $[OUT]/firm/old/cmd
+end
+
+if chk $[ONTYPE] "N3DS"
+    cp -w -n "S:/sector0x96.bin" $[OUT]/sector0x96.bin
+end
+
+sdump -w seeddb.bin
+cp -w -n "0:/gm9/out/seeddb.bin" $[OUT]/seeddb.bin
+rm "0:/gm9/out/seeddb.bin"
+
+set PREVIEW_MODE "threeSD Dumper\nby zhaowenlan1779\n \nSuccess!"
+echo "Successfully dumped necessary\nfiles for threeSD."
+
+@Exit

src/common/common_paths.h

@@ -37,3 +37,4 @@
 #define BOOTROM9 "boot9.bin"
 #define SECRET_SECTOR "sector0x96.bin"
 #define MOVABLE_SED "movable.sed"
+#define SEED_DB "seeddb.bin"

src/common/file_util.cpp

@@ -665,6 +665,10 @@ std::unordered_map<UserPath, std::string> g_paths;
 }
 
 void SetUserPath(const std::string& path) {
+    if (!g_paths.empty()) {
+        g_paths.clear();
+    }
+
     std::string& user_path = g_paths[UserPath::UserDir];
 
     if (!path.empty() && CreateFullPath(path)) {
@@ -717,7 +721,6 @@ const std::string& GetUserPath(UserPath path) {
     return g_paths[path];
 }
 
-
 std::size_t WriteStringToFile(bool text_file, const std::string& filename, std::string_view str) {
     return IOFile(filename, text_file ? "w" : "wb").WriteString(str);
 }

src/core/decryptor.cpp

@@ -59,7 +59,7 @@ bool SDMCDecryptor::DecryptAndWriteFile(const std::string& source,
                                  aes, new CryptoPP::FileSink(destination.c_str(), true)),
                              true);
     } catch (CryptoPP::Exception& e) {
-        LOG_ERROR(Frontend, "Error decrypting and writing file: {}", e.what());
+        LOG_ERROR(Core, "Error decrypting and writing file: {}", e.what());
         return false;
     }
     return true;
@@ -73,7 +73,7 @@ std::vector<u8> SDMCDecryptor::DecryptFile(const std::string& source) const {
 
     FileUtil::IOFile file(root_folder + source, "rb");
     if (!file) {
-        LOG_ERROR(Frontend, "Could not open {}", root_folder + source);
+        LOG_ERROR(Core, "Could not open {}", root_folder + source);
         return {};
     }
 
@@ -81,7 +81,7 @@ std::vector<u8> SDMCDecryptor::DecryptFile(const std::string& source) const {
 
     std::vector<u8> encrypted_data(size);
     if (file.ReadBytes(encrypted_data.data(), size) != size) {
-        LOG_ERROR(Frontend, "Could not read file {}", root_folder + source);
+        LOG_ERROR(Core, "Could not read file {}", root_folder + source);
         return {};
     }
 

src/core/importer.cpp

@@ -2,12 +2,246 @@
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
+#include "common/assert.h"
+#include "common/common_paths.h"
+#include "common/file_util.h"
+#include "core/decryptor.h"
 #include "core/importer.h"
+#include "core/inner_fat.h"
 #include "core/key/key.h"
 
 SDMCImporter::SDMCImporter(const Config& config_) : config(config_) {
-    Key::LoadBootromKeys(config.bootrom_path);
-    Key::LoadMovableSedKeys(config.movable_sed_path);
+    is_good = Init();
 }
 
 SDMCImporter::~SDMCImporter() = default;
+
+bool SDMCImporter::Init() {
+    ASSERT_MSG(config.is_good && !config.sdmc_path.empty() && !config.user_path.empty() &&
+                   !config.bootrom_path.empty() && !config.movable_sed_path.empty(),
+               "Config is not good");
+
+    // Fix paths
+    if (config.sdmc_path.back() != '/' && config.sdmc_path.back() != '\\') {
+        config.sdmc_path += '/';
+    }
+
+    if (config.user_path.back() != '/' && config.user_path.back() != '\\') {
+        config.user_path += '/';
+    }
+
+    Key::ClearKeys();
+    Key::LoadBootromKeys(config.bootrom_path);
+    Key::LoadMovableSedKeys(config.movable_sed_path);
+
+    if (!Key::IsNormalKeyAvailable(Key::SDKey)) {
+        LOG_ERROR(Core, "SDKey is not available");
+        return false;
+    }
+
+    decryptor = std::make_unique<SDMCDecryptor>(config.sdmc_path);
+
+    FileUtil::SetUserPath(config.user_path);
+    return true;
+}
+
+bool SDMCImporter::IsGood() const {
+    return is_good;
+}
+
+bool SDMCImporter::ImportContent(const ContentSpecifier& specifier) {
+    switch (specifier.type) {
+    case ContentType::Application:
+    case ContentType::Update:
+    case ContentType::DLC:
+        return ImportTitle(specifier.id);
+    case ContentType::Savegame:
+        return ImportSavegame(specifier.id);
+    case ContentType::Extdata:
+        return ImportExtdata(specifier.id);
+    case ContentType::Sysdata:
+        return ImportSysdata(specifier.id);
+    default:
+        UNREACHABLE();
+    }
+}
+
+bool SDMCImporter::ImportTitle(u64 id) {
+    const auto path = fmt::format("title/{:08x}/{:08x}/content/", (id >> 32), (id & 0xFFFFFFFF));
+    return FileUtil::ForeachDirectoryEntry(
+        nullptr, config.sdmc_path + path,
+        [this, &path](u64* /*num_entries_out*/, const std::string& directory,
+                      const std::string& virtual_name) {
+            if (FileUtil::IsDirectory(directory + virtual_name)) {
+                return true;
+            }
+            return decryptor->DecryptAndWriteFile(
+                "/" + path + virtual_name,
+                FileUtil::GetUserPath(FileUtil::UserPath::SDMCDir) + path + virtual_name);
+        });
+}
+
+bool SDMCImporter::ImportSavegame(u64 id) {
+    const auto path = fmt::format("title/{:08x}/{:08x}/data/", (id >> 32), (id & 0xFFFFFFFF));
+    SDSavegame save(decryptor->DecryptFile(fmt::format("/{}00000001.sav", path)));
+    if (!save.IsGood()) {
+        return false;
+    }
+
+    return save.Extract(FileUtil::GetUserPath(FileUtil::UserPath::SDMCDir) + path);
+}
+
+bool SDMCImporter::ImportExtdata(u64 id) {
+    const auto path = fmt::format("extdata/{:08x}/{:08x}/", (id >> 32), (id & 0xFFFFFFFF));
+    SDExtdata extdata("/" + path, *decryptor);
+    if (!extdata.IsGood()) {
+        return false;
+    }
+
+    return extdata.Extract(FileUtil::GetUserPath(FileUtil::UserPath::SDMCDir) + path);
+}
+
+bool SDMCImporter::ImportSysdata(u64 id) {
+    switch (id) {
+    case 0: { // boot9.bin
+        const auto target_path = FileUtil::GetUserPath(FileUtil::UserPath::SysDataDir) + BOOTROM9;
+        LOG_INFO(Core, "Copying {} from {} to {}", BOOTROM9, config.bootrom_path, target_path);
+        return FileUtil::Copy(config.bootrom_path, target_path);
+    }
+    case 1: { // safe mode firm
+        // Our GM9 script dumps to different folders for different version (new/old)
+        std::string real_path;
+        bool is_new_3ds = false;
+        if (FileUtil::Exists(config.safe_mode_firm_path + "new/")) {
+            real_path = config.safe_mode_firm_path + "new/";
+            is_new_3ds = true;
+        } else {
+            real_path = config.safe_mode_firm_path + "old/";
+        }
+        return FileUtil::ForeachDirectoryEntry(
+            nullptr, config.safe_mode_firm_path,
+            [is_new_3ds](u64* /*num_entries_out*/, const std::string& directory,
+                         const std::string& virtual_name) {
+                if (FileUtil::IsDirectory(directory + virtual_name)) {
+                    return true;
+                }
+                return FileUtil::Copy(
+                    directory + virtual_name,
+                    fmt::format("{}title/00040138/{}/content/{}",
+                                FileUtil::GetUserPath(FileUtil::UserPath::NANDDir),
+                                (is_new_3ds ? "20000003" : "00000003"), virtual_name));
+            });
+    }
+    case 2: { // seed db
+        const auto target_path = FileUtil::GetUserPath(FileUtil::UserPath::SysDataDir) + SEED_DB;
+        LOG_INFO(Core, "Copying {} from {} to {}", SEED_DB, config.seed_db_path, target_path);
+        return FileUtil::Copy(config.seed_db_path, target_path);
+    }
+    case 3: { // secret sector
+        const auto target_path =
+            FileUtil::GetUserPath(FileUtil::UserPath::SysDataDir) + SECRET_SECTOR;
+        LOG_INFO(Core, "Copying {} from {} to {}", SECRET_SECTOR, config.secret_sector_path,
+                 target_path);
+        return FileUtil::Copy(config.secret_sector_path, target_path);
+    }
+    default:
+        UNREACHABLE_MSG("Unexpected sysdata id {}", id);
+    }
+}
+
+std::vector<ContentSpecifier> SDMCImporter::ListContent() const {
+    std::vector<ContentSpecifier> content_list;
+    ListTitle(content_list);
+    ListExtdata(content_list);
+    ListSysdata(content_list);
+    return content_list;
+}
+
+void SDMCImporter::ListTitle(std::vector<ContentSpecifier>& out) const {
+    const auto ProcessDirectory = [&out, &sdmc_path = config.sdmc_path](ContentType type,
+                                                                        u64 high_id) {
+        FileUtil::ForeachDirectoryEntry(
+            nullptr, fmt::format("{}title/{:08x}/", sdmc_path, high_id),
+            [type, high_id, &out](u64* /*num_entries_out*/, const std::string& directory,
+                                  const std::string& virtual_name) {
+                if (!FileUtil::IsDirectory(directory + virtual_name)) {
+                    return true;
+                }
+
+                const u64 id = (high_id << 32) + std::stoull(virtual_name, nullptr, 16);
+                const auto citra_path = fmt::format(
+                    "{}title/{:08x}/{}/", FileUtil::GetUserPath(FileUtil::UserPath::SDMCDir),
+                    high_id, virtual_name);
+                if (FileUtil::Exists(directory + virtual_name + "/content/")) {
+                    out.push_back({type, id, FileUtil::Exists(citra_path + "content/")});
+                }
+
+                if (type != ContentType::Application) {
+                    return true;
+                }
+                if (FileUtil::Exists(directory + virtual_name + "/data/")) {
+                    out.push_back(
+                        {ContentType::Savegame, id, FileUtil::Exists(citra_path + "data/")});
+                }
+                return true;
+            });
+    };
+
+    ProcessDirectory(ContentType::Application, 0x00040000);
+    ProcessDirectory(ContentType::Update, 0x0004000e);
+    ProcessDirectory(ContentType::DLC, 0x0004008c);
+}
+
+void SDMCImporter::ListExtdata(std::vector<ContentSpecifier>& out) const {
+    FileUtil::ForeachDirectoryEntry(
+        nullptr, fmt::format("{}extdata/00000000/", config.sdmc_path),
+        [&out](u64* /*num_entries_out*/, const std::string& directory,
+               const std::string& virtual_name) {
+            if (!FileUtil::IsDirectory(directory + virtual_name)) {
+                return true;
+            }
+
+            const u64 id = std::stoull(virtual_name, nullptr, 16);
+            const auto citra_path =
+                fmt::format("{}extdata/00000000/{}",
+                            FileUtil::GetUserPath(FileUtil::UserPath::SDMCDir), virtual_name);
+            out.push_back({ContentType::Extdata, id, FileUtil::Exists(citra_path)});
+            return true;
+        });
+}
+
+void SDMCImporter::ListSysdata(std::vector<ContentSpecifier>& out) const {
+#define CHECK_CONTENT(id, var_path, citra_path)                                                    \
+    if (!var_path.empty()) {                                                                       \
+        out.push_back({ContentType::Sysdata, id, FileUtil::Exists(citra_path)});                   \
+    }
+
+    {
+        const auto sysdata_path = FileUtil::GetUserPath(FileUtil::UserPath::SysDataDir);
+        CHECK_CONTENT(0, config.bootrom_path, sysdata_path + BOOTROM9);
+        CHECK_CONTENT(2, config.seed_db_path, sysdata_path + SEED_DB);
+        CHECK_CONTENT(3, config.secret_sector_path, sysdata_path + SECRET_SECTOR);
+    }
+
+    do {
+        if (config.safe_mode_firm_path.empty()) {
+            break;
+        }
+
+        bool is_new = false;
+        if (FileUtil::Exists(config.safe_mode_firm_path + "new/")) {
+            is_new = true;
+        }
+        if (!is_new && !FileUtil::Exists(config.safe_mode_firm_path + "old/")) {
+            LOG_ERROR(Core, "Safe mode firm path specified but not found");
+            break;
+        }
+
+        const auto citra_path = fmt::format("{}title/00040138/{}/content/",
+                                            FileUtil::GetUserPath(FileUtil::UserPath::NANDDir),
+                                            (is_new ? "20000003" : "00000003"));
+        CHECK_CONTENT(1, config.safe_mode_firm_path, citra_path);
+    } while (0);
+
+#undef CHECK_CONTENT
+}

src/core/importer.h

@@ -4,10 +4,13 @@
 
 #pragma once
 
+#include <memory>
 #include <string>
 #include <vector>
 #include "common/common_types.h"
 
+class SDMCDecryptor;
+
 /**
  * Type of an importable content.
  * Applications, updates and DLCs are all considered titles.
@@ -18,6 +21,7 @@ enum class ContentType {
     DLC,
     Savegame,
     Extdata,
+    Sysdata,
 };
 
 /**
@@ -26,22 +30,30 @@ enum class ContentType {
 struct ContentSpecifier {
     ContentType type;
     u64 id;
+    bool already_exists; ///< Tells whether a file already exists in target path.
 };
 
 /**
  * A set of values that are used to initialize the importer.
+ * All paths to directories shall end with a '/' (will be automatically added when not present)
  */
 struct Config {
     std::string sdmc_path; ///< SDMC root path ("Nintendo 3DS/<ID0>/<ID1>")
+    std::string user_path; ///< Target user path of Citra
 
     // Necessary system files keys are loaded from.
     std::string movable_sed_path; ///< Path to movable.sed
-    std::string bootrom_path;     ///< Path to bootrom (boot9.bin)
+    std::string bootrom_path;     ///< Path to bootrom (boot9.bin) (Sysdata 0)
 
     // The following system files are optional for importing and are only copied so that Citra
     // will be able to decrypt imported encrypted ROMs.
-    std::string safe_mode_firm_path; ///< Path to safe mode firm
-    std::string secret_sector_path;  ///< Path to secret sector (New3DS only)
+
+    std::string safe_mode_firm_path; ///< Path to safe mode firm (A folder) (Sysdata 1)
+    std::string seed_db_path;        ///< Path to seeddb.bin (Sysdata 2)
+    std::string secret_sector_path;  ///< Path to secret sector (New3DS only) (Sysdata 3)
+
+    // Whether this config has all necessary information
+    bool is_good;
 };
 
 class SDMCImporter {
@@ -65,10 +77,22 @@ public:
      */
     std::vector<ContentSpecifier> ListContent() const;
 
+    /**
+     * Returns whether the importer is in good state.
+     */
+    bool IsGood() const;
+
 private:
+    bool Init();
     bool ImportTitle(u64 id);
     bool ImportSavegame(u64 id);
     bool ImportExtdata(u64 id);
+    bool ImportSysdata(u64 id);
+    void ListTitle(std::vector<ContentSpecifier>& out) const;
+    void ListExtdata(std::vector<ContentSpecifier>& out) const;
+    void ListSysdata(std::vector<ContentSpecifier>& out) const;
 
+    bool is_good{};
     Config config;
+    std::unique_ptr<SDMCDecryptor> decryptor;
 };

src/core/inner_fat.cpp

@@ -29,7 +29,7 @@ bool InnerFAT::ExtractDirectory(const std::string& path, std::size_t index) cons
     std::string new_path = name.empty() ? path : path + name + "/"; // Name is empty for root
 
     if (!FileUtil::CreateFullPath(new_path)) {
-        LOG_ERROR(Frontend, "Could not create path {}", new_path);
+        LOG_ERROR(Core, "Could not create path {}", new_path);
         return false;
     }
 
@@ -54,7 +54,7 @@ bool InnerFAT::ExtractDirectory(const std::string& path, std::size_t index) cons
 
 bool InnerFAT::WriteMetadata(const std::string& path) const {
     if (!FileUtil::CreateFullPath(path)) {
-        LOG_ERROR(Frontend, "Could not create path {}", path);
+        LOG_ERROR(Core, "Could not create path {}", path);
         return false;
     }
 
@@ -62,11 +62,11 @@ bool InnerFAT::WriteMetadata(const std::string& path) const {
 
     FileUtil::IOFile file(path, "wb");
     if (!file.IsOpen()) {
-        LOG_ERROR(Frontend, "Could not open file {}", path);
+        LOG_ERROR(Core, "Could not open file {}", path);
         return false;
     }
     if (file.WriteBytes(&format_info, sizeof(format_info)) != sizeof(format_info)) {
-        LOG_ERROR(Frontend, "Write data failed (file: {})", path);
+        LOG_ERROR(Core, "Write data failed (file: {})", path);
         return false;
     }
     return true;
@@ -88,7 +88,7 @@ bool SDSavegame::Init() {
     // Read header
     std::memcpy(&header, header_iter, sizeof(header));
     if (header.magic != MakeMagic('S', 'A', 'V', 'E') || header.version != 0x40000) {
-        LOG_ERROR(Frontend, "File is invalid, decryption errors may have happened.");
+        LOG_ERROR(Core, "File is invalid, decryption errors may have happened.");
         return false;
     }
 
@@ -140,7 +140,7 @@ bool SDSavegame::Init() {
 
 bool SDSavegame::ExtractFile(const std::string& path, std::size_t index) const {
     if (!FileUtil::CreateFullPath(path)) {
-        LOG_ERROR(Frontend, "Could not create path {}", path);
+        LOG_ERROR(Core, "Could not create path {}", path);
         return false;
     }
 
@@ -152,7 +152,7 @@ bool SDSavegame::ExtractFile(const std::string& path, std::size_t index) const {
     std::string name{name_data.data()};
     FileUtil::IOFile file(path + name, "wb");
     if (!file.IsOpen()) {
-        LOG_ERROR(Frontend, "Could not open file {}", path + name);
+        LOG_ERROR(Core, "Could not open file {}", path + name);
         return false;
     }
 
@@ -173,7 +173,7 @@ bool SDSavegame::ExtractFile(const std::string& path, std::size_t index) const {
         std::size_t size = fs_info.data_region_block_size * (last_block - block + 1);
         if (file.WriteBytes(data_region.data() + fs_info.data_region_block_size * block, size) !=
             size) {
-            LOG_ERROR(Frontend, "Write data failed (file: {})", path + name);
+            LOG_ERROR(Core, "Write data failed (file: {})", path + name);
             return false;
         }
 
@@ -231,7 +231,7 @@ bool SDExtdata::Init() {
     // Read VSXE file
     auto vsxe_raw = decryptor.DecryptFile(data_path + "00000000/00000001");
     if (vsxe_raw.empty()) {
-        LOG_ERROR(Frontend, "Failed to load or decrypt VSXE");
+        LOG_ERROR(Core, "Failed to load or decrypt VSXE");
         return false;
     }
     DataContainer vsxe_container(vsxe_raw);
@@ -240,7 +240,7 @@ bool SDExtdata::Init() {
     // Read header
     std::memcpy(&header, vsxe.data(), sizeof(header));
     if (header.magic != MakeMagic('V', 'S', 'X', 'E') || header.version != 0x30000) {
-        LOG_ERROR(Frontend, "File is invalid, decryption errors may have happened.");
+        LOG_ERROR(Core, "File is invalid, decryption errors may have happened.");
         return false;
     }
 
@@ -299,7 +299,7 @@ bool SDExtdata::ExtractFile(const std::string& path, std::size_t index) const {
     std::string name{name_data.data()};
     FileUtil::IOFile file(path + name, "wb");
     if (!file) {
-        LOG_ERROR(Frontend, "Could not open file {}", path + name);
+        LOG_ERROR(Core, "Could not open file {}", path + name);
         return false;
     }
 
@@ -311,14 +311,14 @@ bool SDExtdata::ExtractFile(const std::string& path, std::size_t index) const {
 
     auto container_data = decryptor.DecryptFile(device_file_path);
     if (container_data.empty()) { // File does not exist?
-        LOG_WARNING(Frontend, "Ignoring file {}", device_file_path);
+        LOG_WARNING(Core, "Ignoring file {}", device_file_path);
         return true;
     }
 
     DataContainer container(container_data);
     auto data = container.GetIVFCLevel4Data()[0];
     if (file.WriteBytes(data.data(), data.size()) != data.size()) {
-        LOG_ERROR(Frontend, "Write data failed (file: {})", path + name);
+        LOG_ERROR(Core, "Write data failed (file: {})", path + name);
         return false;
     }
 

src/core/key/key.cpp

@@ -186,6 +186,11 @@ void LoadMovableSedKeys(const std::string& path) {
     SetKeyY(0x26, key);
 }
 
+void ClearKeys() {
+    key_slots = {};
+    common_key_y_slots = {};
+}
+
 void SetKeyX(std::size_t slot_id, const AESKey& key) {
     key_slots.at(slot_id).SetKeyX(key);
 }

src/core/key/key.h

@@ -58,6 +58,7 @@ using AESKey = std::array<u8, AES_BLOCK_SIZE>;
 
 void LoadBootromKeys(const std::string& path);
 void LoadMovableSedKeys(const std::string& path);
+void ClearKeys();
 
 void SetKeyX(std::size_t slot_id, const AESKey& key);
 void SetKeyY(std::size_t slot_id, const AESKey& key);