core: Add NAND data importing

dist/threeSDumper.gm9

@@ -117,6 +117,24 @@ decrypt $[OUT]/sysarchives/000400db/00010302.app
 # === Config savegame
 cp -w -n 1:/data/$[SYSID0]/sysdata/00010017/00000000 $[OUT]/config.sav
 
+# === NAND data
+if not find $[OUT]/data NULL
+    mkdir $[OUT]/data
+end
+
+if not find $[OUT]/data/extdata NULL
+    mkdir $[OUT]/data/extdata
+end
+cp -w -n "1:/data/$[SYSID0]/extdata" $[OUT]/data/extdata
+
+if not find $[OUT]/data/sysdata NULL
+    mkdir $[OUT]/data/sysdata
+end
+cp -w -n "1:/data/$[SYSID0]/sysdata" $[OUT]/data/sysdata
+
+# Already dumped above
+rm $[OUT]/data/sysdata/00010017
+
 # === Other system titles
 if not find $[OUT]/title NULL
     mkdir $[OUT]/title
@@ -169,7 +187,7 @@ else
 end
 
 # === Write version
-dumptxt $[OUT]/version.txt 2
+dumptxt $[OUT]/version.txt 3
 
 set PREVIEW_MODE "threeSD Dumper\nby zhaowenlan1779\n \nSuccess!"
 echo "Successfully dumped necessary\nfiles for threeSD."

src/core/importer.cpp

@@ -76,9 +76,17 @@ bool SDMCImporter::ImportContent(const ContentSpecifier& specifier,
     case ContentType::DLC:
         return ImportTitle(specifier, callback);
     case ContentType::Savegame:
+        if ((specifier.id >> 32) == 0) {
+            return ImportNandSavegame(specifier.id, callback);
+        } else {
             return ImportSavegame(specifier.id, callback);
+        }
     case ContentType::Extdata:
+        if ((specifier.id >> 32) == 0) {
             return ImportExtdata(specifier.id, callback);
+        } else {
+            return ImportNandExtdata(specifier.id, callback);
+        }
     case ContentType::SystemArchive:
         return ImportSystemArchive(specifier.id, callback);
     case ContentType::Sysdata:
@@ -182,6 +190,37 @@ bool SDMCImporter::ImportSavegame(u64 id, [[maybe_unused]] const ProgressCallbac
         "Nintendo 3DS/00000000000000000000000000000000/00000000000000000000000000000000/" + path);
 }
 
+bool SDMCImporter::ImportNandSavegame(u64 id, [[maybe_unused]] const ProgressCallback& callback) {
+    const auto path = fmt::format("sysdata/{:08x}/00000000", (id & 0xFFFFFFFF));
+
+    FileUtil::IOFile file(config.nand_data_path + path, "rb");
+    if (!file) {
+        LOG_ERROR(Core, "Could not open file {}", path);
+        return false;
+    }
+
+    std::vector<u8> data(file.GetSize());
+    if (file.ReadBytes(data.data(), data.size()) != data.size()) {
+        LOG_ERROR(Core, "Failed to read from {}", path);
+        return false;
+    }
+
+    DataContainer container(std::move(data));
+    std::vector<std::vector<u8>> container_data;
+    if (!container.GetIVFCLevel4Data(container_data)) {
+        return false;
+    }
+
+    SDSavegame save(std::move(container_data));
+    if (!save.IsGood()) {
+        return false;
+    }
+
+    return save.ExtractDirectory(FileUtil::GetUserPath(FileUtil::UserPath::NANDDir) +
+                                     "data/00000000000000000000000000000000/" + path + "/",
+                                 1);
+}
+
 bool SDMCImporter::ImportExtdata(u64 id, [[maybe_unused]] const ProgressCallback& callback) {
     const auto path = fmt::format("extdata/{:08x}/{:08x}/", (id >> 32), (id & 0xFFFFFFFF));
     SDExtdata extdata("/" + path, *decryptor);
@@ -194,6 +233,17 @@ bool SDMCImporter::ImportExtdata(u64 id, [[maybe_unused]] const ProgressCallback
         "Nintendo 3DS/00000000000000000000000000000000/00000000000000000000000000000000/" + path);
 }
 
+bool SDMCImporter::ImportNandExtdata(u64 id, [[maybe_unused]] const ProgressCallback& callback) {
+    const auto path = fmt::format("extdata/{:08x}/{:08x}/", (id >> 32), (id & 0xFFFFFFFF));
+    SDExtdata extdata(config.nand_data_path + path);
+    if (!extdata.IsGood()) {
+        return false;
+    }
+
+    return extdata.Extract(FileUtil::GetUserPath(FileUtil::UserPath::NANDDir) +
+                           "data/00000000000000000000000000000000/" + path);
+}
+
 bool SDMCImporter::ImportSystemArchive(u64 id, [[maybe_unused]] const ProgressCallback& callback) {
     const auto path = fmt::format("{}{:08x}/{:08x}.app", config.system_archives_path, (id >> 32),
                                   (id & 0xFFFFFFFF));
@@ -364,6 +414,7 @@ std::vector<ContentSpecifier> SDMCImporter::ListContent() const {
     std::vector<ContentSpecifier> content_list;
     ListTitle(content_list);
     ListNandTitle(content_list);
+    ListNandSavegame(content_list);
     ListExtdata(content_list);
     ListSystemArchive(content_list);
     ListSysdata(content_list);
@@ -740,9 +791,9 @@ void SDMCImporter::ListNandTitle(std::vector<ContentSpecifier>& out) const {
     ProcessDirectory(0x00040130);
 }
 
-void SDMCImporter::ListExtdata(std::vector<ContentSpecifier>& out) const {
+void SDMCImporter::ListNandSavegame(std::vector<ContentSpecifier>& out) const {
     FileUtil::ForeachDirectoryEntry(
-        nullptr, fmt::format("{}extdata/00000000/", config.sdmc_path),
+        nullptr, fmt::format("{}sysdata/", config.nand_data_path),
         [&out](u64* /*num_entries_out*/, const std::string& directory,
                const std::string& virtual_name) {
             if (!FileUtil::IsDirectory(directory + virtual_name + "/")) {
@@ -753,16 +804,67 @@ void SDMCImporter::ListExtdata(std::vector<ContentSpecifier>& out) const {
                 return true;
             }
 
+            const auto path = directory + virtual_name + "/00000000";
+
+            // Read the file to test.
+            FileUtil::IOFile file(path, "rb");
+            if (!file) {
+                LOG_ERROR(Core, "Could not open {}", path);
+                return false;
+            }
+
+            std::vector<u8> data(file.GetSize());
+            if (file.ReadBytes(data.data(), data.size()) != data.size()) {
+                LOG_ERROR(Core, "Could not read from {}", path);
+                return false;
+            }
+            DataContainer container(std::move(data));
+            if (!container.IsGood()) {
+                return true;
+            }
+
             const u64 id = std::stoull(virtual_name, nullptr, 16);
             const auto citra_path =
-                fmt::format("{}Nintendo "
+                fmt::format("{}data/00000000000000000000000000000000/sysdata/{}/00000000",
-                            "3DS/00000000000000000000000000000000/00000000000000000000000000000000/"
+                            FileUtil::GetUserPath(FileUtil::UserPath::NANDDir), virtual_name);
-                            "extdata/00000000/{}",
+            out.push_back(
-                            FileUtil::GetUserPath(FileUtil::UserPath::SDMCDir), virtual_name);
+                {ContentType::Savegame, id, FileUtil::Exists(citra_path), FileUtil::GetSize(path)});
-            out.push_back({ContentType::Extdata, id, FileUtil::Exists(citra_path),
+            return true;
+        });
+}
+
+void SDMCImporter::ListExtdata(std::vector<ContentSpecifier>& out) const {
+    const auto ProcessDirectory = [this, &out](u64 id_high, const std::string& path,
+                                               const std::string& citra_path_template) {
+        FileUtil::ForeachDirectoryEntry(
+            nullptr, path,
+            [&out, id_high, citra_path_template](u64* /*num_entries_out*/,
+                                                 const std::string& directory,
+                                                 const std::string& virtual_name) {
+                if (!FileUtil::IsDirectory(directory + virtual_name + "/")) {
+                    return true;
+                }
+
+                if (!std::regex_match(virtual_name, title_regex)) {
+                    return true;
+                }
+
+                const u64 id = std::stoull(virtual_name, nullptr, 16);
+                const auto citra_path = fmt::format(citra_path_template, virtual_name);
+                out.push_back({ContentType::Extdata, (id_high << 32) | id,
+                               FileUtil::Exists(citra_path),
                                FileUtil::GetDirectoryTreeSize(directory + virtual_name + "/")});
                 return true;
             });
+    };
+    ProcessDirectory(0, fmt::format("{}extdata/00000000/", config.sdmc_path),
+                     FileUtil::GetUserPath(FileUtil::UserPath::SDMCDir) +
+                         "Nintendo "
+                         "3DS/00000000000000000000000000000000/00000000000000000000000000000000/"
+                         "extdata/00000000/{}");
+    ProcessDirectory(0x00048000, fmt::format("{}extdata/00048000/", config.nand_data_path),
+                     FileUtil::GetUserPath(FileUtil::UserPath::NANDDir) +
+                         "data/00000000000000000000000000000000/extdata/00048000/{}");
 }
 
 void SDMCImporter::ListSystemArchive(std::vector<ContentSpecifier>& out) const {
@@ -907,19 +1009,31 @@ void SDMCImporter::DeleteNandTitle(u64 id) const {
 }
 
 void SDMCImporter::DeleteSavegame(u64 id) const {
+    if ((id >> 32) == 0) { // NAND
+        FileUtil::DeleteDirRecursively(
+            fmt::format("{}data/00000000000000000000000000000000/sysdata/{:08x}/",
+                        FileUtil::GetUserPath(FileUtil::UserPath::NANDDir), (id & 0xFFFFFFFF)));
+    } else { // SDMC
         FileUtil::DeleteDirRecursively(fmt::format(
             "{}Nintendo "
-        "3DS/00000000000000000000000000000000/00000000000000000000000000000000/title/{:08x}/{:08x}/"
+            "3DS/00000000000000000000000000000000/00000000000000000000000000000000/title/{:08x}/"
-        "data/",
+            "{:08x}/data/",
             FileUtil::GetUserPath(FileUtil::UserPath::SDMCDir), (id >> 32), (id & 0xFFFFFFFF)));
+    }
 }
 
 void SDMCImporter::DeleteExtdata(u64 id) const {
+    if ((id >> 32) == 0) { // SDMC
         FileUtil::DeleteDirRecursively(fmt::format(
             "{}Nintendo "
             "3DS/00000000000000000000000000000000/00000000000000000000000000000000/extdata/{:08x}/"
             "{:08x}/",
             FileUtil::GetUserPath(FileUtil::UserPath::SDMCDir), (id >> 32), (id & 0xFFFFFFFF)));
+    } else { // NAND
+        FileUtil::DeleteDirRecursively(fmt::format(
+            "{}data/00000000000000000000000000000000/extdata/{:08x}/{:08x}/",
+            FileUtil::GetUserPath(FileUtil::UserPath::NANDDir), (id >> 32), (id & 0xFFFFFFFF)));
+    }
 }
 
 void SDMCImporter::DeleteSystemArchive(u64 id) const {
@@ -989,6 +1103,7 @@ std::vector<Config> LoadPresetConfig(std::string mount_point) {
         LOAD_DATA(config_savegame_path, "config.sav");
         LOAD_DATA(system_archives_path, "sysarchives/");
         LOAD_DATA(system_titles_path, "title/");
+        LOAD_DATA(nand_data_path, "data/");
 #undef LOAD_DATA
 
         // Load version

src/core/importer.h

@@ -83,12 +83,13 @@ struct Config {
 
     std::string system_archives_path; ///< Path to system archives.
     std::string system_titles_path;   ///< Path to system titles.
+    std::string nand_data_path;       ///< Path to NAND data. (Extdata and savedata)
 
     int version = 0; ///< Version of the dumper used.
 };
 
 // Version of the current dumper.
-constexpr int CurrentDumperVersion = 2;
+constexpr int CurrentDumperVersion = 3;
 
 class SDMCFile;
 class NCCHContainer;
@@ -170,12 +171,15 @@ private:
     bool ImportTitle(const ContentSpecifier& specifier, const ProgressCallback& callback);
     bool ImportNandTitle(const ContentSpecifier& specifier, const ProgressCallback& callback);
     bool ImportSavegame(u64 id, const ProgressCallback& callback);
+    bool ImportNandSavegame(u64 id, const ProgressCallback& callback);
     bool ImportExtdata(u64 id, const ProgressCallback& callback);
+    bool ImportNandExtdata(u64 id, const ProgressCallback& callback);
     bool ImportSystemArchive(u64 id, const ProgressCallback& callback);
     bool ImportSysdata(u64 id, const ProgressCallback& callback);
 
     void ListTitle(std::vector<ContentSpecifier>& out) const;
     void ListNandTitle(std::vector<ContentSpecifier>& out) const;
+    void ListNandSavegame(std::vector<ContentSpecifier>& out) const;
     void ListExtdata(std::vector<ContentSpecifier>& out) const;
     void ListSystemArchive(std::vector<ContentSpecifier>& out) const;
     void ListSysdata(std::vector<ContentSpecifier>& out) const;

src/core/inner_fat.cpp

@@ -253,20 +253,49 @@ ArchiveFormatInfo SDSavegame::GetFormatInfo() const {
 }
 
 SDExtdata::SDExtdata(std::string data_path_, const SDMCDecryptor& decryptor_)
-    : data_path(std::move(data_path_)), decryptor(decryptor_) {
+    : data_path(std::move(data_path_)), decryptor(&decryptor_) {
 
     if (data_path.back() != '/' && data_path.back() != '\\') {
         data_path += '/';
     }
 
+    use_decryptor = true;
+    is_good = Init();
+}
+
+SDExtdata::SDExtdata(std::string data_path_) : data_path(std::move(data_path_)) {
+    if (data_path.back() != '/' && data_path.back() != '\\') {
+        data_path += '/';
+    }
+
+    use_decryptor = false;
     is_good = Init();
 }
 
 SDExtdata::~SDExtdata() = default;
 
+std::vector<u8> SDExtdata::ReadFile(const std::string& path) const {
+    if (use_decryptor) {
+        return decryptor->DecryptFile(path);
+    } else {
+        FileUtil::IOFile file(path, "rb");
+        if (!file) {
+            LOG_ERROR(Core, "Failed to open {}", path);
+            return {};
+        }
+
+        std::vector<u8> data(file.GetSize());
+        if (file.ReadBytes(data.data(), data.size()) != data.size()) {
+            LOG_ERROR(Core, "Failed to read from {}", path);
+            return {};
+        }
+        return data;
+    }
+}
+
 bool SDExtdata::Init() {
     // Read VSXE file
-    auto vsxe_raw = decryptor.DecryptFile(data_path + "00000000/00000001");
+    auto vsxe_raw = ReadFile(data_path + "00000000/00000001");
     if (vsxe_raw.empty()) {
         LOG_ERROR(Core, "Failed to load or decrypt VSXE");
         return false;
@@ -367,7 +396,7 @@ bool SDExtdata::ExtractFile(const std::string& path, std::size_t index) const {
     std::string device_file_path =
         fmt::format("{}{:08x}/{:08x}", data_path, sub_directory_id, sub_file_id);
 
-    auto container_data = decryptor.DecryptFile(device_file_path);
+    auto container_data = ReadFile(device_file_path);
     if (container_data.empty()) { // File does not exist?
         LOG_WARNING(Core, "Ignoring file {}", device_file_path);
         return true;

src/core/inner_fat.h

@@ -183,17 +183,26 @@ public:
      * @param decryptor Const reference to the SDMCDecryptor.
      */
     explicit SDExtdata(std::string data_path, const SDMCDecryptor& decryptor);
+
+    /**
+     * Loads an Extdata folder without encryption.
+     * @param data_path Path to the DECRYPTED extdata folder
+     */
+    explicit SDExtdata(std::string data_path);
+
     ~SDExtdata() override;
 
     bool Extract(std::string path) const override;
 
 private:
+    std::vector<u8> ReadFile(const std::string& path) const;
     bool Init();
     bool ExtractFile(const std::string& path, std::size_t index) const override;
     ArchiveFormatInfo GetFormatInfo() const override;
 
     std::string data_path;
-    const SDMCDecryptor& decryptor;
+    const SDMCDecryptor* decryptor = nullptr;
+    bool use_decryptor = true;
 };
 
 } // namespace Core