[makerom/ctrool] Implemented ROMFS unicode support. (oschar.c)

ctrtool/ctrtool.vcxproj

@@ -99,6 +99,7 @@
     <ClCompile Include="exheader.c" />
     <ClCompile Include="filepath.c" />
     <ClCompile Include="firm.c" />
+    <ClCompile Include="oschar.c" />
     <ClCompile Include="windows\getopt.c" />
     <ClCompile Include="windows\getopt1.c" />
     <ClCompile Include="ivfc.c" />
@@ -130,6 +131,7 @@
     <ClInclude Include="exheader.h" />
     <ClInclude Include="filepath.h" />
     <ClInclude Include="firm.h" />
+    <ClInclude Include="oschar.h" />
     <ClInclude Include="windows\getopt.h" />
     <ClInclude Include="info.h" />
     <ClInclude Include="ivfc.h" />

ctrtool/ctrtool.vcxproj.filters

@@ -114,6 +114,9 @@
     <ClCompile Include="tinyxml\tinyxmlparser.cpp">
       <Filter>Source Files\tinyxml</Filter>
     </ClCompile>
+    <ClCompile Include="oschar.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="cia.h">
@@ -203,5 +206,8 @@
     <ClInclude Include="tinyxml\tinyxml.h">
       <Filter>Header Files\tinyxml</Filter>
     </ClInclude>
+    <ClInclude Include="oschar.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
   </ItemGroup>
 </Project>

ctrtool/main.c

@@ -102,7 +102,7 @@ int main(int argc, char* argv[])
 	char keysetfname[512] = "keys.xml";
 	keyset tmpkeys;
 	unsigned int checkkeysetfile = 0;
-	
+
 	memset(&ctx, 0, sizeof(toolcontext));
 	ctx.actions = InfoFlag | ExtractFlag;
 	ctx.filetype = FILETYPE_UNKNOWN;

ctrtool/oschar.c

@@ -0,0 +1,204 @@
+#include <stdlib.h>
+#ifndef _WIN32
+#include <iconv.h>
+#endif
+#include "oschar.h"
+
+int os_fstat(const oschar_t *path)
+{
+	struct _osstat st;
+	return os_stat(path, &st);
+}
+
+uint64_t os_fsize(const oschar_t *path)
+{
+	struct _osstat st;
+	if (os_stat(path, &st) != 0)
+		return 0;
+	else
+		return st.st_size;
+}
+
+int os_makedir(const oschar_t *dir)
+{
+#ifdef _WIN32
+	return _wmkdir(dir);
+#else
+	return mkdir(dir, 0777);
+#endif
+}
+
+uint32_t utf16_strlen(const utf16char_t *str)
+{
+	uint32_t i;
+	for (i = 0; str[i] != 0x0; i++);
+	return i;
+}
+
+void utf16_fputs(const utf16char_t *str, FILE *out)
+{
+	oschar_t *_str = os_CopyConvertUTF16Str(str);
+	os_fputs(_str, out);
+	free(_str);
+}
+
+char* strcopy_8to8(const char *src)
+{
+	uint32_t src_len;
+	char *dst;
+
+	if (!src)
+		return NULL;
+
+	src_len = strlen(src);
+
+	// Allocate memory for expanded string
+	dst = calloc(src_len + 1, sizeof(char));
+	if (!dst)
+		return NULL;
+
+	// Copy elements from src into dst
+	strncpy(dst, src, src_len);
+
+	return dst;
+}
+
+utf16char_t* strcopy_8to16(const char *src)
+{
+	uint32_t src_len, i;
+	utf16char_t *dst;
+
+	if (!src)
+		return NULL;
+
+	src_len = strlen(src);
+
+	// Allocate memory for expanded string
+	dst = calloc(src_len + 1, sizeof(utf16char_t));
+	if (!dst)
+		return NULL;
+
+	// Copy elements from src into dst
+	for (i = 0; i < src_len; i++)
+		dst[i] = src[i];
+
+	return dst;
+}
+
+
+utf16char_t* strcopy_16to16(const utf16char_t *src)
+{
+	uint32_t src_len, i;
+	utf16char_t *dst;
+
+	if (!src)
+		return NULL;
+
+	src_len = utf16_strlen(src);
+
+	// Allocate memory for expanded string
+	dst = calloc(src_len + 1, sizeof(utf16char_t));
+	if (!dst)
+		return NULL;
+
+	// Copy elements from src into dst
+	for (i = 0; i < src_len; i++)
+		dst[i] = src[i];
+
+	return dst;
+}
+
+#ifndef _WIN32
+utf16char_t* strcopy_UTF8toUTF16(const char *src)
+{
+	uint32_t src_len, dst_len;
+	size_t in_bytes, out_bytes;
+	utf16char_t *dst;
+	char *in, *out;
+
+	if (!src)
+		return NULL;
+
+	src_len = strlen(src);
+	dst_len = src_len + 1;
+
+	// Allocate memory for string
+	dst = calloc(dst_len, sizeof(utf16char_t));
+	if (!dst)
+		return NULL;
+
+	in = (char*)src;
+	out = (char*)dst;
+	in_bytes = src_len*sizeof(char);
+	out_bytes = dst_len*sizeof(utf16char_t);
+
+	iconv_t cd = iconv_open("UTF-16LE", "UTF-8");
+	iconv(cd, &in, &in_bytes, &out, &out_bytes);
+	return dst;
+}
+
+char* strcopy_UTF16toUTF8(const utf16char_t *src)
+{
+	uint32_t src_len, dst_len;
+	size_t in_bytes, out_bytes;
+	char *dst;
+	char *in, *out;
+
+	if (!src)
+		return NULL;
+
+	src_len = utf16_strlen(src);
+	dst_len = src_len * 2;
+
+	// Allocate memory for string
+	dst = calloc(dst_len, sizeof(char)); // twice the size, as UTF-8 will use up to two bytes for converted UTF16 chars afaik
+	if (!dst)
+		return NULL;
+
+	in = (char*)src;
+	out = (char*)dst;
+	in_bytes = src_len*sizeof(uint16_t);
+	out_bytes = dst_len*sizeof(char);
+
+	iconv_t cd = iconv_open("UTF-8", "UTF-16LE");
+	iconv(cd, &in, &in_bytes, &out, &out_bytes);
+	return dst;
+}
+#endif
+
+oschar_t* os_AppendToPath(const oschar_t *src, const oschar_t *add)
+{
+	uint32_t len;
+	oschar_t *new_path;
+
+	len = os_strlen(src) + os_strlen(add) + 0x10;
+	new_path = calloc(len, sizeof(oschar_t));
+
+#ifdef _WIN32
+	_snwprintf(new_path, len, L"%s%c%s", src, OS_PATH_SEPARATOR, add);
+#else
+	snprintf(new_path, len, "%s%c%s", src, OS_PATH_SEPARATOR, add);
+#endif
+
+	return new_path;
+}
+
+oschar_t* os_AppendUTF16StrToPath(const oschar_t *src, const utf16char_t *add)
+{
+	uint32_t len;
+	oschar_t *new_path, *_add;
+
+	_add = os_CopyConvertUTF16Str(add);
+
+	len = os_strlen(src) + os_strlen(_add) + 0x10;
+	new_path = calloc(len, sizeof(oschar_t));
+
+#ifdef _WIN32
+	_snwprintf(new_path, len, L"%s%c%s", src, OS_PATH_SEPARATOR, _add);
+#else
+	snprintf(new_path, len, "%s%c%s", src, OS_PATH_SEPARATOR, _add);
+#endif
+
+	free(_add);
+	return new_path;
+}

ctrtool/oschar.h

@@ -0,0 +1,97 @@
+#pragma once
+#include <stdio.h>
+#include <string.h>
+#include <inttypes.h>
+#include <sys/stat.h>
+#include <dirent.h>
+#ifdef _WIN32
+#include <direct.h>
+#include <wchar.h>
+#endif
+
+// Nintendo uses UTF16-LE chars for extended ASCII support
+typedef uint16_t utf16char_t;
+
+// Native OS char type for unicode support
+#ifdef _WIN32
+typedef wchar_t oschar_t; // UTF16-LE
+#else
+typedef char oschar_t; // UTF8
+#endif
+
+					   // Simple redirect macros for functions and types
+#ifdef _WIN32
+#define os_strlen wcslen
+#define os_strcmp wcscmp
+#define os_fputs fputws
+
+#define os_CopyStr strcopy_16to16
+#define os_CopyConvertCharStr strcopy_8to16
+#define os_CopyConvertUTF16Str strcopy_16to16
+#define utf16_CopyStr strcopy_16to16
+#define utf16_CopyConvertOsStr strcopy_16to16
+
+#define _osdirent _wdirent
+#define _OSDIR _WDIR
+#define os_readdir _wreaddir
+#define os_opendir _wopendir
+#define os_closedir _wclosedir
+#define os_chdir _wchdir
+
+#define _osstat _stat64
+#define os_stat _wstat64
+
+#define os_fopen _wfopen
+#define OS_MODE_READ L"rb"
+#define OS_MODE_WRITE L"wb"
+#define OS_MODE_EDIT L"rb+"
+#define OS_PATH_SEPARATOR '\\'
+#else
+#define os_strlen strlen
+#define os_strcmp strcmp
+#define os_fputs fputs
+
+#define os_CopyStr strcopy_8to8
+#define os_CopyConvertUTF16Str strcopy_UTF16toUTF8
+#define os_CopyConvertCharStr strcopy_8to8
+#define utf16_CopyStr strcopy_16to16
+#define utf16_CopyConvertOsStr strcopy_UTF8toUTF16
+
+#define _osdirent dirent
+#define _OSDIR DIR
+#define os_readdir readdir
+#define os_opendir opendir
+#define os_closedir closedir
+#define os_chdir chdir
+
+#define _osstat stat
+#define os_stat stat
+
+#define os_fopen fopen
+#define OS_MODE_READ "rb"
+#define OS_MODE_WRITE "wb"
+#define OS_MODE_EDIT "rb+"
+#define OS_PATH_SEPARATOR '/'
+#endif
+
+/* File related */
+int os_fstat(const oschar_t* path);
+uint64_t os_fsize(const oschar_t* path);
+int os_makedir(const oschar_t *dir);
+
+/* UTF16 String property functions */
+uint32_t utf16_strlen(const utf16char_t* str);
+void utf16_fputs(const utf16char_t *str, FILE *out);
+
+/* String Copy and Conversion */
+char* strcopy_8to8(const char *src);
+utf16char_t* strcopy_8to16(const char *src);
+utf16char_t* strcopy_16to16(const utf16char_t *src);
+#ifndef _WIN32
+utf16char_t* strcopy_UTF8toUTF16(const char *src);
+char* strcopy_UTF16toUTF8(const utf16char_t *src);
+#endif
+
+/* String Append and Create */
+oschar_t* os_AppendToPath(const oschar_t *src, const oschar_t *add);
+oschar_t* os_AppendUTF16StrToPath(const oschar_t *src, const utf16char_t *add);

ctrtool/romfs.c

@@ -96,8 +96,13 @@ void romfs_process(romfs_context* ctx, u32 actions)
 	if (actions & InfoFlag)
 		romfs_print(ctx);
 
-	romfs_visit_dir(ctx, 0, 0, actions, settings_get_romfs_dir_path(ctx->usersettings));
+	if (settings_get_romfs_dir_path(ctx->usersettings)->valid)
+		ctx->extractdir = os_CopyConvertCharStr(settings_get_romfs_dir_path(ctx->usersettings)->pathname);
+	else
+		ctx->extractdir = NULL;
 
+	romfs_visit_dir(ctx, 0, 0, actions, ctx->extractdir);
+	free(ctx->extractdir);
 }
 
 int romfs_dirblock_read(romfs_context* ctx, u32 diroffset, u32 dirsize, void* buffer)
@@ -171,12 +176,12 @@ int romfs_fileblock_readentry(romfs_context* ctx, u32 fileoffset, romfs_fileentr
 
 
 
-void romfs_visit_dir(romfs_context* ctx, u32 diroffset, u32 depth, u32 actions, filepath* rootpath)
+void romfs_visit_dir(romfs_context* ctx, u32 diroffset, u32 depth, u32 actions, const oschar_t* rootpath)
 {
 	u32 siblingoffset;
 	u32 childoffset;
 	u32 fileoffset;
-	filepath currentpath;
+	oschar_t* currentpath;
 	romfs_direntry* entry = &ctx->direntry;
 
 
@@ -190,32 +195,39 @@ void romfs_visit_dir(romfs_context* ctx, u32 diroffset, u32 depth, u32 actions,
 //	fwprintf(stdout, L"%ls\n", entry->name);
 
 
-	if (rootpath && rootpath->valid)
+	if (rootpath && os_strlen(rootpath))
 	{
-		filepath_copy(&currentpath, rootpath);
-		filepath_append_utf16(&currentpath, entry->name);
-		if (currentpath.valid)
+		if (utf16_strlen((const utf16char_t*)entry->name) > 0)
+			currentpath = os_AppendUTF16StrToPath(rootpath, (const utf16char_t*)entry->name);
+		else // root dir, use the provided extract path instead of the empty root name.
+			currentpath = os_CopyStr(rootpath);
+
+		if (currentpath)
 		{
-			makedir(currentpath.pathname);
+			os_makedir(currentpath);
 		}
 		else
 		{
-			fprintf(stderr, "Error creating directory in root %s\n", rootpath->pathname);
+			fputs("Error creating directory in root ", stderr);
+			os_fputs(rootpath, stderr);
+			fputs("\n", stderr);
 			return;
 		}
 	}
 	else
 	{
-		filepath_init(&currentpath);
-
+		currentpath = os_CopyConvertUTF16Str((const utf16char_t*)entry->name);
 		if (settings_get_list_romfs_files(ctx->usersettings))
 		{
 			u32 i;
 
 			for(i=0; i<depth; i++)
 				printf(" ");
-			fwprintf(stdout, L"%ls\n", entry->name);
+			os_fputs(currentpath, stdout);
+			fputs("\n", stdout);
 		}
+		free(currentpath);
+		currentpath = NULL;
 	}
 	
 
@@ -224,20 +236,22 @@ void romfs_visit_dir(romfs_context* ctx, u32 diroffset, u32 depth, u32 actions,
 	fileoffset = getle32(entry->fileoffset);
 
 	if (fileoffset != (~0))
-		romfs_visit_file(ctx, fileoffset, depth+1, actions, &currentpath);
+		romfs_visit_file(ctx, fileoffset, depth+1, actions, currentpath);
 
 	if (childoffset != (~0))
-		romfs_visit_dir(ctx, childoffset, depth+1, actions, &currentpath);
+		romfs_visit_dir(ctx, childoffset, depth+1, actions, currentpath);
 
 	if (siblingoffset != (~0))
 		romfs_visit_dir(ctx, siblingoffset, depth, actions, rootpath);
+
+	free(currentpath);
 }
 
 
-void romfs_visit_file(romfs_context* ctx, u32 fileoffset, u32 depth, u32 actions, filepath* rootpath)
+void romfs_visit_file(romfs_context* ctx, u32 fileoffset, u32 depth, u32 actions, const oschar_t* rootpath)
 {
 	u32 siblingoffset = 0;
-	filepath currentpath;
+	oschar_t* currentpath = NULL;
 	romfs_fileentry* entry = &ctx->fileentry;
 
 
@@ -250,55 +264,63 @@ void romfs_visit_file(romfs_context* ctx, u32 fileoffset, u32 depth, u32 actions
 //			getle64(entry->datasize), getle32(entry->unknown));
 //	fwprintf(stdout, L"%ls\n", entry->name);
 
-	if (rootpath && rootpath->valid)
+	if (rootpath && os_strlen(rootpath))
 	{
-		filepath_copy(&currentpath, rootpath);
-		filepath_append_utf16(&currentpath, entry->name);
-		if (currentpath.valid)
+		currentpath = os_AppendUTF16StrToPath(rootpath, (const utf16char_t*)entry->name);
+		if (currentpath)
 		{
-			fprintf(stdout, "Saving %s...\n", currentpath.pathname);
-			romfs_extract_datafile(ctx, getle64(entry->dataoffset), getle64(entry->datasize), &currentpath);
+			fputs("Saving ", stdout);
+			os_fputs(currentpath, stdout);
+			fputs("...\n", stdout);
+			romfs_extract_datafile(ctx, getle64(entry->dataoffset), getle64(entry->datasize), currentpath);
 		}
 		else
 		{
-			fprintf(stderr, "Error creating directory in root %s\n", rootpath->pathname);
+			fputs("Error creating file in root ", stderr);
+			os_fputs(rootpath, stderr);
+			fputs("\n", stderr);
 			return;
 		}
 	}
 	else
 	{
-		filepath_init(&currentpath);
+		currentpath = os_CopyConvertUTF16Str((const utf16char_t*)entry->name);
 		if (settings_get_list_romfs_files(ctx->usersettings))
 		{
 			u32 i;
 
 			for(i=0; i<depth; i++)
 				printf(" ");
-			fwprintf(stdout, L"%ls\n", entry->name);
+			os_fputs(currentpath, stdout);
+			fputs("\n", stdout);
 		}
+		free(currentpath);
+		currentpath = NULL;
 	}
 
 	siblingoffset = getle32(entry->siblingoffset);
 
 	if (siblingoffset != (~0))
 		romfs_visit_file(ctx, siblingoffset, depth, actions, rootpath);
+
+	free(currentpath);
 }
 
-void romfs_extract_datafile(romfs_context* ctx, u64 offset, u64 size, filepath* path)
+void romfs_extract_datafile(romfs_context* ctx, u64 offset, u64 size, const oschar_t* path)
 {
 	FILE* outfile = 0;
 	u32 max;
 	u8 buffer[4096];
 
 
-	if (path == 0 || path->valid == 0)
+	if (path == NULL || os_strlen(path) == 0)
 		goto clean;
 
 	offset += ctx->datablockoffset;
 
 	fseeko64(ctx->file, offset, SEEK_SET);
-	outfile = fopen(path->pathname, "wb");
-	if (outfile == 0)
+	outfile = os_fopen(path, OS_MODE_WRITE);
+	if (outfile == NULL)
 	{
 		fprintf(stderr, "Error opening file for writing\n");
 		goto clean;

ctrtool/romfs.h

@@ -4,7 +4,7 @@
 #include "types.h"
 #include "info.h"
 #include "ctr.h"
-#include "filepath.h"
+#include "oschar.h"
 #include "settings.h"
 #include "ivfc.h"
 
@@ -55,6 +55,7 @@ typedef struct
 typedef struct
 {
 	FILE* file;
+	oschar_t* extractdir;
 	settings* usersettings;
 	u64 offset;
 	u64 size;
@@ -81,9 +82,9 @@ int  romfs_dirblock_read(romfs_context* ctx, u32 diroffset, u32 dirsize, void* b
 int  romfs_dirblock_readentry(romfs_context* ctx, u32 diroffset, romfs_direntry* entry);
 int  romfs_fileblock_read(romfs_context* ctx, u32 fileoffset, u32 filesize, void* buffer);
 int  romfs_fileblock_readentry(romfs_context* ctx, u32 fileoffset, romfs_fileentry* entry);
-void romfs_visit_dir(romfs_context* ctx, u32 diroffset, u32 depth, u32 actions, filepath* rootpath);
-void romfs_visit_file(romfs_context* ctx, u32 fileoffset, u32 depth, u32 actions, filepath* rootpath);
-void romfs_extract_datafile(romfs_context* ctx, u64 offset, u64 size, filepath* path);
+void romfs_visit_dir(romfs_context* ctx, u32 diroffset, u32 depth, u32 actions, const oschar_t* rootpath);
+void romfs_visit_file(romfs_context* ctx, u32 fileoffset, u32 depth, u32 actions, const oschar_t* rootpath);
+void romfs_extract_datafile(romfs_context* ctx, u64 offset, u64 size, const oschar_t* path);
 void romfs_process(romfs_context* ctx, u32 actions);
 void romfs_print(romfs_context* ctx);
 

ctrtool/utils.c

@@ -1,6 +1,8 @@
 
 #include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
+#include <strings.h>
 #include <sys/stat.h>
 #include <sys/types.h>
 #include "utils.h"
@@ -202,4 +204,4 @@ u64 _fsize(const char *filename)
 	else
 		return st.st_size;
 #endif
-}
+}