makerom: fixed romfs generation

makerom/makerom.vcxproj

@@ -67,6 +67,7 @@
     <NMakeCleanCommandLine>make clean</NMakeCleanCommandLine>
     <NMakeReBuildCommandLine>make rebuild</NMakeReBuildCommandLine>
     <NMakePreprocessorDefinitions>WIN32;_DEBUG;$(NMakePreprocessorDefinitions)</NMakePreprocessorDefinitions>
+    <IncludePath>C:\Program Files\mingw-w64\x86_64-5.2.0-win32-seh-rt_v4-rev0\mingw64\x86_64-w64-mingw32\include;$(IncludePath)</IncludePath>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
     <NMakeBuildCommandLine>make</NMakeBuildCommandLine>

makerom/romfs.h

@@ -54,7 +54,7 @@ typedef struct
 	u8 siblingoffset[4];
 	u8 childoffset[4];
 	u8 fileoffset[4];
-	u8 weirdoffset[4]; // this one is weird. it always points to a dir entry, but seems unrelated to the romfs structure.
+	u8 hashoffset[4];
 	u8 namesize[4];
 	//u8 name[ROMFS_MAXNAMESIZE];
 } romfs_direntry; //sizeof(romfs_direntry)  = 0x18
@@ -65,7 +65,7 @@ typedef struct
 	u8 siblingoffset[4];
 	u8 dataoffset[8];
 	u8 datasize[8];
-	u8 weirdoffset[4]; // this one is also weird. it always points to a file entry, but seems unrelated to the romfs structure.
+	u8 hashoffset[4];
 	u8 namesize[4];
 	//u8 name[ROMFS_MAXNAMESIZE];
 } romfs_fileentry; //sizeof(romfs_fileentry)  = 0x20
@@ -86,18 +86,18 @@ typedef struct
 	
 	fs_dir *fs;
 	
-	u32 *dirUTable;
+	u32 *dirHashTable;
-	u32 m_dirUTableEntry;
+	u32 m_dirHashTable;
-	u32 u_dirUTableEntry;
+	u32 u_dirHashTable;
 	
 	u8 *dirTable;
 	u32 dirNum;
 	u32 m_dirTableLen;
 	u32 u_dirTableLen;
 	
-	u32 *fileUTable;
+	u32 *fileHashTable;
-	u32 m_fileUTableEntry;
+	u32 m_fileHashTable;
-	u32 u_fileUTableEntry;
+	u32 u_fileHashTable;
 	
 	u8 *fileTable;
 	u32 fileNum;
@@ -112,16 +112,5 @@ typedef struct
 	ivfc_level level[4];
 } romfs_buildctx;
 
-/*
-typedef struct
-{
-	u8 *output;
-	u64 romfsSize;
-	u64 romfsHeaderSize;
-
-	bool ImportRomfsBinary;
-	FILE *romfsBinary;
-} romfs_buildctx;
-*/
 int SetupRomFs(ncch_settings *ncchset, romfs_buildctx *ctx);
 int BuildRomFs(romfs_buildctx *ctx);

makerom/romfs_gen.c

@@ -11,11 +11,13 @@ const fs_romfs_char ROMFS_EMPTY_PATH[2] = {0x0000, 0x0000};
 bool IsFileWanted(fs_file *file, void *filter_criteria);
 bool IsDirWanted(fs_dir *dir, void *filter_criteria);
 void CalcDirSize(romfs_buildctx *ctx, fs_dir *fs);
-int CalcRomfsSize(romfs_buildctx *ctx);
+void CalcRomfsSize(romfs_buildctx *ctx);
-int AddFileToRomfs(romfs_buildctx *ctx, fs_file *file, u32 parent, u32 sibling);
-int AddDirToRomfs(romfs_buildctx *ctx, fs_dir *fs, u32 parent, u32 sibling);
 int FilterRomFS(fs_dir *fs_raw, fs_dir *fs_filtered, void *filter_criteria);
-int PopulateRomfs(romfs_buildctx *ctx);
+void AddFileToRomfs(romfs_buildctx *ctx, fs_file *file, u32 parent, u32 sibling);
+void AddDirToRomfs(romfs_buildctx *ctx, fs_dir *fs, u32 parent, u32 sibling);
+void AddDirChildrenToRomfs(romfs_buildctx *ctx, fs_dir *fs, u32 parent, u32 dir);
+void PopulateHashTable(romfs_buildctx *ctx);
+void PopulateRomfs(romfs_buildctx *ctx);
 void BuildRomfsHeader(romfs_buildctx *ctx);
 void BuildIvfcHeader(romfs_buildctx *ctx);
 void GenIvfcHashTree(romfs_buildctx *ctx);
@@ -69,7 +71,9 @@ int PrepareBuildRomFsBinary(ncch_settings *ncchset, romfs_buildctx *ctx)
 	free(fs_raw);
 	
 	//printf("leave if no ROMFS needs to be made\n");
-	if(ctx->fs->u_file == 0){
+	// If no wanted directory or file exists in root, don't make romfs
+	// a directory can be wanted if a wanted file exists somewhere under it
+	if(ctx->fs->u_file == 0 && ctx->fs->u_dir == 0){
 		ctx->romfsSize = 0;
 		goto finish;
 	}
@@ -116,8 +120,7 @@ int BuildRomFsBinary(romfs_buildctx *ctx)
 	// Build Romfs
 	ctx->romfsHdr = (romfs_infoheader*)(ctx->level[3].pos);
 	BuildRomfsHeader(ctx);
-	if(PopulateRomfs(ctx) != 0)
+	PopulateRomfs(ctx);
-		return -1;
 	
 	
 	// Finalise by building IVFC hash tree
@@ -180,25 +183,35 @@ void CalcDirSize(romfs_buildctx *ctx, fs_dir *fs)
 	ctx->dirNum += fs->u_dir;
 }
 
-int CalcRomfsSize(romfs_buildctx *ctx)
+u32 GetHashTableCount(u32 num)
+{
+	u32 count = num;
+	if (num < 3)
+		count = 3;
+	else if (count < 19)
+		count |= 1;
+	else {
+		while (count % 2 == 0 || count % 3 == 0 || count % 5 == 0 || count % 7 == 0 || count % 11 == 0 || count % 13 == 0 || count % 17 == 0)
+			count++;
+	}
+	return count;
+}
+
+void CalcRomfsSize(romfs_buildctx *ctx)
 {
 	ctx->dirNum = 1; // root dir
 	//printf("Recursively get FS sizes\n");
 	CalcDirSize(ctx,ctx->fs);
 	
 	//printf("check U tables\n");
-	ctx->u_dirUTableEntry = 0;
+	ctx->u_dirHashTable = 0;
-	ctx->m_dirUTableEntry = 3;
+	ctx->m_dirHashTable = GetHashTableCount(ctx->dirNum);
-	if(ctx->dirNum > 3)
-		ctx->m_dirUTableEntry += align(ctx->dirNum-3,2);
 		
-	ctx->u_fileUTableEntry = 0;
+	ctx->u_fileHashTable = 0;
-	ctx->m_fileUTableEntry = 3;
+	ctx->m_fileHashTable = GetHashTableCount(ctx->fileNum);
-	if(ctx->fileNum > 3)
-		ctx->m_fileUTableEntry += align(ctx->fileNum-3,2);
 	
 	//printf("calc romfs header size\n");
-	u32 romfsHdrSize = align(sizeof(romfs_infoheader) + ctx->m_dirUTableEntry*sizeof(u32) + ctx->m_dirTableLen + ctx->m_fileUTableEntry*sizeof(u32) + ctx->m_fileTableLen,0x10); 
+	u32 romfsHdrSize = align(sizeof(romfs_infoheader) + ctx->m_dirHashTable*sizeof(u32) + ctx->m_dirTableLen + ctx->m_fileHashTable*sizeof(u32) + ctx->m_fileTableLen,0x10); 
 	
 	//printf("predict level sizes\n");
 	ctx->level[3].size = romfsHdrSize + ctx->m_dataLen; // data
@@ -214,7 +227,6 @@ int CalcRomfsSize(romfs_buildctx *ctx)
 		ctx->romfsSize += align(ctx->level[i].size,ROMFS_BLOCK_SIZE);
 		
 	//printf("return from CalcRomfsSize();\n");
-	return 0;
 }
 
 int FilterRomFS(fs_dir *fs_raw, fs_dir *fs_filtered, void *filter_criteria)
@@ -278,10 +290,10 @@ void BuildRomfsHeader(romfs_buildctx *ctx)
 	
 	for(int i = 0; i < 4; i++){
 		if(i == 0){
-			ctx->dirUTable = (u32*)(ctx->level[3].pos + level3_pos);
+			ctx->dirHashTable = (u32*)(ctx->level[3].pos + level3_pos);
 			u32_to_u8(ctx->romfsHdr->section[i].offset,level3_pos,LE);
-			u32_to_u8(ctx->romfsHdr->section[i].size,ctx->m_dirUTableEntry*sizeof(u32),LE);
+			u32_to_u8(ctx->romfsHdr->section[i].size,ctx->m_dirHashTable*sizeof(u32),LE);
-			level3_pos += ctx->m_dirUTableEntry*sizeof(u32);
+			level3_pos += ctx->m_dirHashTable*sizeof(u32);
 		}
 		else if(i == 1 && ctx->m_dirTableLen){
 			ctx->dirTable = ctx->level[3].pos + level3_pos;
@@ -290,10 +302,10 @@ void BuildRomfsHeader(romfs_buildctx *ctx)
 			level3_pos += ctx->m_dirTableLen;
 		}
 		else if(i == 2){
-			ctx->fileUTable = (u32*)(ctx->level[3].pos + level3_pos);
+			ctx->fileHashTable = (u32*)(ctx->level[3].pos + level3_pos);
 			u32_to_u8(ctx->romfsHdr->section[i].offset,level3_pos,LE);
-			u32_to_u8(ctx->romfsHdr->section[i].size,ctx->m_fileUTableEntry*sizeof(u32),LE);
+			u32_to_u8(ctx->romfsHdr->section[i].size,ctx->m_fileHashTable*sizeof(u32),LE);
-			level3_pos += ctx->m_fileUTableEntry*sizeof(u32);
+			level3_pos += ctx->m_fileHashTable*sizeof(u32);
 		}
 		else if(i == 3 && ctx->m_fileTableLen){
 			ctx->fileTable = ctx->level[3].pos + level3_pos;
@@ -310,65 +322,33 @@ void BuildRomfsHeader(romfs_buildctx *ctx)
 	ctx->data = ctx->level[3].pos + align(level3_pos,0x10);
 	u32_to_u8(ctx->romfsHdr->dataoffset,align(level3_pos,0x10),LE);
 	
-	memset(ctx->dirUTable,0xff,ctx->m_dirUTableEntry*sizeof(u32));
+	for (u32 i = 0; i < ctx->m_dirHashTable; i++) {
-	memset(ctx->fileUTable,0xff,ctx->m_fileUTableEntry*sizeof(u32));
+		ctx->dirHashTable[i] = ROMFS_UNUSED_ENTRY;
+	}
 
-	return;
+	for (u32 i = 0; i < ctx->m_fileHashTable; i++) {
-}
+		ctx->fileHashTable[i] = ROMFS_UNUSED_ENTRY;
-
-void AddDirHashKey(romfs_buildctx *ctx, u32 parent, fs_romfs_char* path, u32 dirOffset)
-{
-	u32 hash = CalcPathHash(parent,path,0,fs_u16StrLen(path));
-	u32 index = hash % ctx->m_dirUTableEntry;
-	if(ctx->dirUTable[index] == ROMFS_UNUSED_ENTRY) ctx->dirUTable[index] = dirOffset;
-	else
-	{
-		romfs_direntry * curdir = (romfs_direntry*)(ctx->dirTable + ctx->dirUTable[index]);
-		while(1)
-		{
-			if(*(u32*)curdir->weirdoffset == ROMFS_UNUSED_ENTRY)
-			{
-				*(u32*)curdir->weirdoffset = dirOffset;
-				break;
-			}
-			else
-			{
-				curdir = (romfs_direntry*)(ctx->dirTable + *(u32*)curdir->weirdoffset);
-			}
-		}
 	}
 }
 
-void AddFileHashKey(romfs_buildctx *ctx,u32 parent, fs_romfs_char *path, u32 fileOffset)
+u32 GetFileHashTableIndex(romfs_buildctx *ctx, u32 parent, fs_romfs_char *path)
 {
-	u32 hash = CalcPathHash(parent,path,0,fs_u16StrLen(path));
+	u32 hash = CalcPathHash(parent, path, 0, fs_u16StrLen(path));
-	u32 index = hash % ctx->m_fileUTableEntry;
+	return hash % ctx->m_fileHashTable;
-	if(ctx->fileUTable[index] == ROMFS_UNUSED_ENTRY) ctx->fileUTable[index] = fileOffset;
-	else
-	{
-		romfs_fileentry * curfile = (romfs_fileentry*)(ctx->fileTable + ctx->fileUTable[index]);
-		while(1)
-		{
-			if(*(u32*)curfile->weirdoffset == ROMFS_UNUSED_ENTRY)
-			{
-				*(u32*)curfile->weirdoffset = fileOffset;
-				break;
-			}
-			else
-			{
-				curfile = (romfs_fileentry*)(ctx->fileTable + *(u32*)curfile->weirdoffset);
-			}
-		}
-	}
 }
 
-int AddFileToRomfs(romfs_buildctx *ctx, fs_file *file, u32 parent, u32 sibling)
+u32 GetDirHashTableIndex(romfs_buildctx *ctx, u32 parent, fs_romfs_char* path)
+{
+	u32 hash = CalcPathHash(parent, path, 0, fs_u16StrLen(path));
+	return hash % ctx->m_dirHashTable;
+}
+
+void AddFileToRomfs(romfs_buildctx *ctx, fs_file *file, u32 parent, u32 sibling)
 {
 	romfs_fileentry *entry = (romfs_fileentry*)(ctx->fileTable + ctx->u_fileTableLen);
 	
 	u32_to_u8(entry->parentdiroffset,parent,LE);
 	u32_to_u8(entry->siblingoffset,sibling,LE);	
-	u32_to_u8(entry->weirdoffset,ROMFS_UNUSED_ENTRY,LE);
 	
 	// Import Name
 	u32_to_u8(entry->namesize,file->name_len,LE);
@@ -376,6 +356,12 @@ int AddFileToRomfs(romfs_buildctx *ctx, fs_file *file, u32 parent, u32 sibling)
 	memset(name_pos,0,align(file->name_len,4));
 	memcpy(name_pos,(u8*)file->name,file->name_len);
 
+	// Set Hash Data
+	u32 hashindex = GetFileHashTableIndex(ctx, parent, file->name);
+	u32_to_u8(entry->hashoffset, ctx->fileHashTable[hashindex], LE);
+	ctx->fileHashTable[hashindex] = ctx->u_fileTableLen;
+	
+	
 	// Import Data
 	if(file->size)
 	{
@@ -387,29 +373,31 @@ int AddFileToRomfs(romfs_buildctx *ctx, fs_file *file, u32 parent, u32 sibling)
 		ctx->u_dataLen += file->size; // adding file size
 	}
 	else
-		u64_to_u8(entry->dataoffset,0x40,LE);
+		u64_to_u8(entry->dataoffset,0x00,LE);
 		
-	AddFileHashKey(ctx,parent,file->name,ctx->u_fileTableLen);
+	//AddFileHashKey(ctx,parent,file->name,ctx->u_fileTableLen);
 	ctx->u_fileTableLen += sizeof(romfs_fileentry) + align(file->name_len,4);
-		
-	return 0;
 }
 
-int AddDirToRomfs(romfs_buildctx *ctx, fs_dir *fs, u32 parent, u32 sibling)
+void AddDirToRomfs(romfs_buildctx *ctx, fs_dir *fs, u32 parent, u32 sibling)
 {
 	//wprintf(L"adding %s \n",fs->name);
-	romfs_direntry *entry = (romfs_direntry*)(ctx->dirTable + ctx->u_dirTableLen);
+	u32 offset = ctx->u_dirTableLen;
+	romfs_direntry *entry = (romfs_direntry*)(ctx->dirTable + offset);
 	
 	u32_to_u8(entry->parentoffset,parent,LE);
 	u32_to_u8(entry->siblingoffset,sibling,LE);	
-	u32_to_u8(entry->weirdoffset,ROMFS_UNUSED_ENTRY,LE);
+	u32_to_u8(entry->childoffset, ROMFS_UNUSED_ENTRY, LE);
+	u32_to_u8(entry->fileoffset, ROMFS_UNUSED_ENTRY, LE);
 	
-	u32 Currentdir = ctx->u_dirTableLen;
+	if(offset == 0)
-	
-	if(Currentdir == 0)
 	{
 		u32_to_u8(entry->namesize,0,LE);
-		AddDirHashKey(ctx,parent,(fs_romfs_char*)ROMFS_EMPTY_PATH,ctx->u_dirTableLen);
+
+		u32 index = GetFileHashTableIndex(ctx, parent, (fs_romfs_char*)ROMFS_EMPTY_PATH);
+		u32_to_u8(entry->hashoffset, ctx->dirHashTable[index], LE);
+		ctx->dirHashTable[index] = ctx->u_dirTableLen;
+
 		ctx->u_dirTableLen += sizeof(romfs_direntry);
 	}
 	else
@@ -418,67 +406,87 @@ int AddDirToRomfs(romfs_buildctx *ctx, fs_dir *fs, u32 parent, u32 sibling)
 		u8 *name_pos = (u8*)(ctx->dirTable + ctx->u_dirTableLen + sizeof(romfs_direntry));
 		memset(name_pos,0,(u32)align(fs->name_len,4));
 		memcpy(name_pos,(u8*)fs->name,fs->name_len);
-		AddDirHashKey(ctx,parent,fs->name,ctx->u_dirTableLen);
+
+		u32 index = GetFileHashTableIndex(ctx, parent, fs->name);
+		u32_to_u8(entry->hashoffset, ctx->dirHashTable[index], LE);
+		ctx->dirHashTable[index] = ctx->u_dirTableLen;
+
 		ctx->u_dirTableLen += sizeof(romfs_direntry) + (u32)align(fs->name_len,4);
 	}
 
-	if(fs->u_file)
+	//wprintf(L"added %s \n",fs->name);
-	{
+}
-		u32_to_u8(entry->fileoffset,ctx->u_fileTableLen,LE);
 
-		for(u32 i = 0; i < fs->u_file; i++)
+void AddDirChildrenToRomfs(romfs_buildctx *ctx, fs_dir *fs, u32 parent, u32 dir)
+{
+	//wprintf(L"adding %s \n",fs->name);
+	romfs_direntry *entry = (romfs_direntry*)(ctx->dirTable + dir);
+	
+	if (fs->u_file)
+	{
+		u32_to_u8(entry->fileoffset, ctx->u_fileTableLen, LE);
+
+		for (u32 i = 0; i < fs->u_file; i++)
 		{
 
 			u32 file_sibling = 0;
-			if(i >= fs->u_file-1)
+			if (i >= fs->u_file - 1)
 				file_sibling = ROMFS_UNUSED_ENTRY;
 			else
-				file_sibling = ctx->u_fileTableLen + sizeof(romfs_fileentry) + (u32)align(fs->file[i].name_len,4);
+				file_sibling = ctx->u_fileTableLen + sizeof(romfs_fileentry) + (u32)align(fs->file[i].name_len, 4);
 			//wprintf(L"adding %s (0x%lx)\n",fs->file[i].name,fs->file[i].size);
-			AddFileToRomfs(ctx,&fs->file[i],Currentdir,file_sibling);
+			AddFileToRomfs(ctx, &fs->file[i], dir, file_sibling);
 			//wprintf(L"added %s (0x%lx)\n",fs->file[i].name,fs->file[i].size);
 
 		}
 	}
-	else
-		u32_to_u8(entry->fileoffset,ROMFS_UNUSED_ENTRY,LE);
 
 	//printf("Checking if to add dirs\n");
-	if(fs->u_dir)
+	if (fs->u_dir)
 	{
+		u32 *childs = calloc(fs->u_dir, sizeof(u32));
+		u32 dir_sibling;
+		romfs_direntry *temp_entry;
+
 		//printf(" is adding dirs \n");
-		u32_to_u8(entry->childoffset,ctx->u_dirTableLen,LE);
+		u32_to_u8(entry->childoffset, ctx->u_dirTableLen, LE);
-		fs_dir *dir = (fs_dir*)fs->dir;
+		fs_dir *subdir = (fs_dir*)fs->dir;
-		for(u32 i = 0; i < fs->u_dir; i++)
+		for (u32 i = 0; i < fs->u_dir; i++)
 		{
-			u32 dir_sibling = 0;
+			childs[i] = ctx->u_dirTableLen;
-			romfs_direntry *temp_entry = (romfs_direntry*)(ctx->dirTable + ctx->u_dirTableLen);
+			dir_sibling = 0;
-			if(i >= fs->u_dir-1)
+			temp_entry = (romfs_direntry*)(ctx->dirTable + ctx->u_dirTableLen);
+			
+			if (i >= fs->u_dir - 1)
 				dir_sibling = ROMFS_UNUSED_ENTRY;
 			else
-			{
+				dir_sibling = ctx->u_dirTableLen + sizeof(romfs_direntry) + (u32)align(subdir[i].name_len, 4);
-				//printf(" dir has sibling\n");
+			AddDirToRomfs(ctx, &subdir[i], dir, dir_sibling);
-				dir_sibling = ctx->u_dirTableLen + sizeof(romfs_direntry) + (u32)align(dir[i].name_len,4);
+			/* unnecessisary as the above prediction is correct
-			}
+			if (dir_sibling != ROMFS_UNUSED_ENTRY)
-			AddDirToRomfs(ctx,&dir[i],Currentdir,dir_sibling);
-			if(dir_sibling != ROMFS_UNUSED_ENTRY)
 			{
 				dir_sibling = ctx->u_dirTableLen;//修复同目录文件夹偏移 (Repair the same directory folder offset)
-				u32_to_u8(temp_entry->siblingoffset,dir_sibling,LE);
+				u32_to_u8(temp_entry->siblingoffset, dir_sibling, LE);
 			}
+			*/
 		}
+		for (u32 i = 0; i < fs->u_dir; i++)
+		{
+			AddDirChildrenToRomfs(ctx, &subdir[i], dir, childs[i]);
+		}
+
+		free(childs);
 	}
-	else
+
-		u32_to_u8(entry->childoffset,ROMFS_UNUSED_ENTRY,LE);
 	//printf(" finished adding dirs \n");
 
 	//wprintf(L"added %s \n",fs->name);
-	return 0;
 }
 
-int PopulateRomfs(romfs_buildctx *ctx)
+void PopulateRomfs(romfs_buildctx *ctx)
 {
-	return AddDirToRomfs(ctx,ctx->fs,0x0,ROMFS_UNUSED_ENTRY);
+	AddDirToRomfs(ctx, ctx->fs, 0x0, ROMFS_UNUSED_ENTRY);
+	AddDirChildrenToRomfs(ctx, ctx->fs, 0x0, 0);
 }
 
 void BuildIvfcHeader(romfs_buildctx *ctx)