DarkStore-3DS/DarkStore
1
0
Fork 0
mirror of https://github.com/DarkStore-3DS/DarkStore.git synced 2026-07-06 16:49:11 +00:00
Code Issues Packages Projects Releases 1 Wiki Activity

Merge pull request #17 from mariohackandglitch/master

Browse Source 
Improved download code. (Multithreaded)
This commit is contained in:
StackZ
2020-01-27 01:49:58 +01:00
committed by GitHub
parent 0ca7cf6cb9 c8bfedee76
commit f00334b9da
1 changed files with 182 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files
source/download/download.cpp
+182 -130
View File
@@ -42,7 +42,6 @@
static char* result_buf = NULL; static char* result_buf = NULL;
static size_t result_sz = 0; static size_t result_sz = 0;
static Handle* result_fileHandle = nullptr;
static size_t result_written = 0; static size_t result_written = 0;
std::vector<std::string> _topText; std::vector<std::string> _topText;
std::string jsonName; std::string jsonName;
@@ -63,6 +62,188 @@ bool progressBarType = 0; // 0 = Download | 1 = Extract
extern u32 progressBar; extern u32 progressBar;
extern bool isScriptSelected; extern bool isScriptSelected;
curl_off_t downloadTotal = 1; //Dont initialize with 0 to avoid division by zero later
curl_off_t downloadNow = 0;
static FILE *downfile = NULL;
static size_t file_buffer_pos = 0;
static size_t file_toCommit_size = 0;
static char* g_buffers[2] = { NULL };
static u8 g_index = 0;
static Thread fsCommitThread;
static LightEvent readyToCommit;
static LightEvent waitCommit;
static bool killThread = false;
static bool writeError = false;
#define FILE_ALLOC_SIZE 0x60000
static int curlProgress(CURL *hnd,
curl_off_t dltotal, curl_off_t dlnow,
curl_off_t ultotal, curl_off_t ulnow)
{
downloadTotal = dltotal;
downloadNow = dlnow;
return 0;
}
bool filecommit() {
if (!downfile) return false;
fseek(downfile, 0, SEEK_END);
u32 byteswritten = fwrite(g_buffers[!g_index], 1, file_toCommit_size, downfile);
if (byteswritten != file_toCommit_size) return false;
file_toCommit_size = 0;
return true;
}
static void commitToFileThreadFunc(void* args) {
LightEvent_Signal(&waitCommit);
while (true) {
LightEvent_Wait(&readyToCommit);
LightEvent_Clear(&readyToCommit);
if (killThread) threadExit(0);
writeError = !filecommit();
LightEvent_Signal(&waitCommit);
}
}
static size_t file_handle_data(char *ptr, size_t size, size_t nmemb, void *userdata) {
(void)userdata;
const size_t bsz = size * nmemb;
size_t tofill = 0;
if (writeError) return 0;
if (!g_buffers[g_index]) {
LightEvent_Init(&waitCommit, RESET_STICKY);
LightEvent_Init(&readyToCommit, RESET_STICKY);
s32 prio = 0;
svcGetThreadPriority(&prio, CUR_THREAD_HANDLE);
fsCommitThread = threadCreate(commitToFileThreadFunc, NULL, 0x1000, prio - 1, -2, true);
g_buffers[0] = (char*)memalign(0x1000, FILE_ALLOC_SIZE);
g_buffers[1] = (char*)memalign(0x1000, FILE_ALLOC_SIZE);
if (!fsCommitThread || !g_buffers[0] || !g_buffers[1]) return 0;
}
if (file_buffer_pos + bsz >= FILE_ALLOC_SIZE) {
tofill = FILE_ALLOC_SIZE - file_buffer_pos;
memcpy(g_buffers[g_index] + file_buffer_pos, ptr, tofill);
LightEvent_Wait(&waitCommit);
LightEvent_Clear(&waitCommit);
file_toCommit_size = file_buffer_pos + tofill;
file_buffer_pos = 0;
svcFlushProcessDataCache(CUR_PROCESS_HANDLE, g_buffers[g_index], file_toCommit_size);
g_index = !g_index;
LightEvent_Signal(&readyToCommit);
}
memcpy(g_buffers[g_index] + file_buffer_pos, ptr + tofill, bsz - tofill);
file_buffer_pos += bsz - tofill;
return bsz;
}
Result downloadToFile(std::string url, std::string path) {
Result retcode = 0;
downloadTotal = 1;
downloadNow = 0;
int res;
CURL *hnd;
CURLcode cres;
printf("Downloading from:\n%s\nto:\n%s\n", url.c_str(), path.c_str());
const char* filepath = path.c_str();
void *socubuf = memalign(0x1000, 0x100000);
if (!socubuf) {
retcode = -1;
goto exit;
}
res = socInit((u32*)socubuf, 0x100000);
if (R_FAILED(res)) {
retcode = res;
goto exit;
}
makeDirs(strdup(filepath));
downfile = fopen(filepath, "wb");
if (!downfile) {
retcode = -2;
goto exit;
}
hnd = curl_easy_init();
curl_easy_setopt(hnd, CURLOPT_BUFFERSIZE, FILE_ALLOC_SIZE);
curl_easy_setopt(hnd, CURLOPT_URL, url.c_str());
curl_easy_setopt(hnd, CURLOPT_NOPROGRESS, 0L);
curl_easy_setopt(hnd, CURLOPT_USERAGENT, USER_AGENT);
curl_easy_setopt(hnd, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(hnd, CURLOPT_FAILONERROR, 1L);
curl_easy_setopt(hnd, CURLOPT_ACCEPT_ENCODING, "gzip");
curl_easy_setopt(hnd, CURLOPT_MAXREDIRS, 50L);
curl_easy_setopt(hnd, CURLOPT_XFERINFOFUNCTION, curlProgress);
curl_easy_setopt(hnd, CURLOPT_HTTP_VERSION, (long)CURL_HTTP_VERSION_2TLS);
curl_easy_setopt(hnd, CURLOPT_WRITEFUNCTION, file_handle_data);
curl_easy_setopt(hnd, CURLOPT_SSL_VERIFYPEER, 0L);
curl_easy_setopt(hnd, CURLOPT_VERBOSE, 1L);
curl_easy_setopt(hnd, CURLOPT_STDERR, stdout);
cres = curl_easy_perform(hnd);
curl_easy_cleanup(hnd);
if (cres != CURLE_OK) {
retcode = -cres;
goto exit;
}
LightEvent_Wait(&waitCommit);
LightEvent_Clear(&waitCommit);
file_toCommit_size = file_buffer_pos;
svcFlushProcessDataCache(CUR_PROCESS_HANDLE, g_buffers[g_index], file_toCommit_size);
g_index = !g_index;
if (!filecommit()) {
retcode = -3;
goto exit;
}
fflush(downfile);
exit:
if (fsCommitThread) {
killThread = true;
LightEvent_Signal(&readyToCommit);
threadJoin(fsCommitThread, U64_MAX);
killThread = false;
fsCommitThread = NULL;
}
socExit();
if (socubuf) {
free(socubuf);
}
if (downfile) {
fclose(downfile);
downfile = NULL;
}
if (g_buffers[0]) {
free(g_buffers[0]);
g_buffers[0] = NULL;
}
if (g_buffers[1]) {
free(g_buffers[1]);
g_buffers[1] = NULL;
}
g_index = 0;
file_buffer_pos = 0;
file_toCommit_size = 0;
writeError = false;
return retcode;
}
// following function is from // following function is from
// https://github.com/angelsl/libctrfgh/blob/master/curl_test/src/main.c // https://github.com/angelsl/libctrfgh/blob/master/curl_test/src/main.c
static size_t handle_data(char* ptr, size_t size, size_t nmemb, void* userdata) static size_t handle_data(char* ptr, size_t size, size_t nmemb, void* userdata)
@@ -103,31 +284,6 @@ static size_t handle_data(char* ptr, size_t size, size_t nmemb, void* userdata)
return bsz; return bsz;
} }
static size_t handle_data_to_file(char* ptr, size_t size, size_t nmemb, void* userdata)
{
(void) userdata;
const size_t bsz = size*nmemb;
u32 bytesWritten = 0;
FSFILE_Write(*result_fileHandle, &bytesWritten, result_written, ptr, bsz, 0);
result_written += bsz;
return bsz;
}
curl_off_t downloadTotal = 1; //Dont initialize with 0 to avoid division by zero later
curl_off_t downloadNow = 0;
static int curlProgress(CURL *hnd,
curl_off_t dltotal, curl_off_t dlnow,
curl_off_t ultotal, curl_off_t ulnow)
{
downloadTotal = dltotal;
downloadNow = dlnow;
return 0;
}
static Result setupContext(CURL *hnd, const char * url) static Result setupContext(CURL *hnd, const char * url)
{ {
downloadTotal = 1; downloadTotal = 1;
@@ -149,110 +305,6 @@ static Result setupContext(CURL *hnd, const char * url)
return 0; return 0;
} }
static Result setupContextForDirectToFileDownload(CURL *hnd, const char * url)
{
Result ret = setupContext(hnd, url);
if (ret != 0) {
return ret;
}
curl_easy_setopt(hnd, CURLOPT_WRITEFUNCTION, handle_data_to_file);
return 0;
}
Result downloadToFile(std::string url, std::string path)
{
Result ret = 0;
u64 offset = 0;
u32 bytesWritten = 0;
bool isDownloadToRAM = true;
printf("Downloading from:\n%s\nto:\n%s\n", url.c_str(), path.c_str());
void *socubuf = memalign(0x1000, 0x100000);
if (!socubuf)
{
return -1;
}
ret = socInit((u32*)socubuf, 0x100000);
if (R_FAILED(ret))
{
free(socubuf);
return ret;
}
Handle fileHandle;
ret = openFile(&fileHandle, path.c_str(), true);
if (R_FAILED(ret)) {
printf("Error: couldn't open file to write.\n");
socExit();
free(result_buf);
free(socubuf);
result_buf = NULL;
result_sz = 0;
result_written = 0;
return DL_ERROR_WRITEFILE;
}
result_fileHandle = &fileHandle;
CURL *hnd = curl_easy_init();
ret = setupContext(hnd, url.c_str());
if (downloadTotal > 30000000) {
ret = setupContextForDirectToFileDownload(hnd, url.c_str());
isDownloadToRAM = false;
}
if (ret != 0) {
socExit();
free(result_buf);
free(socubuf);
result_buf = NULL;
result_fileHandle = nullptr;
result_sz = 0;
result_written = 0;
FSFILE_Close(fileHandle);
return ret;
}
u64 startTime = osGetTime();
CURLcode cres = curl_easy_perform(hnd);
curl_easy_cleanup(hnd);
if (cres != CURLE_OK) {
printf("Error in:\ncurl\n");
socExit();
free(result_buf);
free(socubuf);
result_buf = NULL;
result_fileHandle = nullptr;
result_sz = 0;
result_written = 0;
FSFILE_Close(fileHandle);
return -1;
}
if (isDownloadToRAM == true) {
FSFILE_Write(fileHandle, &bytesWritten, offset, result_buf, result_written, 0);
}
u64 endTime = osGetTime();
u64 totalTime = endTime - startTime;
printf("Download took %llu milliseconds.\n", totalTime);
socExit();
free(result_buf);
free(socubuf);
result_buf = NULL;
result_fileHandle = nullptr;
result_sz = 0;
result_written = 0;
FSFILE_Close(fileHandle);
return 0;
}
Result downloadFromRelease(std::string url, std::string asset, std::string path, bool includePrereleases) Result downloadFromRelease(std::string url, std::string asset, std::string path, bool includePrereleases)
{ {
Result ret = 0; Result ret = 0;