From c8bfedee768e84b037b6fbebe4a2869dc140869a Mon Sep 17 00:00:00 2001 From: PabloMK7 Date: Mon, 27 Jan 2020 01:36:19 +0100 Subject: [PATCH] Improved download code. (Multithreaded) --- source/download/download.cpp | 312 ++++++++++++++++++++--------------- 1 file changed, 182 insertions(+), 130 deletions(-) diff --git a/source/download/download.cpp b/source/download/download.cpp index 5e1c0bb..03d89b3 100644 --- a/source/download/download.cpp +++ b/source/download/download.cpp @@ -42,7 +42,6 @@ static char* result_buf = NULL; static size_t result_sz = 0; -static Handle* result_fileHandle = nullptr; static size_t result_written = 0; std::vector _topText; std::string jsonName; @@ -63,6 +62,188 @@ bool progressBarType = 0; // 0 = Download | 1 = Extract extern u32 progressBar; extern bool isScriptSelected; +curl_off_t downloadTotal = 1; //Dont initialize with 0 to avoid division by zero later +curl_off_t downloadNow = 0; + +static FILE *downfile = NULL; +static size_t file_buffer_pos = 0; +static size_t file_toCommit_size = 0; +static char* g_buffers[2] = { NULL }; +static u8 g_index = 0; +static Thread fsCommitThread; +static LightEvent readyToCommit; +static LightEvent waitCommit; +static bool killThread = false; +static bool writeError = false; +#define FILE_ALLOC_SIZE 0x60000 + +static int curlProgress(CURL *hnd, + curl_off_t dltotal, curl_off_t dlnow, + curl_off_t ultotal, curl_off_t ulnow) +{ + downloadTotal = dltotal; + downloadNow = dlnow; + + return 0; +} + +bool filecommit() { + if (!downfile) return false; + fseek(downfile, 0, SEEK_END); + u32 byteswritten = fwrite(g_buffers[!g_index], 1, file_toCommit_size, downfile); + if (byteswritten != file_toCommit_size) return false; + file_toCommit_size = 0; + return true; +} + +static void commitToFileThreadFunc(void* args) { + LightEvent_Signal(&waitCommit); + while (true) { + LightEvent_Wait(&readyToCommit); + LightEvent_Clear(&readyToCommit); + if (killThread) threadExit(0); + writeError = !filecommit(); + LightEvent_Signal(&waitCommit); + } +} + +static size_t file_handle_data(char *ptr, size_t size, size_t nmemb, void *userdata) { + (void)userdata; + const size_t bsz = size * nmemb; + size_t tofill = 0; + if (writeError) return 0; + if (!g_buffers[g_index]) { + + LightEvent_Init(&waitCommit, RESET_STICKY); + LightEvent_Init(&readyToCommit, RESET_STICKY); + + s32 prio = 0; + svcGetThreadPriority(&prio, CUR_THREAD_HANDLE); + fsCommitThread = threadCreate(commitToFileThreadFunc, NULL, 0x1000, prio - 1, -2, true); + + g_buffers[0] = (char*)memalign(0x1000, FILE_ALLOC_SIZE); + g_buffers[1] = (char*)memalign(0x1000, FILE_ALLOC_SIZE); + + if (!fsCommitThread || !g_buffers[0] || !g_buffers[1]) return 0; + } + if (file_buffer_pos + bsz >= FILE_ALLOC_SIZE) { + tofill = FILE_ALLOC_SIZE - file_buffer_pos; + memcpy(g_buffers[g_index] + file_buffer_pos, ptr, tofill); + + LightEvent_Wait(&waitCommit); + LightEvent_Clear(&waitCommit); + file_toCommit_size = file_buffer_pos + tofill; + file_buffer_pos = 0; + svcFlushProcessDataCache(CUR_PROCESS_HANDLE, g_buffers[g_index], file_toCommit_size); + g_index = !g_index; + LightEvent_Signal(&readyToCommit); + } + memcpy(g_buffers[g_index] + file_buffer_pos, ptr + tofill, bsz - tofill); + file_buffer_pos += bsz - tofill; + return bsz; +} + +Result downloadToFile(std::string url, std::string path) { + + Result retcode = 0; + downloadTotal = 1; + downloadNow = 0; + int res; + CURL *hnd; + CURLcode cres; + + printf("Downloading from:\n%s\nto:\n%s\n", url.c_str(), path.c_str()); + const char* filepath = path.c_str(); + + void *socubuf = memalign(0x1000, 0x100000); + if (!socubuf) { + retcode = -1; + goto exit; + } + + res = socInit((u32*)socubuf, 0x100000); + if (R_FAILED(res)) { + retcode = res; + goto exit; + } + + makeDirs(strdup(filepath)); + downfile = fopen(filepath, "wb"); + if (!downfile) { + retcode = -2; + goto exit; + } + + hnd = curl_easy_init(); + curl_easy_setopt(hnd, CURLOPT_BUFFERSIZE, FILE_ALLOC_SIZE); + curl_easy_setopt(hnd, CURLOPT_URL, url.c_str()); + curl_easy_setopt(hnd, CURLOPT_NOPROGRESS, 0L); + curl_easy_setopt(hnd, CURLOPT_USERAGENT, USER_AGENT); + curl_easy_setopt(hnd, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(hnd, CURLOPT_FAILONERROR, 1L); + curl_easy_setopt(hnd, CURLOPT_ACCEPT_ENCODING, "gzip"); + curl_easy_setopt(hnd, CURLOPT_MAXREDIRS, 50L); + curl_easy_setopt(hnd, CURLOPT_XFERINFOFUNCTION, curlProgress); + curl_easy_setopt(hnd, CURLOPT_HTTP_VERSION, (long)CURL_HTTP_VERSION_2TLS); + curl_easy_setopt(hnd, CURLOPT_WRITEFUNCTION, file_handle_data); + curl_easy_setopt(hnd, CURLOPT_SSL_VERIFYPEER, 0L); + curl_easy_setopt(hnd, CURLOPT_VERBOSE, 1L); + curl_easy_setopt(hnd, CURLOPT_STDERR, stdout); + + cres = curl_easy_perform(hnd); + curl_easy_cleanup(hnd); + + if (cres != CURLE_OK) { + retcode = -cres; + goto exit; + } + + LightEvent_Wait(&waitCommit); + LightEvent_Clear(&waitCommit); + + file_toCommit_size = file_buffer_pos; + svcFlushProcessDataCache(CUR_PROCESS_HANDLE, g_buffers[g_index], file_toCommit_size); + g_index = !g_index; + if (!filecommit()) { + retcode = -3; + goto exit; + } + fflush(downfile); + +exit: + if (fsCommitThread) { + killThread = true; + LightEvent_Signal(&readyToCommit); + threadJoin(fsCommitThread, U64_MAX); + killThread = false; + fsCommitThread = NULL; + } + + socExit(); + + if (socubuf) { + free(socubuf); + } + if (downfile) { + fclose(downfile); + downfile = NULL; + } + if (g_buffers[0]) { + free(g_buffers[0]); + g_buffers[0] = NULL; + } + if (g_buffers[1]) { + free(g_buffers[1]); + g_buffers[1] = NULL; + } + g_index = 0; + file_buffer_pos = 0; + file_toCommit_size = 0; + writeError = false; + + return retcode; +} + // following function is from // https://github.com/angelsl/libctrfgh/blob/master/curl_test/src/main.c static size_t handle_data(char* ptr, size_t size, size_t nmemb, void* userdata) @@ -103,31 +284,6 @@ static size_t handle_data(char* ptr, size_t size, size_t nmemb, void* userdata) return bsz; } -static size_t handle_data_to_file(char* ptr, size_t size, size_t nmemb, void* userdata) -{ - (void) userdata; - const size_t bsz = size*nmemb; - - u32 bytesWritten = 0; - FSFILE_Write(*result_fileHandle, &bytesWritten, result_written, ptr, bsz, 0); - - result_written += bsz; - return bsz; -} - -curl_off_t downloadTotal = 1; //Dont initialize with 0 to avoid division by zero later -curl_off_t downloadNow = 0; - -static int curlProgress(CURL *hnd, - curl_off_t dltotal, curl_off_t dlnow, - curl_off_t ultotal, curl_off_t ulnow) -{ - downloadTotal = dltotal; - downloadNow = dlnow; - - return 0; -} - static Result setupContext(CURL *hnd, const char * url) { downloadTotal = 1; @@ -149,110 +305,6 @@ static Result setupContext(CURL *hnd, const char * url) return 0; } -static Result setupContextForDirectToFileDownload(CURL *hnd, const char * url) -{ - Result ret = setupContext(hnd, url); - if (ret != 0) { - return ret; - } - - curl_easy_setopt(hnd, CURLOPT_WRITEFUNCTION, handle_data_to_file); - - return 0; -} - -Result downloadToFile(std::string url, std::string path) -{ - Result ret = 0; - u64 offset = 0; - u32 bytesWritten = 0; - bool isDownloadToRAM = true; - printf("Downloading from:\n%s\nto:\n%s\n", url.c_str(), path.c_str()); - - void *socubuf = memalign(0x1000, 0x100000); - if (!socubuf) - { - return -1; - } - - ret = socInit((u32*)socubuf, 0x100000); - if (R_FAILED(ret)) - { - free(socubuf); - return ret; - } - - Handle fileHandle; - - ret = openFile(&fileHandle, path.c_str(), true); - if (R_FAILED(ret)) { - printf("Error: couldn't open file to write.\n"); - socExit(); - free(result_buf); - free(socubuf); - result_buf = NULL; - result_sz = 0; - result_written = 0; - return DL_ERROR_WRITEFILE; - } - - result_fileHandle = &fileHandle; - - CURL *hnd = curl_easy_init(); - ret = setupContext(hnd, url.c_str()); - if (downloadTotal > 30000000) { - ret = setupContextForDirectToFileDownload(hnd, url.c_str()); - isDownloadToRAM = false; - } - - if (ret != 0) { - socExit(); - free(result_buf); - free(socubuf); - result_buf = NULL; - result_fileHandle = nullptr; - result_sz = 0; - result_written = 0; - FSFILE_Close(fileHandle); - return ret; - } - - u64 startTime = osGetTime(); - - CURLcode cres = curl_easy_perform(hnd); - curl_easy_cleanup(hnd); - - if (cres != CURLE_OK) { - printf("Error in:\ncurl\n"); - socExit(); - free(result_buf); - free(socubuf); - result_buf = NULL; - result_fileHandle = nullptr; - result_sz = 0; - result_written = 0; - FSFILE_Close(fileHandle); - return -1; - } - if (isDownloadToRAM == true) { - FSFILE_Write(fileHandle, &bytesWritten, offset, result_buf, result_written, 0); - } - - u64 endTime = osGetTime(); - u64 totalTime = endTime - startTime; - printf("Download took %llu milliseconds.\n", totalTime); - - socExit(); - free(result_buf); - free(socubuf); - result_buf = NULL; - result_fileHandle = nullptr; - result_sz = 0; - result_written = 0; - FSFILE_Close(fileHandle); - return 0; -} - Result downloadFromRelease(std::string url, std::string asset, std::string path, bool includePrereleases) { Result ret = 0;