From 13f2facdb6b5a556f01e0d04aaa6bda5d2b29417 Mon Sep 17 00:00:00 2001 From: Rémy Raes Date: Fri, 3 Nov 2023 16:32:41 +0100 Subject: Inital native code for verified mod auto-downloading (#545) Allows client to download a mod archive from the Thunderstore API, and extract included mods in the remote mods folder of the current game profile. Not all mods can be automatically downloaded, as it would cause some security issues, and Thunderstore mod name cannot be deduced from actual mod name: to be eligible to auto-downloading, a mod must appear in the list of verified mods. Said list and complete mod verification procedure are described here: https://github.com/R2Northstar/VerifiedMods Exposes two commands to test the feature: - `fetch_verified_mods` retrieves verified mods list from the GitHub organization, and stores it locally; - `download_mod` does the actual mod downloading/extraction job. --- NorthstarDLL/mods/autodownload/moddownloader.cpp | 552 +++++++++++++++++++++++ NorthstarDLL/mods/autodownload/moddownloader.h | 141 ++++++ 2 files changed, 693 insertions(+) create mode 100644 NorthstarDLL/mods/autodownload/moddownloader.cpp create mode 100644 NorthstarDLL/mods/autodownload/moddownloader.h (limited to 'NorthstarDLL/mods') diff --git a/NorthstarDLL/mods/autodownload/moddownloader.cpp b/NorthstarDLL/mods/autodownload/moddownloader.cpp new file mode 100644 index 00000000..9c1489c6 --- /dev/null +++ b/NorthstarDLL/mods/autodownload/moddownloader.cpp @@ -0,0 +1,552 @@ +#include "moddownloader.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +ModDownloader* g_pModDownloader; + +ModDownloader::ModDownloader() +{ + spdlog::info("Mod downloader initialized"); + + // Initialise mods list URI + char* clachar = strstr(GetCommandLineA(), CUSTOM_MODS_URL_FLAG); + if (clachar) + { + std::string url; + int iFlagStringLength = strlen(CUSTOM_MODS_URL_FLAG); + std::string cla = std::string(clachar); + if (strncmp(cla.substr(iFlagStringLength, 1).c_str(), "\"", 1)) + { + int space = cla.find(" "); + url = cla.substr(iFlagStringLength, space - iFlagStringLength); + } + else + { + std::string quote = "\""; + int quote1 = cla.find(quote); + int quote2 = (cla.substr(quote1 + 1)).find(quote); + url = cla.substr(quote1 + 1, quote2); + } + spdlog::info("Found custom verified mods URL in command line argument: {}", url); + modsListUrl = strdup(url.c_str()); + } + else + { + spdlog::info("Custom verified mods URL not found in command line arguments, using default URL."); + modsListUrl = strdup(DEFAULT_MODS_LIST_URL); + } +} + +size_t WriteToString(void* ptr, size_t size, size_t count, void* stream) +{ + ((std::string*)stream)->append((char*)ptr, 0, size * count); + return size * count; +} + +void ModDownloader::FetchModsListFromAPI() +{ + std::thread requestThread( + [this]() + { + CURLcode result; + CURL* easyhandle; + rapidjson::Document verifiedModsJson; + std::string url = modsListUrl; + + curl_global_init(CURL_GLOBAL_ALL); + easyhandle = curl_easy_init(); + std::string readBuffer; + + // Fetching mods list from GitHub repository + curl_easy_setopt(easyhandle, CURLOPT_CUSTOMREQUEST, "GET"); + curl_easy_setopt(easyhandle, CURLOPT_TIMEOUT, 30L); + curl_easy_setopt(easyhandle, CURLOPT_URL, url.c_str()); + curl_easy_setopt(easyhandle, CURLOPT_FAILONERROR, 1L); + curl_easy_setopt(easyhandle, CURLOPT_WRITEDATA, &readBuffer); + curl_easy_setopt(easyhandle, CURLOPT_WRITEFUNCTION, WriteToString); + result = curl_easy_perform(easyhandle); + + if (result == CURLcode::CURLE_OK) + { + spdlog::info("Mods list successfully fetched."); + } + else + { + spdlog::error("Fetching mods list failed: {}", curl_easy_strerror(result)); + goto REQUEST_END_CLEANUP; + } + + // Load mods list into local state + spdlog::info("Loading mods configuration..."); + verifiedModsJson.Parse(readBuffer); + for (auto i = verifiedModsJson.MemberBegin(); i != verifiedModsJson.MemberEnd(); ++i) + { + std::string name = i->name.GetString(); + std::string dependency = i->value["DependencyPrefix"].GetString(); + + std::unordered_map modVersions; + rapidjson::Value& versions = i->value["Versions"]; + assert(versions.IsArray()); + for (auto& attribute : versions.GetArray()) + { + assert(attribute.IsObject()); + std::string version = attribute["Version"].GetString(); + std::string checksum = attribute["Checksum"].GetString(); + modVersions.insert({version, {.checksum = checksum}}); + } + + VerifiedModDetails modConfig = {.dependencyPrefix = dependency, .versions = modVersions}; + verifiedMods.insert({name, modConfig}); + spdlog::info("==> Loaded configuration for mod \"" + name + "\""); + } + + spdlog::info("Done loading verified mods list."); + + REQUEST_END_CLEANUP: + curl_easy_cleanup(easyhandle); + }); + requestThread.detach(); +} + +size_t WriteData(void* ptr, size_t size, size_t nmemb, FILE* stream) +{ + size_t written; + written = fwrite(ptr, size, nmemb, stream); + return written; +} + +void FetchModSync(std::promise>&& p, std::string_view url, fs::path downloadPath) +{ + bool failed = false; + FILE* fp = fopen(downloadPath.generic_string().c_str(), "wb"); + CURLcode result; + CURL* easyhandle; + easyhandle = curl_easy_init(); + + curl_easy_setopt(easyhandle, CURLOPT_TIMEOUT, 30L); + curl_easy_setopt(easyhandle, CURLOPT_URL, url.data()); + curl_easy_setopt(easyhandle, CURLOPT_FAILONERROR, 1L); + curl_easy_setopt(easyhandle, CURLOPT_WRITEDATA, fp); + curl_easy_setopt(easyhandle, CURLOPT_WRITEFUNCTION, WriteData); + result = curl_easy_perform(easyhandle); + + if (result == CURLcode::CURLE_OK) + { + spdlog::info("Mod archive successfully fetched."); + goto REQUEST_END_CLEANUP; + } + else + { + spdlog::error("Fetching mod archive failed: {}", curl_easy_strerror(result)); + failed = true; + goto REQUEST_END_CLEANUP; + } + +REQUEST_END_CLEANUP: + curl_easy_cleanup(easyhandle); + fclose(fp); + p.set_value(failed ? std::optional() : std::optional(downloadPath)); +} + +std::optional ModDownloader::FetchModFromDistantStore(std::string_view modName, std::string_view modVersion) +{ + // Retrieve mod prefix from local mods list, or use mod name as mod prefix if bypass flag is set + std::string modPrefix = strstr(GetCommandLineA(), VERIFICATION_FLAG) ? modName.data() : verifiedMods[modName.data()].dependencyPrefix; + // Build archive distant URI + std::string archiveName = std::format("{}-{}.zip", modPrefix, modVersion.data()); + std::string url = STORE_URL + archiveName; + spdlog::info(std::format("Fetching mod archive from {}", url)); + + // Download destination + std::filesystem::path downloadPath = std::filesystem::temp_directory_path() / archiveName; + spdlog::info(std::format("Downloading archive to {}", downloadPath.generic_string())); + + // Download the actual archive + std::promise> promise; + auto f = promise.get_future(); + std::thread t(&FetchModSync, std::move(promise), std::string_view(url), downloadPath); + t.join(); + return f.get(); +} + +bool ModDownloader::IsModLegit(fs::path modPath, std::string_view expectedChecksum) +{ + if (strstr(GetCommandLineA(), VERIFICATION_FLAG)) + { + spdlog::info("Bypassing mod verification due to flag set up."); + return true; + } + + NTSTATUS status; + BCRYPT_ALG_HANDLE algorithmHandle = NULL; + BCRYPT_HASH_HANDLE hashHandle = NULL; + std::vector hash; + DWORD hashLength = 0; + DWORD resultLength = 0; + std::stringstream ss; + + constexpr size_t bufferSize {1 << 12}; + std::vector buffer(bufferSize, '\0'); + std::ifstream fp(modPath.generic_string(), std::ios::binary); + + // Open an algorithm handle + // This sample passes BCRYPT_HASH_REUSABLE_FLAG with BCryptAlgorithmProvider(...) to load a provider which supports reusable hash + status = BCryptOpenAlgorithmProvider( + &algorithmHandle, // Alg Handle pointer + BCRYPT_SHA256_ALGORITHM, // Cryptographic Algorithm name (null terminated unicode string) + NULL, // Provider name; if null, the default provider is loaded + BCRYPT_HASH_REUSABLE_FLAG); // Flags; Loads a provider which supports reusable hash + if (!NT_SUCCESS(status)) + { + goto cleanup; + } + + // Obtain the length of the hash + status = BCryptGetProperty( + algorithmHandle, // Handle to a CNG object + BCRYPT_HASH_LENGTH, // Property name (null terminated unicode string) + (PBYTE)&hashLength, // Address of the output buffer which recieves the property value + sizeof(hashLength), // Size of the buffer in bytes + &resultLength, // Number of bytes that were copied into the buffer + 0); // Flags + if (!NT_SUCCESS(status)) + { + // goto cleanup; + return false; + } + + // Create a hash handle + status = BCryptCreateHash( + algorithmHandle, // Handle to an algorithm provider + &hashHandle, // A pointer to a hash handle - can be a hash or hmac object + NULL, // Pointer to the buffer that recieves the hash/hmac object + 0, // Size of the buffer in bytes + NULL, // A pointer to a key to use for the hash or MAC + 0, // Size of the key in bytes + 0); // Flags + if (!NT_SUCCESS(status)) + { + goto cleanup; + } + + // Hash archive content + if (!fp.is_open()) + { + spdlog::error("Unable to open archive."); + return false; + } + fp.seekg(0, fp.beg); + while (fp.good()) + { + fp.read(buffer.data(), bufferSize); + std::streamsize bytesRead = fp.gcount(); + if (bytesRead > 0) + { + status = BCryptHashData(hashHandle, (PBYTE)buffer.data(), bytesRead, 0); + if (!NT_SUCCESS(status)) + { + goto cleanup; + } + } + } + + hash = std::vector(hashLength); + + // Obtain the hash of the message(s) into the hash buffer + status = BCryptFinishHash( + hashHandle, // Handle to the hash or MAC object + hash.data(), // A pointer to a buffer that receives the hash or MAC value + hashLength, // Size of the buffer in bytes + 0); // Flags + if (!NT_SUCCESS(status)) + { + goto cleanup; + } + + // Convert hash to string using bytes raw values + ss << std::hex << std::setfill('0'); + for (int i = 0; i < hashLength; i++) + { + ss << std::hex << std::setw(2) << static_cast(hash.data()[i]); + } + + spdlog::info("Expected checksum: {}", expectedChecksum.data()); + spdlog::info("Computed checksum: {}", ss.str()); + return expectedChecksum.compare(ss.str()) == 0; + +cleanup: + if (NULL != hashHandle) + { + BCryptDestroyHash(hashHandle); // Handle to hash/MAC object which needs to be destroyed + } + + if (NULL != algorithmHandle) + { + BCryptCloseAlgorithmProvider( + algorithmHandle, // Handle to the algorithm provider which needs to be closed + 0); // Flags + } + + return false; +} + +bool ModDownloader::IsModAuthorized(std::string_view modName, std::string_view modVersion) +{ + if (strstr(GetCommandLineA(), VERIFICATION_FLAG)) + { + spdlog::info("Bypassing mod verification due to flag set up."); + return true; + } + + if (!verifiedMods.contains(modName.data())) + { + return false; + } + + std::unordered_map versions = verifiedMods[modName.data()].versions; + return versions.count(modVersion.data()) != 0; +} + +void ModDownloader::ExtractMod(fs::path modPath) +{ + unzFile file; + std::string name; + fs::path modDirectory; + + file = unzOpen(modPath.generic_string().c_str()); + if (file == NULL) + { + spdlog::error("Cannot open archive located at {}.", modPath.generic_string()); + goto EXTRACTION_CLEANUP; + } + + unz_global_info64 gi; + int status; + status = unzGetGlobalInfo64(file, &gi); + if (status != UNZ_OK) + { + spdlog::error("Failed getting information from archive (error code: {})", status); + goto EXTRACTION_CLEANUP; + } + + // Mod directory name (removing the ".zip" fom the archive name) + name = modPath.filename().string(); + name = name.substr(0, name.length() - 4); + modDirectory = GetRemoteModFolderPath() / name; + + for (int i = 0; i < gi.number_entry; i++) + { + char zipFilename[256]; + unz_file_info64 fileInfo; + status = unzGetCurrentFileInfo64(file, &fileInfo, zipFilename, sizeof(zipFilename), NULL, 0, NULL, 0); + + // Extract file + { + std::error_code ec; + fs::path fileDestination = modDirectory / zipFilename; + spdlog::info("=> {}", fileDestination.generic_string()); + + // Create parent directory if needed + if (!std::filesystem::exists(fileDestination.parent_path())) + { + spdlog::info("Parent directory does not exist, creating it.", fileDestination.generic_string()); + if (!std::filesystem::create_directories(fileDestination.parent_path(), ec) && ec.value() != 0) + { + spdlog::error("Parent directory ({}) creation failed.", fileDestination.parent_path().generic_string()); + goto EXTRACTION_CLEANUP; + } + } + + // If current file is a directory, create directory... + if (fileDestination.generic_string().back() == '/') + { + // Create directory + if (!std::filesystem::create_directory(fileDestination, ec) && ec.value() != 0) + { + spdlog::error("Directory creation failed: {}", ec.message()); + goto EXTRACTION_CLEANUP; + } + } + // ...else create file + else + { + // Ensure file is in zip archive + if (unzLocateFile(file, zipFilename, 0) != UNZ_OK) + { + spdlog::error("File \"{}\" was not found in archive.", zipFilename); + goto EXTRACTION_CLEANUP; + } + + // Create file + const int bufferSize = 8192; + void* buffer; + int err = UNZ_OK; + FILE* fout = NULL; + + // Open zip file to prepare its extraction + status = unzOpenCurrentFile(file); + if (status != UNZ_OK) + { + spdlog::error("Could not open file {} from archive.", zipFilename); + goto EXTRACTION_CLEANUP; + } + + // Create destination file + fout = fopen(fileDestination.generic_string().c_str(), "wb"); + if (fout == NULL) + { + spdlog::error("Failed creating destination file."); + goto EXTRACTION_CLEANUP; + } + + // Allocate memory for buffer + buffer = (void*)malloc(bufferSize); + if (buffer == NULL) + { + spdlog::error("Error while allocating memory."); + goto EXTRACTION_CLEANUP; + } + + // Extract file to destination + do + { + err = unzReadCurrentFile(file, buffer, bufferSize); + if (err < 0) + { + spdlog::error("error {} with zipfile in unzReadCurrentFile", err); + break; + } + if (err > 0) + { + if (fwrite(buffer, (unsigned)err, 1, fout) != 1) + { + spdlog::error("error in writing extracted file\n"); + err = UNZ_ERRNO; + break; + } + } + } while (err > 0); + + if (err != UNZ_OK) + { + spdlog::error("An error occurred during file extraction (code: {})", err); + goto EXTRACTION_CLEANUP; + } + err = unzCloseCurrentFile(file); + if (err != UNZ_OK) + { + spdlog::error("error {} with zipfile in unzCloseCurrentFile", err); + } + + // Cleanup + if (fout) + fclose(fout); + } + } + + // Go to next file + if ((i + 1) < gi.number_entry) + { + status = unzGoToNextFile(file); + if (status != UNZ_OK) + { + spdlog::error("Error while browsing archive files (error code: {}).", status); + goto EXTRACTION_CLEANUP; + } + } + } + +EXTRACTION_CLEANUP: + if (unzClose(file) != MZ_OK) + { + spdlog::error("Failed closing mod archive after extraction."); + } +} + +void ModDownloader::DownloadMod(std::string modName, std::string modVersion) +{ + // Check if mod can be auto-downloaded + if (!IsModAuthorized(std::string_view(modName), std::string_view(modVersion))) + { + spdlog::warn("Tried to download a mod that is not verified, aborting."); + return; + } + + std::thread requestThread( + [this, modName, modVersion]() + { + fs::path archiveLocation; + + // Download mod archive + std::string expectedHash = verifiedMods[modName].versions[modVersion].checksum; + std::optional fetchingResult = FetchModFromDistantStore(std::string_view(modName), std::string_view(modVersion)); + if (!fetchingResult.has_value()) + { + spdlog::error("Something went wrong while fetching archive, aborting."); + goto REQUEST_END_CLEANUP; + } + archiveLocation = fetchingResult.value(); + if (!IsModLegit(archiveLocation, std::string_view(expectedHash))) + { + spdlog::warn("Archive hash does not match expected checksum, aborting."); + goto REQUEST_END_CLEANUP; + } + + // Extract downloaded mod archive + ExtractMod(archiveLocation); + + REQUEST_END_CLEANUP: + try + { + remove(archiveLocation); + } + catch (const std::exception& a) + { + spdlog::error("Error while removing downloaded archive: {}", a.what()); + } + + spdlog::info("Done downloading {}.", modName); + }); + + requestThread.detach(); +} + +void ConCommandFetchVerifiedMods(const CCommand& args) +{ + g_pModDownloader->FetchModsListFromAPI(); +} + +void ConCommandDownloadMod(const CCommand& args) +{ + if (args.ArgC() < 3) + { + return; + } + + // Split arguments string by whitespaces (https://stackoverflow.com/a/5208977) + std::string buffer; + std::stringstream ss(args.ArgS()); + std::vector tokens; + while (ss >> buffer) + tokens.push_back(buffer); + + std::string modName = tokens[0]; + std::string modVersion = tokens[1]; + g_pModDownloader->DownloadMod(modName, modVersion); +} + +ON_DLL_LOAD_RELIESON("engine.dll", ModDownloader, (ConCommand), (CModule module)) +{ + g_pModDownloader = new ModDownloader(); + RegisterConCommand("fetch_verified_mods", ConCommandFetchVerifiedMods, "fetches verified mods list from GitHub repository", FCVAR_NONE); + RegisterConCommand("download_mod", ConCommandDownloadMod, "downloads a mod from remote store", FCVAR_NONE); +} diff --git a/NorthstarDLL/mods/autodownload/moddownloader.h b/NorthstarDLL/mods/autodownload/moddownloader.h new file mode 100644 index 00000000..edaf090b --- /dev/null +++ b/NorthstarDLL/mods/autodownload/moddownloader.h @@ -0,0 +1,141 @@ +class ModDownloader +{ + private: + const char* VERIFICATION_FLAG = "-disablemodverification"; + const char* CUSTOM_MODS_URL_FLAG = "-customverifiedurl="; + const char* STORE_URL = "https://gcdn.thunderstore.io/live/repository/packages/"; + const char* DEFAULT_MODS_LIST_URL = "https://raw.githubusercontent.com/R2Northstar/VerifiedMods/master/mods.json"; + char* modsListUrl; + + struct VerifiedModVersion + { + std::string checksum; + }; + struct VerifiedModDetails + { + std::string dependencyPrefix; + std::unordered_map versions = {}; + }; + std::unordered_map verifiedMods = {}; + + /** + * Downloads a mod archive from distant store. + * + * This rebuilds the URI of the mod archive using both a predefined store URI + * and the mod dependency string from the `verifiedMods` variable, or using + * input mod name as mod dependency string if bypass flag is set up; fetched + * archive is then stored in a temporary location. + * + * If something went wrong during archive download, this will return an empty + * optional object. + * + * @param modName name of the mod to be downloaded + * @param modVersion version of the mod to be downloaded + * @returns location of the downloaded archive + */ + std::optional FetchModFromDistantStore(std::string_view modName, std::string_view modVersion); + + /** + * Checks whether a mod is verified. + * + * A mod is deemed verified/authorized through a manual validation process that is + * described here: https://github.com/R2Northstar/VerifiedMods; in practice, a mod + * is considered authorized if their name AND exact version appear in the + * `verifiedMods` variable. + * + * @param modName name of the mod to be checked + * @param modVersion version of the mod to be checked, must follow semantic versioning + * @returns whether the mod is authorized and can be auto-downloaded + */ + bool IsModAuthorized(std::string_view modName, std::string_view modVersion); + + /** + * Tells if a mod archive has not been corrupted. + * + * The mod validation procedure includes computing the SHA256 hash of the final + * archive, which is stored in the verified mods list. This hash is used by this + * very method to ensure the archive downloaded from the Internet is the exact + * same that has been manually verified. + * + * @param modPath path of the archive to check + * @param expectedChecksum checksum the archive should have + * @returns whether archive is legit + */ + bool IsModLegit(fs::path modPath, std::string_view expectedChecksum); + + /** + * Extracts a mod archive to the game folder. + * + * This extracts a downloaded mod archive from its original location to the + * current game profile, in the remote mods folder. + * + * @param modPath location of the downloaded archive + * @returns nothing + */ + void ExtractMod(fs::path modPath); + + public: + ModDownloader(); + + /** + * Retrieves the verified mods list from the central authority. + * + * The Northstar auto-downloading feature does NOT allow automatically installing + * all mods for various (notably security) reasons; mods that are candidate to + * auto-downloading are rather listed on a GitHub repository + * (https://raw.githubusercontent.com/R2Northstar/VerifiedMods/master/mods.json), + * which this method gets via a HTTP call to load into local state. + * + * If list fetching fails, local mods list will be initialized as empty, thus + * preventing any mod from being auto-downloaded. + * + * @returns nothing + */ + void FetchModsListFromAPI(); + + /** + * Downloads a given mod from Thunderstore API to local game profile. + * + * @param modName name of the mod to be downloaded + * @param modVersion version of the mod to be downloaded + * @returns nothing + **/ + void DownloadMod(std::string modName, std::string modVersion); + + enum ModInstallState + { + // Normal installation process + DOWNLOADING, + CHECKSUMING, + EXTRACTING, + DONE, // Everything went great, mod can be used in-game + + // Errors + FAILED, // Generic error message, should be avoided as much as possible + FAILED_READING_ARCHIVE, + FAILED_WRITING_TO_DISK, + MOD_FETCHING_FAILED, + MOD_CORRUPTED, // Downloaded archive checksum does not match verified hash + NO_DISK_SPACE_AVAILABLE, + NOT_FOUND // Mod is not currently being auto-downloaded + }; + + struct MOD_STATE + { + ModInstallState state; + int progress; + int total; + float ratio; + } modState = {}; + + /** + * Cancels installation of the mod. + * + * Prevents installation of the mod currently being installed, no matter the install + * progress (downloading, checksuming, extracting), and frees all resources currently + * being used in this purpose. + * + * @returns nothing + */ + void CancelDownload(); +}; -- cgit v1.2.3