From cf8b267d31dc168673e68e336fd5727aad4e7333 Mon Sep 17 00:00:00 2001 From: EladNLG Date: Sat, 22 Jul 2023 23:02:22 +0300 Subject: Safe I/O 2 - Electric Boogaloo! (#404) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Safe I/O initial commit * Formatting * Formatting 2 * God help us all. * fix diffs part 1 * fix diffs 2 * Fix invalid filter file * Update NorthstarLauncher.vcxproj * Remove newline at EOF * fix the damn thing * God help the diffs * diff hate * diff hate the 2nd * Add comments since this code is unreadable Now, this code may look readable, but I barely understand what I did here myself. * Remove char limit and file definitions * Fix build * organization stuff * Fix build * More funcs, more stuff * format fix * oops, this is wrong. * fix a thing * Add max folder size (250MB) and add getallfiles * Fix build also size cap * reduce max folder size to 50MB * he forgor * i give up * aync edition????? * Fix max size bypass * fix it * Remove logs * Add max folder size launch arg * boop * FUCK SHIT * FUCKKKKKKKKKKK * Fix build * Fix crash * Fix error FINAL!!! * fix scriptjson.h not being found * fix scriptjson.h not appearing correctly * shit diffself * format fix * Fix indentation (thank you vscode) * Fix indentation again * adadasa * shit diffself adadasa Fix indentation (thank you vscode) Fix indentation again * format fix * Revert "shit diffself" This reverts commit b49a62b5b36784df1c01663cdec2e4e403279618. * Revert "Merge branch 'safe-io-rewrite' of https://github.com/EladNLG/NorthstarLauncher into safe-io-rewrite" This reverts commit 761eb8a1cb11ccf1dadf55f5be9a08700fe1d560, reversing changes made to 65549b2f799660d7fe7b5a53e81425922b996ea9. * Fix the diff finally * more diff fixes * Fix more diffs * Allow non-ascii characters that aren't NUL * format fix :( * this is untested and I don't wanna update the safe i/o test thing * fix build :( * fix mayhaps? * fix build * Fix lotsa stuff :D * FIxed :D * Rewrite error messages, rename CheckFileName to IsPathSafe * Shout out to clang-format! * oops, wrong error message :3 * SHOUT OUT TO CLANG-FORMAT!!! * Revert "SHOUT OUT TO CLANG-FORMAT!!!" This reverts commit c536384073528b4eee7c330dfab320a5527ff240. * :( * oop * Add failure callback (launcher) * unnecessary comment * holy shit format fix boooooo. BOOOOOO! * restrict path to ASCII * Fix ascii check I hate you all * reformat * Clang-format my beloved :) * Move the thing * Apply suggestions from code review Co-authored-by: Rémy Raes * clang-format no like code suggestions :( * name change * doc'd * - Replace the mutex map with a single mutex for all file actions - Fix Load calls sometimes failing and not calling NSHandleLoadResult - Add NSGetTotalSpaceRemaining which checks how much space a mod has, in KB - Add contstant MAX_FOLDER_SIZE which holds the maximum save folder size, in KB * add safe-io stuff to cmake * surrounds IsPathSafe with a try/catch --------- Co-authored-by: Rémy Raes Co-authored-by: uniboi --- NorthstarDLL/mods/modsavefiles.cpp | 573 +++++++++++++++++++++++++++++++++++++ NorthstarDLL/mods/modsavefiles.h | 16 ++ 2 files changed, 589 insertions(+) create mode 100644 NorthstarDLL/mods/modsavefiles.cpp create mode 100644 NorthstarDLL/mods/modsavefiles.h (limited to 'NorthstarDLL/mods') diff --git a/NorthstarDLL/mods/modsavefiles.cpp b/NorthstarDLL/mods/modsavefiles.cpp new file mode 100644 index 00000000..6676ec34 --- /dev/null +++ b/NorthstarDLL/mods/modsavefiles.cpp @@ -0,0 +1,573 @@ +#include "pch.h" +#include +#include +#include +#include "squirrel/squirrel.h" +#include "util/utils.h" +#include "mods/modmanager.h" +#include "modsavefiles.h" +#include "rapidjson/document.h" +#include "rapidjson/writer.h" +#include "rapidjson/stringbuffer.h" +#include "config/profile.h" +#include "core/tier0.h" +#include "rapidjson/error/en.h" +#include "scripts/scriptjson.h" + +SaveFileManager* g_pSaveFileManager; +int MAX_FOLDER_SIZE = 52428800; // 50MB (50 * 1024 * 1024) +fs::path savePath; + +/// +/// The directory we want the size of. +/// The file we're excluding from the calculation. +/// The size of the contents of the current directory, excluding a specific file. +uintmax_t GetSizeOfFolderContentsMinusFile(fs::path dir, std::string file) +{ + uintmax_t result = 0; + for (const auto& entry : fs::directory_iterator(dir)) + { + if (entry.path().filename() == file) + continue; + // fs::file_size may not work on directories - but does in some cases. + // per cppreference.com, it's "implementation-defined". + try + { + result += fs::file_size(entry.path()); + } + catch (fs::filesystem_error& e) + { + if (entry.is_directory()) + { + result += GetSizeOfFolderContentsMinusFile(entry.path(), ""); + } + } + } + return result; +} + +uintmax_t GetSizeOfFolder(fs::path dir) +{ + uintmax_t result = 0; + for (const auto& entry : fs::directory_iterator(dir)) + { + // fs::file_size may not work on directories - but does in some cases. + // per cppreference.com, it's "implementation-defined". + try + { + result += fs::file_size(entry.path()); + } + catch (fs::filesystem_error& e) + { + if (entry.is_directory()) + { + result += GetSizeOfFolderContentsMinusFile(entry.path(), ""); + } + } + } + return result; +} + +// Saves a file asynchronously. +template void SaveFileManager::SaveFileAsync(fs::path file, std::string contents) +{ + auto mutex = std::ref(fileMutex); + std::thread writeThread( + [mutex, file, contents]() + { + try + { + mutex.get().lock(); + + fs::path dir = file.parent_path(); + // this actually allows mods to go over the limit, but not by much + // the limit is to prevent mods from taking gigabytes of space, + // we don't need to be particularly strict. + if (GetSizeOfFolderContentsMinusFile(dir, file.filename().string()) + contents.length() > MAX_FOLDER_SIZE) + { + // tbh, you're either trying to fill the hard drive or use so much data, you SHOULD be congratulated. + spdlog::error(fmt::format("Mod spamming save requests? Folder limit bypassed despite previous checks. Not saving.")); + mutex.get().unlock(); + return; + } + + std::ofstream fileStr(file); + if (fileStr.fail()) + { + mutex.get().unlock(); + return; + } + + fileStr.write(contents.c_str(), contents.length()); + fileStr.close(); + + mutex.get().unlock(); + // side-note: this causes a leak? + // when a file is added to the map, it's never removed. + // no idea how to fix this - because we have no way to check if there are other threads waiting to use this file(?) + // tried to use m.try_lock(), but it's unreliable, it seems. + } + catch (std::exception ex) + { + spdlog::error("SAVE FAILED!"); + mutex.get().unlock(); + spdlog::error(ex.what()); + } + }); + + writeThread.detach(); +} + +// Loads a file asynchronously. +template int SaveFileManager::LoadFileAsync(fs::path file) +{ + int handle = ++m_iLastRequestHandle; + auto mutex = std::ref(fileMutex); + std::thread readThread( + [mutex, file, handle]() + { + try + { + mutex.get().lock(); + + std::ifstream fileStr(file); + if (fileStr.fail()) + { + spdlog::error("A file was supposed to be loaded but we can't access it?!"); + + g_pSquirrel->AsyncCall("NSHandleLoadResult", handle, false, ""); + mutex.get().unlock(); + return; + } + + std::stringstream stringStream; + stringStream << fileStr.rdbuf(); + + g_pSquirrel->AsyncCall("NSHandleLoadResult", handle, true, stringStream.str()); + + fileStr.close(); + mutex.get().unlock(); + // side-note: this causes a leak? + // when a file is added to the map, it's never removed. + // no idea how to fix this - because we have no way to check if there are other threads waiting to use this file(?) + // tried to use m.try_lock(), but it's unreliable, it seems. + } + catch (std::exception ex) + { + spdlog::error("LOAD FAILED!"); + g_pSquirrel->AsyncCall("NSHandleLoadResult", handle, false, ""); + mutex.get().unlock(); + spdlog::error(ex.what()); + } + }); + + readThread.detach(); + return handle; +} + +// Deletes a file asynchronously. +template void SaveFileManager::DeleteFileAsync(fs::path file) +{ + // P.S. I don't like how we have to async delete calls but we do. + auto m = std::ref(fileMutex); + std::thread deleteThread( + [m, file]() + { + try + { + m.get().lock(); + + fs::remove(file); + + m.get().unlock(); + // side-note: this causes a leak? + // when a file is added to the map, it's never removed. + // no idea how to fix this - because we have no way to check if there are other threads waiting to use this file(?) + // tried to use m.try_lock(), but it's unreliable, it seems. + } + catch (std::exception ex) + { + spdlog::error("DELETE FAILED!"); + m.get().unlock(); + spdlog::error(ex.what()); + } + }); + + deleteThread.detach(); +} + +// Checks if a file contains null characters. +bool ContainsInvalidChars(std::string str) +{ + // we don't allow null characters either, even if they're ASCII characters because idk if people can + // use it to circumvent the file extension suffix. + return std::any_of(str.begin(), str.end(), [](char c) { return c == '\0'; }); +} + +// Checks if the relative path (param) remains inside the mod directory (dir). +// Paths are restricted to ASCII because encoding is fucked and we decided we won't bother. +bool IsPathSafe(const std::string param, fs::path dir) +{ + try + { + auto const normRoot = fs::weakly_canonical(dir); + auto const normChild = fs::weakly_canonical(dir / param); + + auto itr = std::search(normChild.begin(), normChild.end(), normRoot.begin(), normRoot.end()); + // we return if the file is safe (inside the directory) and uses only ASCII chars in the path. + return itr == normChild.begin() && std::none_of( + param.begin(), + param.end(), + [](char c) + { + unsigned char unsignedC = static_cast(c); + return unsignedC > 127 || unsignedC < 0; + }); + } + catch (fs::filesystem_error err) + { + return false; + } +} + +// void NSSaveFile( string file, string data ) +ADD_SQFUNC("void", NSSaveFile, "string file, string data", "", ScriptContext::SERVER | ScriptContext::CLIENT | ScriptContext::UI) +{ + Mod* mod = g_pSquirrel->getcallingmod(sqvm); + if (mod == nullptr) + { + g_pSquirrel->raiseerror(sqvm, "Has to be called from a mod function!"); + return SQRESULT_ERROR; + } + + fs::path dir = savePath / fs::path(mod->m_ModDirectory).filename(); + std::string fileName = g_pSquirrel->getstring(sqvm, 1); + if (!IsPathSafe(fileName, dir)) + { + g_pSquirrel->raiseerror( + sqvm, + fmt::format( + "File name invalid ({})! Make sure it does not contain any non-ASCII character, and results in a path inside your mod's " + "save folder.", + fileName, + mod->Name) + .c_str()); + return SQRESULT_ERROR; + } + + std::string content = g_pSquirrel->getstring(sqvm, 2); + if (ContainsInvalidChars(content)) + { + g_pSquirrel->raiseerror( + sqvm, fmt::format("File contents may not contain NUL/\\0 characters! Make sure your strings are valid!", mod->Name).c_str()); + return SQRESULT_ERROR; + } + + fs::create_directories(dir); + // this actually allows mods to go over the limit, but not by much + // the limit is to prevent mods from taking gigabytes of space, + // this ain't a cloud service. + if (GetSizeOfFolderContentsMinusFile(dir, fileName) + content.length() > MAX_FOLDER_SIZE) + { + g_pSquirrel->raiseerror( + sqvm, + fmt::format( + "The mod {} has reached the maximum folder size.\n\nAsk the mod developer to optimize their data usage," + "or increase the maximum folder size using the -maxfoldersize launch parameter.", + mod->Name) + .c_str()); + return SQRESULT_ERROR; + } + + g_pSaveFileManager->SaveFileAsync(dir / fileName, content); + + return SQRESULT_NULL; +} + +// void NSSaveJSONFile(string file, table data) +ADD_SQFUNC("void", NSSaveJSONFile, "string file, table data", "", ScriptContext::SERVER | ScriptContext::CLIENT | ScriptContext::UI) +{ + Mod* mod = g_pSquirrel->getcallingmod(sqvm); + if (mod == nullptr) + { + g_pSquirrel->raiseerror(sqvm, "Has to be called from a mod function!"); + return SQRESULT_ERROR; + } + + fs::path dir = savePath / fs::path(mod->m_ModDirectory).filename(); + std::string fileName = g_pSquirrel->getstring(sqvm, 1); + if (!IsPathSafe(fileName, dir)) + { + g_pSquirrel->raiseerror( + sqvm, + fmt::format( + "File name invalid ({})! Make sure it does not contain any non-ASCII character, and results in a path inside your mod's " + "save folder.", + fileName, + mod->Name) + .c_str()); + return SQRESULT_ERROR; + } + + // Note - this cannot be done in the async func since the table may get garbage collected. + // This means that especially large tables may still clog up the system. + std::string content = EncodeJSON(sqvm); + if (ContainsInvalidChars(content)) + { + g_pSquirrel->raiseerror( + sqvm, fmt::format("File contents may not contain NUL/\\0 characters! Make sure your strings are valid!", mod->Name).c_str()); + return SQRESULT_ERROR; + } + + fs::create_directories(dir); + // this actually allows mods to go over the limit, but not by much + // the limit is to prevent mods from taking gigabytes of space, + // this ain't a cloud service. + if (GetSizeOfFolderContentsMinusFile(dir, fileName) + content.length() > MAX_FOLDER_SIZE) + { + g_pSquirrel->raiseerror( + sqvm, + fmt::format( + "The mod {} has reached the maximum folder size.\n\nAsk the mod developer to optimize their data usage," + "or increase the maximum folder size using the -maxfoldersize launch parameter.", + mod->Name) + .c_str()); + return SQRESULT_ERROR; + } + + g_pSaveFileManager->SaveFileAsync(dir / fileName, content); + + return SQRESULT_NULL; +} + +// int NS_InternalLoadFile(string file) +ADD_SQFUNC("int", NS_InternalLoadFile, "string file", "", ScriptContext::SERVER | ScriptContext::CLIENT | ScriptContext::UI) +{ + Mod* mod = g_pSquirrel->getcallingmod(sqvm, 1); // the function that called NSLoadFile :) + if (mod == nullptr) + { + g_pSquirrel->raiseerror(sqvm, "Has to be called from a mod function!"); + return SQRESULT_ERROR; + } + + fs::path dir = savePath / fs::path(mod->m_ModDirectory).filename(); + std::string fileName = g_pSquirrel->getstring(sqvm, 1); + if (!IsPathSafe(fileName, dir)) + { + g_pSquirrel->raiseerror( + sqvm, + fmt::format( + "File name invalid ({})! Make sure it does not contain any non-ASCII character, and results in a path inside your mod's " + "save folder.", + fileName, + mod->Name) + .c_str()); + return SQRESULT_ERROR; + } + + g_pSquirrel->pushinteger(sqvm, g_pSaveFileManager->LoadFileAsync(dir / fileName)); + + return SQRESULT_NOTNULL; +} + +// bool NSDoesFileExist(string file) +ADD_SQFUNC("bool", NSDoesFileExist, "string file", "", ScriptContext::SERVER | ScriptContext::CLIENT | ScriptContext::UI) +{ + Mod* mod = g_pSquirrel->getcallingmod(sqvm); + + fs::path dir = savePath / fs::path(mod->m_ModDirectory).filename(); + std::string fileName = g_pSquirrel->getstring(sqvm, 1); + if (!IsPathSafe(fileName, dir)) + { + g_pSquirrel->raiseerror( + sqvm, + fmt::format( + "File name invalid ({})! Make sure it does not contain any non-ASCII character, and results in a path inside your mod's " + "save folder.", + fileName, + mod->Name) + .c_str()); + return SQRESULT_ERROR; + } + + g_pSquirrel->pushbool(sqvm, fs::exists(dir / (fileName))); + return SQRESULT_NOTNULL; +} + +// int NSGetFileSize(string file) +ADD_SQFUNC("int", NSGetFileSize, "string file", "", ScriptContext::SERVER | ScriptContext::CLIENT | ScriptContext::UI) +{ + Mod* mod = g_pSquirrel->getcallingmod(sqvm); + + fs::path dir = savePath / fs::path(mod->m_ModDirectory).filename(); + std::string fileName = g_pSquirrel->getstring(sqvm, 1); + if (!IsPathSafe(fileName, dir)) + { + g_pSquirrel->raiseerror( + sqvm, + fmt::format( + "File name invalid ({})! Make sure it does not contain any non-ASCII character, and results in a path inside your mod's " + "save folder.", + fileName, + mod->Name) + .c_str()); + return SQRESULT_ERROR; + } + try + { + // throws if file does not exist + // we don't want stuff such as "file does not exist, file is unavailable" to be lethal, so we just try/catch fs errors + g_pSquirrel->pushinteger(sqvm, (int)(fs::file_size(dir / fileName) / 1024)); + } + catch (std::filesystem::filesystem_error const& ex) + { + spdlog::error("GET FILE SIZE FAILED! Is the path valid?"); + g_pSquirrel->raiseerror(sqvm, ex.what()); + return SQRESULT_ERROR; + } + return SQRESULT_NOTNULL; +} + +// void NSDeleteFile(string file) +ADD_SQFUNC("void", NSDeleteFile, "string file", "", ScriptContext::SERVER | ScriptContext::CLIENT | ScriptContext::UI) +{ + Mod* mod = g_pSquirrel->getcallingmod(sqvm); + + fs::path dir = savePath / fs::path(mod->m_ModDirectory).filename(); + std::string fileName = g_pSquirrel->getstring(sqvm, 1); + if (!IsPathSafe(fileName, dir)) + { + g_pSquirrel->raiseerror( + sqvm, + fmt::format( + "File name invalid ({})! Make sure it does not contain any non-ASCII character, and results in a path inside your mod's " + "save folder.", + fileName, + mod->Name) + .c_str()); + return SQRESULT_ERROR; + } + + g_pSaveFileManager->DeleteFileAsync(dir / fileName); + return SQRESULT_NOTNULL; +} + +// The param is not optional because that causes issues :) +ADD_SQFUNC("array", NS_InternalGetAllFiles, "string path", "", ScriptContext::CLIENT | ScriptContext::UI | ScriptContext::SERVER) +{ + // depth 1 because this should always get called from Northstar.Custom + Mod* mod = g_pSquirrel->getcallingmod(sqvm, 1); + fs::path dir = savePath / fs::path(mod->m_ModDirectory).filename(); + std::string pathStr = g_pSquirrel->getstring(sqvm, 1); + fs::path path = dir; + if (pathStr != "") + path = dir / pathStr; + if (!IsPathSafe(pathStr, dir)) + { + g_pSquirrel->raiseerror( + sqvm, + fmt::format( + "File name invalid ({})! Make sure it does not contain any non-ASCII character, and results in a path inside your mod's " + "save folder.", + pathStr, + mod->Name) + .c_str()); + return SQRESULT_ERROR; + } + try + { + g_pSquirrel->newarray(sqvm, 0); + for (const auto& entry : fs::directory_iterator(path)) + { + g_pSquirrel->pushstring(sqvm, entry.path().filename().string().c_str()); + g_pSquirrel->arrayappend(sqvm, -2); + } + return SQRESULT_NOTNULL; + } + catch (std::exception ex) + { + spdlog::error("DIR ITERATE FAILED! Is the path valid?"); + g_pSquirrel->raiseerror(sqvm, ex.what()); + return SQRESULT_ERROR; + } +} + +ADD_SQFUNC("bool", NSIsFolder, "string path", "", ScriptContext::CLIENT | ScriptContext::UI | ScriptContext::SERVER) +{ + Mod* mod = g_pSquirrel->getcallingmod(sqvm); + fs::path dir = savePath / fs::path(mod->m_ModDirectory).filename(); + std::string pathStr = g_pSquirrel->getstring(sqvm, 1); + fs::path path = dir; + if (pathStr != "") + path = dir / pathStr; + if (!IsPathSafe(pathStr, dir)) + { + g_pSquirrel->raiseerror( + sqvm, + fmt::format( + "File name invalid ({})! Make sure it does not contain any non-ASCII character, and results in a path inside your mod's " + "save folder.", + pathStr, + mod->Name) + .c_str()); + return SQRESULT_ERROR; + } + try + { + g_pSquirrel->pushbool(sqvm, fs::is_directory(path)); + return SQRESULT_NOTNULL; + } + catch (std::exception ex) + { + spdlog::error("DIR READ FAILED! Is the path valid?"); + spdlog::info(path.string()); + g_pSquirrel->raiseerror(sqvm, ex.what()); + return SQRESULT_ERROR; + } +} + +// side note, expensive. +ADD_SQFUNC("int", NSGetTotalSpaceRemaining, "", "", ScriptContext::CLIENT | ScriptContext::UI | ScriptContext::SERVER) +{ + Mod* mod = g_pSquirrel->getcallingmod(sqvm); + fs::path dir = savePath / fs::path(mod->m_ModDirectory).filename(); + g_pSquirrel->pushinteger(sqvm, (MAX_FOLDER_SIZE - GetSizeOfFolder(dir)) / 1024); + return SQRESULT_NOTNULL; +} + +// ok, I'm just gonna explain what the fuck is going on here because this +// is the pinnacle of my stupidity and I do not want to touch this ever +// again, yet someone will eventually have to maintain this. +template std::string EncodeJSON(HSquirrelVM* sqvm) +{ + // new rapidjson + rapidjson_document doc; + doc.SetObject(); + + // get the SECOND param + SQTable* table = sqvm->_stackOfCurrentFunction[2]._VAL.asTable; + // take the table and copy it's contents over into the rapidjson_document + EncodeJSONTable(table, &doc, doc.GetAllocator()); + + // convert JSON document to string + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + doc.Accept(writer); + + // return the converted string + return buffer.GetString(); +} + +ON_DLL_LOAD("engine.dll", ModSaveFFiles_Init, (CModule module)) +{ + savePath = fs::path(GetNorthstarPrefix()) / "save_data"; + g_pSaveFileManager = new SaveFileManager; + int parm = Tier0::CommandLine()->FindParm("-maxfoldersize"); + if (parm) + MAX_FOLDER_SIZE = std::stoi(Tier0::CommandLine()->GetParm(parm)); +} + +int GetMaxSaveFolderSize() +{ + return MAX_FOLDER_SIZE; +} diff --git a/NorthstarDLL/mods/modsavefiles.h b/NorthstarDLL/mods/modsavefiles.h new file mode 100644 index 00000000..a50fe62c --- /dev/null +++ b/NorthstarDLL/mods/modsavefiles.h @@ -0,0 +1,16 @@ +#pragma once +int GetMaxSaveFolderSize(); +bool ContainsInvalidChars(std::string str); + +class SaveFileManager +{ + public: + template void SaveFileAsync(fs::path file, std::string content); + template int LoadFileAsync(fs::path file); + template void DeleteFileAsync(fs::path file); + // Future proofed in that if we ever get multi-threaded SSDs this code will take advantage of them. + std::mutex fileMutex; + + private: + int m_iLastRequestHandle = 0; +}; -- cgit v1.2.3