Fix issues of restriction of chat message charset (#391)v1.11.4-rc1v1.11.41.11.X

6 files changed, 104 insertions, 52 deletions

diff --git a/NorthstarDLL/NorthstarDLL.vcxproj b/NorthstarDLL/NorthstarDLL.vcxproj
index 3f30a8f5..839d66e6 100644
--- a/NorthstarDLL/NorthstarDLL.vcxproj
+++ b/NorthstarDLL/NorthstarDLL.vcxproj
@@ -443,6 +443,7 @@
     <ClInclude Include="squirrel\squirrelautobind.h" />

     <ClInclude Include="squirrel\squirrelclasstypes.h" />

     <ClInclude Include="squirrel\squirreldatatypes.h" />

+    <ClInclude Include="util\utils.h" />

     <ClInclude Include="util\version.h" />

   </ItemGroup>

   <ItemGroup>

@@ -530,6 +531,7 @@
     <ClCompile Include="squirrel\squirrelautobind.cpp" />

     <ClCompile Include="util\printcommands.cpp" />

     <ClCompile Include="util\printmaps.cpp" />

+    <ClCompile Include="util\utils.cpp" />

     <ClCompile Include="util\version.cpp" />

   </ItemGroup>

   <ItemGroup>

diff --git a/NorthstarDLL/NorthstarDLL.vcxproj.filters b/NorthstarDLL/NorthstarDLL.vcxproj.filters
index 5806c9b4..64fbb6e0 100644
--- a/NorthstarDLL/NorthstarDLL.vcxproj.filters
+++ b/NorthstarDLL/NorthstarDLL.vcxproj.filters
@@ -1167,6 +1167,9 @@
     <ClInclude Include="core\sourceinterface.h">

       <Filter>Header Files\core</Filter>

     </ClInclude>

+    <ClInclude Include="util\utils.h">

+      <Filter>Header Files\util</Filter>

+    </ClInclude>

   </ItemGroup>

   <ItemGroup>

     <None Include="..\include\spdlog\fmt\bundled\LICENSE.rst">

@@ -1402,6 +1405,12 @@
     <ClCompile Include="core\sourceinterface.cpp">

       <Filter>Source Files\core</Filter>

     </ClCompile>

+    <ClCompile Include="client\rejectconnectionfixes.cpp">

+      <Filter>Source Files</Filter>

+    </ClCompile>

+    <ClCompile Include="util\utils.cpp">

+      <Filter>Source Files\util</Filter>

+    </ClCompile>

   </ItemGroup>

   <ItemGroup>

     <MASM Include="audio_asm.asm">

diff --git a/NorthstarDLL/scripts/client/clientchathooks.cpp b/NorthstarDLL/scripts/client/clientchathooks.cpp
index d4fa0c41..a1a9e503 100644
--- a/NorthstarDLL/scripts/client/clientchathooks.cpp
+++ b/NorthstarDLL/scripts/client/clientchathooks.cpp
@@ -1,5 +1,6 @@
 #include "pch.h"
 #include "squirrel/squirrel.h"
+#include "util/utils.h"
 
 #include "server/serverchathooks.h"
 #include "client/localchatwriter.h"
@@ -8,25 +9,6 @@
 
 AUTOHOOK_INIT()
 
-static char* skip_valid_ansi_csi_sgr(char* str)
-{
-	if (*str++ != '\x1B')
-		return NULL;
-	if (*str++ != '[') // CSI
-		return NULL;
-	for (char* c = str; *c; c++)
-	{
-		if (*c >= '0' && *c <= '9')
-			continue;
-		if (*c == ';')
-			continue;
-		if (*c == 'm') // SGR
-			break;
-		return NULL;
-	}
-	return str;
-}
-
 // clang-format off
 AUTOHOOK(CHudChat__AddGameLine, client.dll + 0x22E580,
 void, __fastcall, (void* self, const char* message, int inboxId, bool isTeam, bool isDead))
@@ -49,13 +31,7 @@ void, __fastcall, (void* self, const char* message, int inboxId, bool isTeam, bo
 		payload = message + 1;
 	}
 
-	for (char* c = const_cast<char*>(message); *c; c++)
-	{
-		if (*c == '\x1B' && (c = skip_valid_ansi_csi_sgr(c)))
-			c--;
-		else if (*c <= 9 || (*c >= 12 && *c <= 31))
-			*c = ' ';
-	}
+	NS::Utils::RemoveAsciiControlSequences(const_cast<char*>(message), true);
 
 	SQRESULT result = g_pSquirrel<ScriptContext::CLIENT>->Call(
 		"CHudChat_ProcessMessageStartThread", static_cast<int>(senderId) - 1, payload, isTeam, isDead, type);
diff --git a/NorthstarDLL/server/serverchathooks.cpp b/NorthstarDLL/server/serverchathooks.cpp
index eb0034c0..0ea3c2d2 100644
--- a/NorthstarDLL/server/serverchathooks.cpp
+++ b/NorthstarDLL/server/serverchathooks.cpp
@@ -3,6 +3,7 @@
 #include "shared/exploit_fixes/ns_limits.h"
 #include "squirrel/squirrel.h"
 #include "server/r2server.h"
+#include "util/utils.h"
 
 #include <rapidjson/document.h>
 #include <rapidjson/stringbuffer.h>
@@ -34,38 +35,13 @@ void(__fastcall* MessageWriteByte)(int iValue);
 void(__fastcall* MessageWriteString)(const char* sz);
 void(__fastcall* MessageWriteBool)(bool bValue);
 
-static char* skip_valid_ansi_csi_sgr(char* str)
-{
-	if (*str++ != '\x1B')
-		return NULL;
-	if (*str++ != '[') // CSI
-		return NULL;
-	for (char* c = str; *c; c++)
-	{
-		if (*c >= '0' && *c <= '9')
-			continue;
-		if (*c == ';')
-			continue;
-		if (*c == 'm') // SGR
-			break;
-		return NULL;
-	}
-	return str;
-}
-
 bool bShouldCallSayTextHook = false;
 // clang-format off
 AUTOHOOK(_CServerGameDLL__OnReceivedSayTextMessage, server.dll + 0x1595C0,
 void, __fastcall, (CServerGameDLL* self, unsigned int senderPlayerId, const char* text, bool isTeam))
 // clang-format on
 {
-	for (char* c = const_cast<char*>(text); *c; c++)
-	{
-		if (*c == '\x1B' && (c = skip_valid_ansi_csi_sgr(c)))
-			c--;
-		else if (*c <= 9 || (*c >= 12 && *c <= 31))
-			*c = ' ';
-	}
+	NS::Utils::RemoveAsciiControlSequences(const_cast<char*>(text), true);
 
 	// MiniHook doesn't allow calling the base function outside of anywhere but the hook function.
 	// To allow bypassing the hook, isSkippingHook can be set.
diff --git a/NorthstarDLL/util/utils.cpp b/NorthstarDLL/util/utils.cpp
new file mode 100644
index 00000000..10c5b26d
--- /dev/null
+++ b/NorthstarDLL/util/utils.cpp
@@ -0,0 +1,83 @@
+#include "pch.h"
+#include <ctype.h>
+#include "utils.h"
+
+bool skip_valid_ansi_csi_sgr(char*& str)
+{
+	if (*str++ != '\x1B')
+		return false;
+	if (*str++ != '[') // CSI
+		return false;
+	for (char* c = str; *c; c++)
+	{
+		if (*c >= '0' && *c <= '9')
+			continue;
+		if (*c == ';' || *c == ':')
+			continue;
+		if (*c == 'm') // SGR
+			break;
+		return false;
+	}
+	return true;
+}
+
+void NS::Utils::RemoveAsciiControlSequences(char* str, bool allow_color_codes)
+{
+	for (char *pc = str, c = *pc; c = *pc; pc++)
+	{
+		// skip UTF-8 characters
+		int bytesToSkip = 0;
+		if ((c & 0xE0) == 0xC0)
+			bytesToSkip = 1; // skip 2-byte UTF-8 sequence
+		if ((c & 0xF0) == 0xE0)
+			bytesToSkip = 2; // skip 3-byte UTF-8 sequence
+		if ((c & 0xF8) == 0xF0)
+			bytesToSkip = 3; // skip 4-byte UTF-8 sequence
+		if ((c & 0xFC) == 0xF8)
+			bytesToSkip = 4; // skip 5-byte UTF-8 sequence
+		if ((c & 0xFE) == 0xFC)
+			bytesToSkip = 5; // skip 6-byte UTF-8 sequence
+
+		bool invalid = false;
+		char* orgpc = pc;
+		for (int i = 0; i < bytesToSkip; i++)
+		{
+			char next = pc[1];
+
+			// valid UTF-8 part
+			if ((next & 0xC0) == 0x80)
+			{
+				pc++;
+				continue;
+			}
+
+			// invalid UTF-8 part or encountered \0
+			invalid = true;
+			break;
+		}
+		if (invalid)
+		{
+			// erase the whole "UTF-8" sequence
+			for (char* x = orgpc; x <= pc; x++)
+				if (*x != '\0')
+					*x = ' ';
+				else
+					break;
+		}
+		if (bytesToSkip > 0)
+			continue; // this byte was already handled as UTF-8
+
+		// an invalid control character or an UTF-8 part outside of UTF-8 sequence
+		if ((iscntrl(c) && c != '\n' && c != '\r' && c != '\x1B') || (c & 0x80) != 0)
+		{
+			*pc = ' ';
+			continue;
+		}
+
+		if (c == '\x1B') // separate handling for this escape sequence...
+			if (allow_color_codes && skip_valid_ansi_csi_sgr(pc)) // ...which we allow for color codes...
+				pc--;
+			else // ...but remove it otherwise
+				*pc = ' ';
+	}
+}
diff --git a/NorthstarDLL/util/utils.h b/NorthstarDLL/util/utils.h
new file mode 100644
index 00000000..97b92f18
--- /dev/null
+++ b/NorthstarDLL/util/utils.h
@@ -0,0 +1,6 @@
+#pragma once
+
+namespace NS::Utils
+{
+	void RemoveAsciiControlSequences(char* str, bool allow_color_codes);
+}