properly handle text encoding differences with QueryManager clients

5 files changed, 290 insertions, 59 deletions

diff --git a/src/database_postgres.cc b/src/database_postgres.cc
index 7a676e2..80076c9 100644
--- a/src/database_postgres.cc
+++ b/src/database_postgres.cc
@@ -454,6 +454,9 @@ static int GetResultByteA(PGresult *Result, int Row, int Col, uint8 *Buffer, int
 	return Size;
 }
 
+#if 0
+// NOTE(fusion): DO NOT REMOVE. It is currently not being used so it's switched
+// off to avoid compiler warnings.
 static int GetResultIPAddress(PGresult *Result, int Row, int Col){
 	int IPAddress = 0;
 	int Format = PQfformat(Result, Col);
@@ -524,8 +527,8 @@ static int GetResultIPAddress(PGresult *Result, int Row, int Col){
 		}
 	}
 	return IPAddress;
-
 }
+#endif
 
 static bool ParseTimestamp(int *Dest, const char *String){
 	ASSERT(Dest != NULL && String != NULL);
@@ -633,46 +636,46 @@ static int GetResultTimestamp(PGresult *Result, int Row, int Col){
 	return Timestamp;
 }
 
-static void SkipWhitespace(int *Cursor, const char *String){
-	while(isspace(String[*Cursor])){
-		*Cursor += 1;
+static void SkipWhitespace(int *Position, const char *String){
+	while(isspace(String[*Position])){
+		*Position += 1;
 	}
 }
 
-static bool SkipNextChar(int Ch, int *Cursor, const char *String){
+static bool SkipNextChar(int Ch, int *Position, const char *String){
 	ASSERT(Ch != 0);
-	if(String[*Cursor] != Ch){
+	if(String[*Position] != Ch){
 		return false;
 	}
 
-	*Cursor += 1;
+	*Position += 1;
 	return true;
 }
 
-static bool ReadNextNumber(int *Dest, int *Cursor, const char *String){
-	SkipWhitespace(Cursor, String);
+static bool ReadNextNumber(int *Dest, int *Position, const char *String){
+	SkipWhitespace(Position, String);
 
 	bool Negative = false;
-	if(!isdigit(String[*Cursor])){
-		if(String[*Cursor] != '+' && String[*Cursor] != '-'){
+	if(!isdigit(String[*Position])){
+		if(String[*Position] != '+' && String[*Position] != '-'){
 			return false;
 		}
 
-		if(!isdigit(String[*Cursor + 1])){
+		if(!isdigit(String[*Position + 1])){
 			return false;
 		}
 
-		if(String[*Cursor] == '-'){
+		if(String[*Position] == '-'){
 			Negative = true;
 		}
 
-		*Cursor += 1;
+		*Position += 1;
 	}
 
 	int Number = 0;
-	while(isdigit(String[*Cursor])){
-		Number = (Number * 10) + (String[*Cursor] - '0');
-		*Cursor += 1;
+	while(isdigit(String[*Position])){
+		Number = (Number * 10) + (String[*Position] - '0');
+		*Position += 1;
 	}
 
 	if(Negative){
@@ -683,21 +686,21 @@ static bool ReadNextNumber(int *Dest, int *Cursor, const char *String){
 	return true;
 }
 
-static bool ReadNextWord(char *Dest, int DestCapacity, int *Cursor, const char *String){
+static bool ReadNextWord(char *Dest, int DestCapacity, int *Position, const char *String){
 	ASSERT(DestCapacity > 0);
 
-	SkipWhitespace(Cursor, String);
-	if(!isalpha(String[*Cursor])){
+	SkipWhitespace(Position, String);
+	if(!isalpha(String[*Position])){
 		return false;
 	}
 
 	int WritePos = 0;
-	while(isalpha(String[*Cursor])){
+	while(isalpha(String[*Position])){
 		if(WritePos < DestCapacity){
-			Dest[WritePos] = String[*Cursor];
+			Dest[WritePos] = String[*Position];
 			WritePos += 1;
 		}
-		*Cursor += 1;
+		*Position += 1;
 	}
 
 	bool Result = (WritePos < DestCapacity);
@@ -716,25 +719,25 @@ static bool ParseInterval(int *Dest, const char *String){
 	// is stable enough or it would break other client libraries as well.
 	ASSERT(Dest != NULL && String != NULL);
 	int Interval = 0;
-	int Cursor = 0;
+	int Position = 0;
 	bool Negate = false;
 	while(true){
-		SkipWhitespace(&Cursor, String);
-		if(String[Cursor] == 0){
+		SkipWhitespace(&Position, String);
+		if(String[Position] == 0){
 			break;
 		}
 
 		int Number;
-		if(!ReadNextNumber(&Number, &Cursor, String)){
+		if(!ReadNextNumber(&Number, &Position, String)){
 			// "Expected number"
 			return false;
 		}
 
-		if(SkipNextChar(':', &Cursor, String)){
+		if(SkipNextChar(':', &Position, String)){
 			int Minutes, Seconds;
-			if(!ReadNextNumber(&Minutes, &Cursor, String)
-			|| !SkipNextChar(':', &Cursor, String)
-			|| !ReadNextNumber(&Seconds, &Cursor, String)){
+			if(!ReadNextNumber(&Minutes, &Position, String)
+			|| !SkipNextChar(':', &Position, String)
+			|| !ReadNextNumber(&Seconds, &Position, String)){
 				// "Expected HH:MM:SS.FFFFFF"
 				return false;
 			}
@@ -750,15 +753,15 @@ static bool ParseInterval(int *Dest, const char *String){
 			}
 
 			// NOTE(fusion): Parse microseconds but ignore it.
-			if(SkipNextChar('.', &Cursor, String)){
+			if(SkipNextChar('.', &Position, String)){
 				int Frac;
-				int PrevCursor = Cursor;
-				if(!ReadNextNumber(&Frac, &Cursor, String)){
+				int PrevPosition = Position;
+				if(!ReadNextNumber(&Frac, &Position, String)){
 					// "Expected fractional part"
 					return false;
 				}
 
-				int FracDigits = Cursor - PrevCursor;
+				int FracDigits = Position - PrevPosition;
 				if(FracDigits > 6){
 					// "Too many fractional digits"
 					return false;
@@ -768,7 +771,7 @@ static bool ParseInterval(int *Dest, const char *String){
 			Interval += (Number * 3600 + Minutes * 60 + Seconds);
 		}else{
 			char Unit[16];
-			if(!ReadNextWord(Unit, sizeof(Unit), &Cursor, String)){
+			if(!ReadNextWord(Unit, sizeof(Unit), &Position, String)){
 				// "Expected unit"
 				return false;
 			}
@@ -813,14 +816,14 @@ static bool ParseInterval(int *Dest, const char *String){
 		}
 
 		char Direction[8];
-		if(ReadNextWord(Direction, sizeof(Direction), &Cursor, String)){
+		if(ReadNextWord(Direction, sizeof(Direction), &Position, String)){
 			if(!StringEqCI(Direction, "ago")){
 				// "Invalid interval direction"
 				return false;
 			}
 
-			SkipWhitespace(&Cursor, String);
-			if(String[Cursor] != 0){
+			SkipWhitespace(&Position, String);
+			if(String[Position] != 0){
 				// "Interval direction is expected only at the very end"
 				return false;
 			}
@@ -1184,10 +1187,10 @@ TDatabase *DatabaseOpen(void){
 		"user",
 		"password",
 		"connect_timeout",
-		"client_encoding",
 		"application_name",
 		"sslmode",
 		"sslrootcert",
+		"client_encoding",
 		NULL, // sentinel
 	};
 
@@ -1198,10 +1201,10 @@ TDatabase *DatabaseOpen(void){
 		g_Config.PostgreSQL.User,
 		g_Config.PostgreSQL.Password,
 		g_Config.PostgreSQL.ConnectTimeout,
-		g_Config.PostgreSQL.ClientEncoding,
 		g_Config.PostgreSQL.ApplicationName,
 		g_Config.PostgreSQL.SSLMode,
 		g_Config.PostgreSQL.SSLRootCert,
+		"UTF8",
 		NULL, // sentinel
 	};
 
diff --git a/src/database_sqlite.cc b/src/database_sqlite.cc
index 8523854..5b3bbc0 100644
--- a/src/database_sqlite.cc
+++ b/src/database_sqlite.cc
@@ -855,7 +855,7 @@ bool CharacterNameExists(TDatabase *Database, const char *Name, bool *Exists){
 	}
 
 	*Exists = (ErrorCode == SQLITE_ROW);
-	return false;
+	return true;
 }
 
 bool CreateCharacter(TDatabase *Database, int WorldID, int AccountID, const char *Name, int Sex){
diff --git a/src/query.cc b/src/query.cc
index c4d81a1..afcf664 100644
--- a/src/query.cc
+++ b/src/query.cc
@@ -1346,6 +1346,7 @@ void ProcessCancelHouseTransfer(TDatabase *Database, TQuery *Query){
 	// are kept permanently and this query is used to delete/flag it, in case
 	// the it didn't complete. We might need to refine `FinishHouseTransfers`.
 	//int HouseID = Buffer->Read16();
+	(void)Database;
 	QueryOk(Query);
 }
 
diff --git a/src/querymanager.cc b/src/querymanager.cc
index 5719504..86c2e47 100644
--- a/src/querymanager.cc
+++ b/src/querymanager.cc
@@ -136,6 +136,16 @@ int RoundSecondsToDays(int Seconds){
 	return (Seconds + 86399) / 86400;
 }
 
+uint32 HashString(const char *String){
+	// FNV1a 32-bits
+	uint32 Hash = 0x811C9DC5U;
+	for(int i = 0; String[i] != 0; i += 1){
+		Hash ^= (uint32)String[i];
+		Hash *= 0x01000193U;
+	}
+	return Hash;
+}
+
 bool StringEmpty(const char *String){
 	return String[0] == 0;
 }
@@ -231,14 +241,178 @@ bool StringFormat(char *Dest, int DestCapacity, const char *Format, ...){
 	return Written >= 0 && Written < DestCapacity;
 }
 
-uint32 HashString(const char *String){
-	// FNV1a 32-bits
-	uint32 Hash = 0x811C9DC5U;
-	for(int i = 0; String[i] != 0; i += 1){
-		Hash ^= (uint32)String[i];
-		Hash *= 0x01000193U;
+int UTF8SequenceSize(uint8 LeadingByte){
+	if((LeadingByte & 0x80) == 0){
+		return 1;
+	}else if((LeadingByte & 0xE0) == 0xC0){
+		return 2;
+	}else if((LeadingByte & 0xF0) == 0xE0){
+		return 3;
+	}else if((LeadingByte & 0xF8) == 0xF0){
+		return 4;
+	}else{
+		return 0;
 	}
-	return Hash;
+}
+
+bool UTF8IsTrailingByte(uint8 Byte){
+	return (Byte & 0xC0) == 0x80;
+}
+
+int UTF8EncodedSize(int Codepoint){
+	if(Codepoint < 0){
+		return 0;
+	}else if(Codepoint <= 0x7F){
+		return 1;
+	}else if(Codepoint <= 0x07FF){
+		return 2;
+	}else if(Codepoint <= 0xFFFF){
+		return 3;
+	}else if(Codepoint <= 0x10FFFF){
+		return 4;
+	}else{
+		return 0;
+	}
+}
+
+int UTF8FindNextLeadingByte(const char *Src, int SrcLength){
+	int Offset = 0;
+	while(Offset < SrcLength){
+		// NOTE(fusion): Allow the first byte to be a leading byte, in case we
+		// just want to advance from one leading byte to another.
+		if(Offset > 0 && !UTF8IsTrailingByte(Src[Offset])){
+			break;
+		}
+		Offset += 1;
+	}
+	return Offset;
+}
+
+int UTF8DecodeOne(const uint8 *Src, int SrcLength, int *OutCodepoint){
+	if(SrcLength <= 0){
+		return 0;
+	}
+
+	int Size = UTF8SequenceSize(Src[0]);
+	if(Size <= 0 || Size > SrcLength){
+		return 0;
+	}
+
+	for(int i = 1; i < Size; i += 1){
+		if(!UTF8IsTrailingByte(Src[i])){
+			return 0;
+		}
+	}
+
+	int Codepoint = 0;
+	switch(Size){
+		case 1:{
+			Codepoint = (int)Src[0];
+			break;
+		}
+
+		case 2:{
+			Codepoint = ((int)(Src[0] & 0x1F) <<  6)
+					|   ((int)(Src[1] & 0x3F) <<  0);
+			break;
+		}
+
+		case 3:{
+			Codepoint = ((int)(Src[0] & 0x0F) << 12)
+					|   ((int)(Src[1] & 0x3F) <<  6)
+					|   ((int)(Src[2] & 0x3F) <<  0);
+			break;
+		}
+
+		case 4:{
+			Codepoint = ((int)(Src[0] & 0x07) << 18)
+					|   ((int)(Src[1] & 0x3F) << 12)
+					|   ((int)(Src[2] & 0x3F) <<  6)
+					|   ((int)(Src[3] & 0x3F) <<  0);
+			break;
+		}
+	}
+
+	if(OutCodepoint){
+		*OutCodepoint = Codepoint;
+	}
+
+	return Size;
+}
+
+int UTF8EncodeOne(uint8 *Dest, int DestCapacity, int Codepoint){
+	int Size = UTF8EncodedSize(Codepoint);
+	if(Size > 0 && Size <= DestCapacity){
+		switch(Size){
+			case 1:{
+				Dest[0] = (uint8)Codepoint;
+				break;
+			}
+
+			case 2:{
+				Dest[0] = (uint8)(0xC0 | (0x1F & (Codepoint >>  6)));
+				Dest[1] = (uint8)(0x80 | (0x3F & (Codepoint >>  0)));
+				break;
+			}
+
+			case 3:{
+				Dest[0] = (uint8)(0xE0 | (0x0F & (Codepoint >> 12)));
+				Dest[1] = (uint8)(0x80 | (0x3F & (Codepoint >>  6)));
+				Dest[2] = (uint8)(0x80 | (0x3F & (Codepoint >>  0)));
+				break;
+			}
+
+			case 4:{
+				Dest[0] = (uint8)(0xF0 | (0x07 & (Codepoint >> 18)));
+				Dest[1] = (uint8)(0x80 | (0x3F & (Codepoint >> 12)));
+				Dest[2] = (uint8)(0x80 | (0x3F & (Codepoint >>  6)));
+				Dest[3] = (uint8)(0x80 | (0x3F & (Codepoint >>  0)));
+				break;
+			}
+		}
+	}
+
+	return Size;
+}
+
+// IMPORTANT(fusion): This function WON'T handle null-termination. It'll rather
+// convert any characters, INCLUDING the null-terminator, contained in the src
+// string. Invalid or NON-LATIN1 codepoints are translated into '?'.
+int UTF8ToLatin1(char *Dest, int DestCapacity, const char *Src, int SrcLength){
+	int ReadPos = 0;
+	int WritePos = 0;
+	while(ReadPos < SrcLength){
+		int Codepoint = -1;
+		int Size = UTF8DecodeOne((uint8*)(Src + ReadPos), (SrcLength - ReadPos), &Codepoint);
+		if(Size > 0){
+			ReadPos += Size;
+		}else{
+			ReadPos += UTF8FindNextLeadingByte((Src + ReadPos), (SrcLength - ReadPos));
+		}
+
+		if(WritePos < DestCapacity){
+			if(Codepoint >= 0 && Codepoint <= 0xFF){
+				Dest[WritePos] = (char)Codepoint;
+			}else{
+				Dest[WritePos] = '?';
+			}
+		}
+		WritePos += 1;
+	}
+
+	return WritePos;
+}
+
+// IMPORTANT(fusion): This function WON'T handle null-termination. It'll rather
+// convert any characters, INCLUDING the null-terminator, contained in the src
+// string. Note that LATIN1 characters translates directly into UNICODE codepoints.
+int Latin1ToUTF8(char *Dest, int DestCapacity, const char *Src, int SrcLength){
+	int WritePos = 0;
+	for(int ReadPos = 0; ReadPos < SrcLength; ReadPos += 1){
+		WritePos += UTF8EncodeOne((uint8*)(Dest + WritePos),
+				(DestCapacity - WritePos), (uint8)Src[ReadPos]);
+	}
+	return WritePos;
 }
 
 int HexDigit(int Ch){
@@ -509,8 +683,6 @@ bool ReadConfig(const char *FileName, TConfig *Config){
 			ParseStringBuf(Config->PostgreSQL.Password, Val);
 		}else if(StringEqCI(Key, "PostgreSQL.ConnectTimeout")){
 			ParseStringBuf(Config->PostgreSQL.ConnectTimeout, Val);
-		}else if(StringEqCI(Key, "PostgreSQL.ClientEncoding")){
-			ParseStringBuf(Config->PostgreSQL.ClientEncoding, Val);
 		}else if(StringEqCI(Key, "PostgreSQL.ApplicationName")){
 			ParseStringBuf(Config->PostgreSQL.ApplicationName, Val);
 		}else if(StringEqCI(Key, "PostgreSQL.SSLMode")){
@@ -601,7 +773,6 @@ int main(int argc, const char **argv){
 	StringBufCopy(g_Config.PostgreSQL.User,            "tibia");
 	StringBufCopy(g_Config.PostgreSQL.Password,        "");
 	StringBufCopy(g_Config.PostgreSQL.ConnectTimeout,  "");
-	StringBufCopy(g_Config.PostgreSQL.ClientEncoding,  "UTF8");
 	StringBufCopy(g_Config.PostgreSQL.ApplicationName, "QueryManager");
 	StringBufCopy(g_Config.PostgreSQL.SSLMode,         "");
 	StringBufCopy(g_Config.PostgreSQL.SSLRootCert,     "");
@@ -642,7 +813,6 @@ int main(int argc, const char **argv){
 	LOG("PostgreSQL dbname:                \"%s\"", g_Config.PostgreSQL.DBName);
 	LOG("PostgreSQL user:                  \"%s\"", g_Config.PostgreSQL.User);
 	LOG("PostgreSQL connect_timeout:       \"%s\"", g_Config.PostgreSQL.ConnectTimeout);
-	LOG("PostgreSQL client_encoding:       \"%s\"", g_Config.PostgreSQL.ClientEncoding);
 	LOG("PostgreSQL application_name:      \"%s\"", g_Config.PostgreSQL.ApplicationName);
 	LOG("PostgreSQL sslmode:               \"%s\"", g_Config.PostgreSQL.SSLMode);
 	LOG("PostgreSQL sslrootcert:           \"%s\"", g_Config.PostgreSQL.SSLRootCert);
diff --git a/src/querymanager.hh b/src/querymanager.hh
index 1b90a0d..e33bcc0 100644
--- a/src/querymanager.hh
+++ b/src/querymanager.hh
@@ -111,7 +111,6 @@ struct TConfig{
 		char User[30];
 		char Password[30];
 		char ConnectTimeout[30];
-		char ClientEncoding[30];
 		char ApplicationName[30];
 		char SSLMode[30];
 		char SSLRootCert[100];
@@ -153,6 +152,7 @@ void SleepMS(int DurationMS);
 void CryptoRandom(uint8 *Buffer, int Count);
 int RoundSecondsToDays(int Seconds);
 
+uint32 HashString(const char *String);
 bool StringEmpty(const char *String);
 bool StringEq(const char *A, const char *B);
 bool StringEqCI(const char *A, const char *B);
@@ -162,7 +162,14 @@ bool StringCopyN(char *Dest, int DestCapacity, const char *Src, int SrcLength);
 bool StringCopy(char *Dest, int DestCapacity, const char *Src);
 void StringCopyEllipsis(char *Dest, int DestCapacity, const char *Src);
 bool StringFormat(char *Dest, int DestCapacity, const char *Format, ...) ATTR_PRINTF(3, 4);
-uint32 HashString(const char *String);
+int UTF8SequenceSize(uint8 LeadingByte);
+bool UTF8IsTrailingByte(uint8 Byte);
+int UTF8EncodedSize(int Codepoint);
+int UTF8FindNextLeadingByte(const char *Src, int SrcLength);
+int UTF8DecodeOne(const uint8 *Src, int SrcLength, int *OutCodepoint);
+int UTF8EncodeOne(uint8 *Dest, int DestCapacity, int Codepoint);
+int UTF8ToLatin1(char *Dest, int DestCapacity, const char *Src, int SrcLength);
+int Latin1ToUTF8(char *Dest, int DestCapacity, const char *Src, int SrcLength);
 
 int HexDigit(int Ch);
 int ParseHexString(uint8 *Dest, int DestCapacity, const char *String);
@@ -391,6 +398,7 @@ struct TReadBuffer{
 		return Result;
 	}
 
+#if CLIENT_ENCODING_UTF8
 	void ReadString(char *Dest, int DestCapacity){
 		int Length = (int)this->Read16();
 		if(Length == 0xFFFF){
@@ -398,16 +406,39 @@ struct TReadBuffer{
 		}
 
 		if(Dest != NULL && DestCapacity > 0){
-			if(Length < DestCapacity && this->CanRead(Length)){
+			int Written = 0;
+			if(this->CanRead(Length) && Length < DestCapacity){
 				memcpy(Dest, this->Buffer + this->Position, Length);
-				Dest[Length] = 0;
-			}else{
-				Dest[0] = 0;
+				Written = Length;
 			}
+			memset((Dest + Written), 0, (DestCapacity - Written));
+		}
+
+		this->Position += Length;
+	}
+#else
+	void ReadString(char *Dest, int DestCapacity){
+		int Length = (int)this->Read16();
+		if(Length == 0xFFFF){
+			Length = (int)this->Read32();
+		}
+
+		if(Dest != NULL && DestCapacity > 0){
+			int Written = 0;
+			if(this->CanRead(Length)){
+				const char *Src = (const char*)(this->Buffer + this->Position);
+				Written = Latin1ToUTF8(Dest, DestCapacity, Src, Length);
+				if(Written >= DestCapacity){
+					Written = 0;
+				}
+			}
+
+			memset((Dest + Written), 0, (DestCapacity - Written));
 		}
 
 		this->Position += Length;
 	}
+#endif
 };
 
 struct TWriteBuffer{
@@ -466,6 +497,7 @@ struct TWriteBuffer{
 		this->Position += 4;
 	}
 
+#if CLIENT_ENCODING_UTF8
 	void WriteString(const char *String){
 		int StringLength = 0;
 		if(String != NULL){
@@ -485,6 +517,31 @@ struct TWriteBuffer{
 
 		this->Position += StringLength;
 	}
+#else
+	void WriteString(const char *String){
+		int StringLength = 0;
+		int OutputLength = 0;
+		if(String != NULL){
+			StringLength = (int)strlen(String);
+			OutputLength = UTF8ToLatin1(NULL, 0, String, (int)strlen(String));
+		}
+
+		if(OutputLength < 0xFFFF){
+			this->Write16((uint16)OutputLength);
+		}else{
+			this->Write16(0xFFFF);
+			this->Write32((uint32)OutputLength);
+		}
+
+		if(OutputLength > 0 && this->CanWrite(OutputLength)){
+			int Written = UTF8ToLatin1((char*)(this->Buffer + this->Position),
+					(this->Size - this->Position), String, StringLength);
+			ASSERT(Written == OutputLength);
+		}
+
+		this->Position += OutputLength;
+	}
+#endif
 
 	void Rewrite16(int Position, uint16 Value){
 		if((Position + 2) <= this->Position && !this->Overflowed()){