Fixed UTF8 decoding for non-unicode build;

conclusion: @#%!@#% ASCII/Windows-1252/ISO-8859 must die!

git-svn-id: svn://svn.code.sf.net/p/magicseteditor/code/trunk@621 0fc631ac-6414-0410-93d0-97cfa31319b6

src/gui/set/window.cpp

@@ -402,7 +402,7 @@ void SetWindow::onUpdateUI(wxUpdateUIEvent& ev) {
 	}
 }
 
-static const int FILE_MENU_SIZE_BEFORE_RECENT_SETS = 11; // HACK; we should calculate the position to insert!
+static const int FILE_MENU_SIZE_BEFORE_RECENT_SETS = 12; // HACK; we should calculate the position to insert!
 void SetWindow::updateRecentSets() {
 	wxMenuBar* mb = GetMenuBar();
 	assert(number_of_recent_sets <= (UInt)settings.recent_sets.size()); // the number of recent sets should only increase

src/util/io/reader.cpp

@@ -147,10 +147,35 @@ String read_utf8_line(wxInputStream& input, bool eat_bom, bool until_eof) {
 		Char* result_buf = result.GetWriteBuf(size + 1);
 		wxConvUTF8.MB2WC(result_buf, &buffer[0], size + 1);
 		result.UngetWriteBuf(size);
+		return eat_bom ? decodeUTF8BOM(result) : result;
 	#else
-		result = String(&buffer[0], *wxConvCurrent);
+		// first to wchar, then back to local
+		vector<wchar_t> buf2; buf2.resize(size+1);
+		wxConvUTF8.MB2WC(&buf2[0], &buffer[0], size + 1);
+		// eat BOM?
+		if (eat_bom && buf2[0]==0xFEFF ) {
+			buf2.erase(buf2.begin()); // remove BOM
+		}
+		// convert
+		#ifdef __WXMSW__
+			// size includes null terminator
+			size = ::WideCharToMultiByte(CP_ACP, 0, &buf2[0], -1, nullptr, 0, nullptr, nullptr);
+			Char* result_buf = result.GetWriteBuf(size);
+			::WideCharToMultiByte(CP_ACP, 0, &buf2[0], -1, result_buf, (int)size, nullptr, nullptr);
+			result.UngetWriteBuf(size - 1);
+		#else
+			for (size_t i = 0 ; i < size ; ++i) {
+				wchar_t wc = buf2[i];
+				if (wc < 0xFF) {
+					result += (Char)wc;
+				} else {
+					// not valid in Latin1
+					result += '?';
+				}
+			}
+		#endif
+		return result;
 	#endif
-	return eat_bom ? decodeUTF8BOM(result) : result;
 }
 
 void Reader::readLine(bool in_string) {