overdue unicode cleanup.

our new pdcurses can do utf8 output, let's use it.
convert output on a windows tty to codepage 1252.

scripts/map.lua

@@ -2,7 +2,8 @@ local path = 'scripts'
 if config.install then
 	path = config.install .. '/' .. path
 end
-package.path = package.path .. ';' .. path .. '/?.lua;' .. path .. '/?/init.lua'
+package.path = path .. '/?.lua;' .. path .. '/?/init.lua;' .. package.path
+
 require 'eressea.path'
 require 'eressea'
 require 'eressea.xmlconf'

scripts/run-tests-e2.lua

@@ -10,7 +10,7 @@ path = 'scripts'
 if config.install then
     path = config.install .. '/' .. path
 end
-package.path = package.path .. ';' .. path .. '/?.lua;' .. path .. '/?/init.lua'
+package.path = path .. '/?.lua;' .. path .. '/?/init.lua;' .. package.path
 
 config.rules = 'e2'
 

scripts/run-tests-e3.lua

@@ -10,7 +10,7 @@ path = 'scripts'
 if config.install then
     path = config.install .. '/' .. path
 end
-package.path = package.path .. ';' .. path .. '/?.lua;' .. path .. '/?/init.lua'
+package.path = path .. '/?.lua;' .. path .. '/?/init.lua;' .. package.path
 
 config.rules = 'e3'
 

scripts/run-tests.lua

@@ -10,7 +10,7 @@ path = 'scripts'
 if config.install then
     path = config.install .. '/' .. path
 end
-package.path = package.path .. ';' .. path .. '/?.lua;' .. path .. '/?/init.lua'
+package.path = path .. '/?.lua;' .. path .. '/?/init.lua;' .. package.path
 
 require 'eressea'
 require 'eressea.path'

scripts/run-turn.lua

@@ -192,7 +192,7 @@ local path = 'scripts'
 if config.install then
     path = config.install .. '/' .. path
 end
-package.path = package.path .. ';' .. path .. '/?.lua;' .. path .. '/?/init.lua'
+package.path = path .. '/?.lua;' .. path .. '/?/init.lua;' .. package.path
 require 'eressea'
 require 'eressea.xmlconf' -- read xml data
 

src/gmtool.c

@@ -58,29 +58,14 @@ state *current_state = NULL;
 #define IFL_BUILDINGS (1<<3)
 
 static WINDOW *hstatus;
-
+static int gm_utf8 = 0; /* does our curses support utf-8? */
-#ifdef STDIO_CP
-int gm_codepage = STDIO_CP;
-#else 
-int gm_codepage = -1;
-#endif
 
 static void unicode_remove_diacritics(const char *rp, char *wp) {
     while (*rp) {
-        if (gm_codepage >= 0 && *rp & 0x80) {
+        if (*rp & 0x80) {
             size_t sz = 0;
             unsigned char ch;
-            switch (gm_codepage) {
-            case 1252:
-                unicode_utf8_to_cp1252(&ch, rp, &sz);
-                break;
-            case 437:
-                unicode_utf8_to_cp437(&ch, rp, &sz);
-                break;
-            default:
             unicode_utf8_to_ascii(&ch, rp, &sz);
-                break;
-            }
             rp += sz;
             *wp++ = (char)ch;
         }
@@ -91,8 +76,13 @@ static void unicode_remove_diacritics(const char *rp, char *wp) {
     *wp = 0;
 }
 
-static void simplify(const char *rp, char *wp) {
+static void simplify(const char* rp, char* wp) {
+    if (!gm_utf8) {
         unicode_remove_diacritics(rp, wp);
+    }
+    else if (rp != wp) {
+        strcpy(wp, rp);
+    }
 }
 
 int umvwprintw(WINDOW *win, int y, int x, const char *format, ...) {
@@ -117,6 +107,10 @@ int umvwaddnstr(WINDOW *w, int y, int x, const char * str, int len) {
 
 static void init_curses(void)
 {
+#ifdef PDCURSES
+    /* PDCurses from vcpkg is compiled with UTF8 (and WIDE) support */
+    gm_utf8 = 1;
+#endif
     initscr();
 
     if (has_colors() || force_color) {

src/gmtool.h

@@ -23,7 +23,6 @@ extern "C" {
   void run_mapper(void);
 
   extern int force_color;
-  extern int gm_codepage;
 
   struct state *state_open(void);
   void state_close(struct state *);

src/main.c

@@ -64,7 +64,6 @@ static void load_inifile(void)
 #ifdef USE_CURSES
     /* only one value in the [editor] section */
     force_color = config_get_int("editor.color", force_color);
-    gm_codepage = config_get_int("editor.codepage", gm_codepage);
 #endif
 }
 

src/util/log.c

@@ -16,6 +16,14 @@
 #include <stdarg.h>
 #include <time.h>
 
+#ifdef WIN32
+#include <io.h>
+#define FISATTY(F) (_isatty(_fileno(F)))
+#else
+#include <unistd.h>
+#define FISATTY(F) (isatty(fileno(F)))
+#endif
+
 void errno_check(const char * file, int line) {
     if (errno) {
         log_info("errno is %d (%s) at %s:%d", 
@@ -24,8 +32,8 @@ void errno_check(const char * file, int line) {
     }
 }
 
-#ifdef STDIO_CP
+#if WIN32
-static int stdio_codepage = STDIO_CP;
+static int stdio_codepage = 1252;
 #else
 static int stdio_codepage = 0;
 #endif
@@ -63,7 +71,7 @@ void log_destroy(log_t *handle) {
     }
 }
 
-#define MAXLENGTH 4096          /* because I am lazy, CP437 output is limited to this many chars */
+#define MAXLENGTH 4096          /* because I am lazy, tty output is limited to this many chars */
 #define LOG_MAXBACKUPS 5
 
 static int
@@ -87,7 +95,7 @@ cp_convert(const char *format, unsigned char *buffer, size_t length, int codepag
             return result;
         }
         ++pos;
-        input += length;
+        input += size;
     }
     *pos = 0;
     return 0;
@@ -175,13 +183,16 @@ static void _log_write(FILE * stream, int codepage, const char *format, va_list
     }
 }
 
-static void log_stdio(void *data, int level, const char *module, const char *format, va_list args) {
+static void log_stdio(void* data, int level, const char* module, const char* format, va_list args) {
-    FILE *out = (FILE *)data;
+    FILE* out = (FILE*)data;
-    int codepage = (out == stderr || out == stdout) ? stdio_codepage : 0;
+    int codepage = 0;
-    const char *prefix = log_prefix(level);
+    const char* prefix = log_prefix(level);
     size_t len = strlen(format);
 
     (void)module;
+    if (stdio_codepage && (out == stderr || out == stdout)) {
+        codepage = FISATTY(out) ? stdio_codepage : 0;
+    }
     fprintf(out, "%s: ", prefix);
 
     _log_write(out, codepage, format, args);

src/util/unicode.c

@@ -572,7 +572,8 @@ unicode_utf8_to_cp437(unsigned char *cp_character, const char * utf8_string,
 int unicode_utf8_to_ascii(unsigned char *cp_character, const char * utf8_string,
     size_t *length)
 {
-    int result = unicode_utf8_to_cp437(cp_character, utf8_string, length);
+    wint_t ucs4_character;
+    int result = unicode_utf8_decode(&ucs4_character, utf8_string, length);
     if (result == 0) {
         if (*length > 1) {
             *cp_character = '?';