Merge pull request #340 from avih/win32-unicode-editing

Win32: support unicode editing

7 files changed, 160 insertions, 12 deletions

diff --git a/include/mingw.h b/include/mingw.h
index 232ffadd7..97db2f6a9 100644
--- a/include/mingw.h
+++ b/include/mingw.h
@@ -586,6 +586,18 @@ char *alloc_ext_space(const char *path);
 int add_win32_extension(char *p);
 char *file_is_win32_exe(const char *name);
 
+#if ENABLE_UNICODE_SUPPORT
+/*
+ * windows wchar_t is 16 bit, while linux (and busybox expectation) is 32.
+ * so when (busybox) unicode.h is included, wchar_t is 32 bit.
+ * Without unicode.h, MINGW_BB_WCHAR_T is busybox wide char (32),
+ * and wchar_t is Windows wide char (16).
+ */
+#define MINGW_BB_WCHAR_T uint32_t  /* keep in sync with unicode.h */
+
+MINGW_BB_WCHAR_T *bs_to_slash_u(MINGW_BB_WCHAR_T *p) FAST_FUNC;
+#endif
+
 char *bs_to_slash(char *p) FAST_FUNC;
 void slash_to_bs(char *p) FAST_FUNC;
 size_t remove_cr(char *p, size_t len) FAST_FUNC;
diff --git a/include/unicode.h b/include/unicode.h
index 0317a2151..e894f7148 100644
--- a/include/unicode.h
+++ b/include/unicode.h
@@ -87,6 +87,21 @@ void reinit_unicode(const char *LANG) FAST_FUNC;
 #  undef MB_CUR_MAX
 #  define MB_CUR_MAX 6
 
+#if ENABLE_PLATFORM_MINGW32
+  #undef wint_t
+  #undef mbstate_t
+  #undef mbstowcs
+  #undef wcstombs
+  #undef wcrtomb
+  #undef iswspace
+  #undef iswalnum
+  #undef iswpunct
+  #undef wcwidth
+
+  #undef wchar_t
+  #define wchar_t uint32_t  /* keep in sync with MINGW_BB_WCHAR_T */
+#endif
+
 /* Prevent name collisions */
 #  define wint_t    bb_wint_t
 #  define mbstate_t bb_mbstate_t
diff --git a/libbb/lineedit.c b/libbb/lineedit.c
index a6884c7e0..54f0edef0 100644
--- a/libbb/lineedit.c
+++ b/libbb/lineedit.c
@@ -726,8 +726,15 @@ static void input_forward(void)
 #if !ENABLE_PLATFORM_MINGW32
                 put_cur_glyph_and_inc_cursor();
 #else
+        /*
+         * inc_cursor improves forward cursor movement appearance on
+         * win 7/8 console, but it's broken with unicode wide-glyphs,
+         * e.g. paste and move forward over: echo 开开心心过每一天
+         * so disable inc_cursor when unicode is active (which is only
+         * windows 10+, where inc_cursor is not needed anyway).
+         */
         {
-                if (terminal_mode(FALSE) & VT_INPUT)
+                if (unicode_status == UNICODE_ON)
                         put_cur_glyph_and_inc_cursor();
                 else
                         inc_cursor();
@@ -770,6 +777,11 @@ static void add_match(char *matched, int sensitive)
                  || (!ENABLE_UNICODE_SUPPORT && *p >= 0x7f)
                  || (ENABLE_UNICODE_SUPPORT && *p == 0x7f)
 # else
+                /*
+                 * on Windows, *p > 0x7f is never control:
+                 * without unicode active: these are normal codepage chars.
+                 * with unicode active: these are UTF8 continuation bytes.
+                 */
                  || *p == 0x7f
 # endif
                 ) {
@@ -1318,6 +1330,12 @@ static NOINLINE void input_tab(smallint *lastWasTab)
 # if ENABLE_PLATFORM_MINGW32
         int chosen_index = 0;
         int chosen_sens = FALSE;
+        /*
+         * FIXME: the next three vars are unused with ENABLE_UNICODE_SUPPORT
+         * because the mingw code which uses them to update a tab-completion
+         * prefix to the correct case (e.g. ~/desk<tab> to ~/Desktop/) is
+         * not compiled, and so e.g. ~/desk<tab> completes to ~/desktop/ .
+         */
         unsigned orig_pfx_len;
         char *target;
         const char *source;
@@ -2803,7 +2821,11 @@ int FAST_FUNC read_line_input(line_input_t *st, const char *prompt, char *comman
 #if ENABLE_PLATFORM_MINGW32
                 case CTRL('Z'):
                         command_ps[command_len] = '\0';
+                #if ENABLE_UNICODE_SUPPORT
+                        bs_to_slash_u(command_ps);
+                #else
                         bs_to_slash(command_ps);
+                #endif
                         redraw(cmdedit_y, 0);
                         break;
 #endif
diff --git a/libbb/unicode.c b/libbb/unicode.c
index e98cbbf35..206ec0dcb 100644
--- a/libbb/unicode.c
+++ b/libbb/unicode.c
@@ -69,8 +69,14 @@ void FAST_FUNC init_unicode(void)
 void FAST_FUNC reinit_unicode(const char *LANG)
 {
         unicode_status = UNICODE_OFF;
+#if ENABLE_PLATFORM_MINGW32
+        /* enable unicode only when ACP is UTF8 and the env var is not 'C' */
+        if (GetACP() != CP_UTF8 || (LANG && LANG[0] == 'C' && LANG[1] == 0))
+                return;
+#else
         if (!LANG || !(strstr(LANG, ".utf") || strstr(LANG, ".UTF")))
                 return;
+#endif
         unicode_status = UNICODE_ON;
 }
 
@@ -653,6 +659,9 @@ int FAST_FUNC wcwidth(unsigned ucs)
                         { 0x0A38, 0x0A3A }, { 0x0A3F, 0x0A3F }, { 0xD167, 0xD169 },
                         { 0xD173, 0xD182 }, { 0xD185, 0xD18B }, { 0xD1AA, 0xD1AD },
                         { 0xD242, 0xD244 }
+#if ENABLE_PLATFORM_MINGW32
+                        , { 0xF3FB, 0xF3FF }
+#endif
                 };
                 /* Binary search in table of non-spacing characters in Supplementary Multilingual Plane */
                 if (in_interval_table(ucs ^ 0x10000, combining0x10000, ARRAY_SIZE(combining0x10000) - 1))
@@ -689,6 +698,11 @@ int FAST_FUNC wcwidth(unsigned ucs)
                 || (ucs >= 0xff00 && ucs <= 0xff60) /* Fullwidth Forms */
                 || (ucs >= 0xffe0 && ucs <= 0xffe6)
 #   endif
+#if ENABLE_PLATFORM_MINGW32
+#   if CONFIG_LAST_SUPPORTED_WCHAR >= 0x10000
+                || (ucs >= 0x1f600 && ucs <= 0x1f64f) /* Emoticons */
+#   endif
+#endif
 #   if CONFIG_LAST_SUPPORTED_WCHAR >= 0x20000
                 || ((ucs >> 17) == (2 >> 1)) /* 20000..3ffff: Supplementary and Tertiary Ideographic Planes */
 #   endif
diff --git a/scripts/mk_mingw64u_defconfig b/scripts/mk_mingw64u_defconfig
new file mode 100755
index 000000000..3cca78e5b
--- /dev/null
+++ b/scripts/mk_mingw64u_defconfig
@@ -0,0 +1,35 @@
+#!/bin/sh
+
+configs=$(dirname -- "$0")/../configs
+
+# replace each FOO=bar argument with -e 's/.*FOO.*/FOO=bar/', then sed "$@"
+set_build_opts() {
+    for v; do
+        set -- "$@" -e "s/.*${v%%=*}.*/$v/"
+        shift
+    done
+    sed "$@"
+}
+
+
+# Create unicode configs/mingw64u_defconfig from configs/mingw64_defconfig
+# by flipping some build options to enable:
+# - UTF8 manifest to support unicode on win 10 (filenames, etc).
+# - UTF8 terminal input (shell prompt, read).
+# - UTF8 editing - codepoint awareness (prompt, read):
+#   - Builtin libc unicode functions (mbstowcs etc - no UNICODE_USING_LOCALE).
+#   - Dynamic unicode based on ANSI codepage and ENV (CHECK_UNICODE_IN_ENV).
+#   - Screen-width awareness (COMBINING_WCHARS, WIDE_WCHARS)
+#   - Full unicode range (U+10FFFF - LAST_SUPPORTED_WCHAR=1114111)
+
+set_build_opts \
+    CONFIG_FEATURE_UTF8_MANIFEST=y \
+    CONFIG_FEATURE_UTF8_INPUT=y \
+    CONFIG_UNICODE_SUPPORT=y \
+    CONFIG_FEATURE_CHECK_UNICODE_IN_ENV=y \
+    CONFIG_SUBST_WCHAR=63 \
+    CONFIG_LAST_SUPPORTED_WCHAR=1114111 \
+    CONFIG_UNICODE_COMBINING_WCHARS=y \
+    CONFIG_UNICODE_WIDE_WCHARS=y \
+    < "$configs"/mingw64_defconfig \
+    > "$configs"/mingw64u_defconfig
diff --git a/win32/mingw.c b/win32/mingw.c
index 5e9c71226..dabb2a2e7 100644
--- a/win32/mingw.c
+++ b/win32/mingw.c
@@ -2119,6 +2119,20 @@ char * FAST_FUNC bs_to_slash(char *str)
         return str;
 }
 
+#if ENABLE_UNICODE_SUPPORT
+MINGW_BB_WCHAR_T * FAST_FUNC bs_to_slash_u(MINGW_BB_WCHAR_T *str)
+{
+        MINGW_BB_WCHAR_T *p;
+
+        for (p=str; *p; ++p) {
+                if ( *p == '\\' ) {
+                        *p = '/';
+                }
+        }
+        return str;
+}
+#endif
+
 void FAST_FUNC slash_to_bs(char *p)
 {
         for (; *p; ++p) {
diff --git a/win32/winansi.c b/win32/winansi.c
index bc3e69163..f280177e6 100644
--- a/win32/winansi.c
+++ b/win32/winansi.c
@@ -1284,6 +1284,44 @@ static void maybeEatUpto2ndHalfUp(HANDLE h, DWORD *ph1)
         }
 }
 
+// if the codepoint is a key-down event, remember it, else if
+// it's a key-up event with matching prior down - forget the down,
+// else (up without matching prior key-down) - change it to down.
+// We remember few prior key-down events so that a sequence
+// like X-down Y-down X-up Y-up won't trigger this hack for Y-up.
+// When up is changed into down there won't be further key-up event,
+// but that's OK because the caller ignores key-up events anyway.
+static void maybe_change_up_to_down(wchar_t key, BOOL *isdown)
+{
+        #define DOWN_BUF_SIZ 8
+        static wchar_t downbuf[DOWN_BUF_SIZ] = {0};
+        static int pos = 0;
+
+        if (*isdown) {
+                downbuf[pos++] = key;
+                pos = pos % DOWN_BUF_SIZ;
+                return;
+        }
+
+        // the missing-key-down issue was only observed with unicode values,
+        // so limit this hack to non-ASCII-7 values.
+        // also, launching a new shell/read process from CLI captures
+        // an ENTER-up event without prior down at this new process, which
+        // would otherwise change it to down - creating a wrong ENTER keypress.
+        if (key <= 127)
+                return;
+
+        // key up, try to match a prior down
+        for (int i = 0; i < DOWN_BUF_SIZ; ++i) {
+                if (downbuf[i] == key) {
+                        downbuf[i] = 0;  // "forget" this down
+                        return;
+                }
+        }
+
+        // no prior key-down - replace the up with down
+        *isdown = TRUE;
+}
 
 /*
  * readConsoleInput_utf8 behaves similar enough to ReadConsoleInputA when
@@ -1355,20 +1393,18 @@ BOOL readConsoleInput_utf8(HANDLE h, INPUT_RECORD *r, DWORD len, DWORD *got)
                 srec = *r;
                 codepoint = srec.Event.KeyEvent.uChar.UnicodeChar;
 
-                // At the cmd.exe console (but not windows terminal) we sometimes
-                // get key-up without the prior expected key-down event, sometimes
-                // with UnicodeChar of 0 instead the key-down event. work around it.
-                if (codepoint) {
-                        static wchar_t last_down = 0;
-
-                        if (srec.Event.KeyEvent.bKeyDown)
-                                last_down = codepoint;
-                        else if (codepoint > 127 && codepoint != last_down)
-                                srec.Event.KeyEvent.bKeyDown = TRUE;
-                }
+                // Observed when pasting unicode at cmd.exe console (but not
+                // windows terminal), we sometimes get key-up event without
+                // a prior matching key-down (or with key-down codepoint 0),
+                // so this call would change the up into down in such case.
+                // E.g. pastes fixed by this hack: U+1F600 "😀", or U+0C80 "ಀ"
+                if (codepoint)
+                        maybe_change_up_to_down(codepoint, &srec.Event.KeyEvent.bKeyDown);
 
                 // if it's a 1st (high) surrogate pair half, try to eat upto and
                 // excluding the 2nd (low) half, and combine them into codepoint.
+                // this does not interfere with the missing-key-down workaround
+                // (no issue if the down-buffer has 1st-half-down without up).
                 if (codepoint >= 0xD800 && codepoint <= 0xDBFF)
                         maybeEatUpto2ndHalfUp(h, &codepoint);