From e960b0d69d3f954d50e814a6bc4d6e206bde7f66 Mon Sep 17 00:00:00 2001 From: "Avi Halachmi (:avih)" Date: Tue, 30 Jan 2024 18:44:52 +0200 Subject: win32: UTF8_OUTPUT: recover quicker from bad byte MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When an unexpected value is detected in UTF-8, we should print the placeholder codepoint, and then recover whenever we detect a value which is valid for starting a new UTF-8 codepoint (including ASCII7). However, previously, we only tested recovery at the bytes following the unexpected one, and so if the first unexpected value was also valid for a new codepoint, then didn't rcover it. Now we check for recovery from the first unexpected byte, which, if recoverable, requires both placeholder printout and recovery, so the recovery "unwinding" is modified a bit to allow placeholder. Example of of a sequence which now recovers quicker than before: (where UTF-8 for U+1F600 "😀" is: 0xF0 0x9F 0x98 0x80) printf "\xF0\xF0\x9F\x98\x80A" Previously: ?A Now: ?😀A --- win32/winansi.c | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/win32/winansi.c b/win32/winansi.c index 6142a244c..66b040b31 100644 --- a/win32/winansi.c +++ b/win32/winansi.c @@ -1477,7 +1477,6 @@ static int writeCon_utf8(int fd, const char *u8buf, size_t u8siz) while (c & (0x80 >> topbits)) ++topbits; - process_byte: if (state == 0 && topbits == 0) { // valid ASCII7, state remains 0 codepoint = c; @@ -1494,18 +1493,26 @@ static int writeCon_utf8(int fd, const char *u8buf, size_t u8siz) state = topbits - 1; // remaining bytes after lead continue; - } else if (state >= 0) { - // invalid byte at state 0/1/2/3, add placeholder once - codepoint = CONFIG_SUBST_WCHAR; - state = -1; - } else { - // inside bad sequence (placeholder char already added) - if (topbits == 1 || topbits > 4) - continue; // still bad - // c is valid for state 0, process it with clean slate - state = 0; - goto process_byte; + // already bad (state<0), or unexpected c at state 0-3. + // placeholder is added only at the 1st (state>=0). + // regardless, c may be valid to reprocess as state 0 + // (even when it's the 1st unexpected in state 1/2/3) + int placeholder_done = state < 0; + + if (topbits < 5 && topbits != 1) { + --u8buf; // valid for state 0, reprocess + ++u8siz; + state = 0; + } else { + state = -1; // set/keep bad state + } + + if (placeholder_done) + continue; + + // 1st unexpected char, add placeholder + codepoint = CONFIG_SUBST_WCHAR; } // codepoint is complete -- cgit v1.2.3-55-g6feb