1 files changed, 360 insertions, 0 deletions

diff --git a/scripts/mkwcwidth b/scripts/mkwcwidth
new file mode 100755
index 000000000..515128abb
--- /dev/null
+++ b/scripts/mkwcwidth
@@ -0,0 +1,360 @@
+#!/bin/sh
+#
+# Generate a wcwidth C implementation from Unicode data (tested v7 - v17)
+#
+# The MIT License (MIT)
+#
+# Copyright (C) 2025 Avi Halachmi <avihpit at yahoo.com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+
+# Latest Unicode data source files:
+#   https://www.unicode.org/Public/UCD/latest/ucd/UnicodeData.txt
+#   https://www.unicode.org/Public/UCD/latest/ucd/EastAsianWidth.txt
+#
+# License and term of use in either URL:
+#   http://www.unicode.org/copyright.html
+#   http://www.unicode.org/terms_of_use.html
+
+
+export LC_ALL=C
+self=${0##*/}
+awk=${AWK:-awk} sed=${SED:-sed}
+
+URL_base=https://www.unicode.org/Public/UCD/latest/ucd
+URL_verbase=https://www.unicode.org/Public/%s/ucd
+ud_file=UnicodeData.txt
+eaw_file=EastAsianWidth.txt
+
+# int by default (int is 32 bit and short is 16 on unix/linux/osx/windows, and
+# doesn't require stdint.h - https://en.cppreference.com/w/cpp/language/types)
+u32=${U32:-unsigned}        # or uint32_t
+u16=${U16:-unsigned short}  # or uint16_t
+
+err() { >&2 printf %s\\n "$self: $*"; exit 1; }
+
+case ${1-} in -h | --help)
+        echo "Usage: $self [DL=VERSION] [FATTR]"
+        echo "Print a wcwidth C implementation to stdout."
+        echo
+        echo "Uses the files ./$ud_file and ./$eaw_file ."
+        echo "If the files are missing, the latest will be downloaded."
+        echo
+        echo "If DL=XXX is provided, force-download + overwrite, where XXX is:"
+        echo "- 'latest' -> $URL_base/..."
+        echo "- 'draft', '16.0.0' etc -> $(printf "$URL_verbase" XXX)/..."
+        echo
+        echo "If given, FATTR will be inserted as 'int FATTR wcwidth(...) {...}'."
+        echo "Optional env vars:"
+        echo '  $FN           Function name to generate. Default: wcwidth.'
+        echo '  $U32, $U16    unsigned codepoint, u16 C-types. Default: int/short.'
+        echo '  $AWK, $SED    used programs (word-split, e.g. AWK="busybox awk").'
+        echo
+        echo 'Requires: sh, curl/wget, sed, awk, and few more POSIX utilities.'
+        exit
+esac
+[ "${1-}" = -- ] && shift
+
+DL=
+case ${1-} in DL=*) DL=${1#DL=}; shift; esac
+[ "${DL-}" ] && [ "$DL" != latest ] && URL_base=$(printf "$URL_verbase" "$DL")
+
+FUNC_ATTR=${1:+$1 }
+
+url2file() {  # wget errors on 404-not-found etc, curl requires -f for that
+        >&2 echo "[$1  ->  $2]"
+        rm -f -- "$2.tmp"
+        { { [ "$(which wget)" ] && wget    -O "$2.tmp" "$1"; } ||
+          { [ "$(which curl)" ] && curl -f -o "$2.tmp" "$1"; }
+        } && mv -- "$2.tmp" "$2"
+}
+
+[ -r "$ud_file" ] && [ -r "$eaw_file" ] && [ -z "$DL" ] || {
+        url2file "$URL_base/$ud_file"  "$ud_file"  &&
+        url2file "$URL_base/$eaw_file" "$eaw_file" ||
+                err "can't download $ud_file and/or $eaw_file. abort."
+}
+
+
+# extractors of zero-width and wide-width ranges from the Unicode data files.
+#
+# At https://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c - the mother of most current
+# wcwidth implementations, by Markus Kuhn (Unicode 5.0) - the width is:
+# - 0 for codepoint 0.
+# - -1 for control ranges C0/C1, and DEL (1-31, 128-159, 127)
+# - 0 if U+200B, cat is Me|Mn|Cf but not U+00AD, Hangul Jamo U+1160..U+11FF.
+# - else 2 if East_Asian_Width property is F|W (fullwidth, wide).
+# - else 1.
+#
+# We add as 0-width:
+# - Category Mc (combining, diacritics[-like])
+# - Hangul Jamo Extended-B U+D7B0..U+D7FF (combining, wasn't in Unicode 5.0)
+# - Emoji modifiers U+1F3FB..U+1F3FF (wasn't in Unicode 5.0)
+#
+# Hangul Jamo, Emoji mods are identified at the data automatically by name.
+# U+200B is category Cf since at least 5.0.0 - no need for manual override.
+#
+# Python wcwidth https://github.com/jquast/wcwidth is like us, plus 0-width:
+# - Cat Zl,Zp (2 codepoints: line/para. sep.), but terminals disagree on them.
+# - Few unassigned codepoints at Hangul Jamo Extended-B - it adds the range
+#   manually (includes unassigned), we get (assigned) codepoints at the data.
+# - (U+00AD was 0, fixed to 1 after https://github.com/jquast/wcwidth/issues/8)
+#
+#
+# UnicodeData.txt is lines of:
+#   CODEPOINT;NAME;CATEGORY;(...more) where CODEPOINT is 4-6 hex digits
+# or pairs of:
+#   CODEPOINT;<XNAME, First>;CATEGORY;(...more)  where XNAME is group-ish name
+#   CODEPOINT;<XNAME, Last>;CATEGORY;(...more)
+#
+# EastAsianWidth.txt is lines of:
+#   #... (comment)
+# or
+#   CODEPOINT[..LAST];EAW # <stuff>  (4-6 hex digits, maybe spaces around ";")
+# where EAW is East_Asian_Width property ("F", "W", "A", "N", etc)
+
+# replace First/Last pairs in UnicodeData with one line of FIRST..LAST range.
+# assume that First/Last are at consecutive lines, and XNAME/CAT don't change.
+# (no-op for us at Unicode v16 - doesn't add new wide/zero width codepoints)
+ud_file_ranges() {
+        $awk -F\; '$2 ~ /, First>$/ { printf $1 ".."; next }; 1' < "$ud_file"
+}
+
+# output unsorted/overlapping lines of hex "CODEPOINT[..LAST] WIDTH" (0/1/2)
+raw_ranges() {
+        # wide ranges according to the EAW property
+        $sed -e '/^#/d' -e 's/;/ /' < "$eaw_file" |  # comment lines, ';'
+        $awk '$2 ~ /^[FW]$/ { print $1, 2 }'
+
+        # wide by name (older Unicode had EAW==N for some FULLWIDTH names)
+        ud_file_ranges | $awk -F\; '$2 ~ /FULLWIDTH/ { print $1, 2 }'
+
+        # zero-width ranges (override wide-width)
+        ud_file_ranges | $awk -F\; '
+                $3 ~ /^(Me|Mn|Mc|Cf)$/ ||
+                $2 ~ /EMOJI MODIFIER|HANGUL J[OU]NGSEONG/ { print $1, 0 }'
+
+        # override soft-hyphen as width 1
+        echo 00AD 1
+}
+
+# lines of hex "CODEPOINT[..LAST] WIDTH"  ->  decimal "FIRST LAST WIDTH"
+as_decimal_ranges() {
+        while read range width; do
+                echo $((0x${range%%.*})) $((0x${range##*.})) $width
+        done
+}
+
+# to generate the final ranges list, we need to sort the raw ranges list,
+# ensure zero-width overrides wide-width, join adjacent/overlap ranges,
+# and split big ranges - our C structure uses 15 bits to store LAST-FIRST of
+# each range, and ranges are not allowed to cross plane boundary.
+#
+# we use simple (and slow) approach: codepoint-bucket-store all the individual
+# widths at the input ranges as they come, then scan the whole Unicode range
+# (~ 1M buckets) and output continuous ranges of width 0/2. Therefore, later
+# inputs of codepoint X override earlier, so width 2 should arrive before W 0.
+#
+# there's ~3K input ranges of ~200K codepoints, and the output is 450+ ranges.
+# depending on sys/awk performance, it's 200ms - 2s, up to ~7s on low-end ARM.
+# MERGE_RANGES_C (bottom of this file) is drop-in replacement, runs in few ms.
+
+# sorted lines of "FIRST LAST WIDTH" (FIRST/LAST: hex digits, WIDTH: 0 or 2)
+ranges=$(raw_ranges | as_decimal_ranges | $awk '
+
+    # The final ~ 1M codepoints scan can take few secs, up to 30+s on low end.
+    # so use a simple optimization: divide the unicode range to segments which
+    # do not cross plane boundary (some 2^X, tuned for speed), mark segments
+    # where an input range can change the width at the final scan (its egdes),
+    # then skip unmarked segments at the final output scan, as these segs are
+    # guaranteed to not have any width change. this is effective (x5 speedup)
+    # because most segments are empty or dense. If the input was spread evenly
+    # then it would be more effective to store, sort and iterate exactly these
+    # edges. Currently though, both are same speed, and mark/skip is less code.
+
+    BEGIN { SEG = 128 }  # 128 or 256 seem fastest
+    function mark(i) { segs[int(i/SEG)] = 1 }
+    function skip(i) { return !(int(i/SEG) in segs) }
+
+    # input range $1..$2: width may differ at $1 vs $1-1, and at $2+1 vs $2
+    { mark($1); mark($2+1) }
+
+    # store/overwrite width-1 (1 -> 0) for every codepoint in this range
+    { v=$3-1; for (i=$1; i<=$2; w[i++]=v); }
+
+    function min(a,b) { return a<b ? a : b }
+    END {
+        # scan the full unicode range, print continuous ranges of width 0/2
+        # the SEG skip is SEG-aligned. to disable: use unconditional i++
+        for (i = 0; i <= 1114112; skip(i) ? i+=SEG : i++) {  # 0x110000
+            if (w[i] == wprev && i%65536)  # no change of width or plane
+                continue
+            if (wprev)  # range i0..i-1 is width 0 or 2 - print
+                for (; i0 < i; i0 += 32768)  # split (once) if too big
+                    printf "%06x %06x %d\n", i0, min(i0+32767, i-1), 1+wprev
+            i0 = i
+            wprev = w[i]
+        }
+    }
+')
+
+ranges() { printf %s\\n "$ranges"; }
+
+verify_ranges() {(
+        errv()  { err "verify: bad range: $* -- (0x$ha, 0x$hb, $w)"; }
+        ishex() { case $1 in '' | *[!0-9A-Fa-f]* ) false; esac; }  # LC_ALL=C
+
+        pa=-2 pb=-2 pw=x  # previous range values
+        while read ha hb w; do
+                [ "$w" = 0 ] || [ "$w" = 2 ]  || errv WIDTH is not 0 or 2
+                ishex "$ha" && ishex "$hb"    || errv not hex values
+
+                a=$((0x$ha)) b=$((0x$hb))     #  decimal
+                [ $a -le $b ]                 || errv "FIRST > LAST"
+                [ $b -le 1114111 ]            || errv out of bounds
+                [ $((a>>16)) = $((b>>16)) ]   || errv not same plane
+                [ $((b-a)) -lt 32768 ]        || errv more than 0x8000 values
+                [ $a -gt $pb ]                || errv overlap or not sorted
+
+                # detect adjacent ranges which combined are <= 0x8000 values.
+                # not detecting 3 adjacent 0x5000 ranges, but shouldn't happen
+                [ $w != $pw ] || [ $a != $((pb+1)) ] || [ $((b-pa)) -ge 32768 ] ||
+                     errv small same-width adjacent ranges not combined
+
+                pa=$a pb=$b pw=$w
+        done
+)}
+
+
+as_c_ranges() {
+        while read a b w; do
+                echo "R(0x$a, 0x$b, $w),"
+        done
+}
+
+# ranges -> stdout: array such that a[p], a[p+1] are [from, to) of plane p data
+# (inclusive, exclussive. arr of 18 items - last plane (16) is a[16]..a[17])
+extract_plane_indices() {
+        # for each plane (0..16) print the 1st index at ranges where it appears
+        # or, if missing, index of the next found plane (ranges are same-plane)
+        i=0 pdone=-1  # index at ranges, plane whose index was last printed
+        while read a dummy; do  # a: ranges[i].first (hex, plane(a) == a>>16)
+                while [ $pdone -lt $((0x$a >> 16)) ]; do
+                        printf "$i, "
+                        pdone=$((pdone+1))
+                done  # pdone == plane(a)
+                i=$((i+1))
+        done
+        printf $i  # one past last range. the rest are implicit-init as 0
+}
+
+indent() { $sed -e '2,$s/^/\t\t/' -e 's/\s*$//'; }  # also trim trailing spaces
+
+# only EastAsianWidth.txt has version/date
+ver=$(head -n 1 < "$eaw_file" | $sed -e 's/.*-//' -e 's/.txt//')
+udate=$(head -n 2 < "$eaw_file" | tail -n 1 | $sed -e 's/.*Date: //')
+
+ranges | verify_ranges || exit
+
+# at the C code, p/bot/top/mid can be $idx_t, but faster as 32 bit types
+[ "$(ranges | wc -l)" -lt 65536 ] && idx_t=$u16 || idx_t=$u32
+
+cat << CFUNCTION
+/* wcwidth - Unicode $ver
+ * Copyright (C) 2025 Avi Halachmi <avihpit at yahoo.com>
+ * License: MIT
+ *
+ * Generated by $self on $(date -u -I) using the Unicode files
+ * $ud_file and $eaw_file ($udate)
+ */
+int ${FUNC_ATTR}${FN:-wcwidth}($u32 ucs)
+{
+        /* sorted ranges, "first" is clipped to 16 bit, and its high bits
+         * (plane) are deduced from the "planes" array below.
+         */
+        static const struct range {  /* bitfield order empirically fast */
+                $u16 first:  16;
+                $u16 iswide:  1;
+                $u16 delta:  15;
+        } ranges[] = {
+        #define R(first, last, width) {first & 0xffff, width/2, last-first}
+                $(ranges | as_c_ranges | indent)
+        #undef  R
+        };
+
+        /* planes[p], planes[p+1] are [from, to) at "ranges" for plane p */
+        static const $idx_t planes[18] = {
+                $(ranges | extract_plane_indices | fold -s -w 60 | indent)
+        };
+
+        /******* END OF STATIC DATA *******/
+
+        $u32 p, bot, top;
+
+        /* 0:0, 1..31:-1 (C0), 32..126:1 (isprint), 127..159:-1 (DEL, C1) */
+        if (ucs < 160)
+                return ((ucs + 1) & 127) > 32 ? 1 : ucs ? -1 : 0;
+
+        /* out of range for "planes" (and non-unicode), non-characters. */
+        /* (some also test surrogate halves, but not required by POSIX) */
+        if (ucs > 0x10ffff || (ucs & 0xfffe) == 0xfffe)
+                return -1;
+
+        p = ucs >> 16;
+        ucs &= 0xffff;
+
+        for (bot = planes[p], top = planes[p+1]; bot < top; ) {
+                $u32 mid = (bot + top) / 2;
+                if (ucs < ranges[mid].first)
+                        top = mid;
+                else if (ucs > ranges[mid].first + ranges[mid].delta)
+                        bot = mid + 1;
+                else
+                        return 2 * ranges[mid].iswide;
+        }
+
+        return 1;
+}  /* wcwidth - Unicode $ver */
+CFUNCTION
+
+
+# C drop-in replacement to the awk script which outputs the final ranges list
+: << \MERGE_RANGES_C
+#include <stdio.h>
+int main(void)
+{
+    static char w[0x110000 + 1]; /* = {0} which is width==1 */
+    unsigned a, b, c, i, first;  /* assume 32bit (need 21) */
+
+    while (3 == scanf("%u%u%u", &a, &b, &c))  /* FIRST LAST WIDTH (W:0/1/2) */
+        if (b < 0x110000 && c < 3) while (a <= b) w[a++] = (char)c + 1;
+
+    /* HEXFIRST HEXLAST WIDTH  (W: 0/2, not cross-plane, LAST-FIRST<0x8000) */
+    for (first = 0, i = 1; i <= 0x110000; ++i) {
+        if (w[i] == w[i-1] && i % 0x10000 && i - first < 0x8000)
+            continue;
+        if (w[i-1] & 1)  /* 1 or 3 -> width is 0 or 2 for first..i-1 */
+            printf("%06x %06x %d\n", first, i-1, w[i-1]-1);
+        first = i;
+    }
+
+    return 0;
+}
+MERGE_RANGES_C