ranger-compiler 3.5.1 → 3.5.2

package/dist/lib/evg/EVGCodepoint.rgr

@@ -0,0 +1,316 @@
+; SPDX-License-Identifier: MIT
+
+; EVGCodepoint.rgr — iterate text by codepoint, not by UTF-16 code unit
+;
+; `charAt` returns a UTF-16 code unit. Everything outside the Basic
+; Multilingual Plane — emoji, CJK extensions, historic scripts, most maths
+; symbols — is stored as a SURROGATE PAIR of two units, so a loop written with
+; `charAt` sees one character as two, and neither half is a real codepoint.
+;
+; What that cost before this module existed:
+;
+;   "a😀b"  measured 47.53px instead of ~25px, because the emoji contributed
+;           two .notdef advances rather than one glyph;
+;   the JSON display list came out as CESU-8 (ed a0 bd ed b8 80) rather than
+;           UTF-8 (f0 9f 98 80), which is not valid UTF-8 and no strict parser
+;           will read;
+;   the PDF encoder reported two unencodable characters at U+D83D and U+DE00,
+;           which are not characters at all — they are halves.
+;
+; So: one place that knows how to step through a string properly, used by
+; measurement, line breaking, the rasterizer and the encoders alike.
+;
+; Since docs/plans/PLAN_STRING_INDEXING.md stage 2 the language has the
+; primitive this module was hand-rolling: `(to_chars s)` gives the code
+; points as an `[int]`, the same sequence on every target, in one pass.
+; `toArray`, `charCount` and `count` are that now, and a caller that wants
+; the whole string as codepoints should reach for `to_chars` directly.
+;
+; What stays here is the part `to_chars` cannot express: `codeAt`,
+; `unitsAt`, `utf8CodeAt`, `utf8UnitsAt` and `stringIsBytes` all take or
+; return a STORAGE index -- a caret column, a substring boundary, a position
+; a caller is holding in the target's own unit. Those callers index the
+; string itself, so they need to know how far the next character reaches in
+; that unit, which is a different question from "what are the code points".
+;
+; Note this is about STORAGE, not about what a reader perceives as a character.
+; A grapheme cluster — a flag, a skin-toned emoji, a ZWJ family — is several
+; codepoints and still needs clustering on top of this. This module is the
+; layer below that, and the layer everything else was missing.
+
+class EVGCodepoint {
+
+    ; --- where a line may be broken ---------------------------------------------
+    ;
+    ; A space is the obvious opportunity and was the only one any wrapper here
+    ; knew. It is not the only one, and the gap shows on the commonest kind of
+    ; text there is: a hyphenated compound. "Liiketoiminta-logiikka" is ONE
+    ; word to a space-only wrapper, so a box too narrow for the whole of it has
+    ; nothing to break at and falls through to breaking between letters —
+    ; "Liiketoimint" / "a-logiikka", where every other program gives
+    ; "Liiketoiminta-" / "logiikka".
+    ;
+    ; The break goes AFTER the character, so the hyphen stays on the line it
+    ; belongs to. That is the whole difference between a compound that reads
+    ; and one that looks like a typo.
+    ;
+    ; This is the small, uncontroversial part of UAX #14 — the part every
+    ; implementation agrees on — and deliberately not the rest of it: no line
+    ; breaking around CJK, no numeric-range rules, no soft hyphen. Those want
+    ; a property table, and a wrapper that gets hyphens right is most of the
+    ; distance for the documents these editors open.
+    sfn breaksAfter:boolean (c:int) {
+        ; - hyphen-minus, and the real hyphen nobody types
+        if ((c == 45) || (c == 8208)) {
+            return true
+        }
+        ; – en dash, — em dash
+        if ((c == 8211) || (c == 8212)) {
+            return true
+        }
+        ; / solidus: a path or a date breaks after it, not before
+        if (c == 47) {
+            return true
+        }
+        return false
+    }
+
+    ; A space, a tab — where a break costs nothing because the character it
+    ; breaks at is invisible.
+    sfn isSpace:boolean (c:int) {
+        if (c == 32) {
+            return true
+        }
+        if (c == 9) {
+            return true
+        }
+        return false
+    }
+
+    ; Is a string BYTES here, or UTF-16 code units?
+    ;
+    ; Ranger's `string` does not mean the same thing on every target it
+    ; compiles to. In JavaScript it is UTF-16 code units, so "ä" is one of them
+    ; and `charAt` hands back 228. In C++ it is a `std::string` of bytes, so
+    ; "ä" is the two bytes UTF-8 spells it with and `charAt` hands back 195.
+    ;
+    ; This module's whole job is to say where one character ends and the next
+    ; begins, so it is the one place that has to know — and it asks the
+    ; language rather than the build, which cannot be got wrong by a compiler
+    ; flag. Everything above it — measurement, wrapping, kerning, the
+    ; rasterizer — then walks real characters on both.
+    sfn stringIsBytes:boolean () {
+        return ((strlen "ä") > 1)
+    }
+
+    sfn isHighSurrogate:boolean (u:int) {
+        return ((u >= 55296) && (u <= 56319))
+    }
+
+    sfn isLowSurrogate:boolean (u:int) {
+        return ((u >= 56320) && (u <= 57343))
+    }
+
+    ; The codepoint starting at UTF-16 index `i`.
+    ;
+    ; An unpaired surrogate is returned as-is rather than replaced: text that
+    ; is already damaged should stay recognisable instead of turning into a
+    ; different kind of damage.
+    sfn codeAt:int (s:string i:int) {
+        def u:int (charAt s i)
+        if (EVGCodepoint.stringIsBytes()) {
+            return (EVGCodepoint.utf8CodeAt(s i u))
+        }
+        if (EVGCodepoint.isHighSurrogate(u)) {
+            if ((i + 1) < (strlen s)) {
+                def lo:int (charAt s (i + 1))
+                if (EVGCodepoint.isLowSurrogate(lo)) {
+                    return ((((u - 55296) * 1024) + (lo - 56320)) + 65536)
+                }
+            }
+        }
+        return u
+    }
+
+    ; The codepoint whose UTF-8 lead byte `u` sits at `i`.
+    ;
+    ; A byte that is not a valid lead, or a sequence that runs off the end, is
+    ; returned as itself: text that is already damaged should stay
+    ; recognisable rather than turn into a different kind of damage.
+    sfn utf8CodeAt:int (s:string i:int u:int) {
+        def n:int (strlen s)
+        if (u < 128) {
+            return u
+        }
+        if ((u >= 192) && (u < 224)) {
+            if ((i + 1) < n) {
+                def b1:int (charAt s (i + 1))
+                if (EVGCodepoint.isUtf8Cont(b1)) {
+                    return (((u - 192) * 64) + (b1 - 128))
+                }
+            }
+            return u
+        }
+        if ((u >= 224) && (u < 240)) {
+            if ((i + 2) < n) {
+                def c1:int (charAt s (i + 1))
+                def c2:int (charAt s (i + 2))
+                if ((EVGCodepoint.isUtf8Cont(c1)) && (EVGCodepoint.isUtf8Cont(c2))) {
+                    return ((((u - 224) * 4096) + ((c1 - 128) * 64)) + (c2 - 128))
+                }
+            }
+            return u
+        }
+        if ((u >= 240) && (u < 248)) {
+            if ((i + 3) < n) {
+                def d1:int (charAt s (i + 1))
+                def d2:int (charAt s (i + 2))
+                def d3:int (charAt s (i + 3))
+                if (((EVGCodepoint.isUtf8Cont(d1)) && (EVGCodepoint.isUtf8Cont(d2))) && (EVGCodepoint.isUtf8Cont(d3))) {
+                    return (((((u - 240) * 262144) + ((d1 - 128) * 4096)) + ((d2 - 128) * 64)) + (d3 - 128))
+                }
+            }
+            return u
+        }
+        return u
+    }
+
+    sfn isUtf8Cont:boolean (b:int) {
+        return ((b >= 128) && (b < 192))
+    }
+
+    ; How many storage units the character at `i` occupies — bytes where a
+    ; string is bytes, UTF-16 units where it is units.
+    sfn utf8UnitsAt:int (s:string i:int) {
+        def u:int (charAt s i)
+        def n:int (strlen s)
+        if (u < 128) {
+            return 1
+        }
+        if ((u >= 192) && (u < 224)) {
+            if ((i + 1) < n) {
+                if (EVGCodepoint.isUtf8Cont((charAt s (i + 1)))) {
+                    return 2
+                }
+            }
+            return 1
+        }
+        if ((u >= 224) && (u < 240)) {
+            if ((i + 2) < n) {
+                if ((EVGCodepoint.isUtf8Cont((charAt s (i + 1)))) && (EVGCodepoint.isUtf8Cont((charAt s (i + 2))))) {
+                    return 3
+                }
+            }
+            return 1
+        }
+        if ((u >= 240) && (u < 248)) {
+            if ((i + 3) < n) {
+                def e1:boolean (EVGCodepoint.isUtf8Cont((charAt s (i + 1))))
+                def e2:boolean (EVGCodepoint.isUtf8Cont((charAt s (i + 2))))
+                def e3:boolean (EVGCodepoint.isUtf8Cont((charAt s (i + 3))))
+                if ((e1 && e2) && e3) {
+                    return 4
+                }
+            }
+            return 1
+        }
+        return 1
+    }
+
+    ; How many UTF-16 units the codepoint at `i` occupies: 1, or 2 for a pair.
+    sfn unitsAt:int (s:string i:int) {
+        if (EVGCodepoint.stringIsBytes()) {
+            return (EVGCodepoint.utf8UnitsAt(s i))
+        }
+        def u:int (charAt s i)
+        if (EVGCodepoint.isHighSurrogate(u)) {
+            if ((i + 1) < (strlen s)) {
+                if (EVGCodepoint.isLowSurrogate((charAt s (i + 1)))) {
+                    return 2
+                }
+            }
+        }
+        return 1
+    }
+
+    ; Number of codepoints, which is what "how long is this text" should mean
+    ; anywhere a glyph is about to be looked up.
+    ; `count` is a name the compiler keeps for itself at a call site, so the
+    ; one callers use is `charCount`; both answer the same question.
+    sfn charCount:int (s:string) {
+        return (array_length (to_chars s))
+    }
+
+    sfn count:int (s:string) {
+        return (array_length (to_chars s))
+    }
+
+    ; The string as a codepoint array — convenient where a loop wants random
+    ; access, e.g. a kerning pass that needs the previous codepoint.
+    ;
+    ; This is `to_chars`, and was this module's own decoder until the language
+    ; grew one. The difference is not only tidiness: the old version stepped
+    ; by storage index, and a storage index costs O(n) on Rust and Go, so
+    ; walking a string was quadratic there. `to_chars` is one pass.
+    sfn toArray:[int] (s:string) {
+        return (to_chars s)
+    }
+
+    ; A single codepoint back to a string, re-splitting into a surrogate pair
+    ; when it is above the BMP.
+    sfn toStr:string (cp:int) {
+        if (EVGCodepoint.stringIsBytes()) {
+            ; `strfromcode` writes the whole codepoint as UTF-8 here, including
+            ; the four-byte form. Splitting it into a surrogate pair first and
+            ; encoding each half would produce CESU-8, which is not UTF-8.
+            return (strfromcode cp)
+        }
+        if (cp < 65536) {
+            return (strfromcode cp)
+        }
+        def rel:int (cp - 65536)
+        def hi:int (55296 + (to_int (rel / 1024)))
+        def lo:int (56320 + (rel % 1024))
+        return ((strfromcode hi) + (strfromcode lo))
+    }
+
+    ; Encode to UTF-8 BYTES, one byte per character of the returned string.
+    ;
+    ; This is what has to be written to a file: `buffer_from_string` stores one
+    ; byte per character, so anything else lands on disk mis-encoded. Astral
+    ; codepoints become the proper four-byte form rather than two three-byte
+    ; sequences (CESU-8), which is the bug this replaced.
+    sfn encodeUtf8:string (s:string) {
+        ; Where a string is already bytes, it is already UTF-8.
+        if (EVGCodepoint.stringIsBytes()) {
+            return s
+        }
+        def out:string ""
+        def i:int 0
+        while (i < (strlen s)) {
+            def cp:int (EVGCodepoint.codeAt(s i))
+            i = (i + (EVGCodepoint.unitsAt(s i)))
+            if (cp < 128) {
+                out = out + (strfromcode cp)
+            } {
+                if (cp < 2048) {
+                    out = out + (strfromcode (192 + (to_int (cp / 64))))
+                    out = out + (strfromcode (128 + (cp % 64)))
+                } {
+                    if (cp < 65536) {
+                        out = out + (strfromcode (224 + (to_int (cp / 4096))))
+                        out = out + (strfromcode (128 + ((to_int (cp / 64)) % 64)))
+                        out = out + (strfromcode (128 + (cp % 64)))
+                    } {
+                        out = out + (strfromcode (240 + (to_int (cp / 262144))))
+                        out = out + (strfromcode (128 + ((to_int (cp / 4096)) % 64)))
+                        out = out + (strfromcode (128 + ((to_int (cp / 64)) % 64)))
+                        out = out + (strfromcode (128 + (cp % 64)))
+                    }
+                }
+            }
+            continue
+        }
+        return out
+    }
+}