tjawn 0.0.0-stage → 2.0.0

package/dist/lexer.d.ts

@@ -0,0 +1,62 @@
+/**
+ * A terminal as written in jawn.g: alternatives of items, each a regex
+ * literal (`/.../flags`), a string literal (`"..."`), or the name of
+ * another terminal.  These are transcribed verbatim from the grammar, and
+ * a test checks that they still match it.
+ */
+export interface TerminalDef {
+    priority?: number;
+    alts: string[][];
+}
+export declare const TERMINALS: Record<string, TerminalDef>;
+/** Terminals written as string literals within the grammar's rules. */
+export declare const ANONYMOUS: Record<string, string>;
+/** Terminals skipped between tokens. */
+export declare const IGNORE: readonly string[];
+/**
+ * The terminals the lexer produces, in the order Lark tries them: by
+ * descending priority, then (for those of equal priority, which never
+ * overlap) by descending maximum match length.
+ */
+export declare const ORDER: readonly string[];
+/**
+ * The JavaScript regex source for a terminal, composed as Lark does:
+ * items are concatenated and alternatives grouped.  Regex flags are
+ * applied to every terminal, which is equivalent as an unescaped `.` only
+ * occurs in the grammar's `/s` patterns.
+ */
+export declare function terminalSource(name: string): string;
+export declare const FLAGS = "suy";
+export interface Token {
+    type: string;
+    value: string;
+    start: number;
+    line: number;
+    column: number;
+}
+/** No terminal the parser accepts matches here, but this other one does. */
+export declare class UnexpectedToken {
+    readonly token: Token;
+    constructor(token: Token);
+}
+/** No terminal matches here at all. */
+export declare class UnexpectedCharacters {
+    readonly pos: number;
+    readonly line: number;
+    readonly column: number;
+    constructor(pos: number, line: number, column: number);
+}
+/** A set of terminal names, plus `$END` where the end of input is accepted. */
+export type Context = ReadonlySet<string>;
+export declare class Lexer {
+    readonly text: string;
+    private pos;
+    private line;
+    private column;
+    private last;
+    constructor(text: string);
+    /** The next token, which must be of a type in accept. */
+    next(accept: Context): Token;
+    private match;
+    private advance;
+}

package/dist/lexer.js

@@ -0,0 +1,190 @@
+// A contextual lexer for jawn text, equivalent to the one Lark builds
+// from src/jawn.g for its LALR parser: at each point only the terminals
+// the parser can accept there are considered, tried in priority order,
+// and the first to match wins.
+const R = String.raw;
+export const TERMINALS = {
+    _EMPTYSET: { priority: 15, alts: [[R `/_/`, '_VALUE_END']] },
+    _KEY_FOLLOWS: { alts: [[R `/(?=[ \t\f\r\n]*(?::(?!\/\/)|=))/`]] },
+    KEY_QUOT: { priority: 25, alts: [['_QUOTED', R `/(?=[ \t\f\r\n]*[:=])/`]] },
+    KEY_BARE: {
+        priority: 35,
+        alts: [[R `/(?!(?:[_+-]|[Nn][Uu][Ll][Ll]|[Nn][Ii][Ll]|[Nn][Oo][Nn][Ee]|[Tt][Rr][Uu][Ee]|[Ff][Aa][Ll][Ss][Ee])(?![^\x00-\x20\x7f-\xa0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000"'{}\[\],:=]))[^\x00-\x20\x7f-\xa0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000"'{}\[\],:=]+/`,
+                '_KEY_FOLLOWS']],
+    },
+    TEXT: {
+        priority: 40,
+        alts: [
+            [R `/"""(?:[^"\\]|\\.|"(?!""))*"{0,2}"""/s`, '_NO_ASGN', '_VALUE_END'],
+            [R `/'''(?:[^'\\]|\\.|'(?!''))*'{0,2}'''/s`, '_NO_ASGN', '_VALUE_END'],
+        ],
+    },
+    _NO_ASGN: { alts: [[R `/(?![ \t\f\r\n]*[:=])/`]] },
+    QUOT_STR: { priority: 20, alts: [['_QUOTED', '_NO_ASGN', '_VALUE_END']] },
+    BARE_STR: {
+        priority: 10,
+        alts: [[R `/(?!(?:[_+-]|[:=]+)(?![^\x00-\x20\x7f-\xa0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000"'{}\[\],]))[^\x00-\x20\x7f-\xa0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000"'{}\[\],]+(?<!:)/`,
+                '_VALUE_END']],
+    },
+    _VALUE_END: { alts: [[R `/(?![^ \t\f\r\n,\]}])/`]] },
+    NUMBER: { priority: 30, alts: [[R `/[+-]?[0-9]+(\.[0-9]+)?([eE][+-]?[0-9]+)?/`, '_VALUE_END']] },
+    NULL: { priority: 50, alts: [[R `/(?:[Nn][Uu][Ll][Ll]|[Nn][Ii][Ll]|[Nn][Oo][Nn][Ee])/`, '_VALUE_END']] },
+    TRUE: { priority: 60, alts: [[R `/[Tt][Rr][Uu][Ee]/`, '_VALUE_END']] },
+    FALSE: { priority: 70, alts: [[R `/[Ff][Aa][Ll][Ss][Ee]/`, '_VALUE_END']] },
+    _RSQB: { alts: [[R `/\]/`, '_VALUE_END']] },
+    _RBRACE: { alts: [[R `/\}/`, '_VALUE_END']] },
+    _QUOTED: { alts: [[R `/"(?:[^"\\]|\\.)*"/s`], [R `/'(?:[^'\\]|\\.)*'/s`]] },
+    COMMENT: { priority: 80, alts: [[R `/(?:(?<=[ \t\f\r\n,\[{])|^)\/\/[^\n]*/`]] },
+    _ASGN: { alts: [['":"'], ['"="']] },
+    _SEP: { alts: [['","']] },
+    WS: { alts: [[R `/[ \t\f\r\n]+/`]] },
+};
+/** Terminals written as string literals within the grammar's rules. */
+export const ANONYMOUS = { LBRACE: '{', LSQB: '[' };
+/** Terminals skipped between tokens. */
+export const IGNORE = ['WS', 'COMMENT'];
+/**
+ * The terminals the lexer produces, in the order Lark tries them: by
+ * descending priority, then (for those of equal priority, which never
+ * overlap) by descending maximum match length.
+ */
+export const ORDER = [
+    'COMMENT', 'FALSE', 'TRUE', 'NULL', 'TEXT', 'KEY_BARE', 'NUMBER', 'KEY_QUOT', 'QUOT_STR',
+    '_EMPTYSET', 'BARE_STR', 'WS', '_RBRACE', '_RSQB', '_ASGN', 'LBRACE', 'LSQB', '_SEP',
+];
+function escapeRegExp(s) {
+    return s.replace(/[\\^$.*+?()[\]{}|\/]/g, '\\$&');
+}
+/**
+ * The JavaScript regex source for a terminal, composed as Lark does:
+ * items are concatenated and alternatives grouped.  Regex flags are
+ * applied to every terminal, which is equivalent as an unescaped `.` only
+ * occurs in the grammar's `/s` patterns.
+ */
+export function terminalSource(name) {
+    if (name in ANONYMOUS) {
+        return escapeRegExp(ANONYMOUS[name]);
+    }
+    const def = TERMINALS[name];
+    if (!def) {
+        throw new Error(`Unknown terminal ${name}`);
+    }
+    const alts = def.alts.map((items) => items.map((item) => {
+        if (item.startsWith('/')) {
+            const end = item.lastIndexOf('/');
+            const body = item.slice(1, end);
+            return end < item.length - 1 ? `(?:${body})` : body;
+        }
+        if (item.startsWith('"')) {
+            return escapeRegExp(JSON.parse(item));
+        }
+        return terminalSource(item);
+    }).join(''));
+    return alts.length === 1 ? alts[0] : `(?:${alts.join('|')})`;
+}
+export const FLAGS = 'suy';
+const PATTERNS = new Map(ORDER.map((name) => [name, new RegExp(terminalSource(name), FLAGS)]));
+/** No terminal the parser accepts matches here, but this other one does. */
+export class UnexpectedToken {
+    token;
+    constructor(token) {
+        this.token = token;
+    }
+}
+/** No terminal matches here at all. */
+export class UnexpectedCharacters {
+    pos;
+    line;
+    column;
+    constructor(pos, line, column) {
+        this.pos = pos;
+        this.line = line;
+        this.column = column;
+    }
+}
+// The candidate patterns for each context, in matching order.
+const candidates = new WeakMap();
+function candidatesFor(accept) {
+    let c = candidates.get(accept);
+    if (!c) {
+        c = [...PATTERNS].filter(([name]) => accept.has(name) || IGNORE.includes(name));
+        candidates.set(accept, c);
+    }
+    return c;
+}
+const ROOT = new Set(ORDER);
+// The number of codepoints in s[from:to].
+function codepoints(s, from = 0, to = s.length) {
+    let n = 0;
+    for (let i = from; i < to; i++) {
+        const c = s.charCodeAt(i);
+        if (!(c >= 0xdc00 && c <= 0xdfff && i > from && isHigh(s.charCodeAt(i - 1)))) {
+            n++;
+        }
+    }
+    return n;
+}
+function isHigh(c) {
+    return c >= 0xd800 && c <= 0xdbff;
+}
+export class Lexer {
+    text;
+    pos = 0;
+    // Lines and columns count from 1, columns by codepoint.
+    line = 1;
+    column = 1;
+    last;
+    constructor(text) {
+        this.text = text;
+    }
+    /** The next token, which must be of a type in accept. */
+    next(accept) {
+        for (;;) {
+            if (this.pos >= this.text.length) {
+                // The end of input is located at the last token, as in Lark.
+                const t = this.last;
+                const end = { type: '$END', value: '', start: t?.start ?? 0, line: t?.line ?? 1,
+                    column: t?.column ?? 1 };
+                if (!accept.has('$END')) {
+                    throw new UnexpectedToken(end);
+                }
+                return end;
+            }
+            const t = this.match(candidatesFor(accept));
+            if (!t) {
+                // Matching against every terminal gives a more useful error.
+                const other = this.match(candidatesFor(ROOT));
+                if (other) {
+                    throw new UnexpectedToken(other);
+                }
+                throw new UnexpectedCharacters(this.pos, this.line, this.column);
+            }
+            this.advance(t.value);
+            if (!IGNORE.includes(t.type)) {
+                this.last = t;
+                return t;
+            }
+        }
+    }
+    match(candidates) {
+        for (const [type, re] of candidates) {
+            re.lastIndex = this.pos;
+            const m = re.exec(this.text);
+            if (m) {
+                return { type, value: m[0], start: this.pos, line: this.line, column: this.column };
+            }
+        }
+        return undefined;
+    }
+    advance(value) {
+        const nl = value.lastIndexOf('\n');
+        if (nl >= 0) {
+            this.line += value.split('\n').length - 1;
+            this.column = 1 + codepoints(value, nl + 1);
+        }
+        else {
+            this.column += codepoints(value);
+        }
+        this.pos += value.length;
+    }
+}

package/dist/parser.d.ts

@@ -0,0 +1,7 @@
+import type { Jawn } from './types.js';
+/** Parse a jawn, which is a map. */
+export declare function parse(text: string): Jawn;
+/** Parse a list of jawns, a square-bracketed list of maps. */
+export declare function parseList(text: string): Jawn[];
+/** A string as Python's repr() writes it. */
+export declare function pyRepr(s: string): string;

package/dist/parser.js

@@ -0,0 +1,443 @@
+import { JawnError } from './errors.js';
+import { JawnSet } from './jawnset.js';
+import { Lexer, UnexpectedCharacters, UnexpectedToken } from './lexer.js';
+// The contexts below are the sets of terminals accepted by the states of
+// the LALR parser Lark builds from jawn.g, which the contextual lexer
+// considers.  Where LALR merges states their sets are unions, e.g. after
+// a list element a `}` is lexed, and then rejected by the parser.
+const KEYS = ['KEY_QUOT', 'KEY_BARE'];
+const VALUES = ['NULL', 'TRUE', 'FALSE', 'NUMBER', 'QUOT_STR', 'BARE_STR', 'TEXT', 'LSQB', 'LBRACE'];
+const IS_VALUE = new Set(VALUES);
+function ctx(...names) {
+    return new Set(names.flat());
+}
+const START = ctx(KEYS, 'LBRACE', 'LSQB', '$END');
+const AFTER_KEY = ctx('_ASGN');
+const MAP_VALUE = ctx(VALUES);
+// After a map's opening bracket, where it can't be a set.
+const MAP_OPEN = ctx(KEYS, '_RBRACE');
+// After a map value, whether in a bare or bracketed map, and after its separator.
+const AFTER_VALUE = ctx(KEYS, '_SEP', '_RBRACE', '$END');
+const MAP_NEXT = ctx(KEYS, '_RBRACE', '$END');
+// After an opening bracket that may be a map or a set.
+const BRACE_OPEN = ctx(KEYS, VALUES, '_EMPTYSET', '_RBRACE');
+// After a list or set element, and after their separators.
+const AFTER_ELEM = ctx(VALUES, '_SEP', '_RBRACE', '_RSQB');
+const LIST_NEXT = ctx(VALUES, '_RSQB');
+const SET_NEXT = ctx(VALUES, '_RBRACE');
+const AFTER_EMPTYSET = ctx('_SEP', '_RBRACE');
+const SET_CLOSE = ctx('_RBRACE');
+// Within a list of jawns.
+const JAWNS_NEXT = ctx('LBRACE', '_RSQB');
+const AFTER_JAWN = ctx('LBRACE', '_SEP', '_RSQB');
+const END = ctx('$END');
+/** An error found in converting a value, reported at its token. */
+class SourceError {
+    msg;
+    token;
+    constructor(msg, token) {
+        this.msg = msg;
+        this.token = token;
+    }
+}
+// Whitespace per the spec; other Unicode whitespace is not whitespace.
+const WS = ' \t\f\r\n';
+function lstrip(s) {
+    let i = 0;
+    while (i < s.length && WS.includes(s[i])) {
+        i++;
+    }
+    return s.slice(i);
+}
+function isBlank(s) {
+    return lstrip(s) === '';
+}
+// Escapes per the spec, those of JSON less \b plus \' and \U; a
+// backslash beginning anything else is kept.
+// A \u surrogate pair is combined into one codepoint.
+const ESCAPE = /\\(?:u([dD][89abAB][0-9a-fA-F]{2})\\u([dD][c-fC-F][0-9a-fA-F]{2})|u([0-9a-fA-F]{4})|U([0-9a-fA-F]{8})|(.))/gsu;
+const SIMPLE = new Map([['n', '\n'], ['t', '\t'], ['r', '\r'], ['f', '\f'],
+    ['"', '"'], ["'", "'"], ['\\', '\\'], ['/', '/']]);
+function unescape(s) {
+    return s.replace(ESCAPE, (m, hi, lo, u4, u8, c) => {
+        if (hi) {
+            return String.fromCodePoint(0x10000 + ((parseInt(hi, 16) - 0xd800) << 10)
+                + (parseInt(lo, 16) - 0xdc00));
+        }
+        if (u4 || u8) {
+            // Out of range codepoints and lone surrogates are kept as written.
+            const cp = parseInt((u4 || u8), 16);
+            return cp <= 0x10ffff && !(cp >= 0xd800 && cp <= 0xdfff) ? String.fromCodePoint(cp) : m;
+        }
+        return SIMPLE.get(c) ?? m;
+    });
+}
+// The content of a quoted string or key.  CRLF linebreaks are
+// normalized, as in texts, before escapes so that an escaped `\r\n`
+// survives.
+function quoted(s) {
+    return unescape(s.slice(1, -1).replaceAll('\r\n', '\n'));
+}
+function text(s) {
+    // CRLF linebreaks are normalized so that CRs don't trail lines.
+    const lines = s.slice(3, -3).replaceAll('\r\n', '\n').split('\n');
+    // Content on the opening line is only stripped of its leading
+    // whitespace and doesn't count toward the common indentation.
+    const head = [];
+    if (!isBlank(lines[0])) {
+        head.push(lstrip(lines.shift()));
+    }
+    while (lines.length > 0 && head.length === 0 && isBlank(lines[0])) {
+        lines.shift();
+    }
+    while (lines.length > 0 && isBlank(lines[lines.length - 1])) {
+        lines.pop();
+    }
+    // Remove the leading whitespace common to all non-blank lines;
+    // blank lines don't count toward it and become empty.
+    const indents = lines.filter((l) => !isBlank(l)).map((l) => l.slice(0, l.length - lstrip(l).length));
+    const prefix = commonPrefix(indents);
+    const res = head.concat(lines.map((l) => isBlank(l) ? '' : l.slice(prefix.length)));
+    // Dedent before unescaping so escaped linebreaks don't affect it.
+    return unescape(res.join('\n'));
+}
+function commonPrefix(ss) {
+    if (ss.length === 0) {
+        return '';
+    }
+    let p = ss[0];
+    for (const s of ss) {
+        let i = 0;
+        while (i < p.length && i < s.length && p[i] === s[i]) {
+            i++;
+        }
+        p = p.slice(0, i);
+    }
+    return p;
+}
+function setKey(o, k, v) {
+    if (k === '__proto__') {
+        Object.defineProperty(o, k, { value: v, writable: true, enumerable: true, configurable: true });
+    }
+    else {
+        o[k] = v;
+    }
+}
+class Parser {
+    lexer;
+    // Checks made as the LALR parser reduces a value, which it does only
+    // once it has lexed the token following that value; a syntax error
+    // there takes precedence.
+    pending = [];
+    constructor(text) {
+        this.lexer = new Lexer(text);
+    }
+    next(accept) {
+        const t = this.lexer.next(accept);
+        this.flush();
+        return t;
+    }
+    flush() {
+        const pending = this.pending;
+        this.pending = [];
+        for (const check of pending) {
+            check();
+        }
+    }
+    /** A jawn or a list of jawns, as the grammar's start rule permits both. */
+    parse() {
+        const t = this.next(START);
+        if (t.type === 'LBRACE') {
+            const m = this.map(this.next(MAP_OPEN), '_RBRACE');
+            this.next(END);
+            return m;
+        }
+        if (t.type === 'LSQB') {
+            const l = this.jawns();
+            this.next(END);
+            return l;
+        }
+        const m = this.map(t, '$END');
+        this.flush();
+        return m;
+    }
+    jawns() {
+        const items = [];
+        let t = this.next(JAWNS_NEXT);
+        while (t.type !== '_RSQB') {
+            if (t.type !== 'LBRACE') {
+                throw new UnexpectedToken(t);
+            }
+            items.push(this.map(this.next(MAP_OPEN), '_RBRACE'));
+            t = this.next(AFTER_JAWN);
+            if (t.type === '_SEP') {
+                t = this.next(JAWNS_NEXT);
+            }
+        }
+        return items;
+    }
+    /**
+     * A map's pairs, from its first token through its closing one.  Like
+     * other values, it's checked once the token following it is lexed.
+     */
+    map(t, close) {
+        const o = {};
+        let duplicate;
+        let duplicateKey = '';
+        while (t.type !== close) {
+            if (!KEYS.includes(t.type)) {
+                throw new UnexpectedToken(t);
+            }
+            const kt = t;
+            const k = t.type === 'KEY_BARE' ? t.value : quoted(t.value);
+            this.next(AFTER_KEY);
+            let v;
+            [v, t] = this.value(this.next(MAP_VALUE), AFTER_VALUE);
+            if (!Object.hasOwn(o, k)) {
+                setKey(o, k, v);
+            }
+            else if (!duplicate) {
+                duplicate = kt;
+                duplicateKey = k;
+            }
+            if (t.type === '_SEP') {
+                t = this.next(MAP_NEXT);
+            }
+        }
+        if (duplicate) {
+            const d = duplicate;
+            this.pending.push(() => {
+                throw new SourceError(`Duplicate key ${pyRepr(duplicateKey)}`, d);
+            });
+        }
+        return o;
+    }
+    /**
+     * A value from its first token, along with the token following it,
+     * which is lexed in context follow.
+     */
+    value(t, follow) {
+        let v;
+        switch (t.type) {
+            case 'NULL':
+                v = null;
+                break;
+            case 'TRUE':
+                v = true;
+                break;
+            case 'FALSE':
+                v = false;
+                break;
+            case 'NUMBER':
+                v = Number(t.value);
+                if (!Number.isFinite(v)) {
+                    this.pending.push(() => {
+                        throw new SourceError(`Number ${t.value} out of range`, t);
+                    });
+                }
+                break;
+            case 'QUOT_STR':
+                v = quoted(t.value);
+                break;
+            case 'BARE_STR':
+                v = t.value;
+                break;
+            case 'TEXT':
+                v = text(t.value);
+                break;
+            case 'LSQB':
+                v = this.list();
+                break;
+            case 'LBRACE':
+                v = this.brace();
+                break;
+            default:
+                throw new UnexpectedToken(t);
+        }
+        return [v, this.next(follow)];
+    }
+    list() {
+        const items = [];
+        let t = this.next(LIST_NEXT);
+        while (t.type !== '_RSQB') {
+            if (!IS_VALUE.has(t.type)) {
+                throw new UnexpectedToken(t);
+            }
+            let v;
+            [v, t] = this.value(t, AFTER_ELEM);
+            items.push(v);
+            if (t.type === '_SEP') {
+                t = this.next(LIST_NEXT);
+            }
+        }
+        return items;
+    }
+    /**
+     * A map or set: a map if its first element is a key, and otherwise a
+     * set.
+     */
+    brace() {
+        let t = this.next(BRACE_OPEN);
+        if (t.type === '_RBRACE' || KEYS.includes(t.type)) {
+            return this.map(t, '_RBRACE');
+        }
+        const s = new JawnSet();
+        if (t.type === '_EMPTYSET') {
+            t = this.next(AFTER_EMPTYSET);
+            if (t.type === '_SEP') {
+                this.next(SET_CLOSE);
+            }
+            return s;
+        }
+        while (t.type !== '_RBRACE') {
+            if (!IS_VALUE.has(t.type)) {
+                throw new UnexpectedToken(t);
+            }
+            let v;
+            [v, t] = this.value(t, AFTER_ELEM);
+            s.add(v);
+            if (t.type === '_SEP') {
+                t = this.next(SET_NEXT);
+            }
+        }
+        return s;
+    }
+}
+function parseText(s) {
+    // A leading byte order mark, as some editors write, is ignored.
+    if (s.startsWith('\ufeff')) {
+        s = s.slice(1);
+    }
+    try {
+        return new Parser(s).parse();
+    }
+    catch (e) {
+        if (e instanceof UnexpectedToken || e instanceof UnexpectedCharacters) {
+            const pos = e instanceof UnexpectedToken ? e.token.start : e.pos;
+            throw new JawnError(`Unexpected ${describe(e, s)}\n${context(s, pos)}`);
+        }
+        if (e instanceof SourceError) {
+            const t = e.token;
+            throw new JawnError(`${e.msg} on line ${t.line}, column ${t.column}\n${context(s, t.start)}`);
+        }
+        throw e;
+    }
+}
+/** Parse a jawn, which is a map. */
+export function parse(text) {
+    const r = parseText(text);
+    if (Array.isArray(r)) {
+        throw new JawnError('A jawn is a map, not a list; use parseList to read a list of jawns');
+    }
+    return r;
+}
+/** Parse a list of jawns, a square-bracketed list of maps. */
+export function parseList(text) {
+    const r = parseText(text);
+    if (!Array.isArray(r)) {
+        throw new JawnError('Expected a list of jawns, a square-bracketed list of maps');
+    }
+    return r;
+}
+// Error messages, which match those of the Python implementation.
+// A run of characters up to the next separator, for describing values
+// that aren't separated from what follows them.
+const RUN = /.[^ \t\f\r\n,\]}]*/suy;
+function describe(e, s) {
+    if (e instanceof UnexpectedCharacters) {
+        // A run with a quote or bracket past its start, or a closing
+        // bracket at its start, is values missing a separator.
+        RUN.lastIndex = e.pos;
+        const run = [...RUN.exec(s)[0]];
+        if (run.length > 1 && (']}'.includes(run[0]) || run.slice(1).some((c) => `"'{[`.includes(c)))) {
+            return `${pyRepr(run.join(''))} on line ${e.line}, column ${e.column};`
+                + ' values must be separated by whitespace or a comma';
+        }
+        return `character ${pyRepr(run[0])} on line ${e.line}, column ${e.column}`;
+    }
+    const t = e.token;
+    return `${describeToken(t)} on line ${t.line}, column ${t.column}`;
+}
+function describeToken(t) {
+    if (t.type === '$END') {
+        return 'end of input';
+    }
+    if (KEYS.includes(t.type)) {
+        return `key ${pyRepr(t.value)}`;
+    }
+    return pyRepr(t.value);
+}
+// The line around a position with a caret beneath it, as Lark does,
+// spanning 40 codepoints either side.
+function context(s, pos, span = 40) {
+    const before = s.slice(back(s, pos, span), pos).split('\n').pop();
+    const after = s.slice(pos, forward(s, pos, span)).split('\n', 1)[0];
+    return before + after + '\n' + ' '.repeat(expandedLength(before)) + '^';
+}
+function back(s, pos, n) {
+    for (; n > 0 && pos > 0; n--) {
+        pos -= pos > 1 && isLow(s.charCodeAt(pos - 1)) && isHigh(s.charCodeAt(pos - 2)) ? 2 : 1;
+    }
+    return pos;
+}
+function forward(s, pos, n) {
+    for (; n > 0 && pos < s.length; n--) {
+        pos += isHigh(s.charCodeAt(pos)) && isLow(s.charCodeAt(pos + 1)) ? 2 : 1;
+    }
+    return pos;
+}
+function isHigh(c) {
+    return c >= 0xd800 && c <= 0xdbff;
+}
+function isLow(c) {
+    return c >= 0xdc00 && c <= 0xdfff;
+}
+// The length of s with tabs expanded to multiples of 8 columns, as
+// Python's str.expandtabs().
+function expandedLength(s) {
+    let length = 0;
+    let column = 0;
+    for (const c of s) {
+        if (c === '\t') {
+            const n = 8 - (column % 8);
+            length += n;
+            column += n;
+        }
+        else {
+            length++;
+            column = c === '\n' || c === '\r' ? 0 : column + 1;
+        }
+    }
+    return length;
+}
+const NONPRINTABLE = /^[\p{Cc}\p{Cf}\p{Cs}\p{Co}\p{Cn}\p{Zl}\p{Zp}\p{Zs}]$/u;
+/** A string as Python's repr() writes it. */
+export function pyRepr(s) {
+    const q = s.includes("'") && !s.includes('"') ? '"' : "'";
+    let r = q;
+    for (const c of s) {
+        if (c === q || c === '\\') {
+            r += '\\' + c;
+        }
+        else if (c === '\t') {
+            r += '\\t';
+        }
+        else if (c === '\n') {
+            r += '\\n';
+        }
+        else if (c === '\r') {
+            r += '\\r';
+        }
+        else if (c !== ' ' && NONPRINTABLE.test(c)) {
+            const cp = c.codePointAt(0);
+            const hex = cp.toString(16);
+            r += cp < 0x100 ? '\\x' + hex.padStart(2, '0')
+                : cp < 0x10000 ? '\\u' + hex.padStart(4, '0')
+                    : '\\U' + hex.padStart(8, '0');
+        }
+        else {
+            r += c;
+        }
+    }
+    return r + q;
+}