tex2typst 0.3.23 → 0.3.25

package/src/typst-parser.ts

@@ -1,6 +1,10 @@
 
 import { array_find } from "./generic";
-import { TYPST_NONE, TypstLrData, TypstNamedParams, TypstNode, TypstSupsubData, TypstToken, TypstTokenType } from "./types";
+import { TypstFraction, TypstFuncCall, TypstGroup, TypstLeftright, TypstLeftRightData, TypstMarkupFunc, TypstMatrixLike, TypstNode, TypstSupsub, TypstTerminal } from "./typst-types";
+import { TypstNamedParams } from "./typst-types";
+import { TypstSupsubData } from "./typst-types";
+import { TypstToken } from "./typst-types";
+import { TypstTokenType } from "./typst-types";
 import { tokenize_typst } from "./typst-tokenizer";
 import { assert, isalpha } from "./util";
 
@@ -15,7 +19,6 @@ function eat_primes(tokens: TypstToken[], start: number): number {
     return pos - start;
 }
 
-
 function _find_closing_match(tokens: TypstToken[], start: number,
         leftBrackets: TypstToken[], rightBrackets: TypstToken[]): number {
     assert(tokens[start].isOneOf(leftBrackets));
@@ -24,7 +27,7 @@ function _find_closing_match(tokens: TypstToken[], start: number,
 
     while (count > 0) {
         if (pos >= tokens.length) {
-            throw new Error('Unmatched brackets');
+            throw new Error('Unmatched brackets or parentheses');
         }
         if (tokens[pos].isOneOf(rightBrackets)) {
             count -= 1;
@@ -58,8 +61,10 @@ function find_closing_delim(tokens: TypstToken[], start: number): number {
 
 
 function find_closing_parenthesis(nodes: TypstNode[], start: number): number {
-    const left_parenthesis = new TypstNode('atom', '(');
-    const right_parenthesis = new TypstNode('atom', ')');
+    const left_parenthesis = new TypstToken(TypstTokenType.ELEMENT, '(').toNode();
+    const right_parenthesis = new TypstToken(TypstTokenType.ELEMENT, ')').toNode();
+
+
 
     assert(nodes[start].eq(left_parenthesis));
 
@@ -68,7 +73,7 @@ function find_closing_parenthesis(nodes: TypstNode[], start: number): number {
 
     while (count > 0) {
         if (pos >= nodes.length) {
-            throw new Error('Unmatched brackets');
+            throw new Error("Unmatched '('");
         }
         if (nodes[pos].eq(left_parenthesis)) {
             count += 1;
@@ -84,18 +89,18 @@ function find_closing_parenthesis(nodes: TypstNode[], start: number): number {
 function primes(num: number): TypstNode[] {
     const res: TypstNode[] = [];
     for (let i = 0; i < num; i++) {
-        res.push(new TypstNode('atom', "'"));
+        res.push(new TypstToken(TypstTokenType.ELEMENT, "'").toNode());
     }
     return res;
 }
 
-const DIV = new TypstNode('atom', '/');
+const DIV = new TypstToken(TypstTokenType.ELEMENT, '/').toNode();
 
 
 
 function next_non_whitespace(nodes: TypstNode[], start: number): TypstNode | null {
     let pos = start;
-    while (pos < nodes.length && nodes[pos].type === 'whitespace') {
+    while (pos < nodes.length && (nodes[pos].head.type === TypstTokenType.SPACE || nodes[pos].head.type === TypstTokenType.NEWLINE)) {
         pos++;
     }
     return pos === nodes.length ? null : nodes[pos];
@@ -106,7 +111,7 @@ function trim_whitespace_around_operators(nodes: TypstNode[]): TypstNode[] {
     const res: TypstNode[] = [];
     for (let i = 0; i < nodes.length; i++) {
         const current = nodes[i];
-        if (current.type === 'whitespace') {
+        if (current.head.type === TypstTokenType.SPACE || current.head.type === TypstTokenType.NEWLINE) {
             if(after_operator) {
                 continue;
             }
@@ -127,8 +132,8 @@ function trim_whitespace_around_operators(nodes: TypstNode[]): TypstNode[] {
 function process_operators(nodes: TypstNode[], parenthesis = false): TypstNode {
     nodes = trim_whitespace_around_operators(nodes);
 
-    const opening_bracket = new TypstNode('atom', '(');
-    const closing_bracket = new TypstNode('atom', ')');
+    const opening_bracket = LEFT_PARENTHESES.toNode();
+    const closing_bracket = RIGHT_PARENTHESES.toNode();
 
     const stack: TypstNode[] = [];
 
@@ -155,35 +160,44 @@ function process_operators(nodes: TypstNode[], parenthesis = false): TypstNode {
             }
 
             if(stack.length > 0 && stack[stack.length-1].eq(DIV)) {
-                const denominator = current_tree;
+                let denominator = current_tree;
                 if(args.length === 0) {
                     throw new TypstParserError("Unexpected '/' operator, no numerator before it");
                 }
-                const numerator = args.pop()!;
+                let numerator = args.pop()!;
 
-                if(denominator.type === 'group' && denominator.content === 'parenthesis') {
-                    denominator.content = '';
+                if(denominator.type === 'leftright') {
+                    denominator = (denominator as TypstLeftright).body;
                 }
-                if(numerator.type === 'group' && numerator.content === 'parenthesis') {
-                    numerator.content = '';
+                if(numerator.type === 'leftright') {
+                    numerator = (numerator as TypstLeftright).body;
                 }
 
-                args.push(new TypstNode('fraction', '', [numerator, denominator]));
+                args.push(new TypstFraction([numerator, denominator]));
                 stack.pop(); // drop the '/' operator
             } else {
                 args.push(current_tree);
             }
         }
     }
+    const body = args.length === 1? args[0]: new TypstGroup(args);
     if(parenthesis) {
-        return new TypstNode('group', 'parenthesis', args);
+        return new TypstLeftright(null, { body: body, left: LEFT_PARENTHESES, right: RIGHT_PARENTHESES } as TypstLeftRightData);
     } else {
-        if(args.length === 1) {
-            return args[0];
-        } else {
-            return new TypstNode('group', '', args);
-        }
+        return body;
+    }
+}
+
+function parse_named_params(groups: TypstGroup[]): TypstNamedParams {
+    const COLON = new TypstToken(TypstTokenType.ELEMENT, ':').toNode();
+
+    const np: TypstNamedParams = {};
+    for (const group of groups) {
+        assert(group.items.length == 3);
+        assert(group.items[1].eq(COLON));
+        np[group.items[0].toString()] = new TypstTerminal(new TypstToken(TypstTokenType.LITERAL, group.items[2].toString()));
     }
+    return np;
 }
 
 export class TypstParserError extends Error {
@@ -208,6 +222,7 @@ const VERTICAL_BAR = new TypstToken(TypstTokenType.ELEMENT, '|');
 const COMMA = new TypstToken(TypstTokenType.ELEMENT, ',');
 const SEMICOLON = new TypstToken(TypstTokenType.ELEMENT, ';');
 const SINGLE_SPACE = new TypstToken(TypstTokenType.SPACE, ' ');
+const CONTROL_AND = new TypstToken(TypstTokenType.CONTROL, '&');
 
 export class TypstParser {
     space_sensitive: boolean;
@@ -230,11 +245,11 @@ export class TypstParser {
         while (pos < end) {
             const [res, newPos] = this.parseNextExpr(tokens, pos);
             pos = newPos;
-            if (res.type === 'whitespace') {
-                if (!this.space_sensitive && res.content.replace(/ /g, '').length === 0) {
+            if (res.head.type === TypstTokenType.SPACE || res.head.type === TypstTokenType.NEWLINE) {
+                if (!this.space_sensitive && res.head.value.replace(/ /g, '').length === 0) {
                     continue;
                 }
-                if (!this.newline_sensitive && res.content === '\n') {
+                if (!this.newline_sensitive && res.head.value === '\n') {
                     continue;
                 }
             }
@@ -261,7 +276,7 @@ export class TypstParser {
 
         const num_base_prime = eat_primes(tokens, pos);
         if (num_base_prime > 0) {
-            base = new TypstNode('group', '', [base].concat(primes(num_base_prime)));
+            base = new TypstGroup([base].concat(primes(num_base_prime)));
             pos += num_base_prime;
         }
         if (pos < tokens.length && tokens[pos].eq(SUB_SYMBOL)) {
@@ -277,14 +292,8 @@ export class TypstParser {
         }
 
         if (sub !== null || sup !== null) {
-            const res: TypstSupsubData = { base };
-            if (sub) {
-                res.sub = sub;
-            }
-            if (sup) {
-                res.sup = sup;
-            }
-            return [new TypstNode('supsub', '', [], res), pos];
+            const res: TypstSupsubData = { base, sup, sub };
+            return [new TypstSupsub(res), pos];
         } else {
             return [base, pos];
         }
@@ -301,7 +310,7 @@ export class TypstParser {
         }
         const num_prime = eat_primes(tokens, end);
         if (num_prime > 0) {
-            node = new TypstNode('group', '', [node].concat(primes(num_prime)));
+            node = new TypstGroup([node].concat(primes(num_prime)));
             end += num_prime;
         }
         return [node, end];
@@ -320,24 +329,34 @@ export class TypstParser {
         if ([TypstTokenType.ELEMENT, TypstTokenType.SYMBOL].includes(firstToken.type)) {
             if (start + 1 < tokens.length && tokens[start + 1].eq(LEFT_PARENTHESES)) {
                 if(firstToken.value === 'mat') {
-                    const [matrix, named_params, newPos] = this.parseGroupsOfArguments(tokens, start + 1);
-                    const mat = new TypstNode('matrix', '', [], matrix);
+                    const [matrix, named_params, newPos] = this.parseMatrix(tokens, start + 1, SEMICOLON, COMMA);
+                    const mat = new TypstMatrixLike(firstToken, matrix);
                     mat.setOptions(named_params);
                     return [mat, newPos];
                 }
                 if(firstToken.value === 'cases') {
-                    const [cases, named_params, newPos] = this.parseGroupsOfArguments(tokens, start + 1, COMMA);
-                    const casesNode = new TypstNode('cases', '', [], cases);
+                    const [cases, named_params, newPos] = this.parseMatrix(tokens, start + 1, COMMA, CONTROL_AND);
+                    const casesNode = new TypstMatrixLike(firstToken, cases);
                     casesNode.setOptions(named_params);
                     return [casesNode, newPos];
                 }
                 if (firstToken.value === 'lr') {
-                    const [args, newPos, lrData] = this.parseLrArguments(tokens, start + 1);
-                    const func_call = new TypstNode('funcCall', firstToken.value, args, lrData);
-                    return [func_call, newPos];
+                    return this.parseLrArguments(tokens, start + 1);
+                }
+                if (['#heading', '#text'].includes(firstToken.value)) {
+                    const [args, newPos] = this.parseArguments(tokens, start + 1);
+                    const named_params = parse_named_params(args as TypstGroup[]);
+                    assert(tokens[newPos].eq(LEFT_BRACKET));
+                    const DOLLAR = new TypstToken(TypstTokenType.ELEMENT, '$');
+                    const end = _find_closing_match(tokens, newPos + 1, [DOLLAR], [DOLLAR]);
+                    const [group, _] = this.parseGroup(tokens, newPos + 2, end);
+                    assert(tokens[end + 1].eq(RIGHT_BRACKET));
+                    const markup_func = new TypstMarkupFunc(firstToken, [group]);
+                    markup_func.setOptions(named_params);
+                    return [markup_func, end + 2];
                 }
                 const [args, newPos] = this.parseArguments(tokens, start + 1);
-                const func_call = new TypstNode('funcCall', firstToken.value, args);
+                const func_call = new TypstFuncCall(firstToken, args);
                 return [func_call, newPos];
             }
         }
@@ -348,33 +367,32 @@ export class TypstParser {
     // start: the position of the left parentheses
     parseArguments(tokens: TypstToken[], start: number): [TypstNode[], number] {
         const end = find_closing_match(tokens, start);
-        return [this.parseCommaSeparatedArguments(tokens, start + 1, end), end + 1];
+        return [this.parseArgumentsWithSeparator(tokens, start + 1, end, COMMA), end + 1];
     }
 
     // start: the position of the left parentheses
-    parseLrArguments(tokens: TypstToken[], start: number): [TypstNode[], number, TypstLrData] {
+    parseLrArguments(tokens: TypstToken[], start: number): [TypstNode, number] {
+        const lr_token = tokens[start];
         if (tokens[start + 1].isOneOf([LEFT_PARENTHESES, LEFT_BRACKET, LEFT_CURLY_BRACKET, VERTICAL_BAR])) {
             const end = find_closing_match(tokens, start);
             const inner_start = start + 1;
             const inner_end = find_closing_delim(tokens, inner_start);
-            const inner_args= this.parseCommaSeparatedArguments(tokens, inner_start + 1, inner_end);
+            const inner_args= this.parseArgumentsWithSeparator(tokens, inner_start + 1, inner_end, COMMA);
             return [
-                inner_args,
+                new TypstLeftright(lr_token, { body: new TypstGroup(inner_args), left: tokens[inner_start], right: tokens[inner_end]}),
                 end + 1,
-                {leftDelim: tokens[inner_start].value, rightDelim: tokens[inner_end].value} as TypstLrData
             ];
         } else {
             const [args, end] = this.parseArguments(tokens, start);
             return [
-                args,
+                new TypstLeftright(lr_token, { body: new TypstGroup(args), left: null, right: null }),
                 end,
-                {leftDelim: null, rightDelim: null} as TypstLrData,
             ];
         }
     }
 
     // start: the position of the left parentheses
-    parseGroupsOfArguments(tokens: TypstToken[], start: number, newline_token = SEMICOLON): [TypstNode[][], TypstNamedParams, number] {
+    parseMatrix(tokens: TypstToken[], start: number, rowSepToken: TypstToken, cellSepToken: TypstToken): [TypstNode[][], TypstNamedParams, number] {
         const end = find_closing_match(tokens, start);
         tokens = tokens.slice(0, end);
 
@@ -384,16 +402,16 @@ export class TypstParser {
         let pos = start + 1;
         while (pos < end) {
             while(pos < end) {
-                let next_stop = array_find(tokens, newline_token, pos);
+                let next_stop = array_find(tokens, rowSepToken, pos);
                 if (next_stop === -1) {
                     next_stop = end;
                 }
 
-                let row = this.parseCommaSeparatedArguments(tokens, pos, next_stop);
+                let row = this.parseArgumentsWithSeparator(tokens, pos, next_stop, cellSepToken);
                 let np: TypstNamedParams = {};
 
                 function extract_named_params(arr: TypstNode[]): [TypstNode[], TypstNamedParams] {
-                    const COLON = new TypstNode('atom', ':');
+                    const COLON = new TypstToken(TypstTokenType.ELEMENT, ':').toNode();
                     const np: TypstNamedParams = {};
 
                     const to_delete: number[] = [];
@@ -402,18 +420,18 @@ export class TypstParser {
                             continue;
                         }
 
-                        const g = arr[i];
-                        const pos_colon = array_find(g.args!, COLON);
+                        const g = arr[i] as TypstGroup;
+                        const pos_colon = array_find(g.items, COLON);
                         if(pos_colon === -1 || pos_colon === 0) {
                             continue;
                         }
                         to_delete.push(i);
-                        const param_name = g.args![pos_colon - 1];
-                        if(param_name.eq(new TypstNode('symbol', 'delim'))) {
-                            if(g.args!.length !== 3) {
+                        const param_name = g.items[pos_colon - 1];
+                        if(param_name.eq(new TypstToken(TypstTokenType.SYMBOL, 'delim').toNode())) {
+                            if(g.items.length !== 3) {
                                 throw new TypstParserError('Invalid number of arguments for delim');
                             }
-                            np['delim'] = g.args![pos_colon + 1];
+                            np['delim'] = g.items[pos_colon + 1];
                         } else {
                             throw new TypstParserError('Not implemented for other named parameters');
                         }
@@ -435,13 +453,13 @@ export class TypstParser {
     }
 
     // start: the position of the first token of arguments
-    parseCommaSeparatedArguments(tokens: TypstToken[], start: number, end: number): TypstNode[] {
+    parseArgumentsWithSeparator(tokens: TypstToken[], start: number, end: number, sepToken: TypstToken): TypstNode[] {
         const args: TypstNode[] = [];
         let pos = start;
         while (pos < end) {
             let nodes: TypstNode[] = [];
             while(pos < end) {
-                if(tokens[pos].eq(COMMA)) {
+                if(tokens[pos].eq(sepToken)) {
                     pos += 1;
                     break;
                 } else if(tokens[pos].eq(SINGLE_SPACE)) {

package/src/typst-tokenizer.ts

@@ -1,85 +1,86 @@
-import { TypstToken, TypstTokenType } from "./types";
-import { reverseShorthandMap } from "./typst-shorthands";
-import { JSLex, Scanner } from "./jslex";
-
-const TYPST_SHORTHANDS = Array.from(reverseShorthandMap.keys());
-
-
-function generate_regex_for_shorthands(): string {
-    const regex_list = TYPST_SHORTHANDS.map((s) => {
-        s = s.replaceAll('|', '\\|');
-        s = s.replaceAll('.', '\\.');
-        s = s.replaceAll('[', '\\[');
-        s = s.replaceAll(']', '\\]');
-        return s;
-    });
-    return `(${regex_list.join('|')})`;
-}
-
-
-const REGEX_SHORTHANDS = generate_regex_for_shorthands();
-
-const rules_map = new Map<string, (a: Scanner<TypstToken>) => TypstToken | TypstToken[]>([
-    [String.raw`//[^\n]*`, (s) => new TypstToken(TypstTokenType.COMMENT, s.text()!.substring(2))],
-    [String.raw`/`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
-    [String.raw`[_^&]`, (s) => new TypstToken(TypstTokenType.CONTROL, s.text()!)],
-    [String.raw`\r?\n`, (_s) => new TypstToken(TypstTokenType.NEWLINE, "\n")],
-    [String.raw`\s+`, (s) => new TypstToken(TypstTokenType.SPACE, s.text()!)],
-    [String.raw`\\[$&#_]`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
-    [String.raw`\\\n`, (s) => {
-        return [
-            new TypstToken(TypstTokenType.CONTROL, "\\"),
-            new TypstToken(TypstTokenType.NEWLINE, "\n"),
-        ]
-    }],
-    [String.raw`\\\s`, (s) => {
-        return [
-            new TypstToken(TypstTokenType.CONTROL, "\\"),
-            new TypstToken(TypstTokenType.SPACE, " "),
-        ]
-    }],
-    // this backslash is dummy and will be ignored in later stages
-    [String.raw`\\\S`, (_s) => new TypstToken(TypstTokenType.CONTROL, "")],
-    [
-        String.raw`"([^"]|(\\"))*"`,
-        (s) => {
-            const text = s.text()!.substring(1, s.text()!.length - 1);
-            // replace all escape characters with their actual characters
-            text.replaceAll('\\"', '"');
-            return new TypstToken(TypstTokenType.TEXT, text);
-        }
-    ],
-    [
-        REGEX_SHORTHANDS,
-        (s) => {
-            const shorthand = s.text()!;
-            const symbol = reverseShorthandMap.get(shorthand)!;
-            return new TypstToken(TypstTokenType.SYMBOL, symbol);
-        }
-    ],
-    [String.raw`[0-9]+(\.[0-9]+)?`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
-    [String.raw`[+\-*/=\'<>!.,;?()\[\]|]`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
-    [String.raw`#h\((.+?)\)`, (s) => {
-        const match = s.reMatchArray()!;
-        return [
-            new TypstToken(TypstTokenType.SYMBOL, "#h"),
-            new TypstToken(TypstTokenType.ELEMENT, "("),
-            new TypstToken(TypstTokenType.LITERAL, match[1]),
-            new TypstToken(TypstTokenType.ELEMENT, ")"),
-        ];
-    }],
-    [String.raw`[a-zA-Z\.]+`, (s) => {
-        return new TypstToken(s.text()!.length === 1? TypstTokenType.ELEMENT: TypstTokenType.SYMBOL, s.text()!);
-    }],
-    [String.raw`#none`, (s) => new TypstToken(TypstTokenType.NONE, s.text()!)],
-    [String.raw`.`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
-]);
-
-const spec = {
-    "start": rules_map
-};
-
-export function tokenize_typst(input: string): TypstToken[] {
-    const lexer = new JSLex<TypstToken>(spec);
-    return lexer.collect(input);
-}
+import { TypstToken } from "./typst-types";
+import { TypstTokenType } from "./typst-types";
+import { reverseShorthandMap } from "./typst-shorthands";
+import { JSLex, Scanner } from "./jslex";
+
+const TYPST_SHORTHANDS = Array.from(reverseShorthandMap.keys());
+
+
+function generate_regex_for_shorthands(): string {
+    const regex_list = TYPST_SHORTHANDS.map((s) => {
+        s = s.replaceAll('|', '\\|');
+        s = s.replaceAll('.', '\\.');
+        s = s.replaceAll('[', '\\[');
+        s = s.replaceAll(']', '\\]');
+        return s;
+    });
+    return `(${regex_list.join('|')})`;
+}
+
+
+const REGEX_SHORTHANDS = generate_regex_for_shorthands();
+
+const rules_map = new Map<string, (a: Scanner<TypstToken>) => TypstToken | TypstToken[]>([
+    [String.raw`//[^\n]*`, (s) => new TypstToken(TypstTokenType.COMMENT, s.text()!.substring(2))],
+    [String.raw`/`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
+    [String.raw`[_^&]`, (s) => new TypstToken(TypstTokenType.CONTROL, s.text()!)],
+    [String.raw`\r?\n`, (_s) => new TypstToken(TypstTokenType.NEWLINE, "\n")],
+    [String.raw`\s+`, (s) => new TypstToken(TypstTokenType.SPACE, s.text()!)],
+    [String.raw`\\[$&#_]`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
+    [String.raw`\\\n`, (s) => {
+        return [
+            new TypstToken(TypstTokenType.CONTROL, "\\"),
+            new TypstToken(TypstTokenType.NEWLINE, "\n"),
+        ]
+    }],
+    [String.raw`\\\s`, (s) => {
+        return [
+            new TypstToken(TypstTokenType.CONTROL, "\\"),
+            new TypstToken(TypstTokenType.SPACE, " "),
+        ]
+    }],
+    // this backslash is dummy and will be ignored in later stages
+    [String.raw`\\\S`, (_s) => new TypstToken(TypstTokenType.CONTROL, "")],
+    [
+        String.raw`"([^"]|(\\"))*"`,
+        (s) => {
+            const text = s.text()!.substring(1, s.text()!.length - 1);
+            // replace all escape characters with their actual characters
+            text.replaceAll('\\"', '"');
+            return new TypstToken(TypstTokenType.TEXT, text);
+        }
+    ],
+    [
+        REGEX_SHORTHANDS,
+        (s) => {
+            const shorthand = s.text()!;
+            const symbol = reverseShorthandMap.get(shorthand)!;
+            return new TypstToken(TypstTokenType.SYMBOL, symbol);
+        }
+    ],
+    [String.raw`[0-9]+(\.[0-9]+)?`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
+    [String.raw`[+\-*/=\'<>!.,;?()\[\]|]`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
+    [String.raw`#h\((.+?)\)`, (s) => {
+        const match = s.reMatchArray()!;
+        return [
+            new TypstToken(TypstTokenType.SYMBOL, "#h"),
+            new TypstToken(TypstTokenType.ELEMENT, "("),
+            new TypstToken(TypstTokenType.LITERAL, match[1]),
+            new TypstToken(TypstTokenType.ELEMENT, ")"),
+        ];
+    }],
+    [String.raw`#none`, (s) => new TypstToken(TypstTokenType.NONE, s.text()!)],
+    [String.raw`#?[a-zA-Z\.]+`, (s) => {
+        return new TypstToken(s.text()!.length === 1? TypstTokenType.ELEMENT: TypstTokenType.SYMBOL, s.text()!);
+    }],
+    [String.raw`.`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
+]);
+
+const spec = {
+    "start": rules_map
+};
+
+export function tokenize_typst(input: string): TypstToken[] {
+    const lexer = new JSLex<TypstToken>(spec);
+    return lexer.collect(input);
+}