tex2typst 0.6.0 → 0.6.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tex2typst",
-  "version": "0.6.0",
+  "version": "0.6.2",
   "description": "JavaScript library for converting TeX code to Typst",
   "type": "module",
   "main": "dist/index.js",

package/src/convert.ts

@@ -87,9 +87,6 @@ function tex_token_to_typst(token: TexToken, options: Tex2TypstOptions): TypstTo
             if (token.value === '\\\\') {
                 // \\ -> \
                 return new TypstToken(TypstTokenType.CONTROL, '\\');
-            } else if (token.value === '\\!') {
-                // \! -> #h(-math.thin.amount)
-                return new TypstToken(TypstTokenType.SYMBOL, '#h(-math.thin.amount)');
             } else if (token.value === '~') {
                 // ~ -> space.nobreak
                 const typst_symbol = symbolMap.get('~')!;
@@ -215,6 +212,13 @@ export function convert_tex_node_to_typst(abstractNode: TexNode, options: Tex2Ty
     switch (abstractNode.type) {
         case 'terminal': {
             const node = abstractNode as TexTerminal;
+            // \! -> #h(-math.thin.amount)
+            if (node.head.eq(new TexToken(TexTokenType.CONTROL, '\\!'))) {
+                return new TypstFuncCall(
+                    new TypstToken(TypstTokenType.SYMBOL, '#h'),
+                    [new TypstToken(TypstTokenType.LITERAL, '-math.thin.amount').toNode()]
+                );
+            }
             return tex_token_to_typst(node.head, options).toNode();
         }
         case 'text': {
@@ -752,25 +756,6 @@ export function convert_typst_node_to_tex(abstractNode: TypstNode, options: Typs
         case 'terminal': {
             const node = abstractNode as TypstTerminal;
             if (node.head.type === TypstTokenType.SYMBOL) {
-                // special hook for eq.def
-                if (node.head.value === 'eq.def') {
-                    return new TexFuncCall(new TexToken(TexTokenType.COMMAND, '\\overset'), [
-                        new TexText(new TexToken(TexTokenType.LITERAL, 'def')),
-                        new TexToken(TexTokenType.ELEMENT, '=').toNode()
-                    ]);
-                }
-                // special hook for comma
-                if(node.head.value === 'comma') {
-                    return new TexToken(TexTokenType.ELEMENT, ',').toNode();
-                }
-                // special hook for dif
-                if(node.head.value === 'dif') {
-                    return new TexFuncCall(new TexToken(TexTokenType.COMMAND, '\\mathrm'), [new TexToken(TexTokenType.ELEMENT, 'd').toNode()]);
-                }
-                // special hook for hyph and hyph.minus
-                if(node.head.value === 'hyph' || node.head.value === 'hyph.minus') {
-                    return new TexText(new TexToken(TexTokenType.LITERAL, '-'));
-                }
                 // special hook for mathbb{R} <-- RR
                 if(/^([A-Z])\1$/.test(node.head.value)) {
                     return new TexFuncCall(new TexToken(TexTokenType.COMMAND, '\\mathbb'), [

package/src/index.ts

@@ -8,6 +8,7 @@ import { parseTypst } from "./typst-parser";
 import { TexWriter } from "./tex-writer";
 import { shorthandMap } from "./typst-shorthands";
 import { expand_tex_predefined_macros } from "./tex-semantic-analysis";
+import { expand_typst_predefined_variables } from "./typst-semantic-analyais";
 
 
 export function tex2typst(tex: string, options: Partial<Tex2TypstOptions> = {}): string {
@@ -53,7 +54,8 @@ export function typst2tex(typst: string, options: Partial<Typst2TexOptions> = {}
     }
 
     const typstTree = parseTypst(typst);
-    const texTree = convert_typst_node_to_tex(typstTree, opt);
+    const preprocessedTypstTree = expand_typst_predefined_variables(typstTree);
+    const texTree = convert_typst_node_to_tex(preprocessedTypstTree, opt);
     const writer = new TexWriter();
     writer.append(texTree);
     return writer.finalize();

package/src/lex.ts

@@ -0,0 +1,189 @@
+/**
+ * Last modified: 2026-05-30
+ * Adapted from jslex - A lexer in JavaScript. https://github.com/jimbojw/jslex
+ * Licensed under MIT license
+ */
+
+export type ScannerCallback<T> = (a: ScannerState) => ScanResult<T>;
+
+
+type TypeEOF = null;
+const EOF: TypeEOF = null;
+
+interface IRule<T> {
+    re: RegExp;
+    action: ScannerCallback<T>;
+}
+
+interface IMatch<T> {
+    index: number;
+    rule: IRule<T>;
+    reMatchArray: RegExpMatchArray;
+}
+
+/**
+ * Utility function for comparing two matches.
+ * @param {object} m1 Left-hand side match.
+ * @param {object} m2 Right-hand side match.
+ * @return {int} Difference between the matches.
+ */
+function matchcompare<T>(m1: IMatch<T>, m2: IMatch<T>): number {
+    const m1_len = m1.reMatchArray[0].length;
+    const m2_len = m2.reMatchArray[0].length;
+    if(m2_len !== m1_len) {
+        return m2_len - m1_len;
+    } else {
+        return m1.index - m2.index;
+    }
+}
+
+enum ScanResultStatus {
+    ACCEPTED = 0,
+    REJECTED = 1,
+    ERROR = 2,
+}
+
+export class ScanResult<T> {
+    public result: T | T[];
+    public status: ScanResultStatus;
+    constructor(status: ScanResultStatus, result: T | T[]) {
+        this.result = result;
+        this.status = status;
+    }
+
+
+    public static Accepted<U>(result: U | U[]): ScanResult<U> {
+        return new ScanResult(ScanResultStatus.ACCEPTED, result);
+    }
+
+    public static Rejected<U>(): ScanResult<U> {
+        return new ScanResult(ScanResultStatus.REJECTED, []);
+    }
+
+    public static Error<U>(message: string): ScanResult<U> {
+        return new ScanResult(ScanResultStatus.ERROR, []);
+    }
+}
+
+interface ScannerState {
+    pos: number;
+    text: string;
+    reMatchArray: RegExpMatchArray;
+}
+
+export class Scanner<T> {
+    private readonly _input: string;
+    private readonly rules: IRule<T>[];
+
+
+    // position within input stream
+    private _pos: number = 0;
+
+
+    constructor(input: string, rules: IRule<T>[]) {
+        this._input = input;
+        this.rules = rules;
+    }
+
+
+    /**
+     * Scan method to be returned to caller - grabs the next token and fires appropriate calback.
+     * @return {T} The next token extracted from the stream.
+     */
+    public scan(): T | T[] | TypeEOF {
+        if(this._pos >= this._input.length) {
+            return EOF;
+        }
+
+        const str = this._input.substring(this._pos);
+        const rules = this.rules;
+        const matches: IMatch<T>[] = [];
+        for (let i = 0; i < rules.length; i++) {
+            const rule = rules[i];
+            const mt = str.match(rule.re);
+            if (mt !== null && mt[0].length > 0) {
+                matches.push({
+                    index: i,
+                    rule: rule,
+                    reMatchArray: mt,
+                });
+            }
+        }
+        if (matches.length === 0) {
+            throw new Error("No match found for input '" + str + "'");
+        }
+        matches.sort(matchcompare);
+
+        for (const m of matches) {
+            const matched_text = m.reMatchArray[0];
+            const result = m.rule.action({
+                pos: this._pos,
+                text: matched_text,
+                reMatchArray: m.reMatchArray,
+            });
+            if (result.status === ScanResultStatus.ACCEPTED) {
+                this._pos += matched_text.length;
+                return result.result;
+            }
+        }
+        throw new Error("No match found for input '" + str + "'");
+    }
+}
+
+export class JSLex<T> {
+    public readonly rules: IRule<T>[] = [];
+
+
+    constructor(ruleMap: Map<string, ScannerCallback<T>>) {
+        for (const [k, v] of ruleMap.entries()) {
+            let re: RegExp;
+            try {
+                // FIXME: e.g. "neg|norm" becomes /^neg|norm/,
+                // but what we really want is /^(neg|norm)/ .
+                // This will cause error when tokenize input like "...norm..."
+                re = new RegExp('^' + k);
+            } catch (err) {
+                throw "Invalid regexp '" + k + "' (" + (err as Error).message + ")";
+            }
+            this.rules.push({
+                re: re,
+                action: v
+            });
+        }
+    }
+
+
+    /**
+     * Similar to lex's yylex() function, consumes all input, calling calback for each token.
+     * @param {string} input Text to lex.
+     * @param {function} callback Function to execute for each token.
+     */
+    public lex(input: string, callback: (arg0: T | T[]) => void) {
+        const scanner = new Scanner(input, this.rules);
+        while (true) {
+            const token = scanner.scan();
+            if (token === EOF) {
+                break;
+            }
+            callback(token);
+        }
+    }
+
+    /**
+     * Consumes all input, collecting tokens along the way.
+     * @param {string} input Text to lex.
+     * @return {array} List of tokens, may contain an Error at the end.
+     */
+    public collect(input: string): T[] {
+        const tokens: T[] = [];
+        const callback = function(item: T | T[]) {
+            if (Array.isArray(item)) {
+                tokens.push(...item);
+            } else {
+                tokens.push(item);
+            }
+        };
+        this.lex(input, callback);
+        return tokens;
+    }
+};

package/src/map.ts

@@ -52,7 +52,6 @@ const symbolMap = new Map<string, string>([
     ['neq', 'eq.not'],
     ['dot', 'dot'],
     ['ddot', 'dot.double'],
-    ['doteq', 'dot(eq)'],
     ['dots', 'dots.h'],
     ['vdots', 'dots.v'],
     ['ddots', 'dots.down'],
@@ -1136,7 +1135,6 @@ const reverseSymbolMap = new Map<string, string>();
 for(const [key, value] of Array.from(symbolMap.entries()).reverse()) {
     reverseSymbolMap.set(value, key);
 }
-reverseSymbolMap.set('oo', 'infty');
 
 // force override some one-to-multiple mappings
 const typst_to_tex_map = new Map<string, string>([

package/src/tex-semantic-analysis.ts

@@ -1,37 +1,39 @@
-import { parseTex } from "./tex-parser";
-import { TexNode, TexTerminal, TexToken, TexTokenType } from "./tex-types";
-
-const TEX_PREDEFINED_MACROS: Map<string, string> = new Map([
-// https://github.com/KaTeX/KaTeX/blob/434d4b8aef4c3311ebfd3405a9f0cce18ead953b/src/macros.ts#L351-L367
-    ["\\varGamma", "\\mathit{\\Gamma}"],
-    ["\\varDelta", "\\mathit{\\Delta}"],
-    ["\\varTheta", "\\mathit{\\Theta}"],
-    ["\\varLambda", "\\mathit{\\Lambda}"],
-    ["\\varXi", "\\mathit{\\Xi}"],
-    ["\\varPi", "\\mathit{\\Pi}"],
-    ["\\varSigma", "\\mathit{\\Sigma}"],
-    ["\\varUpsilon", "\\mathit{\\Upsilon}"],
-    ["\\varPhi", "\\mathit{\\Phi}"],
-    ["\\varPsi", "\\mathit{\\Psi}"],
-    ["\\varOmega", "\\mathit{\\Omega}"],
-]);
-
-function _expand_tex_predefined_macros(node: TexNode): TexNode {
-    switch (node.type) {
-        case "terminal": {
-            if (node.head.type === TexTokenType.COMMAND) {
-                if (TEX_PREDEFINED_MACROS.has(node.head.value)) {
-                    const target_str = TEX_PREDEFINED_MACROS.get(node.head.value)!;
-                    return parseTex(target_str);
-                }
-            }
-        }
-        case "funcCall":
-        default:
-            return node;
-    }
-}
-
-export function expand_tex_predefined_macros(node: TexNode): TexNode {
-    return node.bottomTopTraversalTransform(_expand_tex_predefined_macros);
-}
+import { parseTex } from "./tex-parser";
+import { TexNode, TexTokenType } from "./tex-types";
+
+const TEX_PREDEFINED_MACROS: Map<string, string> = new Map([
+// https://github.com/KaTeX/KaTeX/blob/434d4b8aef4c3311ebfd3405a9f0cce18ead953b/src/macros.ts#L351-L367
+    ["\\varGamma", "\\mathit{\\Gamma}"],
+    ["\\varDelta", "\\mathit{\\Delta}"],
+    ["\\varTheta", "\\mathit{\\Theta}"],
+    ["\\varLambda", "\\mathit{\\Lambda}"],
+    ["\\varXi", "\\mathit{\\Xi}"],
+    ["\\varPi", "\\mathit{\\Pi}"],
+    ["\\varSigma", "\\mathit{\\Sigma}"],
+    ["\\varUpsilon", "\\mathit{\\Upsilon}"],
+    ["\\varPhi", "\\mathit{\\Phi}"],
+    ["\\varPsi", "\\mathit{\\Psi}"],
+    ["\\varOmega", "\\mathit{\\Omega}"],
+
+    ["\\doteq", "\\dot{=}"],
+]);
+
+function _expand_tex_predefined_macros(node: TexNode): TexNode {
+    switch (node.type) {
+        case "terminal": {
+            if (node.head.type === TexTokenType.COMMAND) {
+                if (TEX_PREDEFINED_MACROS.has(node.head.value)) {
+                    const target_str = TEX_PREDEFINED_MACROS.get(node.head.value)!;
+                    return parseTex(target_str);
+                }
+            }
+        }
+        case "funcCall":
+        default:
+            return node;
+    }
+}
+
+export function expand_tex_predefined_macros(node: TexNode): TexNode {
+    return node.bottomTopTraversalTransform(_expand_tex_predefined_macros);
+}

package/src/tex-tokenizer.ts

@@ -1,5 +1,5 @@
 import { TexToken, TexTokenType } from "./tex-types";
-import { JSLex, Scanner } from "./jslex";
+import { JSLex, ScannerCallback, ScanResult } from "./lex";
 
 export const TEX_UNARY_COMMANDS = [
     'sqrt',
@@ -70,12 +70,12 @@ function unescape(str: string): string {
     return str;
 }
 
-const rules_map = new Map<string, (a: Scanner<TexToken>) => TexToken | TexToken[]>([
+const rules_map = new Map<string, ScannerCallback<TexToken>>([
     // match `\begin{array}{cc}`
     [
         String.raw`\\begin{(array|subarry)}{(.+?)}`, (s) => {
-            const match = s.reMatchArray()!;
-            return [
+            const match = s.reMatchArray;
+            return ScanResult.Accepted([
                 new TexToken(TexTokenType.COMMAND, '\\begin'),
                 new TexToken(TexTokenType.CONTROL, '{'),
                 new TexToken(TexTokenType.LITERAL, match[1]),
@@ -83,74 +83,69 @@ const rules_map = new Map<string, (a: Scanner<TexToken>) => TexToken | TexToken[
                 new TexToken(TexTokenType.CONTROL, '{'),
                 new TexToken(TexTokenType.LITERAL, match[2]),
                 new TexToken(TexTokenType.CONTROL, '}'),
-            ]
+            ]);
         }
     ],
     [
         String.raw`\\(text|operatorname\*?|textcolor|begin|end|hspace|array)\s*{(.+?)}`, (s) => {
-            const match = s.reMatchArray()!;
-            return [
+            const match = s.reMatchArray;
+            return ScanResult.Accepted([
                 new TexToken(TexTokenType.COMMAND, '\\' + match[1]),
                 new TexToken(TexTokenType.CONTROL, '{'),
                 new TexToken(TexTokenType.LITERAL, unescape(match[2])),
                 new TexToken(TexTokenType.CONTROL, '}')
-            ]
+            ]);
         }
     ],
-    [String.raw`%[^\n]*`, (s) => new TexToken(TexTokenType.COMMENT, s.text()!.substring(1))],
-    [String.raw`[{}_^&]`, (s) => new TexToken(TexTokenType.CONTROL, s.text()!)],
-    [String.raw`\\[\\,:;!> ]`, (s) => new TexToken(TexTokenType.CONTROL, s.text()!)],
-    [String.raw`~`, (s) => new TexToken(TexTokenType.CONTROL, s.text()!)],
-    [String.raw`\r?\n`, (_s) => new TexToken(TexTokenType.NEWLINE, "\n")],
-    [String.raw`\s+`, (s) => new TexToken(TexTokenType.SPACE, s.text()!)],
-    [String.raw`\\[{}%$&#_|]`, (s) => new TexToken(TexTokenType.ELEMENT, s.text()!)],
+    [String.raw`%[^\n]*`, (s) => ScanResult.Accepted(new TexToken(TexTokenType.COMMENT, s.text.substring(1)))],
+    [String.raw`[{}_^&]`, (s) => ScanResult.Accepted([new TexToken(TexTokenType.CONTROL, s.text)])],
+    [String.raw`\\[\\,:;!> ]`, (s) => ScanResult.Accepted([new TexToken(TexTokenType.CONTROL, s.text)])],
+    [String.raw`~`, (s) => ScanResult.Accepted([new TexToken(TexTokenType.CONTROL, s.text)])],
+    [String.raw`\r?\n`, (_s) => ScanResult.Accepted([new TexToken(TexTokenType.NEWLINE, "\n")])],
+    [String.raw`\s+`, (s) => ScanResult.Accepted([new TexToken(TexTokenType.SPACE, s.text)])],
+    [String.raw`\\[{}%$&#_|]`, (s) => ScanResult.Accepted([new TexToken(TexTokenType.ELEMENT, s.text)])],
     // e.g. match `\frac13`, `\frac1 b`, `\frac a b`
     [String.raw`(\\[a-zA-Z]+)(\s*\d|\s+[a-zA-Z])\s*([0-9a-zA-Z])`, (s) => {
-        const match = s.reMatchArray()!;
+        const match = s.reMatchArray;
         const command = match![1];
         if (TEX_BINARY_COMMANDS.includes(command.substring(1))) {
             const arg1 = match[2].trimStart();
             const arg2 = match[3];
-            return [
+            return ScanResult.Accepted([
                 new TexToken(TexTokenType.COMMAND, command),
                 new TexToken(TexTokenType.ELEMENT, arg1),
                 new TexToken(TexTokenType.ELEMENT, arg2),
-            ];
+            ]);
         } else {
-            s.reject();
-            return [];
+            return ScanResult.Rejected();
         }
     }],
     // e.g. match `\sqrt3`, `\sqrt a`
     [String.raw`(\\[a-zA-Z]+)(\s*\d|\s+[a-zA-Z])`, (s) => {
-        const match = s.reMatchArray()!;
+        const match = s.reMatchArray;
         const command = match[1];
         if (TEX_UNARY_COMMANDS.includes(command.substring(1))) {
             const arg1 = match[2].trimStart();
-            return [
+            return ScanResult.Accepted([
                 new TexToken(TexTokenType.COMMAND, command),
                 new TexToken(TexTokenType.ELEMENT, arg1),
-            ];
+            ]);
         } else {
-            s.reject();
-            return [];
+            return ScanResult.Rejected();
         }
     }],
-    [String.raw`\\[a-zA-Z]+`, (s) => new TexToken(TexTokenType.COMMAND, s.text()!)],
+    [String.raw`\\[a-zA-Z]+`, (s) => ScanResult.Accepted(new TexToken(TexTokenType.COMMAND, s.text))],
     // Numbers like "123", "3.14"
-    [String.raw`[0-9]+(\.[0-9]+)?`, (s) => new TexToken(TexTokenType.ELEMENT, s.text()!)],
-    [String.raw`[a-zA-Z]`, (s) => new TexToken(TexTokenType.ELEMENT, s.text()!)],
-    [String.raw`[+\-*/='<>!.,;:?()\[\]|]`, (s) => new TexToken(TexTokenType.ELEMENT, s.text()!)],
+    [String.raw`[0-9]+(\.[0-9]+)?`, (s) => ScanResult.Accepted(new TexToken(TexTokenType.ELEMENT, s.text))],
+    [String.raw`[a-zA-Z]`, (s) => ScanResult.Accepted(new TexToken(TexTokenType.ELEMENT, s.text))],
+    [String.raw`[+\-*/='<>!.,;:?()\[\]|]`, (s) => ScanResult.Accepted(new TexToken(TexTokenType.ELEMENT, s.text))],
     // non-ASCII characters
-    [String.raw`[^\x00-\x7F]`, (s) => new TexToken(TexTokenType.ELEMENT, s.text()!)],
-    [String.raw`.`, (s) => new TexToken(TexTokenType.UNKNOWN, s.text()!)],
+    [String.raw`[^\x00-\x7F]`, (s) => ScanResult.Accepted(new TexToken(TexTokenType.ELEMENT, s.text))],
+    [String.raw`.`, (s) => ScanResult.Accepted([new TexToken(TexTokenType.UNKNOWN, s.text)])],
 ]);
 
-const spec = {
-    "start": rules_map
-};
 
 export function tokenize_tex(input: string): TexToken[] {
-    const lexer = new JSLex<TexToken>(spec);
+    const lexer = new JSLex<TexToken>(rules_map);
     return lexer.collect(input);
 }

package/src/typst-semantic-analyais.ts

@@ -0,0 +1,38 @@
+import { parseTypst } from "./typst-parser";
+import { TypstNode, TypstTokenType } from "./typst-types";
+
+
+const TEX_PREDEFINED_VARIABLES: Map<string, string> = new Map([
+    ["dif", "upright(d)"],
+    ["eq.def", 'limits(=)^"def"'],
+    ["oo", "infinity"],
+    ["comma", ","],
+    ["hyph", '"-"'],
+    ["hyph.minus", '"-"'],
+
+    /*
+    ["AA", "bb(A)"],
+    ["BB", "bb(B)"],
+    ["CC", "bb(C)"],
+    */
+]);
+
+function _expand_typst_predefined_variables(node: TypstNode): TypstNode {
+    switch (node.type) {
+        case "terminal": {
+            if (node.head.type === TypstTokenType.SYMBOL) {
+                if (TEX_PREDEFINED_VARIABLES.has(node.head.value)) {
+                    const target_str = TEX_PREDEFINED_VARIABLES.get(node.head.value)!;
+                    return parseTypst(target_str);
+                }
+            }
+        }
+        case "funcCall":
+        default:
+            return node;
+    }
+}
+
+export function expand_typst_predefined_variables(node: TypstNode): TypstNode {
+    return node.bottomTopTraversalTransform(_expand_typst_predefined_variables);
+}

package/src/typst-tokenizer.ts

@@ -1,7 +1,7 @@
 import { TypstToken } from "./typst-types";
 import { TypstTokenType } from "./typst-types";
 import { reverseShorthandMap } from "./typst-shorthands";
-import { JSLex, Scanner } from "./jslex";
+import { JSLex, ScannerCallback, ScanResult } from "./lex";
 
 const TYPST_SHORTHANDS = Array.from(reverseShorthandMap.keys());
 
@@ -20,67 +20,71 @@ function generate_regex_for_shorthands(): string {
 
 const REGEX_SHORTHANDS = generate_regex_for_shorthands();
 
-const rules_map = new Map<string, (a: Scanner<TypstToken>) => TypstToken | TypstToken[]>([
-    [String.raw`//[^\n]*`, (s) => new TypstToken(TypstTokenType.COMMENT, s.text()!.substring(2))],
-    [String.raw`/`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
-    [String.raw`[_^&]`, (s) => new TypstToken(TypstTokenType.CONTROL, s.text()!)],
-    [String.raw`\r?\n`, (_s) => new TypstToken(TypstTokenType.NEWLINE, "\n")],
-    [String.raw`\s+`, (s) => new TypstToken(TypstTokenType.SPACE, s.text()!)],
-    [String.raw`\\[$&#_]`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
-    [String.raw`\\\n`, (s) => {
-        return [
-            new TypstToken(TypstTokenType.CONTROL, "\\"),
-            new TypstToken(TypstTokenType.NEWLINE, "\n"),
-        ]
-    }],
+const rules_map = new Map<string, ScannerCallback<TypstToken>>([
+    [String.raw`//[^\n]*`, (s) => ScanResult.Accepted(new TypstToken(TypstTokenType.COMMENT, s.text.substring(2)))],
+    [String.raw`/`, (s) => ScanResult.Accepted(new TypstToken(TypstTokenType.ELEMENT, s.text))],
+    [String.raw`[_^&]`, (s) => ScanResult.Accepted(new TypstToken(TypstTokenType.CONTROL, s.text))],
+    [String.raw`\r?\n`, (_s) => ScanResult.Accepted(new TypstToken(TypstTokenType.NEWLINE, "\n"))],
+    [String.raw`\s+`, (s) => ScanResult.Accepted(new TypstToken(TypstTokenType.SPACE, s.text))],
+    [String.raw`\\[$&#_]`, (s) => ScanResult.Accepted(new TypstToken(TypstTokenType.ELEMENT, s.text))],
+    [
+        String.raw`\\\n`,
+        (s) => ScanResult.Accepted([
+                new TypstToken(TypstTokenType.CONTROL, "\\"),
+                new TypstToken(TypstTokenType.NEWLINE, "\n")
+            ])
+    ],
     [String.raw`\\\s`, (s) => {
-        return [
+        return ScanResult.Accepted([
             new TypstToken(TypstTokenType.CONTROL, "\\"),
             new TypstToken(TypstTokenType.SPACE, " "),
-        ]
+        ])
     }],
     // this backslash is dummy and will be ignored in later stages
-    [String.raw`\\\S`, (_s) => new TypstToken(TypstTokenType.CONTROL, "")],
+    [String.raw`\\\S`, (_s) => ScanResult.Accepted(new TypstToken(TypstTokenType.CONTROL, ""))],
     [
         String.raw`"([^"]|(\\"))*"`,
         (s) => {
-            const text = s.text()!.substring(1, s.text()!.length - 1);
+            const text = s.text.substring(1, s.text.length - 1);
             // replace all escape characters with their actual characters
             text.replaceAll('\\"', '"');
-            return new TypstToken(TypstTokenType.TEXT, text);
+            return ScanResult.Accepted(new TypstToken(TypstTokenType.TEXT, text));
         }
     ],
     [
         REGEX_SHORTHANDS,
         (s) => {
-            const shorthand = s.text()!;
+            const shorthand = s.text;
             const symbol = reverseShorthandMap.get(shorthand)!;
-            return new TypstToken(TypstTokenType.SYMBOL, symbol);
+            return ScanResult.Accepted(new TypstToken(TypstTokenType.SYMBOL, symbol));
         }
     ],
-    [String.raw`[0-9]+(\.[0-9]+)?`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
-    [String.raw`[+\-*/=\'<>!.,;?()\[\]|]`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
-    [String.raw`#h\((.+?)\)`, (s) => {
-        const match = s.reMatchArray()!;
-        return [
-            new TypstToken(TypstTokenType.SYMBOL, "#h"),
-            new TypstToken(TypstTokenType.ELEMENT, "("),
-            new TypstToken(TypstTokenType.LITERAL, match[1]),
-            new TypstToken(TypstTokenType.ELEMENT, ")"),
-        ];
-    }],
-    [String.raw`#none`, (s) => new TypstToken(TypstTokenType.NONE, s.text()!)],
+    [ String.raw`[0-9]+(\.[0-9]+)?`, (s) => ScanResult.Accepted(new TypstToken(TypstTokenType.ELEMENT, s.text)) ],
+    [ String.raw`[+\-*/=\'<>!.,;?()\[\]|]`, (s) => ScanResult.Accepted(new TypstToken(TypstTokenType.ELEMENT, s.text)) ],
+    [
+        String.raw`#h\((.+?)\)`,
+        (s) => {
+            const match = s.reMatchArray;
+            return ScanResult.Accepted([
+                new TypstToken(TypstTokenType.SYMBOL, "#h"),
+                new TypstToken(TypstTokenType.ELEMENT, "("),
+                new TypstToken(TypstTokenType.LITERAL, match[1]),
+                new TypstToken(TypstTokenType.ELEMENT, ")"),
+            ]);
+        }
+    ],
+    [String.raw`#none`, (s) => ScanResult.Accepted(new TypstToken(TypstTokenType.NONE, s.text))],
+    [
+        String.raw`#none`,
+        (s) => ScanResult.Accepted(new TypstToken(TypstTokenType.NONE, s.text)),
+    ],
     [String.raw`#?[a-zA-Z\.]+`, (s) => {
-        return new TypstToken(s.text()!.length === 1? TypstTokenType.ELEMENT: TypstTokenType.SYMBOL, s.text()!);
+        return ScanResult.Accepted(new TypstToken(s.text.length === 1? TypstTokenType.ELEMENT: TypstTokenType.SYMBOL, s.text));
     }],
-    [String.raw`.`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
+    [String.raw`.`, (s) => ScanResult.Accepted(new TypstToken(TypstTokenType.ELEMENT, s.text))],
 ]);
 
-const spec = {
-    "start": rules_map
-};
-
 export function tokenize_typst(input: string): TypstToken[] {
-    const lexer = new JSLex<TypstToken>(spec);
+    const lexer = new JSLex<TypstToken>(rules_map);
     return lexer.collect(input);
 }