tex2typst 0.6.1 → 0.6.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tex2typst",
-  "version": "0.6.1",
+  "version": "0.6.2",
   "description": "JavaScript library for converting TeX code to Typst",
   "type": "module",
   "main": "dist/index.js",

package/src/convert.ts

@@ -87,9 +87,6 @@ function tex_token_to_typst(token: TexToken, options: Tex2TypstOptions): TypstTo
             if (token.value === '\\\\') {
                 // \\ -> \
                 return new TypstToken(TypstTokenType.CONTROL, '\\');
-            } else if (token.value === '\\!') {
-                // \! -> #h(-math.thin.amount)
-                return new TypstToken(TypstTokenType.SYMBOL, '#h(-math.thin.amount)');
             } else if (token.value === '~') {
                 // ~ -> space.nobreak
                 const typst_symbol = symbolMap.get('~')!;
@@ -215,6 +212,13 @@ export function convert_tex_node_to_typst(abstractNode: TexNode, options: Tex2Ty
     switch (abstractNode.type) {
         case 'terminal': {
             const node = abstractNode as TexTerminal;
+            // \! -> #h(-math.thin.amount)
+            if (node.head.eq(new TexToken(TexTokenType.CONTROL, '\\!'))) {
+                return new TypstFuncCall(
+                    new TypstToken(TypstTokenType.SYMBOL, '#h'),
+                    [new TypstToken(TypstTokenType.LITERAL, '-math.thin.amount').toNode()]
+                );
+            }
             return tex_token_to_typst(node.head, options).toNode();
         }
         case 'text': {

package/src/lex.ts

@@ -1,14 +1,14 @@
 /**
- * Last modified: 2026-04-09
+ * Last modified: 2026-05-30
  * Adapted from jslex - A lexer in JavaScript. https://github.com/jimbojw/jslex
  * Licensed under MIT license
  */
 
-export type ScannerCallback<T> = (a: Scanner<T>) => T | T[];
+export type ScannerCallback<T> = (a: ScannerState) => ScanResult<T>;
 
-interface ILexSpec<T> {
-    [key: string]: Map<string, ScannerCallback<T>>;
-}
+
+type TypeEOF = null;
+const EOF: TypeEOF = null;
 
 interface IRule<T> {
     re: RegExp;
@@ -21,10 +21,6 @@ interface IMatch<T> {
     reMatchArray: RegExpMatchArray;
 }
 
-
-// End of File marker
-const EOF = {};
-
 /**
  * Utility function for comparing two matches.
  * @param {object} m1 Left-hand side match.
@@ -41,146 +37,66 @@ function matchcompare<T>(m1: IMatch<T>, m2: IMatch<T>): number {
     }
 }
 
-export class Scanner<T> {
-    private _input: string;
-    private _lexer: JSLex<T>;
-
-    // position within input stream
-    private _pos: number = 0;
-
-    // current line number
-    private _line: number = 0;
-
-    // current column number
-    private _col: number = 0;
-
-    private _offset: number = 0;
-    private _less: number | null = null;
-    private _go: boolean = false;
-    private _newstate: string | null = null;
-    private _state: string;
-
-    private _text: string | null = null;
-    private _leng: number | null = null;
-    private _reMatchArray: RegExpMatchArray | null = null;
-
-    constructor(input: string, lexer: JSLex<T>) {
-        this._input = input;
-        this._lexer = lexer;
-        this._state = lexer.states[0];
-    }
+enum ScanResultStatus {
+    ACCEPTED = 0,
+    REJECTED = 1,
+    ERROR = 2,
+}
 
-    /**
-     * Analogous to yytext and yyleng in lex - will be set during scan.
-     */
-    public text(): string | null {
-        return this._text;
+export class ScanResult<T> {
+    public result: T | T[];
+    public status: ScanResultStatus;
+    constructor(status: ScanResultStatus, result: T | T[]) {
+        this.result = result;
+        this.status = status;
     }
 
-    public leng(): number | null {
-        return this._leng;
-    }
 
-    public reMatchArray(): RegExpMatchArray | null {
-        return this._reMatchArray;
+    public static Accepted<U>(result: U | U[]): ScanResult<U> {
+        return new ScanResult(ScanResultStatus.ACCEPTED, result);
     }
 
-    /**
-     * Position of in stream, line number and column number of match.
-     */
-    public pos(): number {
-        return this._pos;
+    public static Rejected<U>(): ScanResult<U> {
+        return new ScanResult(ScanResultStatus.REJECTED, []);
     }
 
-    public line(): number {
-        return this._line;
+    public static Error<U>(message: string): ScanResult<U> {
+        return new ScanResult(ScanResultStatus.ERROR, []);
     }
+}
 
-    public column(): number {
-        return this._col;
-    }
+interface ScannerState {
+    pos: number;
+    text: string;
+    reMatchArray: RegExpMatchArray;
+}
 
-    /**
-     * Analogous to input() in lex.
-     * @return {string} The next character in the stream.
-     */
-    public input(): string {
-        return this._input.charAt(this._pos + this._leng! + this._offset++);
-    }
+export class Scanner<T> {
+    private readonly _input: string;
+    private readonly rules: IRule<T>[];
 
-    /**
-     * Similar to unput() in lex, but does not allow modifying the stream.
-     * @return {int} The offset position after the operation.
-     */
-    public unput(): number {
-        return this._offset = this._offset > 0 ? this._offset-- : 0;
-    }
 
-    /**
-     * Analogous to yyless(n) in lex - retains the first n characters from this pattern, and returns
-     * the rest to the input stream, such that they will be used in the next pattern-matching operation.
-     * @param {int} n Number of characters to retain.
-     * @return {int} Length of the stream after the operation has completed.
-     */
-    public less(n: number): number {
-        this._less = n;
-        this._offset = 0;
-        this._text = this._text!.substring(0, n);
-        return this._leng = this._text.length;
-    }
+    // position within input stream
+    private _pos: number = 0;
 
-    /**
-     * Like less(), but instead of retaining the first n characters, it chops off the last n.
-     * @param {int} n Number of characters to chop.
-     * @return {int} Length of the stream after the operation has completed.
-     */
-    public pushback(n: number): number {
-        return this.less(this._leng! - n);
-    }
 
-    /**
-     * Similar to REJECT in lex, except it doesn't break the current execution context.
-     * TIP: reject() should be the last instruction in a spec callback.
-     */
-    public reject(): void {
-        this._go = true;
+    constructor(input: string, rules: IRule<T>[]) {
+        this._input = input;
+        this.rules = rules;
     }
 
-    /**
-     * Analogous to BEGIN in lex - sets the named state (start condition).
-     * @param {string|int} state Name of state to switch to, or ordinal number (0 is first, etc).
-     * @return {string} The new state on successful switch, throws exception on failure.
-     */
-    public begin(state: string | number): string {
-        if (this._lexer.specification[state]) {
-            return this._newstate = state as string;
-        }
-        const s = this._lexer.states[parseInt(state as string)];
-        if (s) {
-            return this._newstate = s;
-        }
-        throw "Unknown state '" + state + "' requested";
-    }
-
-    /**
-     * Simple accessor for reading in the current state.
-     * @return {string} The current state.
-     */
-    public state(): string {
-        return this._state;
-    }
 
     /**
      * Scan method to be returned to caller - grabs the next token and fires appropriate calback.
      * @return {T} The next token extracted from the stream.
      */
-    public scan(): T | T[] {
+    public scan(): T | T[] | TypeEOF {
         if(this._pos >= this._input.length) {
-            return EOF as T;
+            return EOF;
         }
 
         const str = this._input.substring(this._pos);
-        const rules = this._lexer.specification[this._state];
+        const rules = this.rules;
         const matches: IMatch<T>[] = [];
         for (let i = 0; i < rules.length; i++) {
             const rule = rules[i];
@@ -197,86 +113,45 @@ export class Scanner<T> {
             throw new Error("No match found for input '" + str + "'");
         }
         matches.sort(matchcompare);
-        this._go = true;
 
-        let result: T | T[];
-        let matched_text: string;
-        for (let j = 0, n = matches.length; j < n && this._go; j++) {
-            this._offset = 0;
-            this._less = null;
-            this._go = false;
-            this._newstate = null;
-            const m = matches[j];
-            matched_text = m.reMatchArray[0];
-            this._text = matched_text;
-            this._leng = matched_text.length;
-            this._reMatchArray = m.reMatchArray;
-            result = m.rule.action(this);
-            if (this._newstate && this._newstate != this._state) {
-                this._state = this._newstate;
-                break;
+        for (const m of matches) {
+            const matched_text = m.reMatchArray[0];
+            const result = m.rule.action({
+                pos: this._pos,
+                text: matched_text,
+                reMatchArray: m.reMatchArray,
+            });
+            if (result.status === ScanResultStatus.ACCEPTED) {
+                this._pos += matched_text.length;
+                return result.result;
             }
         }
-        const text = this._less === null ? matched_text! : matched_text!.substring(0, this._less);
-        const len = text.length;
-        this._pos += len + this._offset;
-
-        const nlm = text.match(/\n/g);
-        if (nlm !== null) {
-            this._line += nlm.length;
-            this._col = len - text.lastIndexOf("\n") - 1;
-        } else {
-            this._col += len;
-        }
-        return result!;
+        throw new Error("No match found for input '" + str + "'");
     }
 }
 
 export class JSLex<T> {
-    public states: string[];
-    public specification: Record<string, IRule<T>[]>;
-
-    constructor(spec: ILexSpec<T>) {
-        this.states = Object.keys(spec);
-        this.specification = {};
-
-        // build out internal representation of the provided spec
-        for (const s of this.states) {
-            // e.g. s = "start"
-            const rule_map = spec[s] as Map<string, (arg0: Scanner<T>) => T | T[]>;
-
-            if (s in this.specification) {
-                throw "Duplicate state declaration encountered for state '" + s + "'";
-            }
-
-            this.specification[s] = [] as IRule<T>[];
-
-            for (const [k,v] of rule_map.entries()) {
-                let re: RegExp;
-                try {
-                    // FIXME: e.g. "neg|norm" becomes /^neg|norm/,
-                    // but what we really want is /^(neg|norm)/ .
-                    // This will cause error when tokenize input like "...norm..."
-                    re = new RegExp('^' + k);
-                } catch (err) {
-                    throw "Invalid regexp '" + k + "' in state '" + s + "' (" + (err as Error).message + ")";
-                }
-                this.specification[s].push({
-                    re: re,
-                    action: v
-                });
+    public readonly rules: IRule<T>[] = [];
+
+
+    constructor(ruleMap: Map<string, ScannerCallback<T>>) {
+        for (const [k, v] of ruleMap.entries()) {
+            let re: RegExp;
+            try {
+                // FIXME: e.g. "neg|norm" becomes /^neg|norm/,
+                // but what we really want is /^(neg|norm)/ .
+                // This will cause error when tokenize input like "...norm..."
+                re = new RegExp('^' + k);
+            } catch (err) {
+                throw "Invalid regexp '" + k + "' (" + (err as Error).message + ")";
             }
+            this.rules.push({
+                re: re,
+                action: v
+            });
         }
     }
 
-    /**
-     * Scanner function - makes a new scanner object which is used to get tokens one at a time.
-     * @param {string} input Input text to tokenize.
-     * @return {function} Scanner function.
-     */
-    public scanner(input: string): Scanner<T> {
-        return new Scanner(input, this);
-    }
 
     /**
      * Similar to lex's yylex() function, consumes all input, calling calback for each token.
@@ -284,15 +159,13 @@ export class JSLex<T> {
      * @param {function} callback Function to execute for each token.
      */
     public lex(input: string, callback: (arg0: T | T[]) => void) {
-        const scanner = this.scanner(input);
+        const scanner = new Scanner(input, this.rules);
         while (true) {
             const token = scanner.scan();
             if (token === EOF) {
-                return;
-            }
-            if (token !== undefined) {
-                callback(token);
+                break;
             }
+            callback(token);
         }
     }
 

package/src/tex-tokenizer.ts

@@ -1,5 +1,5 @@
 import { TexToken, TexTokenType } from "./tex-types";
-import { JSLex, Scanner, ScannerCallback } from "./lex";
+import { JSLex, ScannerCallback, ScanResult } from "./lex";
 
 export const TEX_UNARY_COMMANDS = [
     'sqrt',
@@ -74,8 +74,8 @@ const rules_map = new Map<string, ScannerCallback<TexToken>>([
     // match `\begin{array}{cc}`
     [
         String.raw`\\begin{(array|subarry)}{(.+?)}`, (s) => {
-            const match = s.reMatchArray()!;
-            return [
+            const match = s.reMatchArray;
+            return ScanResult.Accepted([
                 new TexToken(TexTokenType.COMMAND, '\\begin'),
                 new TexToken(TexTokenType.CONTROL, '{'),
                 new TexToken(TexTokenType.LITERAL, match[1]),
@@ -83,74 +83,69 @@ const rules_map = new Map<string, ScannerCallback<TexToken>>([
                 new TexToken(TexTokenType.CONTROL, '{'),
                 new TexToken(TexTokenType.LITERAL, match[2]),
                 new TexToken(TexTokenType.CONTROL, '}'),
-            ]
+            ]);
         }
     ],
     [
         String.raw`\\(text|operatorname\*?|textcolor|begin|end|hspace|array)\s*{(.+?)}`, (s) => {
-            const match = s.reMatchArray()!;
-            return [
+            const match = s.reMatchArray;
+            return ScanResult.Accepted([
                 new TexToken(TexTokenType.COMMAND, '\\' + match[1]),
                 new TexToken(TexTokenType.CONTROL, '{'),
                 new TexToken(TexTokenType.LITERAL, unescape(match[2])),
                 new TexToken(TexTokenType.CONTROL, '}')
-            ]
+            ]);
         }
     ],
-    [String.raw`%[^\n]*`, (s) => new TexToken(TexTokenType.COMMENT, s.text()!.substring(1))],
-    [String.raw`[{}_^&]`, (s) => new TexToken(TexTokenType.CONTROL, s.text()!)],
-    [String.raw`\\[\\,:;!> ]`, (s) => new TexToken(TexTokenType.CONTROL, s.text()!)],
-    [String.raw`~`, (s) => new TexToken(TexTokenType.CONTROL, s.text()!)],
-    [String.raw`\r?\n`, (_s) => new TexToken(TexTokenType.NEWLINE, "\n")],
-    [String.raw`\s+`, (s) => new TexToken(TexTokenType.SPACE, s.text()!)],
-    [String.raw`\\[{}%$&#_|]`, (s) => new TexToken(TexTokenType.ELEMENT, s.text()!)],
+    [String.raw`%[^\n]*`, (s) => ScanResult.Accepted(new TexToken(TexTokenType.COMMENT, s.text.substring(1)))],
+    [String.raw`[{}_^&]`, (s) => ScanResult.Accepted([new TexToken(TexTokenType.CONTROL, s.text)])],
+    [String.raw`\\[\\,:;!> ]`, (s) => ScanResult.Accepted([new TexToken(TexTokenType.CONTROL, s.text)])],
+    [String.raw`~`, (s) => ScanResult.Accepted([new TexToken(TexTokenType.CONTROL, s.text)])],
+    [String.raw`\r?\n`, (_s) => ScanResult.Accepted([new TexToken(TexTokenType.NEWLINE, "\n")])],
+    [String.raw`\s+`, (s) => ScanResult.Accepted([new TexToken(TexTokenType.SPACE, s.text)])],
+    [String.raw`\\[{}%$&#_|]`, (s) => ScanResult.Accepted([new TexToken(TexTokenType.ELEMENT, s.text)])],
     // e.g. match `\frac13`, `\frac1 b`, `\frac a b`
     [String.raw`(\\[a-zA-Z]+)(\s*\d|\s+[a-zA-Z])\s*([0-9a-zA-Z])`, (s) => {
-        const match = s.reMatchArray()!;
+        const match = s.reMatchArray;
         const command = match![1];
         if (TEX_BINARY_COMMANDS.includes(command.substring(1))) {
             const arg1 = match[2].trimStart();
             const arg2 = match[3];
-            return [
+            return ScanResult.Accepted([
                 new TexToken(TexTokenType.COMMAND, command),
                 new TexToken(TexTokenType.ELEMENT, arg1),
                 new TexToken(TexTokenType.ELEMENT, arg2),
-            ];
+            ]);
         } else {
-            s.reject();
-            return [];
+            return ScanResult.Rejected();
         }
     }],
     // e.g. match `\sqrt3`, `\sqrt a`
     [String.raw`(\\[a-zA-Z]+)(\s*\d|\s+[a-zA-Z])`, (s) => {
-        const match = s.reMatchArray()!;
+        const match = s.reMatchArray;
         const command = match[1];
         if (TEX_UNARY_COMMANDS.includes(command.substring(1))) {
             const arg1 = match[2].trimStart();
-            return [
+            return ScanResult.Accepted([
                 new TexToken(TexTokenType.COMMAND, command),
                 new TexToken(TexTokenType.ELEMENT, arg1),
-            ];
+            ]);
         } else {
-            s.reject();
-            return [];
+            return ScanResult.Rejected();
         }
     }],
-    [String.raw`\\[a-zA-Z]+`, (s) => new TexToken(TexTokenType.COMMAND, s.text()!)],
+    [String.raw`\\[a-zA-Z]+`, (s) => ScanResult.Accepted(new TexToken(TexTokenType.COMMAND, s.text))],
     // Numbers like "123", "3.14"
-    [String.raw`[0-9]+(\.[0-9]+)?`, (s) => new TexToken(TexTokenType.ELEMENT, s.text()!)],
-    [String.raw`[a-zA-Z]`, (s) => new TexToken(TexTokenType.ELEMENT, s.text()!)],
-    [String.raw`[+\-*/='<>!.,;:?()\[\]|]`, (s) => new TexToken(TexTokenType.ELEMENT, s.text()!)],
+    [String.raw`[0-9]+(\.[0-9]+)?`, (s) => ScanResult.Accepted(new TexToken(TexTokenType.ELEMENT, s.text))],
+    [String.raw`[a-zA-Z]`, (s) => ScanResult.Accepted(new TexToken(TexTokenType.ELEMENT, s.text))],
+    [String.raw`[+\-*/='<>!.,;:?()\[\]|]`, (s) => ScanResult.Accepted(new TexToken(TexTokenType.ELEMENT, s.text))],
     // non-ASCII characters
-    [String.raw`[^\x00-\x7F]`, (s) => new TexToken(TexTokenType.ELEMENT, s.text()!)],
-    [String.raw`.`, (s) => new TexToken(TexTokenType.UNKNOWN, s.text()!)],
+    [String.raw`[^\x00-\x7F]`, (s) => ScanResult.Accepted(new TexToken(TexTokenType.ELEMENT, s.text))],
+    [String.raw`.`, (s) => ScanResult.Accepted([new TexToken(TexTokenType.UNKNOWN, s.text)])],
 ]);
 
-const spec = {
-    "start": rules_map
-};
 
 export function tokenize_tex(input: string): TexToken[] {
-    const lexer = new JSLex<TexToken>(spec);
+    const lexer = new JSLex<TexToken>(rules_map);
     return lexer.collect(input);
 }

package/src/typst-tokenizer.ts

@@ -1,7 +1,7 @@
 import { TypstToken } from "./typst-types";
 import { TypstTokenType } from "./typst-types";
 import { reverseShorthandMap } from "./typst-shorthands";
-import { JSLex, Scanner, ScannerCallback } from "./lex";
+import { JSLex, ScannerCallback, ScanResult } from "./lex";
 
 const TYPST_SHORTHANDS = Array.from(reverseShorthandMap.keys());
 
@@ -21,66 +21,70 @@ function generate_regex_for_shorthands(): string {
 const REGEX_SHORTHANDS = generate_regex_for_shorthands();
 
 const rules_map = new Map<string, ScannerCallback<TypstToken>>([
-    [String.raw`//[^\n]*`, (s) => new TypstToken(TypstTokenType.COMMENT, s.text()!.substring(2))],
-    [String.raw`/`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
-    [String.raw`[_^&]`, (s) => new TypstToken(TypstTokenType.CONTROL, s.text()!)],
-    [String.raw`\r?\n`, (_s) => new TypstToken(TypstTokenType.NEWLINE, "\n")],
-    [String.raw`\s+`, (s) => new TypstToken(TypstTokenType.SPACE, s.text()!)],
-    [String.raw`\\[$&#_]`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
-    [String.raw`\\\n`, (s) => {
-        return [
-            new TypstToken(TypstTokenType.CONTROL, "\\"),
-            new TypstToken(TypstTokenType.NEWLINE, "\n"),
-        ]
-    }],
+    [String.raw`//[^\n]*`, (s) => ScanResult.Accepted(new TypstToken(TypstTokenType.COMMENT, s.text.substring(2)))],
+    [String.raw`/`, (s) => ScanResult.Accepted(new TypstToken(TypstTokenType.ELEMENT, s.text))],
+    [String.raw`[_^&]`, (s) => ScanResult.Accepted(new TypstToken(TypstTokenType.CONTROL, s.text))],
+    [String.raw`\r?\n`, (_s) => ScanResult.Accepted(new TypstToken(TypstTokenType.NEWLINE, "\n"))],
+    [String.raw`\s+`, (s) => ScanResult.Accepted(new TypstToken(TypstTokenType.SPACE, s.text))],
+    [String.raw`\\[$&#_]`, (s) => ScanResult.Accepted(new TypstToken(TypstTokenType.ELEMENT, s.text))],
+    [
+        String.raw`\\\n`,
+        (s) => ScanResult.Accepted([
+                new TypstToken(TypstTokenType.CONTROL, "\\"),
+                new TypstToken(TypstTokenType.NEWLINE, "\n")
+            ])
+    ],
     [String.raw`\\\s`, (s) => {
-        return [
+        return ScanResult.Accepted([
             new TypstToken(TypstTokenType.CONTROL, "\\"),
             new TypstToken(TypstTokenType.SPACE, " "),
-        ]
+        ])
     }],
     // this backslash is dummy and will be ignored in later stages
-    [String.raw`\\\S`, (_s) => new TypstToken(TypstTokenType.CONTROL, "")],
+    [String.raw`\\\S`, (_s) => ScanResult.Accepted(new TypstToken(TypstTokenType.CONTROL, ""))],
     [
         String.raw`"([^"]|(\\"))*"`,
         (s) => {
-            const text = s.text()!.substring(1, s.text()!.length - 1);
+            const text = s.text.substring(1, s.text.length - 1);
             // replace all escape characters with their actual characters
             text.replaceAll('\\"', '"');
-            return new TypstToken(TypstTokenType.TEXT, text);
+            return ScanResult.Accepted(new TypstToken(TypstTokenType.TEXT, text));
         }
     ],
     [
         REGEX_SHORTHANDS,
         (s) => {
-            const shorthand = s.text()!;
+            const shorthand = s.text;
             const symbol = reverseShorthandMap.get(shorthand)!;
-            return new TypstToken(TypstTokenType.SYMBOL, symbol);
+            return ScanResult.Accepted(new TypstToken(TypstTokenType.SYMBOL, symbol));
         }
     ],
-    [String.raw`[0-9]+(\.[0-9]+)?`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
-    [String.raw`[+\-*/=\'<>!.,;?()\[\]|]`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
-    [String.raw`#h\((.+?)\)`, (s) => {
-        const match = s.reMatchArray()!;
-        return [
-            new TypstToken(TypstTokenType.SYMBOL, "#h"),
-            new TypstToken(TypstTokenType.ELEMENT, "("),
-            new TypstToken(TypstTokenType.LITERAL, match[1]),
-            new TypstToken(TypstTokenType.ELEMENT, ")"),
-        ];
-    }],
-    [String.raw`#none`, (s) => new TypstToken(TypstTokenType.NONE, s.text()!)],
+    [ String.raw`[0-9]+(\.[0-9]+)?`, (s) => ScanResult.Accepted(new TypstToken(TypstTokenType.ELEMENT, s.text)) ],
+    [ String.raw`[+\-*/=\'<>!.,;?()\[\]|]`, (s) => ScanResult.Accepted(new TypstToken(TypstTokenType.ELEMENT, s.text)) ],
+    [
+        String.raw`#h\((.+?)\)`,
+        (s) => {
+            const match = s.reMatchArray;
+            return ScanResult.Accepted([
+                new TypstToken(TypstTokenType.SYMBOL, "#h"),
+                new TypstToken(TypstTokenType.ELEMENT, "("),
+                new TypstToken(TypstTokenType.LITERAL, match[1]),
+                new TypstToken(TypstTokenType.ELEMENT, ")"),
+            ]);
+        }
+    ],
+    [String.raw`#none`, (s) => ScanResult.Accepted(new TypstToken(TypstTokenType.NONE, s.text))],
+    [
+        String.raw`#none`,
+        (s) => ScanResult.Accepted(new TypstToken(TypstTokenType.NONE, s.text)),
+    ],
     [String.raw`#?[a-zA-Z\.]+`, (s) => {
-        return new TypstToken(s.text()!.length === 1? TypstTokenType.ELEMENT: TypstTokenType.SYMBOL, s.text()!);
+        return ScanResult.Accepted(new TypstToken(s.text.length === 1? TypstTokenType.ELEMENT: TypstTokenType.SYMBOL, s.text));
     }],
-    [String.raw`.`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
+    [String.raw`.`, (s) => ScanResult.Accepted(new TypstToken(TypstTokenType.ELEMENT, s.text))],
 ]);
 
-const spec = {
-    "start": rules_map
-};
-
 export function tokenize_typst(input: string): TypstToken[] {
-    const lexer = new JSLex<TypstToken>(spec);
+    const lexer = new JSLex<TypstToken>(rules_map);
     return lexer.collect(input);
 }