tex2typst 0.2.13 → 0.2.15

package/src/{parser.ts → tex-parser.ts}

@@ -1,5 +1,5 @@
 import { symbolMap } from "./map";
-import { TexNode, TexSupsubData, TokenType } from "./types";
+import { TexNode, TexSupsubData, TexTokenType } from "./types";
 
 
 const UNARY_COMMANDS = [
@@ -45,21 +45,21 @@ const BINARY_COMMANDS = [
 ]
 
 
-export class Token {
-    type: TokenType;
+export class TexToken {
+    type: TexTokenType;
     value: string;
 
-    constructor(type: TokenType, value: string) {
+    constructor(type: TexTokenType, value: string) {
         this.type = type;
         this.value = value;
     }
 
-    public eq(token: Token): boolean {
+    public eq(token: TexToken): boolean {
         return this.type === token.type && this.value === token.value;
     }
 }
 
-const EMPTY_NODE: TexNode = { type: 'empty', content: '' };
+const EMPTY_NODE: TexNode = new TexNode('empty', '');
 
 function assert(condition: boolean, message: string = ''): void {
     if (!condition) {
@@ -77,10 +77,10 @@ function get_command_param_num(command: string): number {
     }
 }
 
-const LEFT_CURLY_BRACKET: Token = new Token(TokenType.CONTROL, '{');
-const RIGHT_CURLY_BRACKET: Token = new Token(TokenType.CONTROL, '}');
+const LEFT_CURLY_BRACKET: TexToken = new TexToken(TexTokenType.CONTROL, '{');
+const RIGHT_CURLY_BRACKET: TexToken = new TexToken(TexTokenType.CONTROL, '}');
 
-function find_closing_curly_bracket(tokens: Token[], start: number): number {
+function find_closing_curly_bracket(tokens: TexToken[], start: number): number {
     assert(tokens[start].eq(LEFT_CURLY_BRACKET));
     let count = 1;
     let pos = start + 1;
@@ -100,10 +100,10 @@ function find_closing_curly_bracket(tokens: Token[], start: number): number {
     return pos - 1;
 }
 
-const LEFT_SQUARE_BRACKET: Token = new Token(TokenType.ELEMENT, '[');
-const RIGHT_SQUARE_BRACKET: Token = new Token(TokenType.ELEMENT, ']');
+const LEFT_SQUARE_BRACKET: TexToken = new TexToken(TexTokenType.ELEMENT, '[');
+const RIGHT_SQUARE_BRACKET: TexToken = new TexToken(TexTokenType.ELEMENT, ']');
 
-function find_closing_square_bracket(tokens: Token[], start: number): number {
+function find_closing_square_bracket(tokens: TexToken[], start: number): number {
     assert(tokens[start].eq(LEFT_SQUARE_BRACKET));
     let count = 1;
     let pos = start + 1;
@@ -132,29 +132,29 @@ function isdigit(char: string): boolean {
     return '0123456789'.includes(char);
 }
 
-function eat_whitespaces(tokens: Token[], start: number): Token[] {
+function eat_whitespaces(tokens: TexToken[], start: number): TexToken[] {
     let pos = start;
-    while (pos < tokens.length && [TokenType.WHITESPACE, TokenType.NEWLINE].includes(tokens[pos].type)) {
+    while (pos < tokens.length && [TexTokenType.SPACE, TexTokenType.NEWLINE].includes(tokens[pos].type)) {
         pos++;
     }
     return tokens.slice(start, pos);
 }
 
 
-function eat_parenthesis(tokens: Token[], start: number): Token | null {
+function eat_parenthesis(tokens: TexToken[], start: number): TexToken | null {
     const firstToken = tokens[start];
-    if (firstToken.type === TokenType.ELEMENT && ['(', ')', '[', ']', '|', '\\{', '\\}'].includes(firstToken.value)) {
+    if (firstToken.type === TexTokenType.ELEMENT && ['(', ')', '[', ']', '|', '\\{', '\\}'].includes(firstToken.value)) {
         return firstToken;
-    } else if (firstToken.type === TokenType.COMMAND && ['lfloor', 'rfloor', 'lceil', 'rceil', 'langle', 'rangle'].includes(firstToken.value.slice(1))) {
+    } else if (firstToken.type === TexTokenType.COMMAND && ['lfloor', 'rfloor', 'lceil', 'rceil', 'langle', 'rangle'].includes(firstToken.value.slice(1))) {
         return firstToken;
     } else {
         return null;
     }
 }
 
-function eat_primes(tokens: Token[], start: number): number {
+function eat_primes(tokens: TexToken[], start: number): number {
     let pos = start;
-    while (pos < tokens.length && tokens[pos].eq(new Token(TokenType.ELEMENT, "'"))) {
+    while (pos < tokens.length && tokens[pos].eq(new TexToken(TexTokenType.ELEMENT, "'"))) {
         pos += 1;
     }
     return pos - start;
@@ -170,10 +170,10 @@ function eat_command_name(latex: string, start: number): string {
 }
 
 
-const LEFT_COMMAND: Token = new Token(TokenType.COMMAND, '\\left');
-const RIGHT_COMMAND: Token = new Token(TokenType.COMMAND, '\\right');
+const LEFT_COMMAND: TexToken = new TexToken(TexTokenType.COMMAND, '\\left');
+const RIGHT_COMMAND: TexToken = new TexToken(TexTokenType.COMMAND, '\\right');
 
-function find_closing_right_command(tokens: Token[], start: number): number {
+function find_closing_right_command(tokens: TexToken[], start: number): number {
     let count = 1;
     let pos = start;
 
@@ -193,11 +193,11 @@ function find_closing_right_command(tokens: Token[], start: number): number {
 }
 
 
-const BEGIN_COMMAND: Token = new Token(TokenType.COMMAND, '\\begin');
-const END_COMMAND: Token = new Token(TokenType.COMMAND, '\\end');
+const BEGIN_COMMAND: TexToken = new TexToken(TexTokenType.COMMAND, '\\begin');
+const END_COMMAND: TexToken = new TexToken(TexTokenType.COMMAND, '\\end');
 
 
-function find_closing_end_command(tokens: Token[], start: number): number {
+function find_closing_end_command(tokens: TexToken[], start: number): number {
     let count = 1;
     let pos = start;
 
@@ -241,20 +241,20 @@ function find_closing_curly_bracket_char(latex: string, start: number): number {
 }
 
 
-export function tokenize(latex: string): Token[] {
-    const tokens: Token[] = [];
+export function tokenize(latex: string): TexToken[] {
+    const tokens: TexToken[] = [];
     let pos = 0;
 
     while (pos < latex.length) {
         const firstChar = latex[pos];
-        let token: Token;
+        let token: TexToken;
         switch (firstChar) {
             case '%': {
                 let newPos = pos + 1;
                 while (newPos < latex.length && latex[newPos] !== '\n') {
                     newPos += 1;
                 }
-                token = new Token(TokenType.COMMENT, latex.slice(pos + 1, newPos));
+                token = new TexToken(TexTokenType.COMMENT, latex.slice(pos + 1, newPos));
                 pos = newPos;
                 break;
             }
@@ -263,19 +263,19 @@ export function tokenize(latex: string): Token[] {
             case '_':
             case '^':
             case '&':
-                token = new Token(TokenType.CONTROL, firstChar);
+                token = new TexToken(TexTokenType.CONTROL, firstChar);
                 pos++;
                 break;
             case '\n':
-                token = new Token(TokenType.NEWLINE, firstChar);
+                token = new TexToken(TexTokenType.NEWLINE, firstChar);
                 pos++;
                 break;
             case '\r': {
                 if (pos + 1 < latex.length && latex[pos + 1] === '\n') {
-                    token = new Token(TokenType.NEWLINE, '\n');
+                    token = new TexToken(TexTokenType.NEWLINE, '\n');
                     pos += 2;
                 } else {
-                    token = new Token(TokenType.NEWLINE, '\n');
+                    token = new TexToken(TexTokenType.NEWLINE, '\n');
                     pos ++;
                 }
                 break;
@@ -285,7 +285,7 @@ export function tokenize(latex: string): Token[] {
                 while (newPos < latex.length && latex[newPos] === ' ') {
                     newPos += 1;
                 }
-                token = new Token(TokenType.WHITESPACE, latex.slice(pos, newPos));
+                token = new TexToken(TexTokenType.SPACE, latex.slice(pos, newPos));
                 pos = newPos;
                 break;
             }
@@ -295,12 +295,12 @@ export function tokenize(latex: string): Token[] {
                 }
                 const firstTwoChars = latex.slice(pos, pos + 2);
                 if (['\\\\', '\\,'].includes(firstTwoChars)) {
-                    token = new Token(TokenType.CONTROL, firstTwoChars);
+                    token = new TexToken(TexTokenType.CONTROL, firstTwoChars);
                 } else if (['\\{','\\}', '\\%', '\\$', '\\&', '\\#', '\\_', '\\|'].includes(firstTwoChars)) {
-                    token = new Token(TokenType.ELEMENT, firstTwoChars);
+                    token = new TexToken(TexTokenType.ELEMENT, firstTwoChars);
                 } else {
                     const command = eat_command_name(latex, pos + 1);
-                    token = new Token(TokenType.COMMAND, '\\' + command);
+                    token = new TexToken(TexTokenType.COMMAND, '\\' + command);
                 }
                 pos += token.value.length;
                 break;
@@ -311,13 +311,13 @@ export function tokenize(latex: string): Token[] {
                     while (newPos < latex.length && isdigit(latex[newPos])) {
                         newPos += 1;
                     }
-                    token = new Token(TokenType.ELEMENT, latex.slice(pos, newPos));
+                    token = new TexToken(TexTokenType.ELEMENT, latex.slice(pos, newPos));
                 } else if (isalpha(firstChar)) {
-                    token = new Token(TokenType.ELEMENT, firstChar);
+                    token = new TexToken(TexTokenType.ELEMENT, firstChar);
                 } else if ('+-*/=\'<>!.,;?()[]|'.includes(firstChar)) {
-                    token = new Token(TokenType.ELEMENT, firstChar)
+                    token = new TexToken(TexTokenType.ELEMENT, firstChar)
                 } else {
-                    token = new Token(TokenType.UNKNOWN, firstChar);
+                    token = new TexToken(TexTokenType.UNKNOWN, firstChar);
                 }
                 pos += token.value.length;
             }
@@ -325,11 +325,11 @@ export function tokenize(latex: string): Token[] {
 
         tokens.push(token);
 
-        if (token.type === TokenType.COMMAND && ['\\text', '\\operatorname', '\\begin', '\\end'].includes(token.value)) {
+        if (token.type === TexTokenType.COMMAND && ['\\text', '\\operatorname', '\\begin', '\\end'].includes(token.value)) {
             if (pos >= latex.length || latex[pos] !== '{') {
                 throw new LatexParserError(`No content for ${token.value} command`);
             }
-            tokens.push(new Token(TokenType.CONTROL, '{'));
+            tokens.push(new TexToken(TexTokenType.CONTROL, '{'));
             const posClosingBracket = find_closing_curly_bracket_char(latex, pos);
             pos++;
             let textInside = latex.slice(pos, posClosingBracket);
@@ -338,8 +338,8 @@ export function tokenize(latex: string): Token[] {
             for (const char of chars) {
                 textInside = textInside.replaceAll('\\' + char, char);
             }
-            tokens.push(new Token(TokenType.TEXT, textInside));
-            tokens.push(new Token(TokenType.CONTROL, '}'));
+            tokens.push(new TexToken(TexTokenType.TEXT, textInside));
+            tokens.push(new TexToken(TexTokenType.CONTROL, '}'));
             pos = posClosingBracket + 1;
         }
     }
@@ -357,8 +357,8 @@ export class LatexParserError extends Error {
 
 type ParseResult = [TexNode, number];
 
-const SUB_SYMBOL:Token = new Token(TokenType.CONTROL, '_');
-const SUP_SYMBOL:Token = new Token(TokenType.CONTROL, '^');
+const SUB_SYMBOL:TexToken = new TexToken(TexTokenType.CONTROL, '_');
+const SUP_SYMBOL:TexToken = new TexToken(TexTokenType.CONTROL, '^');
 
 export class LatexParser {
     space_sensitive: boolean;
@@ -369,7 +369,7 @@ export class LatexParser {
         this.newline_sensitive = newline_sensitive;
     }
 
-    parse(tokens: Token[]): TexNode {
+    parse(tokens: TexToken[]): TexNode {
         const results: TexNode[] = [];
         let pos = 0;
         while (pos < tokens.length) {
@@ -379,11 +379,13 @@ export class LatexParser {
             while (pos < tokens.length) {
                 const [res, newPos] = this.parseNextExpr(tokens, pos);
                 pos = newPos;
-                if (!this.space_sensitive && res.type === 'whitespace') {
-                    continue;
-                }
-                if (!this.newline_sensitive && res.type === 'newline') {
-                    continue;
+                if(res.type === 'whitespace') {
+                    if (!this.space_sensitive && res.content.replace(/ /g, '').length === 0) {
+                        continue;
+                    }
+                    if (!this.newline_sensitive && res.content === '\n') {
+                        continue;
+                    }
                 }
                 if (res.type === 'control' && res.content === '&') {
                     throw new LatexParserError('Unexpected & outside of an alignment');
@@ -396,7 +398,7 @@ export class LatexParser {
             } else if (results.length === 1) {
                 return results[0];
             } else {
-                return { type: 'ordgroup', content: '', args: results };
+                return new TexNode('ordgroup', '', results);
             }
         }
 
@@ -406,11 +408,11 @@ export class LatexParser {
         } else if (results.length === 1) {
             return results[0];
         } else {
-            return { type: 'ordgroup', content: '', args: results };
+            return new TexNode('ordgroup', '', results);
         }
     }
 
-    parseNextExpr(tokens: Token[], start: number): ParseResult {
+    parseNextExpr(tokens: TexToken[], start: number): ParseResult {
         let [base, pos] = this.parseNextExprWithoutSupSub(tokens, start);
         let sub: TexNode | null = null;
         let sup: TexNode | null = null;
@@ -447,9 +449,9 @@ export class LatexParser {
                 res.sub = sub;
             }
             if (num_prime > 0) {
-                res.sup = { type: 'ordgroup', content: '', args:  [] };
+                res.sup = new TexNode('ordgroup', '',  []);
                 for (let i = 0; i < num_prime; i++) {
-                    res.sup.args!.push({ type: 'element', content: "'" });
+                    res.sup.args!.push(new TexNode('element', "'"));
                 }
                 if (sup) {
                     res.sup.args!.push(sup);
@@ -460,27 +462,26 @@ export class LatexParser {
             } else if (sup) {
                 res.sup = sup;
             }
-            return [{type: 'supsub',  content: '', data: res }, pos];
+            return [new TexNode('supsub', '', [], res), pos];
         } else {
             return [base, pos];
         }
     }
 
-    parseNextExprWithoutSupSub(tokens: Token[], start: number): ParseResult {
+    parseNextExprWithoutSupSub(tokens: TexToken[], start: number): ParseResult {
         const firstToken = tokens[start];
         const tokenType = firstToken.type;
         switch (tokenType) {
-            case TokenType.ELEMENT:
-                return [{ type: 'element', content: firstToken.value }, start + 1];
-            case TokenType.TEXT:
-                return [{ type: 'text', content: firstToken.value }, start + 1];
-            case TokenType.COMMENT:
-                return [{ type: 'comment', content: firstToken.value }, start + 1];
-            case TokenType.WHITESPACE:
-                return [{ type: 'whitespace', content: firstToken.value }, start + 1];
-            case TokenType.NEWLINE:
-                return [{ type: 'newline', content: firstToken.value }, start + 1];
-            case TokenType.COMMAND:
+            case TexTokenType.ELEMENT:
+                return [new TexNode('element', firstToken.value), start + 1];
+            case TexTokenType.TEXT:
+                return [new TexNode('text', firstToken.value), start + 1];
+            case TexTokenType.COMMENT:
+                return [new TexNode('comment', firstToken.value), start + 1];
+            case TexTokenType.SPACE:
+            case TexTokenType.NEWLINE:
+                return [new TexNode('whitespace', firstToken.value), start + 1];
+            case TexTokenType.COMMAND:
                 if (firstToken.eq(BEGIN_COMMAND)) {
                     return this.parseBeginEndExpr(tokens, start);
                 } else if (firstToken.eq(LEFT_COMMAND)) {
@@ -488,7 +489,7 @@ export class LatexParser {
                 } else {
                     return this.parseCommandExpr(tokens, start);
                 }
-            case TokenType.CONTROL:
+            case TexTokenType.CONTROL:
                 const controlChar = firstToken.value;
                 switch (controlChar) {
                     case '{':
@@ -498,9 +499,9 @@ export class LatexParser {
                     case '}':
                         throw new LatexParserError("Unmatched '}'");
                     case '\\\\':
-                        return [{ type: 'control', content: '\\\\' }, start + 1];
+                        return [new TexNode('control', '\\\\'), start + 1];
                     case '\\,':
-                        return [{ type: 'control', content: '\\,' }, start + 1];
+                        return [new TexNode('control', '\\,'), start + 1];
                     case '_': {
                         return [ EMPTY_NODE, start];
                     }
@@ -508,7 +509,7 @@ export class LatexParser {
                         return [ EMPTY_NODE, start];
                     }
                     case '&':
-                        return [{ type: 'control', content: '&' }, start + 1];
+                        return [new TexNode('control', '&'), start + 1];
                     default:
                         throw new LatexParserError('Unknown control sequence');
                 }
@@ -517,8 +518,8 @@ export class LatexParser {
         }
     }
 
-    parseCommandExpr(tokens: Token[], start: number): ParseResult {
-        assert(tokens[start].type === TokenType.COMMAND);
+    parseCommandExpr(tokens: TexToken[], start: number): ParseResult {
+        assert(tokens[start].type === TexTokenType.COMMAND);
 
         const command = tokens[start].value; // command name starts with a \
 
@@ -533,9 +534,9 @@ export class LatexParser {
         switch (paramNum) {
             case 0:
                 if (!symbolMap.has(command.slice(1))) {
-                    return [{ type: 'unknownMacro', content: command }, pos];
+                    return [new TexNode('unknownMacro', command), pos];
                 }
-                return [{ type: 'symbol', content: command }, pos];
+                return [new TexNode('symbol', command), pos];
             case 1: {
                 if (command === '\\sqrt' && pos < tokens.length && tokens[pos].eq(LEFT_SQUARE_BRACKET)) {
                     const posLeftSquareBracket = pos;
@@ -543,31 +544,31 @@ export class LatexParser {
                     const exprInside = tokens.slice(posLeftSquareBracket + 1, posRightSquareBracket);
                     const exponent = this.parse(exprInside);
                     const [arg1, newPos] = this.parseNextExprWithoutSupSub(tokens, posRightSquareBracket + 1);
-                    return [{ type: 'unaryFunc', content: command, args: [arg1], data: exponent }, newPos];
+                    return [new TexNode('unaryFunc', command, [arg1], exponent), newPos];
                 } else if (command === '\\text') {
                     if (pos + 2 >= tokens.length) {
                         throw new LatexParserError('Expecting content for \\text command');
                     }
                     assert(tokens[pos].eq(LEFT_CURLY_BRACKET));
-                    assert(tokens[pos + 1].type === TokenType.TEXT);
+                    assert(tokens[pos + 1].type === TexTokenType.TEXT);
                     assert(tokens[pos + 2].eq(RIGHT_CURLY_BRACKET));
                     const text = tokens[pos + 1].value;
-                    return [{ type: 'text', content: text }, pos + 3];
+                    return [new TexNode('text', text), pos + 3];
                 }
                 let [arg1, newPos] = this.parseNextExprWithoutSupSub(tokens, pos);
-                return [{ type: 'unaryFunc', content: command, args: [arg1] }, newPos];
+                return [new TexNode('unaryFunc', command, [arg1]), newPos];
             }
             case 2: {
                 const [arg1, pos1] = this.parseNextExprWithoutSupSub(tokens, pos);
                 const [arg2, pos2] = this.parseNextExprWithoutSupSub(tokens, pos1);
-                return [{ type: 'binaryFunc', content: command, args: [arg1, arg2] }, pos2];
+                return [new TexNode('binaryFunc', command, [arg1, arg2]), pos2];
             }
             default:
                 throw new Error( 'Invalid number of parameters');
         }
     }
 
-    parseLeftRightExpr(tokens: Token[], start: number): ParseResult {
+    parseLeftRightExpr(tokens: TexToken[], start: number): ParseResult {
         assert(tokens[start].eq(LEFT_COMMAND));
 
         let pos = start + 1;
@@ -604,20 +605,20 @@ export class LatexParser {
         const exprInside = tokens.slice(exprInsideStart, exprInsideEnd);
         const body = this.parse(exprInside);
         const args: TexNode[] = [
-            { type: 'element', content: leftDelimiter.value },
+            new TexNode('element', leftDelimiter.value),
             body,
-            { type: 'element', content: rightDelimiter.value }
+            new TexNode('element', rightDelimiter.value)
         ]
-        const res: TexNode = { type: 'leftright', content: '', args: args };
+        const res = new TexNode('leftright', '', args);
         return [res, pos];
     }
 
-    parseBeginEndExpr(tokens: Token[], start: number): ParseResult {
+    parseBeginEndExpr(tokens: TexToken[], start: number): ParseResult {
         assert(tokens[start].eq(BEGIN_COMMAND));
 
         let pos = start + 1;
         assert(tokens[pos].eq(LEFT_CURLY_BRACKET));
-        assert(tokens[pos + 1].type === TokenType.TEXT);
+        assert(tokens[pos + 1].type === TexTokenType.TEXT);
         assert(tokens[pos + 2].eq(RIGHT_CURLY_BRACKET));
         const envName = tokens[pos + 1].value;
         pos += 3;
@@ -634,7 +635,7 @@ export class LatexParser {
         pos = endIdx + 1;
         
         assert(tokens[pos].eq(LEFT_CURLY_BRACKET));
-        assert(tokens[pos + 1].type === TokenType.TEXT);
+        assert(tokens[pos + 1].type === TexTokenType.TEXT);
         assert(tokens[pos + 2].eq(RIGHT_CURLY_BRACKET));
         if (tokens[pos + 1].value !== envName) {
             throw new LatexParserError('Mismatched \\begin and \\end environments');
@@ -642,37 +643,43 @@ export class LatexParser {
         pos += 3;
     
         const exprInside = tokens.slice(exprInsideStart, exprInsideEnd);
-        // ignore whitespaces and '\n' before \end{envName}
-        while(exprInside.length > 0 && [TokenType.WHITESPACE, TokenType.NEWLINE].includes(exprInside[exprInside.length - 1].type)) {
+        // ignore spaces and '\n' before \end{envName}
+        while(exprInside.length > 0 && [TexTokenType.SPACE, TexTokenType.NEWLINE].includes(exprInside[exprInside.length - 1].type)) {
             exprInside.pop();
         }
         const body = this.parseAligned(exprInside);
-        const res: TexNode = { type: 'beginend', content: envName, data: body };
+        const res = new TexNode('beginend', envName, [], body);
         return [res, pos];
     }
 
-    parseAligned(tokens: Token[]): TexNode[][] {
+    parseAligned(tokens: TexToken[]): TexNode[][] {
         let pos = 0;
         const allRows: TexNode[][] = [];
         let row: TexNode[] = [];
         allRows.push(row);
-        let group: TexNode = { type: 'ordgroup', content: '', args: [] };
+        let group = new TexNode('ordgroup', '', []);
         row.push(group);
 
         while (pos < tokens.length) {
             const [res, newPos] = this.parseNextExpr(tokens, pos);
             pos = newPos;
+
             if (res.type === 'whitespace') {
-                continue;
-            } else if (res.type === 'newline' && !this.newline_sensitive) {
-                continue;
-            } else if (res.type === 'control' && res.content === '\\\\') {
+                if (!this.space_sensitive && res.content.replace(/ /g, '').length === 0) {
+                    continue;
+                }
+                if (!this.newline_sensitive && res.content === '\n') {
+                    continue;
+                }
+            }
+            
+            if (res.type === 'control' && res.content === '\\\\') {
                 row = [];
-                group = { type: 'ordgroup', content: '', args: [] };
+                group = new TexNode('ordgroup', '', []);
                 row.push(group);
                 allRows.push(row);
             } else if (res.type === 'control' && res.content === '&') {
-                group = { type: 'ordgroup', content: '', args: [] };
+                group = new TexNode('ordgroup', '', []);
                 row.push(group);
             } else {
                 group.args!.push(res);
@@ -683,14 +690,14 @@ export class LatexParser {
 }
 
 // Remove all whitespace before or after _ or ^
-function passIgnoreWhitespaceBeforeScriptMark(tokens: Token[]): Token[] {
-    const is_script_mark = (token: Token) => token.eq(SUB_SYMBOL) || token.eq(SUP_SYMBOL);
-    let out_tokens: Token[] = [];
+function passIgnoreWhitespaceBeforeScriptMark(tokens: TexToken[]): TexToken[] {
+    const is_script_mark = (token: TexToken) => token.eq(SUB_SYMBOL) || token.eq(SUP_SYMBOL);
+    let out_tokens: TexToken[] = [];
     for (let i = 0; i < tokens.length; i++) {
-        if (tokens[i].type === TokenType.WHITESPACE && i + 1 < tokens.length && is_script_mark(tokens[i + 1])) {
+        if (tokens[i].type === TexTokenType.SPACE && i + 1 < tokens.length && is_script_mark(tokens[i + 1])) {
             continue;
         }
-        if (tokens[i].type === TokenType.WHITESPACE && i - 1 >= 0 && is_script_mark(tokens[i - 1])) {
+        if (tokens[i].type === TexTokenType.SPACE && i - 1 >= 0 && is_script_mark(tokens[i - 1])) {
             continue;
         }
         out_tokens.push(tokens[i]);
@@ -699,10 +706,10 @@ function passIgnoreWhitespaceBeforeScriptMark(tokens: Token[]): Token[] {
 }
 
 // expand custom tex macros
-function passExpandCustomTexMacros(tokens: Token[], customTexMacros: {[key: string]: string}): Token[] {
-    let out_tokens: Token[] = [];
+function passExpandCustomTexMacros(tokens: TexToken[], customTexMacros: {[key: string]: string}): TexToken[] {
+    let out_tokens: TexToken[] = [];
     for (const token of tokens) {
-        if (token.type === TokenType.COMMAND && customTexMacros[token.value]) {
+        if (token.type === TexTokenType.COMMAND && customTexMacros[token.value]) {
             const expanded_tokens = tokenize(customTexMacros[token.value]);
             out_tokens = out_tokens.concat(expanded_tokens);
         } else {