tex2typst 0.2.13 → 0.2.16

package/src/writer.ts

@@ -1,5 +1,5 @@
 import { symbolMap } from "./map";
-import { TexNode, TexSqrtData, TexSupsubData, TypstNode, TypstSupsubData } from "./types";
+import { TexNode, TexSqrtData, TexSupsubData, TypstNode, TypstSupsubData, TypstToken, TypstTokenType } from "./types";
 
 
 // symbols that are supported by Typst but not by KaTeX
@@ -19,9 +19,6 @@ function is_delimiter(c: TypstNode): boolean {
     return c.type === 'atom' && ['(', ')', '[', ']', '{', '}', '|', '⌊', '⌋', '⌈', '⌉'].includes(c.content);
 }
 
-function text_node_shallow_eq(a: TexNode, b: TexNode): boolean {
-    return (a.type === b.type) && (a.content === b.content);
-}
 
 // \overset{X}{Y} -> op(Y, limits: #true)^X
 // and with special case \overset{\text{def}}{=} -> eq.def
@@ -29,48 +26,51 @@ function convert_overset(node: TexNode): TypstNode {
     const [sup, base] = node.args!;
 
     const is_def = (n: TexNode): boolean => {
-        if(n.type === 'text' && n.content === 'def') {
+        if (n.eq_shallow(new TexNode('text', 'def'))) {
             return true;
         }
         // \overset{def}{=} is also considered as eq.def
-        if(n.type === 'ordgroup' && n.args!.length === 3) {
+        if (n.type === 'ordgroup' && n.args!.length === 3) {
             const [a1, a2, a3] = n.args!;
-            const d: TexNode = { type: 'element', content: 'd' };
-            const e: TexNode = { type: 'element', content: 'e' };
-            const f: TexNode = { type: 'element', content: 'f' };
-            if(text_node_shallow_eq(a1, d) && text_node_shallow_eq(a2, e) && text_node_shallow_eq(a3, f)) {
+            const d = new TexNode('element', 'd');
+            const e = new TexNode('element', 'e');
+            const f = new TexNode('element', 'f');
+            if (a1.eq_shallow(d) && a2.eq_shallow(e) && a3.eq_shallow(f)) {
                 return true;
             }
         }
         return false;
     };
-    const is_eq = (n: TexNode): boolean => (n.type === 'element' && n.content === '=');
-    if(is_def(sup) && is_eq(base)) {
-        return {
-            type: 'symbol',
-            content: 'eq.def',
-        };
+    const is_eq = (n: TexNode): boolean => n.eq_shallow(new TexNode('element', '='));
+    if (is_def(sup) && is_eq(base)) {
+        return new TypstNode('symbol', 'eq.def');
     }
-    const op_call: TypstNode = {
-        type: 'unaryFunc',
-        content: 'op',
-        args: [convertTree(base)],
-        options: { limits: '#true' },
-    };
-    return {
-        type: 'supsub',
-        content: '',
-        data: {
+    const op_call = new TypstNode(
+        'unaryFunc',
+        'op',
+        [convertTree(base)]
+    );
+    op_call.setOptions({ limits: '#true' });
+    return new TypstNode(
+        'supsub',
+        '',
+        [],
+        {
             base: op_call,
             sup: convertTree(sup),
-        },
-    }
+        }
+    );
 }
 
+const TYPST_LEFT_PARENTHESIS: TypstToken = new TypstToken(TypstTokenType.ATOM, '(');
+const TYPST_RIGHT_PARENTHESIS: TypstToken = new TypstToken(TypstTokenType.ATOM, ')');
+const TYPST_COMMA: TypstToken = new TypstToken(TypstTokenType.ATOM, ',');
+const TYPST_NEWLINE: TypstToken = new TypstToken(TypstTokenType.SYMBOL, '\n');
+
 export class TypstWriterError extends Error {
-    node: TexNode | TypstNode;
+    node: TexNode | TypstNode | TypstToken;
 
-    constructor(message: string, node: TexNode | TypstNode) {
+    constructor(message: string, node: TexNode | TypstNode | TypstToken) {
         super(message);
         this.name = "TypstWriterError";
         this.node = node;
@@ -80,73 +80,91 @@ export class TypstWriterError extends Error {
 export class TypstWriter {
     private nonStrict: boolean;
     private preferTypstIntrinsic: boolean;
+    private keepSpaces: boolean;
 
     protected buffer: string = "";
-    protected queue: TypstNode[] = [];
+    protected queue: TypstToken[] = [];
 
-    private needSpaceAfterSingleItemScript = false;
     private insideFunctionDepth = 0;
 
-    constructor(nonStrict: boolean, preferTypstIntrinsic: boolean) {
+    constructor(nonStrict: boolean, preferTypstIntrinsic: boolean, keepSpaces: boolean) {
         this.nonStrict = nonStrict;
         this.preferTypstIntrinsic = preferTypstIntrinsic;
+        this.keepSpaces = keepSpaces;
     }
 
 
-    private writeBuffer(str: string) {
-        if (this.needSpaceAfterSingleItemScript && /^[0-9a-zA-Z\(]/.test(str)) {
-            this.buffer += ' ';
-        } else {
-            let no_need_space = false;
-            // starting clause
-            no_need_space ||= /[\(\|]$/.test(this.buffer) && /^\w/.test(str);
-            // putting punctuation
-            no_need_space ||= /^[}()_^,;!\|]$/.test(str);
-            // putting a prime
-            no_need_space ||= str === "'";
-            // continue a number
-            no_need_space ||= /[0-9]$/.test(this.buffer) && /^[0-9]/.test(str);
-            // leading sign
-            no_need_space ||= /[\(\[{]\s*(-|\+)$/.test(this.buffer) || this.buffer === "-" || this.buffer === "+";
-            // new line
-            no_need_space ||= str.startsWith('\n');
-            // buffer is empty
-            no_need_space ||= this.buffer === "";
-            // other cases
-            no_need_space ||= /[\s_^{\(]$/.test(this.buffer);
-            if(!no_need_space) {
-                this.buffer += ' ';
-            }
+    private writeBuffer(token: TypstToken) {
+        const str = token.content;
+
+        if (str === '') {
+            return;
         }
 
-        if (this.needSpaceAfterSingleItemScript) {
-            this.needSpaceAfterSingleItemScript = false;
+        let no_need_space = false;
+        // starting clause
+        no_need_space ||= /[\(\|]$/.test(this.buffer) && /^\w/.test(str);
+        // putting punctuation
+        no_need_space ||= /^[}()_^,;!\|]$/.test(str);
+        // putting a prime
+        no_need_space ||= str === "'";
+        // continue a number
+        no_need_space ||= /[0-9]$/.test(this.buffer) && /^[0-9]/.test(str);
+        // leading sign. e.g. produce "+1" instead of " +1"
+        no_need_space ||= /[\(\[{]\s*(-|\+)$/.test(this.buffer) || this.buffer === "-" || this.buffer === "+";
+        // new line
+        no_need_space ||= str.startsWith('\n');
+        // buffer is empty
+        no_need_space ||= this.buffer === "";
+        // str is starting with a space itself
+        no_need_space ||= /^\s/.test(str);
+        // other cases
+        no_need_space ||= /[\s_^{\(]$/.test(this.buffer);
+        if (!no_need_space) {
+            this.buffer += ' ';
         }
 
         this.buffer += str;
     }
 
-    public append(node: TypstNode) {
+    // Serialize a tree of TypstNode into a list of TypstToken
+    public serialize(node: TypstNode) {
         switch (node.type) {
             case 'empty':
                 break;
             case 'atom': {
                 if (node.content === ',' && this.insideFunctionDepth > 0) {
-                    this.queue.push({ type: 'symbol', content: 'comma' });
+                    this.queue.push(new TypstToken(TypstTokenType.SYMBOL, 'comma'));
                 } else {
-                    this.queue.push({ type: 'atom', content: node.content });
+                    this.queue.push(new TypstToken(TypstTokenType.ATOM, node.content));
                 }
                 break;
             }
             case 'symbol':
+                this.queue.push(new TypstToken(TypstTokenType.SYMBOL, node.content));
+                break;
             case 'text':
+                this.queue.push(new TypstToken(TypstTokenType.TEXT, `"${node.content}"`));
+                break;
             case 'comment':
-            case 'newline':
-                this.queue.push(node);
+                this.queue.push(new TypstToken(TypstTokenType.COMMENT, `//${node.content}`));
+                break;
+            case 'whitespace':
+                for (const c of node.content) {
+                    if (c === ' ') {
+                        if (this.keepSpaces) {
+                            this.queue.push(new TypstToken(TypstTokenType.SPACE, c));
+                        }
+                    } else if (c === '\n') {
+                        this.queue.push(new TypstToken(TypstTokenType.SYMBOL, c));
+                    } else {
+                        throw new TypstWriterError(`Unexpected whitespace character: ${c}`, node);
+                    }
+                }
                 break;
             case 'group':
                 for (const item of node.args!) {
-                    this.append(item);
+                    this.serialize(item);
                 }
                 break;
             case 'supsub': {
@@ -160,49 +178,49 @@ export class TypstWriter {
                     // e.g. 
                     // y_1' -> y'_1
                     // y_{a_1}' -> y'_{a_1}
-                    this.queue.push({ type: 'atom', content: '\''});
+                    this.queue.push(new TypstToken(TypstTokenType.ATOM, '\''));
                     trailing_space_needed = false;
                 }
                 if (sub) {
-                    this.queue.push({ type: 'atom', content: '_'});
+                    this.queue.push(new TypstToken(TypstTokenType.ATOM, '_'));
                     trailing_space_needed = this.appendWithBracketsIfNeeded(sub);
                 }
                 if (sup && !has_prime) {
-                    this.queue.push({ type: 'atom', content: '^'});
+                    this.queue.push(new TypstToken(TypstTokenType.ATOM, '^'));
                     trailing_space_needed = this.appendWithBracketsIfNeeded(sup);
                 }
                 if (trailing_space_needed) {
-                    this.queue.push({ type: 'softSpace', content: ''});
+                    this.queue.push(new TypstToken(TypstTokenType.CONTROL, ' '));
                 }
                 break;
             }
             case 'binaryFunc': {
-                const func_symbol: TypstNode = { type: 'symbol', content: node.content };
+                const func_symbol: TypstToken = new TypstToken(TypstTokenType.SYMBOL, node.content);
                 const [arg0, arg1] = node.args!;
                 this.queue.push(func_symbol);
-                this.insideFunctionDepth ++;
-                this.queue.push({ type: 'atom', content: '('});
-                this.append(arg0);
-                this.queue.push({ type: 'atom', content: ','});
-                this.append(arg1);
-                this.queue.push({ type: 'atom', content: ')'});
-                this.insideFunctionDepth --;
+                this.insideFunctionDepth++;
+                this.queue.push(TYPST_LEFT_PARENTHESIS);
+                this.serialize(arg0);
+                this.queue.push(new TypstToken(TypstTokenType.ATOM, ','));
+                this.serialize(arg1);
+                this.queue.push(TYPST_RIGHT_PARENTHESIS);
+                this.insideFunctionDepth--;
                 break;
             }
             case 'unaryFunc': {
-                const func_symbol: TypstNode = { type: 'symbol', content: node.content };
+                const func_symbol: TypstToken = new TypstToken(TypstTokenType.SYMBOL, node.content);
                 const arg0 = node.args![0];
                 this.queue.push(func_symbol);
-                this.insideFunctionDepth ++;
-                this.queue.push({ type: 'atom', content: '('});
-                this.append(arg0);
-                if(node.options) {
+                this.insideFunctionDepth++;
+                this.queue.push(TYPST_LEFT_PARENTHESIS);
+                this.serialize(arg0);
+                if (node.options) {
                     for (const [key, value] of Object.entries(node.options)) {
-                        this.queue.push({ type: 'symbol', content: `, ${key}: ${value}`});
+                        this.queue.push(new TypstToken(TypstTokenType.SYMBOL, `, ${key}: ${value}`));
                     }
                 }
-                this.queue.push({ type: 'atom', content: ')'});
-                this.insideFunctionDepth --;
+                this.queue.push(TYPST_RIGHT_PARENTHESIS);
+                this.insideFunctionDepth--;
                 break;
             }
             case 'align': {
@@ -210,50 +228,54 @@ export class TypstWriter {
                 matrix.forEach((row, i) => {
                     row.forEach((cell, j) => {
                         if (j > 0) {
-                            this.queue.push({ type: 'atom', content: '&' });
+                            this.queue.push(new TypstToken(TypstTokenType.ATOM, '&'));
                         }
-                        this.append(cell);
+                        this.serialize(cell);
                     });
                     if (i < matrix.length - 1) {
-                        this.queue.push({ type: 'symbol', content: '\\' });
+                        this.queue.push(new TypstToken(TypstTokenType.SYMBOL, '\\'));
                     }
                 });
                 break;
             }
             case 'matrix': {
                 const matrix = node.data as TypstNode[][];
-                this.queue.push({ type: 'symbol', content: 'mat' });
-                this.insideFunctionDepth ++; 
-                this.queue.push({ type: 'atom', content: '('});
-                this.queue.push({type: 'symbol', content: 'delim: #none, '});
+                this.queue.push(new TypstToken(TypstTokenType.SYMBOL, 'mat'));
+                this.insideFunctionDepth++;
+                this.queue.push(TYPST_LEFT_PARENTHESIS);
+                if (node.options) {
+                    for (const [key, value] of Object.entries(node.options)) {
+                        this.queue.push(new TypstToken(TypstTokenType.SYMBOL, `${key}: ${value}, `));
+                    }
+                }
                 matrix.forEach((row, i) => {
                     row.forEach((cell, j) => {
                         // There is a leading & in row
                         // if (cell.type === 'ordgroup' && cell.args!.length === 0) {
-                            // this.queue.push({ type: 'atom', content: ',' });
-                            // return;
+                        // this.queue.push(new TypstNode('atom', ','));
+                        // return;
                         // }
                         // if (j == 0 && cell.type === 'newline' && cell.content === '\n') {
-                            // return;
+                        // return;
                         // }
-                        this.append(cell);
+                        this.serialize(cell);
                         // cell.args!.forEach((n) => this.append(n));
                         if (j < row.length - 1) {
-                            this.queue.push({ type: 'atom', content: ',' });
+                            this.queue.push(new TypstToken(TypstTokenType.ATOM, ','));
                         } else {
                             if (i < matrix.length - 1) {
-                                this.queue.push({ type: 'atom', content: ';' });
+                                this.queue.push(new TypstToken(TypstTokenType.ATOM, ';'));
                             }
                         }
                     });
                 });
-                this.queue.push({ type: 'atom', content: ')'});
-                this.insideFunctionDepth --;
+                this.queue.push(TYPST_RIGHT_PARENTHESIS);
+                this.insideFunctionDepth--;
                 break;
             }
             case 'unknown': {
                 if (this.nonStrict) {
-                    this.queue.push({ type: 'symbol', content: node.content });
+                    this.queue.push(new TypstToken(TypstTokenType.SYMBOL, node.content));
                 } else {
                     throw new TypstWriterError(`Unknown macro: ${node.content}`, node);
                 }
@@ -276,44 +298,35 @@ export class TypstWriter {
         }
 
         if (need_to_wrap) {
-            this.queue.push({ type: 'atom', content: '(' });
-            this.append(node);
-            this.queue.push({ type: 'atom', content: ')' });
+            this.queue.push(TYPST_LEFT_PARENTHESIS);
+            this.serialize(node);
+            this.queue.push(TYPST_RIGHT_PARENTHESIS);
         } else {
-            this.append(node);
+            this.serialize(node);
         }
 
         return !need_to_wrap;
     }
 
     protected flushQueue() {
-        this.queue.forEach((node) => {
-            let str = "";
-            switch (node.type) {
-                case 'atom':
-                case 'symbol':
-                    str = node.content;
-                    break;
-                case 'text':
-                    str = `"${node.content}"`;
-                    break;
-                case 'softSpace':
-                    this.needSpaceAfterSingleItemScript = true;
-                    str = '';
-                    break;
-                case 'comment':
-                    str = `//${node.content}`;
-                    break;
-                case 'newline':
-                    str = '\n';
-                    break;
-                default:
-                    throw new TypstWriterError(`Unexpected node type to stringify: ${node.type}`, node)
-            }
-            if (str !== '') {
-                this.writeBuffer(str);
+        const SOFT_SPACE = new TypstToken(TypstTokenType.CONTROL, ' ');
+
+        // delete soft spaces if they are not needed
+        for(let i = 0; i < this.queue.length; i++) {
+            let token = this.queue[i];
+            if (token.eq(SOFT_SPACE)) {
+                if (i === this.queue.length - 1) {
+                    this.queue[i].content = '';
+                } else if (this.queue[i + 1].isOneOf([TYPST_RIGHT_PARENTHESIS, TYPST_COMMA, TYPST_NEWLINE])) {
+                    this.queue[i].content = '';
+                }
             }
+        }
+
+        this.queue.forEach((token) => {
+            this.writeBuffer(token)
         });
+
         this.queue = [];
     }
 
@@ -348,48 +361,50 @@ export class TypstWriter {
     }
 }
 
+// Convert a tree of TexNode into a tree of TypstNode
 export function convertTree(node: TexNode): TypstNode {
     switch (node.type) {
         case 'empty':
+            return new TypstNode('empty', '');
         case 'whitespace':
-            return { type: 'empty', content: '' };
+            return new TypstNode('whitespace', node.content);
         case 'ordgroup':
-            return {
-                type: 'group',
-                content: '',
-                args: node.args!.map(convertTree),
-            };
+            return new TypstNode(
+                'group',
+                '',
+                node.args!.map(convertTree),
+            );
         case 'element':
-            return { type: 'atom', content: convertToken(node.content) };
+            return new TypstNode('atom', convertToken(node.content));
         case 'symbol':
-            return { type: 'symbol', content: convertToken(node.content) };
+            return new TypstNode('symbol', convertToken(node.content));
         case 'text':
-            return { type: 'text', content: node.content };
+            return new TypstNode('text', node.content);
         case 'comment':
-            return { type: 'comment', content: node.content };
+            return new TypstNode('comment', node.content);
         case 'supsub': {
             let { base, sup, sub } = node.data as TexSupsubData;
 
             // Special logic for overbrace
             if (base && base.type === 'unaryFunc' && base.content === '\\overbrace' && sup) {
-                return {
-                    type: 'binaryFunc',
-                    content: 'overbrace',
-                    args: [convertTree(base.args![0]), convertTree(sup)],
-                };
+                return new TypstNode(
+                    'binaryFunc',
+                    'overbrace',
+                    [convertTree(base.args![0]), convertTree(sup)],
+                );
             } else if (base && base.type === 'unaryFunc' && base.content === '\\underbrace' && sub) {
-                return {
-                    type: 'binaryFunc',
-                    content: 'underbrace',
-                    args: [convertTree(base.args![0]), convertTree(sub)],
-                };
+                return new TypstNode(
+                    'binaryFunc',
+                    'underbrace',
+                    [convertTree(base.args![0]), convertTree(sub)],
+                );
             }
 
             const data: TypstSupsubData = {
                 base: convertTree(base),
             };
             if (data.base.type === 'empty') {
-                data.base = { type: 'text', content: '' };
+                data.base = new TypstNode('text', '');
             }
 
             if (sup) {
@@ -400,74 +415,67 @@ export function convertTree(node: TexNode): TypstNode {
                 data.sub = convertTree(sub);
             }
 
-            return {
-                type: 'supsub',
-                content: '',
-                data: data,
-            };
+            return new TypstNode('supsub', '', [], data);
         }
         case 'leftright': {
             const [left, body, right] = node.args!;
             // These pairs will be handled by Typst compiler by default. No need to add lr()
-            const group: TypstNode = {
-                type: 'group',
-                content: '',
-                args: node.args!.map(convertTree),
-            };                
+            const group: TypstNode = new TypstNode(
+                'group',
+                '',
+                node.args!.map(convertTree),
+            );
             if ([
-                    "[]", "()", "\\{\\}",
-                    "\\lfloor\\rfloor",
-                    "\\lceil\\rceil",
-                    "\\lfloor\\rceil",
-                ].includes(left.content + right.content)) {
+                "[]", "()", "\\{\\}",
+                "\\lfloor\\rfloor",
+                "\\lceil\\rceil",
+                "\\lfloor\\rceil",
+            ].includes(left.content + right.content)) {
                 return group;
             }
-            return {
-                type: 'unaryFunc',
-                content: 'lr',
-                args: [group],
-            };
+            return new TypstNode(
+                'unaryFunc',
+                'lr',
+                [group],
+            );
         }
         case 'binaryFunc': {
             if (node.content === '\\overset') {
                 return convert_overset(node);
             }
-            return {
-                type: 'binaryFunc',
-                content: convertToken(node.content),
-                args: node.args!.map(convertTree),
-            };
+            return new TypstNode(
+                'binaryFunc',
+                convertToken(node.content),
+                node.args!.map(convertTree),
+            );
         }
         case 'unaryFunc': {
             const arg0 = convertTree(node.args![0]);
             // \sqrt{3}{x} -> root(3, x)
             if (node.content === '\\sqrt' && node.data) {
                 const data = convertTree(node.data as TexSqrtData); // the number of times to take the root
-                return {
-                    type: 'binaryFunc',
-                    content: 'root',
-                    args: [data, arg0],
-                };
+                return new TypstNode(
+                    'binaryFunc',
+                    'root',
+                    [data, arg0],
+                );
             }
             // \mathbf{a} -> upright(mathbf(a))
             if (node.content === '\\mathbf') {
-                const inner: TypstNode = {
-                    type: 'unaryFunc',
-                    content: 'bold',
-                    args: [arg0],
-                };
-                return {
-                    type: 'unaryFunc',
-                    content: 'upright',
-                    args: [inner],
-                };
+                const inner: TypstNode = new TypstNode(
+                    'unaryFunc',
+                    'bold',
+                    [arg0],
+                );
+                return new TypstNode(
+                    'unaryFunc',
+                    'upright',
+                    [inner],
+                );
             }
             // \mathbb{R} -> RR
             if (node.content === '\\mathbb' && arg0.type === 'atom' && /^[A-Z]$/.test(arg0.content)) {
-                return {
-                    type: 'symbol',
-                    content: arg0.content + arg0.content,
-                };
+                return new TypstNode('symbol', arg0.content + arg0.content);
             }
             // \operatorname{opname} -> op("opname")
             if (node.content === '\\operatorname') {
@@ -478,54 +486,43 @@ export function convertTree(node: TexNode): TypstNode {
                 const text = body[0].content;
 
                 if (TYPST_INTRINSIC_SYMBOLS.includes(text)) {
-                    return {
-                        type: 'symbol',
-                        content: text,
-                    };
+                    return new TypstNode('symbol', text);
                 } else {
-                    return {
-                        type: 'unaryFunc',
-                        content: 'op',
-                        args: [{ type: 'text', content: text }],
-                    };
+                    return new TypstNode(
+                        'unaryFunc',
+                        'op',
+                        [new TypstNode('text', text)],
+                    );
                 }
             }
 
             // generic case
-            return {
-                type: 'unaryFunc',
-                content: convertToken(node.content),
-                args: node.args!.map(convertTree),
-            };
+            return new TypstNode(
+                'unaryFunc',
+                convertToken(node.content),
+                node.args!.map(convertTree),
+            );
         }
-        case 'newline':
-            return { type: 'newline', content: '\n' };
         case 'beginend': {
             const matrix = node.data as TexNode[][];
             const data = matrix.map((row) => row.map(convertTree));
 
             if (node.content!.startsWith('align')) {
                 // align, align*, alignat, alignat*, aligned, etc.
-                return {
-                    type: 'align',
-                    content: '',
-                    data: data,
-                };
+                return new TypstNode( 'align', '', [], data);
             } else {
-                return {
-                    type: 'matrix',
-                    content: 'mat',
-                    data: data,
-                };
+                const res = new TypstNode('matrix', '', [], data);
+                res.setOptions({'delim': '#none'});
+                return res;
             }
         }
         case 'unknownMacro':
-            return { type: 'unknown', content: convertToken(node.content) };
+            return new TypstNode('unknown', convertToken(node.content));
         case 'control':
             if (node.content === '\\\\') {
-                return { type: 'symbol', content: '\\' };
+                return new TypstNode('symbol', '\\');
             } else if (node.content === '\\,') {
-                return { type: 'symbol', content: 'thin' };
+                return new TypstNode('symbol', 'thin');
             } else {
                 throw new TypstWriterError(`Unknown control sequence: ${node.content}`, node);
             }
@@ -542,7 +539,9 @@ function convertToken(token: string): string {
         return '\\/';
     } else if (token === '\\|') {
         // \| in LaTeX is double vertical bar looks like ||
-        return 'parallel'; 
+        return 'parallel';
+    } else if (token === '\\colon') {
+        return ':';
     } else if (token === '\\\\') {
         return '\\';
     } else if (['\\$', '\\#', '\\&', '\\_'].includes(token)) {