tex2typst 0.3.18 → 0.3.20

package/src/map.ts

@@ -7,6 +7,9 @@ const symbolMap = new Map<string, string>([
     [':', 'med'],
     [';', 'thick'],
 
+    /* textual operators */
+    ['ln', 'ln'],
+    ['log', 'log'],
     ['cos', 'cos'],
     ['sin', 'sin'],
     ['tan', 'tan'],

package/src/tex-parser.ts

@@ -1,54 +1,8 @@
 import { symbolMap } from "./map";
 import { TexNode, TexSupsubData, TexToken, TexTokenType } from "./types";
 import { assert } from "./util";
-import { JSLex, Scanner } from "./jslex";
 import { array_find } from "./generic";
-
-const UNARY_COMMANDS = [
-    'sqrt',
-    'text',
-
-    'bar',
-    'bold',
-    'boldsymbol',
-    'ddot',
-    'dot',
-    'hat',
-    'mathbb',
-    'mathbf',
-    'mathcal',
-    'mathfrak',
-    'mathit',
-    'mathrm',
-    'mathscr',
-    'mathsf',
-    'mathtt',
-    'operatorname',
-    'overbrace',
-    'overline',
-    'pmb',
-    'rm',
-    'tilde',
-    'underbrace',
-    'underline',
-    'vec',
-    'widehat',
-    'widetilde',
-    'overleftarrow',
-    'overrightarrow',
-    'hspace',
-]
-
-const BINARY_COMMANDS = [
-    'frac',
-    'tfrac',
-    'binom',
-    'dbinom',
-    'dfrac',
-    'tbinom',
-    'overset',
-    'underset',
-]
+import { TEX_BINARY_COMMANDS, TEX_UNARY_COMMANDS, tokenize_tex } from "./tex-tokenizer";
 
 const IGNORED_COMMANDS = [
     'bigl', 'bigr',
@@ -60,9 +14,9 @@ const IGNORED_COMMANDS = [
 const EMPTY_NODE: TexNode = new TexNode('empty', '');
 
 function get_command_param_num(command: string): number {
-    if (UNARY_COMMANDS.includes(command)) {
+    if (TEX_UNARY_COMMANDS.includes(command)) {
         return 1;
-    } else if (BINARY_COMMANDS.includes(command)) {
+    } else if (TEX_BINARY_COMMANDS.includes(command)) {
         return 2;
     } else {
         return 0;
@@ -143,93 +97,6 @@ function find_closing_end_command(tokens: TexToken[], start: number): number {
 }
 
 
-function unescape(str: string): string {
-    const chars = ['{', '}', '\\', '$', '&', '#', '_', '%'];
-    for (const char of chars) {
-        str = str.replaceAll('\\' + char, char);
-    }
-    return str;
-}
-
-const rules_map = new Map<string, (a: Scanner<TexToken>) => TexToken | TexToken[]>([
-    [
-        String.raw`\\(text|operatorname|begin|end|hspace){.+?}`, (s) => {
-            const text = s.text()!;
-            const command = text.substring(0, text.indexOf('{'));
-            const text_inside = text.substring(text.indexOf('{') + 1, text.lastIndexOf('}'));
-            return [
-                new TexToken(TexTokenType.COMMAND, command),
-                new TexToken(TexTokenType.CONTROL, '{'),
-                new TexToken(TexTokenType.TEXT, unescape(text_inside)),
-                new TexToken(TexTokenType.CONTROL, '}')
-            ]
-        }
-    ],
-    [String.raw`%[^\n]*`, (s) => new TexToken(TexTokenType.COMMENT, s.text()!.substring(1))],
-    [String.raw`[{}_^&]`, (s) => new TexToken(TexTokenType.CONTROL, s.text()!)],
-    [String.raw`\\[\\,:;! ]`, (s) => new TexToken(TexTokenType.CONTROL, s.text()!)],
-    [String.raw`\r?\n`, (_s) => new TexToken(TexTokenType.NEWLINE, "\n")],
-    [String.raw`\s+`, (s) => new TexToken(TexTokenType.SPACE, s.text()!)],
-    [String.raw`\\[{}%$&#_|]`, (s) => new TexToken(TexTokenType.ELEMENT, s.text()!)],
-    [String.raw`(\\[a-zA-Z]+)(\s*\d|\s+[a-zA-Z])\s*([0-9a-zA-Z])`, (s) => {
-        const text = s.text()!;
-        const regex = RegExp(String.raw`(\\[a-zA-Z]+)(\s*\d|\s+[a-zA-Z])\s*([0-9a-zA-Z])`);
-        const match = text.match(regex);
-        assert(match !== null);
-        const command = match![1];
-        if (BINARY_COMMANDS.includes(command.substring(1))) {
-            const arg1 = match![2].trimStart();
-            const arg2 = match![3];
-            return [
-                new TexToken(TexTokenType.COMMAND, command),
-                new TexToken(TexTokenType.ELEMENT, arg1),
-                new TexToken(TexTokenType.ELEMENT, arg2),
-            ];
-        } else {
-            s.reject();
-            return [];
-        }
-   }],
-    [String.raw`(\\[a-zA-Z]+)(\s*\d|\s+[a-zA-Z])`, (s) => {
-        const text = s.text()!;
-        const regex = RegExp(String.raw`(\\[a-zA-Z]+)(\s*\d|\s+[a-zA-Z])`);
-        const match = text.match(regex);
-        assert(match !== null);
-        const command = match![1];
-        if (UNARY_COMMANDS.includes(command.substring(1))) {
-            const arg1 = match![2].trimStart();
-            return [
-                new TexToken(TexTokenType.COMMAND, command),
-                new TexToken(TexTokenType.ELEMENT, arg1),
-            ];
-        } else {
-            s.reject();
-            return [];
-        }
-    }],
-    [String.raw`\\[a-zA-Z]+`, (s) => {
-        const command = s.text()!;
-        return [ new TexToken(TexTokenType.COMMAND, command), ];
-    }],
-    // Numbers like "123", "3.14"
-    [String.raw`[0-9]+(\.[0-9]+)?`, (s) => new TexToken(TexTokenType.ELEMENT, s.text()!)],
-    [String.raw`[a-zA-Z]`, (s) => new TexToken(TexTokenType.ELEMENT, s.text()!)],
-    [String.raw`[+\-*/='<>!.,;:?()\[\]|]`, (s) => new TexToken(TexTokenType.ELEMENT, s.text()!)],
-    // non-ASCII characters
-    [String.raw`[^\x00-\x7F]`, (s) => new TexToken(TexTokenType.ELEMENT, s.text()!)],
-    [String.raw`.`, (s) => new TexToken(TexTokenType.UNKNOWN, s.text()!)],
-]);
-
-const spec = {
-    "start": rules_map
-};
-
-export function tokenize_tex(input: string): TexToken[] {
-    const lexer = new JSLex<TexToken>(spec);
-    return lexer.collect(input);
-}
-
-
 export class LatexParserError extends Error {
     constructor(message: string) {
         super(message);
@@ -395,7 +262,7 @@ export class LatexParser {
                     case '}':
                         throw new LatexParserError("Unmatched '}'");
                     case '\\\\':
-                    case '\\!':    
+                    case '\\!':
                     case '\\,':
                     case '\\:':
                     case '\\;':
@@ -554,7 +421,7 @@ export class LatexParser {
         while (pos < tokens.length) {
             const whitespaceCount = eat_whitespaces(tokens, pos).length;
             pos += whitespaceCount;
-            
+
             if (pos >= tokens.length || !tokens[pos].eq(LEFT_CURLY_BRACKET)) {
                 break;
             }

package/src/tex-tokenizer.ts

@@ -0,0 +1,138 @@
+import { TexToken, TexTokenType } from "./types";
+import { assert } from "./util";
+import { JSLex, Scanner } from "./jslex";
+
+export const TEX_UNARY_COMMANDS = [
+    'sqrt',
+    'text',
+
+    'bar',
+    'bold',
+    'boldsymbol',
+    'ddot',
+    'dot',
+    'hat',
+    'mathbb',
+    'mathbf',
+    'mathcal',
+    'mathfrak',
+    'mathit',
+    'mathrm',
+    'mathscr',
+    'mathsf',
+    'mathtt',
+    'operatorname',
+    'overbrace',
+    'overline',
+    'pmb',
+    'rm',
+    'tilde',
+    'underbrace',
+    'underline',
+    'vec',
+    'widehat',
+    'widetilde',
+    'overleftarrow',
+    'overrightarrow',
+    'hspace',
+]
+
+export const TEX_BINARY_COMMANDS = [
+    'frac',
+    'tfrac',
+    'binom',
+    'dbinom',
+    'dfrac',
+    'tbinom',
+    'overset',
+    'underset',
+]
+
+
+function unescape(str: string): string {
+    const chars = ['{', '}', '\\', '$', '&', '#', '_', '%'];
+    for (const char of chars) {
+        str = str.replaceAll('\\' + char, char);
+    }
+    return str;
+}
+
+const rules_map = new Map<string, (a: Scanner<TexToken>) => TexToken | TexToken[]>([
+    [
+        String.raw`\\(text|operatorname|begin|end|hspace){.+?}`, (s) => {
+            const text = s.text()!;
+            const command = text.substring(0, text.indexOf('{'));
+            const text_inside = text.substring(text.indexOf('{') + 1, text.lastIndexOf('}'));
+            return [
+                new TexToken(TexTokenType.COMMAND, command),
+                new TexToken(TexTokenType.CONTROL, '{'),
+                new TexToken(TexTokenType.TEXT, unescape(text_inside)),
+                new TexToken(TexTokenType.CONTROL, '}')
+            ]
+        }
+    ],
+    [String.raw`%[^\n]*`, (s) => new TexToken(TexTokenType.COMMENT, s.text()!.substring(1))],
+    [String.raw`[{}_^&]`, (s) => new TexToken(TexTokenType.CONTROL, s.text()!)],
+    [String.raw`\\[\\,:;! ]`, (s) => new TexToken(TexTokenType.CONTROL, s.text()!)],
+    [String.raw`\r?\n`, (_s) => new TexToken(TexTokenType.NEWLINE, "\n")],
+    [String.raw`\s+`, (s) => new TexToken(TexTokenType.SPACE, s.text()!)],
+    [String.raw`\\[{}%$&#_|]`, (s) => new TexToken(TexTokenType.ELEMENT, s.text()!)],
+    // e.g. match `\frac13`, `\frac1 b`, `\frac a b`
+    [String.raw`(\\[a-zA-Z]+)(\s*\d|\s+[a-zA-Z])\s*([0-9a-zA-Z])`, (s) => {
+        const text = s.text()!;
+        const regex = RegExp(String.raw`(\\[a-zA-Z]+)(\s*\d|\s+[a-zA-Z])\s*([0-9a-zA-Z])`);
+        const match = text.match(regex);
+        assert(match !== null);
+        const command = match![1];
+        if (TEX_BINARY_COMMANDS.includes(command.substring(1))) {
+            const arg1 = match![2].trimStart();
+            const arg2 = match![3];
+            return [
+                new TexToken(TexTokenType.COMMAND, command),
+                new TexToken(TexTokenType.ELEMENT, arg1),
+                new TexToken(TexTokenType.ELEMENT, arg2),
+            ];
+        } else {
+            s.reject();
+            return [];
+        }
+    }],
+    // e.g. match `\sqrt3`, `\sqrt a`
+    [String.raw`(\\[a-zA-Z]+)(\s*\d|\s+[a-zA-Z])`, (s) => {
+        const text = s.text()!;
+        const regex = RegExp(String.raw`(\\[a-zA-Z]+)(\s*\d|\s+[a-zA-Z])`);
+        const match = text.match(regex);
+        assert(match !== null);
+        const command = match![1];
+        if (TEX_UNARY_COMMANDS.includes(command.substring(1))) {
+            const arg1 = match![2].trimStart();
+            return [
+                new TexToken(TexTokenType.COMMAND, command),
+                new TexToken(TexTokenType.ELEMENT, arg1),
+            ];
+        } else {
+            s.reject();
+            return [];
+        }
+    }],
+    [String.raw`\\[a-zA-Z]+`, (s) => {
+        const command = s.text()!;
+        return [ new TexToken(TexTokenType.COMMAND, command), ];
+    }],
+    // Numbers like "123", "3.14"
+    [String.raw`[0-9]+(\.[0-9]+)?`, (s) => new TexToken(TexTokenType.ELEMENT, s.text()!)],
+    [String.raw`[a-zA-Z]`, (s) => new TexToken(TexTokenType.ELEMENT, s.text()!)],
+    [String.raw`[+\-*/='<>!.,;:?()\[\]|]`, (s) => new TexToken(TexTokenType.ELEMENT, s.text()!)],
+    // non-ASCII characters
+    [String.raw`[^\x00-\x7F]`, (s) => new TexToken(TexTokenType.ELEMENT, s.text()!)],
+    [String.raw`.`, (s) => new TexToken(TexTokenType.UNKNOWN, s.text()!)],
+]);
+
+const spec = {
+    "start": rules_map
+};
+
+export function tokenize_tex(input: string): TexToken[] {
+    const lexer = new JSLex<TexToken>(spec);
+    return lexer.collect(input);
+}

package/src/types.ts

@@ -393,11 +393,10 @@ export const TYPST_FALSE: TypstPrimitiveValue = false;
  * ATTENTION:
  * Don't use any options except those explicitly documented in
  *   https://github.com/qwinsi/tex2typst/blob/main/docs/api-reference.md
- * Any undocumented options may break in the future!
+ * Any undocumented options may be not working at present or break in the future!
  */
 export interface Tex2TypstOptions {
     nonStrict?: boolean; /** default is true */
-    preferTypstIntrinsic?: boolean; /** default is true */
     preferShorthands?: boolean; /** default is true */
     keepSpaces?: boolean; /** default is false */
     fracToSlash?: boolean; /** default is true */

package/src/typst-parser.ts

@@ -1,11 +1,9 @@
 
 import { array_find } from "./generic";
 import { TYPST_NONE, TypstLrData, TypstNamedParams, TypstNode, TypstSupsubData, TypstToken, TypstTokenType } from "./types";
+import { tokenize_typst } from "./typst-tokenizer";
 import { assert, isalpha } from "./util";
-import { reverseShorthandMap } from "./typst-shorthands";
-import { JSLex, Scanner } from "./jslex";
 
-const TYPST_SHORTHANDS = Array.from(reverseShorthandMap.keys());
 
 // TODO: In Typst, y' ' is not the same as y''.
 // The parser should be able to parse the former correctly.
@@ -18,77 +16,6 @@ function eat_primes(tokens: TypstToken[], start: number): number {
 }
 
 
-function generate_regex_for_shorthands(): string {
-    const regex_list = TYPST_SHORTHANDS.map((s) => {
-        s = s.replaceAll('|', '\\|');
-        s = s.replaceAll('.', '\\.');
-        s = s.replaceAll('[', '\\[');
-        s = s.replaceAll(']', '\\]');
-        return s;
-    });
-    return `(${regex_list.join('|')})`;
-}
-
-
-const REGEX_SHORTHANDS = generate_regex_for_shorthands();
-
-const rules_map = new Map<string, (a: Scanner<TypstToken>) => TypstToken | TypstToken[]>([
-    [String.raw`//[^\n]*`, (s) => new TypstToken(TypstTokenType.COMMENT, s.text()!.substring(2))],
-    [String.raw`/`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
-    [String.raw`[_^&]`, (s) => new TypstToken(TypstTokenType.CONTROL, s.text()!)],
-    [String.raw`\r?\n`, (_s) => new TypstToken(TypstTokenType.NEWLINE, "\n")],
-    [String.raw`\s+`, (s) => new TypstToken(TypstTokenType.SPACE, s.text()!)],
-    [String.raw`\\[$&#_]`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
-    [String.raw`\\\n`, (s) => {
-        return [
-            new TypstToken(TypstTokenType.CONTROL, "\\"),
-            new TypstToken(TypstTokenType.NEWLINE, "\n"),
-        ]
-    }],
-    [String.raw`\\\s`, (s) => {
-        return [
-            new TypstToken(TypstTokenType.CONTROL, "\\"),
-            new TypstToken(TypstTokenType.SPACE, " "),
-        ]
-    }],
-    // this backslash is dummy and will be ignored in later stages
-    [String.raw`\\\S`, (_s) => new TypstToken(TypstTokenType.CONTROL, "")],
-    [
-        String.raw`"([^"]|(\\"))*"`,
-        (s) => {
-            const text = s.text()!.substring(1, s.text()!.length - 1);
-            // replace all escape characters with their actual characters
-            text.replaceAll('\\"', '"');
-            return new TypstToken(TypstTokenType.TEXT, text);
-        }
-    ],
-    [
-        REGEX_SHORTHANDS,
-        (s) => {
-            const shorthand = s.text()!;
-            const symbol = reverseShorthandMap.get(shorthand)!;
-            return new TypstToken(TypstTokenType.SYMBOL, symbol);
-        }
-    ],
-    [String.raw`[0-9]+(\.[0-9]+)?`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
-    [String.raw`[+\-*/=\'<>!.,;?()\[\]|]`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
-    [String.raw`[a-zA-Z\.]+`, (s) => {
-        return new TypstToken(s.text()!.length === 1? TypstTokenType.ELEMENT: TypstTokenType.SYMBOL, s.text()!);
-    }],
-    [String.raw`#none`, (s) => new TypstToken(TypstTokenType.NONE, s.text()!)],
-    [String.raw`.`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
-]);
-
-const spec = {
-    "start": rules_map
-};
-
-export function tokenize_typst(input: string): TypstToken[] {
-    const lexer = new JSLex<TypstToken>(spec);
-    return lexer.collect(input);
-}
-
-
 function _find_closing_match(tokens: TypstToken[], start: number,
         leftBrackets: TypstToken[], rightBrackets: TypstToken[]): number {
     assert(tokens[start].isOneOf(leftBrackets));

package/src/typst-tokenizer.ts

@@ -0,0 +1,76 @@
+import { TypstToken, TypstTokenType } from "./types";
+import { reverseShorthandMap } from "./typst-shorthands";
+import { JSLex, Scanner } from "./jslex";
+
+const TYPST_SHORTHANDS = Array.from(reverseShorthandMap.keys());
+
+
+function generate_regex_for_shorthands(): string {
+    const regex_list = TYPST_SHORTHANDS.map((s) => {
+        s = s.replaceAll('|', '\\|');
+        s = s.replaceAll('.', '\\.');
+        s = s.replaceAll('[', '\\[');
+        s = s.replaceAll(']', '\\]');
+        return s;
+    });
+    return `(${regex_list.join('|')})`;
+}
+
+
+const REGEX_SHORTHANDS = generate_regex_for_shorthands();
+
+const rules_map = new Map<string, (a: Scanner<TypstToken>) => TypstToken | TypstToken[]>([
+    [String.raw`//[^\n]*`, (s) => new TypstToken(TypstTokenType.COMMENT, s.text()!.substring(2))],
+    [String.raw`/`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
+    [String.raw`[_^&]`, (s) => new TypstToken(TypstTokenType.CONTROL, s.text()!)],
+    [String.raw`\r?\n`, (_s) => new TypstToken(TypstTokenType.NEWLINE, "\n")],
+    [String.raw`\s+`, (s) => new TypstToken(TypstTokenType.SPACE, s.text()!)],
+    [String.raw`\\[$&#_]`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
+    [String.raw`\\\n`, (s) => {
+        return [
+            new TypstToken(TypstTokenType.CONTROL, "\\"),
+            new TypstToken(TypstTokenType.NEWLINE, "\n"),
+        ]
+    }],
+    [String.raw`\\\s`, (s) => {
+        return [
+            new TypstToken(TypstTokenType.CONTROL, "\\"),
+            new TypstToken(TypstTokenType.SPACE, " "),
+        ]
+    }],
+    // this backslash is dummy and will be ignored in later stages
+    [String.raw`\\\S`, (_s) => new TypstToken(TypstTokenType.CONTROL, "")],
+    [
+        String.raw`"([^"]|(\\"))*"`,
+        (s) => {
+            const text = s.text()!.substring(1, s.text()!.length - 1);
+            // replace all escape characters with their actual characters
+            text.replaceAll('\\"', '"');
+            return new TypstToken(TypstTokenType.TEXT, text);
+        }
+    ],
+    [
+        REGEX_SHORTHANDS,
+        (s) => {
+            const shorthand = s.text()!;
+            const symbol = reverseShorthandMap.get(shorthand)!;
+            return new TypstToken(TypstTokenType.SYMBOL, symbol);
+        }
+    ],
+    [String.raw`[0-9]+(\.[0-9]+)?`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
+    [String.raw`[+\-*/=\'<>!.,;?()\[\]|]`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
+    [String.raw`[a-zA-Z\.]+`, (s) => {
+        return new TypstToken(s.text()!.length === 1? TypstTokenType.ELEMENT: TypstTokenType.SYMBOL, s.text()!);
+    }],
+    [String.raw`#none`, (s) => new TypstToken(TypstTokenType.NONE, s.text()!)],
+    [String.raw`.`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
+]);
+
+const spec = {
+    "start": rules_map
+};
+
+export function tokenize_typst(input: string): TypstToken[] {
+    const lexer = new JSLex<TypstToken>(spec);
+    return lexer.collect(input);
+}

package/src/typst-writer.ts

@@ -11,6 +11,8 @@ const TYPST_RIGHT_PARENTHESIS: TypstToken = new TypstToken(TypstTokenType.ELEMEN
 const TYPST_COMMA: TypstToken = new TypstToken(TypstTokenType.ELEMENT, ',');
 const TYPST_NEWLINE: TypstToken = new TypstToken(TypstTokenType.SYMBOL, '\n');
 
+const SOFT_SPACE = new TypstToken(TypstTokenType.CONTROL, ' ');
+
 function typst_primitive_to_string(value: TypstPrimitiveValue) {
     switch (typeof value) {
         case 'string':
@@ -177,7 +179,7 @@ export class TypstWriter {
                     trailing_space_needed = this.appendWithBracketsIfNeeded(sup);
                 }
                 if (trailing_space_needed) {
-                    this.queue.push(new TypstToken(TypstTokenType.CONTROL, ' '));
+                    this.queue.push(SOFT_SPACE);
                 }
                 break;
             }
@@ -208,7 +210,16 @@ export class TypstWriter {
             }
             case 'fraction': {
                 const [numerator, denominator] = node.args!;
-                this.appendWithBracketsIfNeeded(numerator);
+                const pos = this.queue.length;
+                const no_wrap = this.appendWithBracketsIfNeeded(numerator);
+
+                // This is a dirty hack to force `C \frac{xy}{z}`to translate to `C (x y)/z` instead of `C(x y)/z`
+                // To solve this properly, we should implement a Typst formatter
+                const wrapped = !no_wrap;
+                if (wrapped) {
+                    this.queue.splice(pos, 0, SOFT_SPACE);
+                }
+
                 this.queue.push(new TypstToken(TypstTokenType.ELEMENT, '/'));
                 this.appendWithBracketsIfNeeded(denominator);
                 break;
@@ -332,20 +343,25 @@ export class TypstWriter {
     }
 
     protected flushQueue() {
-        const SOFT_SPACE = new TypstToken(TypstTokenType.CONTROL, ' ');
+        const dummy_token = new TypstToken(TypstTokenType.SYMBOL, '');
 
         // delete soft spaces if they are not needed
         for(let i = 0; i < this.queue.length; i++) {
             let token = this.queue[i];
             if (token.eq(SOFT_SPACE)) {
-                if (i === this.queue.length - 1) {
-                    this.queue[i].value = '';
-                } else if (this.queue[i + 1].isOneOf([TYPST_RIGHT_PARENTHESIS, TYPST_COMMA, TYPST_NEWLINE])) {
-                    this.queue[i].value = '';
+                const to_delete = (i === 0)
+                                || (i === this.queue.length - 1)
+                                || (this.queue[i - 1].type === TypstTokenType.SPACE)
+                                || this.queue[i - 1].isOneOf([TYPST_LEFT_PARENTHESIS, TYPST_NEWLINE])
+                                || this.queue[i + 1].isOneOf([TYPST_RIGHT_PARENTHESIS, TYPST_COMMA, TYPST_NEWLINE]);
+                if (to_delete) {
+                    this.queue[i] = dummy_token;
                 }
             }
         }
 
+        this.queue = this.queue.filter((token) => !token.eq(dummy_token));
+
         this.queue.forEach((token) => {
             this.writeBuffer(token)
         });