tex2typst 0.3.29 → 0.4.1

package/src/typst-parser.ts

@@ -6,7 +6,7 @@ import { TypstSupsubData } from "./typst-types";
 import { TypstToken } from "./typst-types";
 import { TypstTokenType } from "./typst-types";
 import { tokenize_typst } from "./typst-tokenizer";
-import { assert, isalpha } from "./util";
+import { assert, isalpha } from "./utils";
 
 
 // TODO: In Typst, y' ' is not the same as y''.
@@ -49,41 +49,37 @@ function find_closing_match(tokens: TypstToken[], start: number): number {
     );
 }
 
-function find_closing_delim(tokens: TypstToken[], start: number): number {
-    return _find_closing_match(
-        tokens,
-        start,
-        TypstToken.LEFT_DELIMITERS,
-        TypstToken.RIGHT_DELIMITERS
-    );
-}
-
 
+function extract_named_params(arr: TypstNode[]): [TypstNode[], TypstNamedParams] {
+    const COLON = new TypstToken(TypstTokenType.ELEMENT, ':').toNode();
+    const np: TypstNamedParams = {};
 
-function find_closing_parenthesis(nodes: TypstNode[], start: number): number {
-    const left_parenthesis = new TypstToken(TypstTokenType.ELEMENT, '(').toNode();
-    const right_parenthesis = new TypstToken(TypstTokenType.ELEMENT, ')').toNode();
-
-
-
-    assert(nodes[start].eq(left_parenthesis));
-
-    let count = 1;
-    let pos = start + 1;
+    const to_delete: number[] = [];
+    for(let i = 0; i < arr.length; i++) {
+        if(arr[i].type !== 'group') {
+            continue;
+        }
 
-    while (count > 0) {
-        if (pos >= nodes.length) {
-            throw new Error("Unmatched '('");
+        const g = arr[i] as TypstGroup;
+        const pos_colon = array_find(g.items, COLON);
+        if(pos_colon === -1 || pos_colon === 0) {
+            continue;
         }
-        if (nodes[pos].eq(left_parenthesis)) {
-            count += 1;
-        } else if (nodes[pos].eq(right_parenthesis)) {
-            count -= 1;
+        to_delete.push(i);
+        const param_name = g.items[pos_colon - 1];
+        if(param_name.eq(new TypstToken(TypstTokenType.SYMBOL, 'delim').toNode())) {
+            if(g.items.length !== 3) {
+                throw new TypstParserError('Invalid number of arguments for delim');
+            }
+            np['delim'] = g.items[pos_colon + 1];
+        } else {
+            throw new TypstParserError('Not implemented for other named parameters');
         }
-        pos += 1;
     }
-
-    return pos - 1;
+    for(let i = to_delete.length - 1; i >= 0; i--) {
+        arr.splice(to_delete[i], 1);
+    }
+    return [arr, np];
 }
 
 function primes(num: number): TypstNode[] {
@@ -129,36 +125,18 @@ function trim_whitespace_around_operators(nodes: TypstNode[]): TypstNode[] {
     return res;
 }
 
-function process_operators(nodes: TypstNode[], parenthesis = false): TypstNode {
+function process_operators(nodes: TypstNode[]): TypstNode {
     nodes = trim_whitespace_around_operators(nodes);
 
-    const opening_bracket = LEFT_PARENTHESES.toNode();
-    const closing_bracket = RIGHT_PARENTHESES.toNode();
-
     const stack: TypstNode[] = [];
 
     const args: TypstNode[] = [];
     let pos = 0;
     while (pos < nodes.length) {
-        const current = nodes[pos];
-        if (current.eq(closing_bracket)) {
-            throw new TypstParserError("Unexpected ')'");
-        } else if(current.eq(DIV)) {
-            stack.push(current);
-            pos++;
+        const current_tree = nodes[pos];
+        if(current_tree.eq(DIV)) {
+            stack.push(current_tree);
         } else {
-            let current_tree: TypstNode;
-            if(current.eq(opening_bracket)) {
-                // the expression is a group wrapped in parenthesis
-                const pos_closing = find_closing_parenthesis(nodes, pos);
-                current_tree = process_operators(nodes.slice(pos + 1, pos_closing), true);
-                pos = pos_closing + 1;
-            } else {
-                // the expression is just a single item
-                current_tree = current;
-                pos++;
-            }
-
             if(stack.length > 0 && stack[stack.length-1].eq(DIV)) {
                 let denominator = current_tree;
                 if(args.length === 0) {
@@ -179,13 +157,9 @@ function process_operators(nodes: TypstNode[], parenthesis = false): TypstNode {
                 args.push(current_tree);
             }
         }
+        pos++;
     }
-    const body = args.length === 1? args[0]: new TypstGroup(args);
-    if(parenthesis) {
-        return new TypstLeftright(null, { body: body, left: LEFT_PARENTHESES, right: RIGHT_PARENTHESES } as TypstLeftRightData);
-    } else {
-        return body;
-    }
+    return args.length === 1? args[0]: new TypstGroup(args);
 }
 
 function parse_named_params(groups: TypstGroup[]): TypstNamedParams {
@@ -220,9 +194,14 @@ const LEFT_CURLY_BRACKET: TypstToken = new TypstToken(TypstTokenType.ELEMENT, '{
 const RIGHT_CURLY_BRACKET: TypstToken = new TypstToken(TypstTokenType.ELEMENT, '}');
 const COMMA = new TypstToken(TypstTokenType.ELEMENT, ',');
 const SEMICOLON = new TypstToken(TypstTokenType.ELEMENT, ';');
-const SINGLE_SPACE = new TypstToken(TypstTokenType.SPACE, ' ');
 const CONTROL_AND = new TypstToken(TypstTokenType.CONTROL, '&');
 
+
+interface TexParseEnv {
+    spaceSensitive: boolean;
+    newlineSensitive: boolean;
+}
+
 export class TypstParser {
     space_sensitive: boolean;
     newline_sensitive: boolean;
@@ -237,35 +216,8 @@ export class TypstParser {
         return tree;
     }
 
-    parseGroup(tokens: TypstToken[], start: number, end: number, parentheses = false): TypstParseResult {
-        const results: TypstNode[] = [];
-        let pos = start;
-
-        while (pos < end) {
-            const [res, newPos] = this.parseNextExpr(tokens, pos);
-            pos = newPos;
-            if (res.head.type === TypstTokenType.SPACE || res.head.type === TypstTokenType.NEWLINE) {
-                if (!this.space_sensitive && res.head.value.replace(/ /g, '').length === 0) {
-                    continue;
-                }
-                if (!this.newline_sensitive && res.head.value === '\n') {
-                    continue;
-                }
-            }
-            results.push(res);
-        }
-
-        let node: TypstNode;
-        if(parentheses) {
-            node = process_operators(results, true);
-        } else {
-            if (results.length === 1) {
-                node = results[0];
-            } else {
-                node = process_operators(results);
-            }
-        }
-        return [node, end + 1];
+    parseGroup(tokens: TypstToken[], start: number, end: number): TypstParseResult {
+        return this.parseUntil(tokens.slice(start, end), 0, null);
     }
 
     parseNextExpr(tokens: TypstToken[], start: number): TypstParseResult {
@@ -298,12 +250,51 @@ export class TypstParser {
         }
     }
 
+    // return pos: (position of stopToken) + 1
+    // pos will be -1 if stopToken is not found
+    parseUntil(tokens: TypstToken[], start: number, stopToken: TypstToken | null, env: Partial<TexParseEnv> = {}): TypstParseResult {
+        if (env.spaceSensitive === undefined) {
+            env.spaceSensitive = this.space_sensitive;
+        }
+        if (env.newlineSensitive === undefined) {
+            env.newlineSensitive = this.newline_sensitive;
+        }
+
+        const results: TypstNode[] = [];
+        let pos = start;
+
+        while (pos < tokens.length) {
+            if (stopToken !== null && tokens[pos].eq(stopToken)) {
+                break;
+            }
+            const [res, newPos] = this.parseNextExpr(tokens, pos);
+            pos = newPos;
+            if (res.head.type === TypstTokenType.SPACE || res.head.type === TypstTokenType.NEWLINE) {
+                if (!env.spaceSensitive && res.head.value.replace(/ /g, '').length === 0) {
+                    continue;
+                }
+                if (!env.newlineSensitive && res.head.value === '\n') {
+                    continue;
+                }
+            }
+            results.push(res);
+        }
+        if (pos >= tokens.length && stopToken !== null) {
+            return [TypstToken.NONE.toNode(), -1];
+        }
+
+        const node = process_operators(results);
+        return [node, pos + 1];
+    }
+
     parseSupOrSub(tokens: TypstToken[], start: number): TypstParseResult {
         let node: TypstNode;
         let end: number;
         if(tokens[start].eq(LEFT_PARENTHESES)) {
-            const pos_closing = find_closing_match(tokens, start);
-            [node, end] = this.parseGroup(tokens, start + 1, pos_closing);
+            [node, end] = this.parseUntil(tokens, start + 1, RIGHT_PARENTHESES);
+            if (end === -1) {
+                throw new Error("Unmatched '('");
+            }
         } else {
             [node, end] = this.parseNextExprWithoutSupSub(tokens, start);
         }
@@ -319,8 +310,12 @@ export class TypstParser {
         const firstToken = tokens[start];
         const node = firstToken.toNode();
         if(firstToken.eq(LEFT_PARENTHESES)) {
-            const pos_closing = find_closing_match(tokens, start);
-            return this.parseGroup(tokens, start + 1, pos_closing, true);
+            const [body, end] = this.parseUntil(tokens, start + 1, RIGHT_PARENTHESES);
+            if (end === -1) {
+                throw new Error("Unmatched '('");
+            }
+            const res = new TypstLeftright(null, { body: body, left: LEFT_PARENTHESES, right: RIGHT_PARENTHESES } as TypstLeftRightData);
+            return [res, end];
         }
         if(firstToken.type === TypstTokenType.ELEMENT && !isalpha(firstToken.value[0])) {
             return [node, start + 1];
@@ -371,29 +366,32 @@ export class TypstParser {
 
     // start: the position of the left parentheses
     parseLrArguments(tokens: TypstToken[], start: number): [TypstNode, number] {
-        const lr_token = tokens[start];
+        const lr_token = new TypstToken(TypstTokenType.SYMBOL, 'lr');
         const end = find_closing_match(tokens, start);
-        if (tokens[start + 1].isOneOf(TypstToken.LEFT_DELIMITERS)) {
-            const inner_start = start + 1;
-            const inner_end = find_closing_delim(tokens, inner_start);
-            const [inner_args, _]= this.parseGroup(tokens, inner_start + 1, inner_end);
-            return [
-                new TypstLeftright(lr_token, { body: inner_args, left: tokens[inner_start], right: tokens[inner_end]}),
-                end + 1,
-            ];
-        } else {
-            const [inner_args, _] = this.parseGroup(tokens, start + 1, end - 1);
-            return [
-                new TypstLeftright(lr_token, { body: inner_args, left: null, right: null }),
-                end + 1,
-            ];
+
+        let left: TypstToken | null = null;
+        let right: TypstToken | null = null;
+        let inner_start = start + 1;
+        let inner_end = end;
+        if (inner_end > inner_start && tokens[inner_start].isOneOf(TypstToken.LEFT_DELIMITERS)) {
+            left = tokens[inner_start];
+            inner_start += 1;
         }
+        if (inner_end - 1 > inner_start && tokens[inner_end - 1].isOneOf(TypstToken.RIGHT_DELIMITERS)) {
+            right = tokens[inner_end - 1];
+            inner_end -= 1;
+        }
+
+        const [inner_args, _] = this.parseGroup(tokens, inner_start, inner_end);
+        return [
+            new TypstLeftright(lr_token, { body: inner_args, left: left, right: right }),
+            end + 1,
+        ];
     }
 
     // start: the position of the left parentheses
     parseMatrix(tokens: TypstToken[], start: number, rowSepToken: TypstToken, cellSepToken: TypstToken): [TypstNode[][], TypstNamedParams, number] {
         const end = find_closing_match(tokens, start);
-        tokens = tokens.slice(0, end);
 
         const matrix: TypstNode[][] = [];
         let named_params: TypstNamedParams = {};
@@ -402,45 +400,13 @@ export class TypstParser {
         while (pos < end) {
             while(pos < end) {
                 let next_stop = array_find(tokens, rowSepToken, pos);
-                if (next_stop === -1) {
+                if (next_stop === -1 || next_stop > end) {
                     next_stop = end;
                 }
 
                 let row = this.parseArgumentsWithSeparator(tokens, pos, next_stop, cellSepToken);
                 let np: TypstNamedParams = {};
 
-                function extract_named_params(arr: TypstNode[]): [TypstNode[], TypstNamedParams] {
-                    const COLON = new TypstToken(TypstTokenType.ELEMENT, ':').toNode();
-                    const np: TypstNamedParams = {};
-
-                    const to_delete: number[] = [];
-                    for(let i = 0; i < arr.length; i++) {
-                        if(arr[i].type !== 'group') {
-                            continue;
-                        }
-
-                        const g = arr[i] as TypstGroup;
-                        const pos_colon = array_find(g.items, COLON);
-                        if(pos_colon === -1 || pos_colon === 0) {
-                            continue;
-                        }
-                        to_delete.push(i);
-                        const param_name = g.items[pos_colon - 1];
-                        if(param_name.eq(new TypstToken(TypstTokenType.SYMBOL, 'delim').toNode())) {
-                            if(g.items.length !== 3) {
-                                throw new TypstParserError('Invalid number of arguments for delim');
-                            }
-                            np['delim'] = g.items[pos_colon + 1];
-                        } else {
-                            throw new TypstParserError('Not implemented for other named parameters');
-                        }
-                    }
-                    for(let i = to_delete.length - 1; i >= 0; i--) {
-                        arr.splice(to_delete[i], 1);
-                    }
-                    return [arr, np];
-                }
-
                 [row, np] = extract_named_params(row);
                 matrix.push(row);
                 Object.assign(named_params, np);
@@ -455,29 +421,17 @@ export class TypstParser {
     parseArgumentsWithSeparator(tokens: TypstToken[], start: number, end: number, sepToken: TypstToken): TypstNode[] {
         const args: TypstNode[] = [];
         let pos = start;
-        while (pos < end) {
-            let nodes: TypstNode[] = [];
-            while(pos < end) {
-                if(tokens[pos].eq(sepToken)) {
-                    pos += 1;
-                    break;
-                } else if(tokens[pos].eq(SINGLE_SPACE)) {
-                    pos += 1;
-                    continue;
-                }
-                const [argItem, newPos] = this.parseNextExpr(tokens, pos);
-                pos = newPos;
-                nodes.push(argItem);
-            }
 
+        while (pos < end) {
             let arg: TypstNode;
-            if (nodes.length === 1) {
-                arg = nodes[0];
-            } else {
-                arg = process_operators(nodes);
+            let newPos: number;
+            const env = { spaceSensitive: false, newlineSensitive: true };
+            [arg, newPos] = this.parseUntil(tokens.slice(0, end), pos, sepToken, env);
+            if (newPos == -1) {
+                [arg, newPos] = this.parseUntil(tokens.slice(0, end), pos, null, env);
             }
-
             args.push(arg);
+            pos = newPos;
         }
         return args;
     }

package/tests/cheat-sheet.test.ts

@@ -1,42 +0,0 @@
-import path from 'path';
-import toml from 'toml';
-import fs from 'node:fs';
-import { describe, it, test, expect } from 'vitest';
-import { tex2typst, symbolMap } from '../src';
-
-interface CheatSheet {
-    math_commands: { [key: string]: string };
-    math_symbols: { [key: string]: string };
-}
-
-describe('cheat sheet', () => {
-    const cheatSheetFile = path.join(__dirname, 'cheat-sheet.toml');
-    const text_content = fs.readFileSync(cheatSheetFile, { encoding: 'utf-8' });
-    const data = toml.parse(text_content) as CheatSheet;
-
-    test('math_commands', () => {
-        expect(data.math_commands).toBeDefined();
-
-
-        for (const [key, value] of Object.entries(data.math_commands)) {
-            const input = `\\${key}{x}{y}`;
-            const expected1 = `${value} x y`;
-            const expected2 = `${value}(x) y`;
-            const expected3 = `${value}(x, y)`;
-            const result = tex2typst(input, {preferShorthands: false});
-            expect([expected1, expected2, expected3]).toContain(result);
-        }
-    });
-
-    test('math_symbols', () => {
-        expect(data.math_symbols).toBeDefined();
-
-        for (const [key, value] of Object.entries(data.math_symbols)) {
-            const input = `\\${key}`;
-            const expected = value;
-            const result = tex2typst(input, {preferShorthands: false});
-            expect(result).toBe(expected);
-            expect(symbolMap.get(key)).toBe(expected);
-        }
-    });
-});