tex2typst 0.3.23 → 0.3.24

package/src/typst-parser.ts

@@ -1,6 +1,10 @@
 
 import { array_find } from "./generic";
-import { TYPST_NONE, TypstLrData, TypstNamedParams, TypstNode, TypstSupsubData, TypstToken, TypstTokenType } from "./types";
+import { TypstCases, TypstFraction, TypstFuncCall, TypstGroup, TypstLeftright, TypstLeftRightData, TypstMatrix, TypstNode, TypstSupsub } from "./typst-types";
+import { TypstNamedParams } from "./typst-types";
+import { TypstSupsubData } from "./typst-types";
+import { TypstToken } from "./typst-types";
+import { TypstTokenType } from "./typst-types";
 import { tokenize_typst } from "./typst-tokenizer";
 import { assert, isalpha } from "./util";
 
@@ -24,7 +28,7 @@ function _find_closing_match(tokens: TypstToken[], start: number,
 
     while (count > 0) {
         if (pos >= tokens.length) {
-            throw new Error('Unmatched brackets');
+            throw new Error('Unmatched brackets or parentheses');
         }
         if (tokens[pos].isOneOf(rightBrackets)) {
             count -= 1;
@@ -58,8 +62,10 @@ function find_closing_delim(tokens: TypstToken[], start: number): number {
 
 
 function find_closing_parenthesis(nodes: TypstNode[], start: number): number {
-    const left_parenthesis = new TypstNode('atom', '(');
-    const right_parenthesis = new TypstNode('atom', ')');
+    const left_parenthesis = new TypstToken(TypstTokenType.ELEMENT, '(').toNode();
+    const right_parenthesis = new TypstToken(TypstTokenType.ELEMENT, ')').toNode();
+
+
 
     assert(nodes[start].eq(left_parenthesis));
 
@@ -68,7 +74,7 @@ function find_closing_parenthesis(nodes: TypstNode[], start: number): number {
 
     while (count > 0) {
         if (pos >= nodes.length) {
-            throw new Error('Unmatched brackets');
+            throw new Error("Unmatched '('");
         }
         if (nodes[pos].eq(left_parenthesis)) {
             count += 1;
@@ -84,18 +90,18 @@ function find_closing_parenthesis(nodes: TypstNode[], start: number): number {
 function primes(num: number): TypstNode[] {
     const res: TypstNode[] = [];
     for (let i = 0; i < num; i++) {
-        res.push(new TypstNode('atom', "'"));
+        res.push(new TypstToken(TypstTokenType.ELEMENT, "'").toNode());
     }
     return res;
 }
 
-const DIV = new TypstNode('atom', '/');
+const DIV = new TypstToken(TypstTokenType.ELEMENT, '/').toNode();
 
 
 
 function next_non_whitespace(nodes: TypstNode[], start: number): TypstNode | null {
     let pos = start;
-    while (pos < nodes.length && nodes[pos].type === 'whitespace') {
+    while (pos < nodes.length && (nodes[pos].head.type === TypstTokenType.SPACE || nodes[pos].head.type === TypstTokenType.NEWLINE)) {
         pos++;
     }
     return pos === nodes.length ? null : nodes[pos];
@@ -106,7 +112,7 @@ function trim_whitespace_around_operators(nodes: TypstNode[]): TypstNode[] {
     const res: TypstNode[] = [];
     for (let i = 0; i < nodes.length; i++) {
         const current = nodes[i];
-        if (current.type === 'whitespace') {
+        if (current.head.type === TypstTokenType.SPACE || current.head.type === TypstTokenType.NEWLINE) {
             if(after_operator) {
                 continue;
             }
@@ -127,8 +133,8 @@ function trim_whitespace_around_operators(nodes: TypstNode[]): TypstNode[] {
 function process_operators(nodes: TypstNode[], parenthesis = false): TypstNode {
     nodes = trim_whitespace_around_operators(nodes);
 
-    const opening_bracket = new TypstNode('atom', '(');
-    const closing_bracket = new TypstNode('atom', ')');
+    const opening_bracket = LEFT_PARENTHESES.toNode();
+    const closing_bracket = RIGHT_PARENTHESES.toNode();
 
     const stack: TypstNode[] = [];
 
@@ -155,20 +161,20 @@ function process_operators(nodes: TypstNode[], parenthesis = false): TypstNode {
             }
 
             if(stack.length > 0 && stack[stack.length-1].eq(DIV)) {
-                const denominator = current_tree;
+                let denominator = current_tree;
                 if(args.length === 0) {
                     throw new TypstParserError("Unexpected '/' operator, no numerator before it");
                 }
-                const numerator = args.pop()!;
+                let numerator = args.pop()!;
 
-                if(denominator.type === 'group' && denominator.content === 'parenthesis') {
-                    denominator.content = '';
+                if(denominator.type === 'leftright') {
+                    denominator = new TypstGroup(denominator.args!);
                 }
-                if(numerator.type === 'group' && numerator.content === 'parenthesis') {
-                    numerator.content = '';
+                if(numerator.type === 'leftright') {
+                    numerator = new TypstGroup(numerator.args!);
                 }
 
-                args.push(new TypstNode('fraction', '', [numerator, denominator]));
+                args.push(new TypstFraction([numerator, denominator]));
                 stack.pop(); // drop the '/' operator
             } else {
                 args.push(current_tree);
@@ -176,12 +182,12 @@ function process_operators(nodes: TypstNode[], parenthesis = false): TypstNode {
         }
     }
     if(parenthesis) {
-        return new TypstNode('group', 'parenthesis', args);
+        return new TypstLeftright(null, args, { left: LEFT_PARENTHESES, right: RIGHT_PARENTHESES } as TypstLeftRightData);
     } else {
         if(args.length === 1) {
             return args[0];
         } else {
-            return new TypstNode('group', '', args);
+            return new TypstGroup(args);
         }
     }
 }
@@ -208,6 +214,7 @@ const VERTICAL_BAR = new TypstToken(TypstTokenType.ELEMENT, '|');
 const COMMA = new TypstToken(TypstTokenType.ELEMENT, ',');
 const SEMICOLON = new TypstToken(TypstTokenType.ELEMENT, ';');
 const SINGLE_SPACE = new TypstToken(TypstTokenType.SPACE, ' ');
+const CONTROL_AND = new TypstToken(TypstTokenType.CONTROL, '&');
 
 export class TypstParser {
     space_sensitive: boolean;
@@ -230,11 +237,11 @@ export class TypstParser {
         while (pos < end) {
             const [res, newPos] = this.parseNextExpr(tokens, pos);
             pos = newPos;
-            if (res.type === 'whitespace') {
-                if (!this.space_sensitive && res.content.replace(/ /g, '').length === 0) {
+            if (res.head.type === TypstTokenType.SPACE || res.head.type === TypstTokenType.NEWLINE) {
+                if (!this.space_sensitive && res.head.value.replace(/ /g, '').length === 0) {
                     continue;
                 }
-                if (!this.newline_sensitive && res.content === '\n') {
+                if (!this.newline_sensitive && res.head.value === '\n') {
                     continue;
                 }
             }
@@ -261,7 +268,7 @@ export class TypstParser {
 
         const num_base_prime = eat_primes(tokens, pos);
         if (num_base_prime > 0) {
-            base = new TypstNode('group', '', [base].concat(primes(num_base_prime)));
+            base = new TypstGroup([base].concat(primes(num_base_prime)));
             pos += num_base_prime;
         }
         if (pos < tokens.length && tokens[pos].eq(SUB_SYMBOL)) {
@@ -277,14 +284,8 @@ export class TypstParser {
         }
 
         if (sub !== null || sup !== null) {
-            const res: TypstSupsubData = { base };
-            if (sub) {
-                res.sub = sub;
-            }
-            if (sup) {
-                res.sup = sup;
-            }
-            return [new TypstNode('supsub', '', [], res), pos];
+            const res: TypstSupsubData = { base, sup, sub };
+            return [new TypstSupsub(res), pos];
         } else {
             return [base, pos];
         }
@@ -301,7 +302,7 @@ export class TypstParser {
         }
         const num_prime = eat_primes(tokens, end);
         if (num_prime > 0) {
-            node = new TypstNode('group', '', [node].concat(primes(num_prime)));
+            node = new TypstGroup([node].concat(primes(num_prime)));
             end += num_prime;
         }
         return [node, end];
@@ -320,24 +321,22 @@ export class TypstParser {
         if ([TypstTokenType.ELEMENT, TypstTokenType.SYMBOL].includes(firstToken.type)) {
             if (start + 1 < tokens.length && tokens[start + 1].eq(LEFT_PARENTHESES)) {
                 if(firstToken.value === 'mat') {
-                    const [matrix, named_params, newPos] = this.parseGroupsOfArguments(tokens, start + 1);
-                    const mat = new TypstNode('matrix', '', [], matrix);
+                    const [matrix, named_params, newPos] = this.parseMatrix(tokens, start + 1, SEMICOLON, COMMA);
+                    const mat = new TypstMatrix(matrix);
                     mat.setOptions(named_params);
                     return [mat, newPos];
                 }
                 if(firstToken.value === 'cases') {
-                    const [cases, named_params, newPos] = this.parseGroupsOfArguments(tokens, start + 1, COMMA);
-                    const casesNode = new TypstNode('cases', '', [], cases);
+                    const [cases, named_params, newPos] = this.parseMatrix(tokens, start + 1, COMMA, CONTROL_AND);
+                    const casesNode = new TypstCases(cases);
                     casesNode.setOptions(named_params);
                     return [casesNode, newPos];
                 }
                 if (firstToken.value === 'lr') {
-                    const [args, newPos, lrData] = this.parseLrArguments(tokens, start + 1);
-                    const func_call = new TypstNode('funcCall', firstToken.value, args, lrData);
-                    return [func_call, newPos];
+                    return this.parseLrArguments(tokens, start + 1);
                 }
                 const [args, newPos] = this.parseArguments(tokens, start + 1);
-                const func_call = new TypstNode('funcCall', firstToken.value, args);
+                const func_call = new TypstFuncCall(firstToken, args);
                 return [func_call, newPos];
             }
         }
@@ -348,33 +347,32 @@ export class TypstParser {
     // start: the position of the left parentheses
     parseArguments(tokens: TypstToken[], start: number): [TypstNode[], number] {
         const end = find_closing_match(tokens, start);
-        return [this.parseCommaSeparatedArguments(tokens, start + 1, end), end + 1];
+        return [this.parseArgumentsWithSeparator(tokens, start + 1, end, COMMA), end + 1];
     }
 
     // start: the position of the left parentheses
-    parseLrArguments(tokens: TypstToken[], start: number): [TypstNode[], number, TypstLrData] {
+    parseLrArguments(tokens: TypstToken[], start: number): [TypstNode, number] {
+        const lr_token = tokens[start];
         if (tokens[start + 1].isOneOf([LEFT_PARENTHESES, LEFT_BRACKET, LEFT_CURLY_BRACKET, VERTICAL_BAR])) {
             const end = find_closing_match(tokens, start);
             const inner_start = start + 1;
             const inner_end = find_closing_delim(tokens, inner_start);
-            const inner_args= this.parseCommaSeparatedArguments(tokens, inner_start + 1, inner_end);
+            const inner_args= this.parseArgumentsWithSeparator(tokens, inner_start + 1, inner_end, COMMA);
             return [
-                inner_args,
+                new TypstLeftright(lr_token, inner_args, {left: tokens[inner_start], right: tokens[inner_end]}),
                 end + 1,
-                {leftDelim: tokens[inner_start].value, rightDelim: tokens[inner_end].value} as TypstLrData
             ];
         } else {
             const [args, end] = this.parseArguments(tokens, start);
             return [
-                args,
+                new TypstLeftright(lr_token, args, { left: null, right: null }),
                 end,
-                {leftDelim: null, rightDelim: null} as TypstLrData,
             ];
         }
     }
 
     // start: the position of the left parentheses
-    parseGroupsOfArguments(tokens: TypstToken[], start: number, newline_token = SEMICOLON): [TypstNode[][], TypstNamedParams, number] {
+    parseMatrix(tokens: TypstToken[], start: number, rowSepToken: TypstToken, cellSepToken: TypstToken): [TypstNode[][], TypstNamedParams, number] {
         const end = find_closing_match(tokens, start);
         tokens = tokens.slice(0, end);
 
@@ -384,16 +382,16 @@ export class TypstParser {
         let pos = start + 1;
         while (pos < end) {
             while(pos < end) {
-                let next_stop = array_find(tokens, newline_token, pos);
+                let next_stop = array_find(tokens, rowSepToken, pos);
                 if (next_stop === -1) {
                     next_stop = end;
                 }
 
-                let row = this.parseCommaSeparatedArguments(tokens, pos, next_stop);
+                let row = this.parseArgumentsWithSeparator(tokens, pos, next_stop, cellSepToken);
                 let np: TypstNamedParams = {};
 
                 function extract_named_params(arr: TypstNode[]): [TypstNode[], TypstNamedParams] {
-                    const COLON = new TypstNode('atom', ':');
+                    const COLON = new TypstToken(TypstTokenType.ELEMENT, ':').toNode();
                     const np: TypstNamedParams = {};
 
                     const to_delete: number[] = [];
@@ -409,7 +407,7 @@ export class TypstParser {
                         }
                         to_delete.push(i);
                         const param_name = g.args![pos_colon - 1];
-                        if(param_name.eq(new TypstNode('symbol', 'delim'))) {
+                        if(param_name.eq(new TypstToken(TypstTokenType.SYMBOL, 'delim').toNode())) {
                             if(g.args!.length !== 3) {
                                 throw new TypstParserError('Invalid number of arguments for delim');
                             }
@@ -435,13 +433,13 @@ export class TypstParser {
     }
 
     // start: the position of the first token of arguments
-    parseCommaSeparatedArguments(tokens: TypstToken[], start: number, end: number): TypstNode[] {
+    parseArgumentsWithSeparator(tokens: TypstToken[], start: number, end: number, sepToken: TypstToken): TypstNode[] {
         const args: TypstNode[] = [];
         let pos = start;
         while (pos < end) {
             let nodes: TypstNode[] = [];
             while(pos < end) {
-                if(tokens[pos].eq(COMMA)) {
+                if(tokens[pos].eq(sepToken)) {
                     pos += 1;
                     break;
                 } else if(tokens[pos].eq(SINGLE_SPACE)) {

package/src/typst-tokenizer.ts

@@ -1,85 +1,86 @@
-import { TypstToken, TypstTokenType } from "./types";
-import { reverseShorthandMap } from "./typst-shorthands";
-import { JSLex, Scanner } from "./jslex";
-
-const TYPST_SHORTHANDS = Array.from(reverseShorthandMap.keys());
-
-
-function generate_regex_for_shorthands(): string {
-    const regex_list = TYPST_SHORTHANDS.map((s) => {
-        s = s.replaceAll('|', '\\|');
-        s = s.replaceAll('.', '\\.');
-        s = s.replaceAll('[', '\\[');
-        s = s.replaceAll(']', '\\]');
-        return s;
-    });
-    return `(${regex_list.join('|')})`;
-}
-
-
-const REGEX_SHORTHANDS = generate_regex_for_shorthands();
-
-const rules_map = new Map<string, (a: Scanner<TypstToken>) => TypstToken | TypstToken[]>([
-    [String.raw`//[^\n]*`, (s) => new TypstToken(TypstTokenType.COMMENT, s.text()!.substring(2))],
-    [String.raw`/`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
-    [String.raw`[_^&]`, (s) => new TypstToken(TypstTokenType.CONTROL, s.text()!)],
-    [String.raw`\r?\n`, (_s) => new TypstToken(TypstTokenType.NEWLINE, "\n")],
-    [String.raw`\s+`, (s) => new TypstToken(TypstTokenType.SPACE, s.text()!)],
-    [String.raw`\\[$&#_]`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
-    [String.raw`\\\n`, (s) => {
-        return [
-            new TypstToken(TypstTokenType.CONTROL, "\\"),
-            new TypstToken(TypstTokenType.NEWLINE, "\n"),
-        ]
-    }],
-    [String.raw`\\\s`, (s) => {
-        return [
-            new TypstToken(TypstTokenType.CONTROL, "\\"),
-            new TypstToken(TypstTokenType.SPACE, " "),
-        ]
-    }],
-    // this backslash is dummy and will be ignored in later stages
-    [String.raw`\\\S`, (_s) => new TypstToken(TypstTokenType.CONTROL, "")],
-    [
-        String.raw`"([^"]|(\\"))*"`,
-        (s) => {
-            const text = s.text()!.substring(1, s.text()!.length - 1);
-            // replace all escape characters with their actual characters
-            text.replaceAll('\\"', '"');
-            return new TypstToken(TypstTokenType.TEXT, text);
-        }
-    ],
-    [
-        REGEX_SHORTHANDS,
-        (s) => {
-            const shorthand = s.text()!;
-            const symbol = reverseShorthandMap.get(shorthand)!;
-            return new TypstToken(TypstTokenType.SYMBOL, symbol);
-        }
-    ],
-    [String.raw`[0-9]+(\.[0-9]+)?`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
-    [String.raw`[+\-*/=\'<>!.,;?()\[\]|]`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
-    [String.raw`#h\((.+?)\)`, (s) => {
-        const match = s.reMatchArray()!;
-        return [
-            new TypstToken(TypstTokenType.SYMBOL, "#h"),
-            new TypstToken(TypstTokenType.ELEMENT, "("),
-            new TypstToken(TypstTokenType.LITERAL, match[1]),
-            new TypstToken(TypstTokenType.ELEMENT, ")"),
-        ];
-    }],
-    [String.raw`[a-zA-Z\.]+`, (s) => {
-        return new TypstToken(s.text()!.length === 1? TypstTokenType.ELEMENT: TypstTokenType.SYMBOL, s.text()!);
-    }],
-    [String.raw`#none`, (s) => new TypstToken(TypstTokenType.NONE, s.text()!)],
-    [String.raw`.`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
-]);
-
-const spec = {
-    "start": rules_map
-};
-
-export function tokenize_typst(input: string): TypstToken[] {
-    const lexer = new JSLex<TypstToken>(spec);
-    return lexer.collect(input);
-}
+import { TypstToken } from "./typst-types";
+import { TypstTokenType } from "./typst-types";
+import { reverseShorthandMap } from "./typst-shorthands";
+import { JSLex, Scanner } from "./jslex";
+
+const TYPST_SHORTHANDS = Array.from(reverseShorthandMap.keys());
+
+
+function generate_regex_for_shorthands(): string {
+    const regex_list = TYPST_SHORTHANDS.map((s) => {
+        s = s.replaceAll('|', '\\|');
+        s = s.replaceAll('.', '\\.');
+        s = s.replaceAll('[', '\\[');
+        s = s.replaceAll(']', '\\]');
+        return s;
+    });
+    return `(${regex_list.join('|')})`;
+}
+
+
+const REGEX_SHORTHANDS = generate_regex_for_shorthands();
+
+const rules_map = new Map<string, (a: Scanner<TypstToken>) => TypstToken | TypstToken[]>([
+    [String.raw`//[^\n]*`, (s) => new TypstToken(TypstTokenType.COMMENT, s.text()!.substring(2))],
+    [String.raw`/`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
+    [String.raw`[_^&]`, (s) => new TypstToken(TypstTokenType.CONTROL, s.text()!)],
+    [String.raw`\r?\n`, (_s) => new TypstToken(TypstTokenType.NEWLINE, "\n")],
+    [String.raw`\s+`, (s) => new TypstToken(TypstTokenType.SPACE, s.text()!)],
+    [String.raw`\\[$&#_]`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
+    [String.raw`\\\n`, (s) => {
+        return [
+            new TypstToken(TypstTokenType.CONTROL, "\\"),
+            new TypstToken(TypstTokenType.NEWLINE, "\n"),
+        ]
+    }],
+    [String.raw`\\\s`, (s) => {
+        return [
+            new TypstToken(TypstTokenType.CONTROL, "\\"),
+            new TypstToken(TypstTokenType.SPACE, " "),
+        ]
+    }],
+    // this backslash is dummy and will be ignored in later stages
+    [String.raw`\\\S`, (_s) => new TypstToken(TypstTokenType.CONTROL, "")],
+    [
+        String.raw`"([^"]|(\\"))*"`,
+        (s) => {
+            const text = s.text()!.substring(1, s.text()!.length - 1);
+            // replace all escape characters with their actual characters
+            text.replaceAll('\\"', '"');
+            return new TypstToken(TypstTokenType.TEXT, text);
+        }
+    ],
+    [
+        REGEX_SHORTHANDS,
+        (s) => {
+            const shorthand = s.text()!;
+            const symbol = reverseShorthandMap.get(shorthand)!;
+            return new TypstToken(TypstTokenType.SYMBOL, symbol);
+        }
+    ],
+    [String.raw`[0-9]+(\.[0-9]+)?`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
+    [String.raw`[+\-*/=\'<>!.,;?()\[\]|]`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
+    [String.raw`#h\((.+?)\)`, (s) => {
+        const match = s.reMatchArray()!;
+        return [
+            new TypstToken(TypstTokenType.SYMBOL, "#h"),
+            new TypstToken(TypstTokenType.ELEMENT, "("),
+            new TypstToken(TypstTokenType.LITERAL, match[1]),
+            new TypstToken(TypstTokenType.ELEMENT, ")"),
+        ];
+    }],
+    [String.raw`[a-zA-Z\.]+`, (s) => {
+        return new TypstToken(s.text()!.length === 1? TypstTokenType.ELEMENT: TypstTokenType.SYMBOL, s.text()!);
+    }],
+    [String.raw`#none`, (s) => new TypstToken(TypstTokenType.NONE, s.text()!)],
+    [String.raw`.`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
+]);
+
+const spec = {
+    "start": rules_map
+};
+
+export function tokenize_typst(input: string): TypstToken[] {
+    const lexer = new JSLex<TypstToken>(spec);
+    return lexer.collect(input);
+}