tex2typst 0.3.27-beta.1 → 0.3.28

package/src/tex-parser.ts

@@ -1,6 +1,6 @@
 import { TexBeginEnd, TexFuncCall, TexLeftRight, TexNode, TexGroup, TexSupSub, TexSupsubData, TexText, TexToken, TexTokenType } from "./tex-types";
 import { assert } from "./util";
-import { array_find } from "./generic";
+import { array_find, array_join } from "./generic";
 import { TEX_BINARY_COMMANDS, TEX_UNARY_COMMANDS, tokenize_tex } from "./tex-tokenizer";
 
 const IGNORED_COMMANDS = [
@@ -42,7 +42,7 @@ function eat_parenthesis(tokens: TexToken[], start: number): TexToken | null {
     const firstToken = tokens[start];
     if (firstToken.type === TexTokenType.ELEMENT && ['(', ')', '[', ']', '|', '\\{', '\\}', '.', '\\|'].includes(firstToken.value)) {
         return firstToken;
-    } else if (firstToken.type === TexTokenType.COMMAND && ['lfloor', 'rfloor', 'lceil', 'rceil', 'langle', 'rangle'].includes(firstToken.value.slice(1))) {
+    } else if (firstToken.type === TexTokenType.COMMAND && ['lfloor', 'rfloor', 'lceil', 'rceil', 'langle', 'rangle', 'lparen', 'rparen', 'lbrace', 'rbrace'].includes(firstToken.value.slice(1))) {
         return firstToken;
     } else {
         return null;
@@ -58,49 +58,29 @@ function eat_primes(tokens: TexToken[], start: number): number {
 }
 
 
-function find_closing_match(tokens: TexToken[], start: number, leftToken: TexToken, rightToken: TexToken): number {
-    assert(tokens[start].eq(leftToken));
-    let count = 1;
-    let pos = start + 1;
-
-    while (count > 0) {
-        if (pos >= tokens.length) {
-            return -1;
-        }
-        if (tokens[pos].eq(leftToken)) {
-            count += 1;
-        } else if (tokens[pos].eq(rightToken)) {
-            count -= 1;
-        }
-        pos += 1;
-    }
-
-    return pos - 1;
-}
-
 
 const LEFT_COMMAND: TexToken = new TexToken(TexTokenType.COMMAND, '\\left');
 const RIGHT_COMMAND: TexToken = new TexToken(TexTokenType.COMMAND, '\\right');
 
-function find_closing_right_command(tokens: TexToken[], start: number): number {
-    return find_closing_match(tokens, start, LEFT_COMMAND, RIGHT_COMMAND);
-}
-
-
 const BEGIN_COMMAND: TexToken = new TexToken(TexTokenType.COMMAND, '\\begin');
 const END_COMMAND: TexToken = new TexToken(TexTokenType.COMMAND, '\\end');
 
-
-function find_closing_end_command(tokens: TexToken[], start: number): number {
-    return find_closing_match(tokens, start, BEGIN_COMMAND, END_COMMAND);
-}
-
+const CONTROL_LINEBREAK = new TexToken(TexTokenType.CONTROL, '\\\\');
 
 export class LatexParserError extends Error {
     constructor(message: string) {
         super(message);
         this.name = 'LatexParserError';
     }
+
+    static readonly UNMATCHED_LEFT_BRACE = new LatexParserError("Unmatched '\\{'");
+    static readonly UNMATCHED_RIGHT_BRACE = new LatexParserError("Unmatched '\\}'");
+    static readonly UNMATCHED_LEFT_BRACKET = new LatexParserError("Unmatched '\\['");
+    static readonly UNMATCHED_RIGHT_BRACKET = new LatexParserError("Unmatched '\\]'");
+    static readonly UNMATCHED_COMMAND_BEGIN = new LatexParserError("Unmatched '\\begin'");
+    static readonly UNMATCHED_COMMAND_END = new LatexParserError("Unmatched '\\end'");
+    static readonly UNMATCHED_COMMAND_LEFT = new LatexParserError("Unmatched '\\left'");
+    static readonly UNMATCHED_COMMAND_RIGHT = new LatexParserError("Unmatched '\\right'");
 }
 
 
@@ -110,8 +90,12 @@ const SUB_SYMBOL:TexToken = new TexToken(TexTokenType.CONTROL, '_');
 const SUP_SYMBOL:TexToken = new TexToken(TexTokenType.CONTROL, '^');
 
 export class LatexParser {
-    space_sensitive: boolean;
-    newline_sensitive: boolean;
+    public space_sensitive: boolean;
+    public newline_sensitive: boolean;
+
+    // how many levels of \begin{...} \end{...} are we currently in
+    public alignmentDepth: number = 0;
+
 
     constructor(space_sensitive: boolean = false, newline_sensitive: boolean = true) {
         this.space_sensitive = space_sensitive;
@@ -123,25 +107,35 @@ export class LatexParser {
         const idx = array_find(tokens, token_displaystyle);
         if (idx === -1) {
             // no \displaystyle, normal execution path
-            const [tree, _] = this.parseGroup(tokens, 0, tokens.length);
-            return tree;
+            return this.parseGroup(tokens.slice(0));
         } else if (idx === 0) {
             // \displaystyle at the beginning. Wrap the whole thing in \displaystyle
-            const [tree, _] = this.parseGroup(tokens, 1, tokens.length);
+            const tree = this.parseGroup(tokens.slice(1));
             return new TexFuncCall(token_displaystyle, [tree]);
         } else {
             // \displaystyle somewhere in the middle. Split the expression to two parts
-            const [tree1, _1] = this.parseGroup(tokens, 0, idx);
-            const [tree2, _2] = this.parseGroup(tokens, idx + 1, tokens.length);
+            const tree1 = this.parseGroup(tokens.slice(0, idx));
+            const tree2 = this.parseGroup(tokens.slice(idx + 1, tokens.length));
             const display = new TexFuncCall(token_displaystyle, [tree2]);
             return new TexGroup([tree1, display]);
         }
     }
 
-    parseGroup(tokens: TexToken[], start: number, end: number): ParseResult {
+    parseGroup(tokens: TexToken[]): TexNode {
+        const [tree, _] = this.parseClosure(tokens, 0, null);
+        return tree;
+    }
+
+    // return pos: (position of closingToken) + 1
+    // pos will be -1 if closingToken is not found
+    parseClosure(tokens: TexToken[], start: number, closingToken: TexToken | null): ParseResult {
         const results: TexNode[] = [];
         let pos = start;
-        while (pos < end) {
+        while (pos < tokens.length) {
+            if (closingToken !== null && tokens[pos].eq(closingToken)) {
+                break;
+            }
+
             const [res, newPos] = this.parseNextExpr(tokens, pos);
             pos = newPos;
             if(res.head.type === TexTokenType.SPACE || res.head.type === TexTokenType.NEWLINE) {
@@ -152,11 +146,11 @@ export class LatexParser {
                     continue;
                 }
             }
-            if (res.head.eq(new TexToken(TexTokenType.CONTROL, '&'))) {
-                throw new LatexParserError('Unexpected & outside of an alignment');
-            }
             results.push(res);
         }
+        if (pos >= tokens.length && closingToken !== null) {
+            return [EMPTY_NODE, -1];
+        }
 
         let node: TexNode;
         if (results.length === 1) {
@@ -164,7 +158,7 @@ export class LatexParser {
         } else {
             node = new TexGroup(results);
         }
-        return [node, end + 1];
+        return [node, pos + 1];
     }
 
     parseNextExpr(tokens: TexToken[], start: number): ParseResult {
@@ -175,25 +169,28 @@ export class LatexParser {
 
         num_prime += eat_primes(tokens, pos);
         pos += num_prime;
-        if (pos < tokens.length && tokens[pos].eq(SUB_SYMBOL)) {
-            [sub, pos] = this.parseNextExprWithoutSupSub(tokens, pos + 1);
-            num_prime += eat_primes(tokens, pos);
-            pos += num_prime;
-            if (pos < tokens.length && tokens[pos].eq(SUP_SYMBOL)) {
+        if (pos < tokens.length) {
+            const next_token = tokens[pos];
+            if (next_token.eq(SUB_SYMBOL)) {
+                [sub, pos] = this.parseNextExprWithoutSupSub(tokens, pos + 1);
+                num_prime += eat_primes(tokens, pos);
+                pos += num_prime;
+                if (pos < tokens.length && tokens[pos].eq(SUP_SYMBOL)) {
+                    [sup, pos] = this.parseNextExprWithoutSupSub(tokens, pos + 1);
+                    if (eat_primes(tokens, pos) > 0) {
+                        throw new LatexParserError('Double superscript');
+                    }
+                }
+            } else if (next_token.eq(SUP_SYMBOL)) {
                 [sup, pos] = this.parseNextExprWithoutSupSub(tokens, pos + 1);
                 if (eat_primes(tokens, pos) > 0) {
                     throw new LatexParserError('Double superscript');
                 }
-            }
-        } else if (pos < tokens.length && tokens[pos].eq(SUP_SYMBOL)) {
-            [sup, pos] = this.parseNextExprWithoutSupSub(tokens, pos + 1);
-            if (eat_primes(tokens, pos) > 0) {
-                throw new LatexParserError('Double superscript');
-            }
-            if (pos < tokens.length && tokens[pos].eq(SUB_SYMBOL)) {
-                [sub, pos] = this.parseNextExprWithoutSupSub(tokens, pos + 1);
-                if (eat_primes(tokens, pos) > 0) {
-                    throw new LatexParserError('Double superscript');
+                if (pos < tokens.length && tokens[pos].eq(SUB_SYMBOL)) {
+                    [sub, pos] = this.parseNextExprWithoutSupSub(tokens, pos + 1);
+                    if (eat_primes(tokens, pos) > 0) {
+                        throw new LatexParserError('Double superscript');
+                    }
                 }
             }
         }
@@ -223,7 +220,7 @@ export class LatexParser {
 
     parseNextExprWithoutSupSub(tokens: TexToken[], start: number): ParseResult {
         if (start >= tokens.length) {
-            return [EMPTY_NODE, start];
+            throw new LatexParserError("Unexpected end of input");
         }
         const firstToken = tokens[start];
         switch (firstToken.type) {
@@ -240,8 +237,12 @@ export class LatexParser {
                 }
                 if (firstToken.eq(BEGIN_COMMAND)) {
                     return this.parseBeginEndExpr(tokens, start);
+                } else if(firstToken.eq(END_COMMAND)) {
+                    throw LatexParserError.UNMATCHED_COMMAND_END;
                 } else if (firstToken.eq(LEFT_COMMAND)) {
                     return this.parseLeftRightExpr(tokens, start);
+                } else if (firstToken.eq(RIGHT_COMMAND)) {
+                    throw LatexParserError.UNMATCHED_COMMAND_RIGHT;
                 } else {
                     return this.parseCommandExpr(tokens, start);
                 }
@@ -249,13 +250,13 @@ export class LatexParser {
                 const controlChar = firstToken.value;
                 switch (controlChar) {
                     case '{':
-                        const posClosingBracket = find_closing_match(tokens, start, LEFT_CURLY_BRACKET, RIGHT_CURLY_BRACKET);
-                        if(posClosingBracket === -1) {
-                            throw new LatexParserError("Unmatched '{'");
+                        const [group, newPos] = this.parseClosure(tokens, start + 1, RIGHT_CURLY_BRACKET);
+                        if (newPos === -1) {
+                            throw LatexParserError.UNMATCHED_LEFT_BRACE;
                         }
-                        return this.parseGroup(tokens, start + 1, posClosingBracket);
+                        return [group, newPos];
                     case '}':
-                        throw new LatexParserError("Unmatched '}'");
+                        throw LatexParserError.UNMATCHED_RIGHT_BRACE;
                     case '\\\\':
                     case '\\!':
                     case '\\,':
@@ -266,8 +267,12 @@ export class LatexParser {
                         return [firstToken.toNode(), start + 1];
                     case '_':
                     case '^':
+                        // e.g. "_1" or "^2" are valid LaTeX math expressions
                         return [ EMPTY_NODE, start];
                     case '&':
+                        if (this.alignmentDepth <= 0) {
+                            throw new LatexParserError('Unexpected & outside of an alignment');
+                        }
                         return [firstToken.toNode(), start + 1];
                     default:
                         throw new LatexParserError('Unknown control sequence');
@@ -285,11 +290,6 @@ export class LatexParser {
 
         let pos = start + 1;
 
-        if (['left', 'right', 'begin', 'end'].includes(command.slice(1))) {
-            throw new LatexParserError('Unexpected command: ' + command);
-        }
-
-
         const paramNum = get_command_param_num(command.slice(1));
         switch (paramNum) {
             case 0:
@@ -301,14 +301,12 @@ export class LatexParser {
                     throw new LatexParserError('Expecting argument for ' + command);
                 }
                 if (command === '\\sqrt' && pos < tokens.length && tokens[pos].eq(LEFT_SQUARE_BRACKET)) {
-                    const posLeftSquareBracket = pos;
-                    const posRightSquareBracket = find_closing_match(tokens, pos, LEFT_SQUARE_BRACKET, RIGHT_SQUARE_BRACKET);
-                    if (posRightSquareBracket === -1) {
-                        throw new LatexParserError('No matching right square bracket for [');
+                    const [exponent, newPos1] = this.parseClosure(tokens, pos + 1, RIGHT_SQUARE_BRACKET);
+                    if (newPos1 === -1) {
+                        throw LatexParserError.UNMATCHED_LEFT_BRACKET;
                     }
-                    const [exponent, _] = this.parseGroup(tokens, posLeftSquareBracket + 1, posRightSquareBracket);
-                    const [arg1, newPos] = this.parseNextArg(tokens, posRightSquareBracket + 1);
-                    return [new TexFuncCall(command_token, [arg1], exponent), newPos];
+                    const [arg1, newPos2] = this.parseNextArg(tokens, newPos1);
+                    return [new TexFuncCall(command_token, [arg1], exponent), newPos2];
                 } else if (command === '\\text') {
                     if (pos + 2 >= tokens.length) {
                         throw new LatexParserError('Expecting content for \\text command');
@@ -318,6 +316,14 @@ export class LatexParser {
                     assert(tokens[pos + 2].eq(RIGHT_CURLY_BRACKET));
                     const literal = tokens[pos + 1];
                     return [new TexText(literal), pos + 3];
+                } else if (command === '\\displaylines') {
+                    assert(tokens[pos].eq(LEFT_CURLY_BRACKET));
+                    const [matrix, newPos] = this.parseAligned(tokens, pos + 1, RIGHT_CURLY_BRACKET);
+                    if (newPos === -1) {
+                        throw LatexParserError.UNMATCHED_LEFT_BRACE;
+                    }
+                    const group = new TexGroup(array_join(matrix, CONTROL_LINEBREAK.toNode()));
+                    return [new TexFuncCall(command_token, [group]), newPos];
                 }
                 let [arg1, newPos] = this.parseNextArg(tokens, pos);
                 return [new TexFuncCall(command_token, [arg1]), newPos];
@@ -328,7 +334,7 @@ export class LatexParser {
                 return [new TexFuncCall(command_token, [arg1, arg2]), pos2];
             }
             default:
-                throw new Error( 'Invalid number of parameters');
+                throw new Error('Invalid number of parameters');
         }
     }
 
@@ -363,7 +369,7 @@ export class LatexParser {
         pos += eat_whitespaces(tokens, pos).length;
 
         if (pos >= tokens.length) {
-            throw new LatexParserError('Expecting delimiter after \\left');
+            throw new LatexParserError('Expecting a delimiter after \\left');
         }
 
         const leftDelimiter = eat_parenthesis(tokens, pos);
@@ -371,17 +377,16 @@ export class LatexParser {
             throw new LatexParserError('Invalid delimiter after \\left');
         }
         pos++;
-        const exprInsideStart = pos;
-        const idx = find_closing_right_command(tokens, start);
+
+        const [body, idx] = this.parseClosure(tokens, pos, RIGHT_COMMAND);
         if (idx === -1) {
-            throw new LatexParserError('No matching \\right');
+            throw LatexParserError.UNMATCHED_COMMAND_LEFT;
         }
-        const exprInsideEnd = idx;
-        pos = idx + 1;
+        pos = idx;
 
         pos += eat_whitespaces(tokens, pos).length;
         if (pos >= tokens.length) {
-            throw new LatexParserError('Expecting \\right after \\left');
+            throw new LatexParserError('Expecting a delimiter after \\right');
         }
 
         const rightDelimiter = eat_parenthesis(tokens, pos);
@@ -390,7 +395,6 @@ export class LatexParser {
         }
         pos++;
 
-        const [body, _] = this.parseGroup(tokens, exprInsideStart, exprInsideEnd);
         const left = leftDelimiter.value === '.'? null: leftDelimiter;
         const right = rightDelimiter.value === '.'? null: rightDelimiter;
         const res = new TexLeftRight({body: body, left: left, right: right});
@@ -414,38 +418,34 @@ export class LatexParser {
             [data, pos] = this.parseNextArg(tokens, pos);
         }
 
-        pos += eat_whitespaces(tokens, pos).length; // ignore whitespaces and '\n' after \begin{envName}
-
-
-        const exprInsideStart = pos;
-
-        const endIdx = find_closing_end_command(tokens, start);
+        const [body, endIdx] = this.parseAligned(tokens, pos, END_COMMAND);
         if (endIdx === -1) {
-            throw new LatexParserError('No matching \\end');
+            throw LatexParserError.UNMATCHED_COMMAND_BEGIN;
         }
-        const exprInsideEnd = endIdx;
-        pos = endIdx + 1;
+
+        pos = endIdx;
 
         assert(tokens[pos].eq(LEFT_CURLY_BRACKET));
         assert(tokens[pos + 1].type === TexTokenType.LITERAL);
         assert(tokens[pos + 2].eq(RIGHT_CURLY_BRACKET));
         if (tokens[pos + 1].value !== envName) {
-            throw new LatexParserError('Mismatched \\begin and \\end environments');
+            throw new LatexParserError('\\begin and \\end environments mismatch');
         }
         pos += 3;
 
-        const exprInside = tokens.slice(exprInsideStart, exprInsideEnd);
-        // ignore spaces and '\n' before \end{envName}
-        while(exprInside.length > 0 && [TexTokenType.SPACE, TexTokenType.NEWLINE].includes(exprInside[exprInside.length - 1].type)) {
-            exprInside.pop();
-        }
-        const body = this.parseAligned(exprInside);
         const res = new TexBeginEnd(new TexToken(TexTokenType.LITERAL, envName), body, data);
         return [res, pos];
     }
 
-    parseAligned(tokens: TexToken[]): TexNode[][] {
-        let pos = 0;
+    // return pos: (position of closingToken) + 1
+    // pos will be -1 if closingToken is not found
+    parseAligned(tokens: TexToken[], start: number, closingToken: TexToken): [TexNode[][], number] {
+        this.alignmentDepth++;
+
+        let pos = start;
+        // ignore whitespaces and '\n' after \begin{envName}
+        pos += eat_whitespaces(tokens, pos).length;
+
         const allRows: TexNode[][] = [];
         let row: TexNode[] = [];
         allRows.push(row);
@@ -453,6 +453,10 @@ export class LatexParser {
         row.push(group);
 
         while (pos < tokens.length) {
+            if (tokens[pos].eq(closingToken)) {
+                break;
+            }
+
             const [res, newPos] = this.parseNextExpr(tokens, pos);
             pos = newPos;
 
@@ -477,7 +481,24 @@ export class LatexParser {
                 group.items.push(res);
             }
         }
-        return allRows;
+
+        if (pos >= tokens.length) {
+            return [[], -1];
+        }
+
+        // ignore spaces and '\n' before \end{envName}
+        if (allRows.length > 0 && allRows[allRows.length - 1].length > 0) {
+            const last_cell = allRows[allRows.length - 1][allRows[allRows.length - 1].length - 1];
+            if (last_cell.type === 'ordgroup') {
+                const last_cell_items = (last_cell as TexGroup).items;
+                while(last_cell_items.length > 0 && [TexTokenType.SPACE, TexTokenType.NEWLINE].includes(last_cell_items[last_cell_items.length - 1].head.type)) {
+                    last_cell_items.pop();
+                }
+            }
+        }
+
+        this.alignmentDepth--;
+        return [allRows, pos + 1];
     }
 }
 
@@ -511,7 +532,7 @@ function passExpandCustomTexMacros(tokens: TexToken[], customTexMacros: {[key: s
     return out_tokens;
 }
 
-export function parseTex(tex: string, customTexMacros: {[key: string]: string}): TexNode {
+export function parseTex(tex: string, customTexMacros: {[key: string]: string} = {}): TexNode {
     const parser = new LatexParser();
     let tokens = tokenize_tex(tex);
     tokens = passIgnoreWhitespaceBeforeScriptMark(tokens);

package/src/tex-tokenizer.ts

@@ -35,6 +35,12 @@ export const TEX_UNARY_COMMANDS = [
     'overrightarrow',
     'hspace',
     'substack',
+    'set',
+    'displaylines',
+    'mathinner',
+    'mathrel',
+    'mathbin',
+    'mathop',
 ]
 
 export const TEX_BINARY_COMMANDS = [

package/src/tex2typst.ts

@@ -4,7 +4,5 @@
 
 import { tex2typst, typst2tex } from './index';
 
-if(typeof window !== 'undefined') {
-    (window as any).tex2typst = tex2typst;
-    (window as any).typst2tex = typst2tex;
-}
+(window as any).tex2typst = tex2typst;
+(window as any).typst2tex = typst2tex;

package/src/typst-parser.ts

@@ -53,8 +53,8 @@ function find_closing_delim(tokens: TypstToken[], start: number): number {
     return _find_closing_match(
         tokens,
         start,
-        [LEFT_PARENTHESES, LEFT_BRACKET, LEFT_CURLY_BRACKET, VERTICAL_BAR],
-        [RIGHT_PARENTHESES, RIGHT_BRACKET, RIGHT_CURLY_BRACKET, VERTICAL_BAR]
+        TypstToken.LEFT_DELIMITERS,
+        TypstToken.RIGHT_DELIMITERS
     );
 }
 
@@ -218,7 +218,6 @@ const LEFT_BRACKET: TypstToken = new TypstToken(TypstTokenType.ELEMENT, '[');
 const RIGHT_BRACKET: TypstToken = new TypstToken(TypstTokenType.ELEMENT, ']');
 const LEFT_CURLY_BRACKET: TypstToken = new TypstToken(TypstTokenType.ELEMENT, '{');
 const RIGHT_CURLY_BRACKET: TypstToken = new TypstToken(TypstTokenType.ELEMENT, '}');
-const VERTICAL_BAR = new TypstToken(TypstTokenType.ELEMENT, '|');
 const COMMA = new TypstToken(TypstTokenType.ELEMENT, ',');
 const SEMICOLON = new TypstToken(TypstTokenType.ELEMENT, ';');
 const SINGLE_SPACE = new TypstToken(TypstTokenType.SPACE, ' ');
@@ -373,20 +372,20 @@ export class TypstParser {
     // start: the position of the left parentheses
     parseLrArguments(tokens: TypstToken[], start: number): [TypstNode, number] {
         const lr_token = tokens[start];
-        if (tokens[start + 1].isOneOf([LEFT_PARENTHESES, LEFT_BRACKET, LEFT_CURLY_BRACKET, VERTICAL_BAR])) {
-            const end = find_closing_match(tokens, start);
+        const end = find_closing_match(tokens, start);
+        if (tokens[start + 1].isOneOf(TypstToken.LEFT_DELIMITERS)) {
             const inner_start = start + 1;
             const inner_end = find_closing_delim(tokens, inner_start);
-            const inner_args= this.parseArgumentsWithSeparator(tokens, inner_start + 1, inner_end, COMMA);
+            const [inner_args, _]= this.parseGroup(tokens, inner_start + 1, inner_end);
             return [
-                new TypstLeftright(lr_token, { body: new TypstGroup(inner_args), left: tokens[inner_start], right: tokens[inner_end]}),
+                new TypstLeftright(lr_token, { body: inner_args, left: tokens[inner_start], right: tokens[inner_end]}),
                 end + 1,
             ];
         } else {
-            const [args, end] = this.parseArguments(tokens, start);
+            const [inner_args, _] = this.parseGroup(tokens, start + 1, end - 1);
             return [
-                new TypstLeftright(lr_token, { body: new TypstGroup(args), left: null, right: null }),
-                end,
+                new TypstLeftright(lr_token, { body: inner_args, left: null, right: null }),
+                end + 1,
             ];
         }
     }