tex2typst 0.3.22 → 0.3.23

package/src/tex-parser.ts

@@ -231,8 +231,8 @@ export class LatexParser {
         switch (firstToken.type) {
             case TexTokenType.ELEMENT:
                 return [new TexNode('element', firstToken.value), start + 1];
-            case TexTokenType.TEXT:
-                return [new TexNode('text', firstToken.value), start + 1];
+            case TexTokenType.LITERAL:
+                return [new TexNode('literal', firstToken.value), start + 1];
             case TexTokenType.COMMENT:
                 return [new TexNode('comment', firstToken.value), start + 1];
             case TexTokenType.SPACE:
@@ -321,7 +321,7 @@ export class LatexParser {
                         throw new LatexParserError('Expecting content for \\text command');
                     }
                     assert(tokens[pos].eq(LEFT_CURLY_BRACKET));
-                    assert(tokens[pos + 1].type === TexTokenType.TEXT);
+                    assert(tokens[pos + 1].type === TexTokenType.LITERAL);
                     assert(tokens[pos + 2].eq(RIGHT_CURLY_BRACKET));
                     const text = tokens[pos + 1].value;
                     return [new TexNode('text', text), pos + 3];
@@ -412,24 +412,25 @@ export class LatexParser {
 
         let pos = start + 1;
         assert(tokens[pos].eq(LEFT_CURLY_BRACKET));
-        assert(tokens[pos + 1].type === TexTokenType.TEXT);
+        assert(tokens[pos + 1].type === TexTokenType.LITERAL);
         assert(tokens[pos + 2].eq(RIGHT_CURLY_BRACKET));
         const envName = tokens[pos + 1].value;
         pos += 3;
 
-        const args: TexNode[] = [];
-        while (pos < tokens.length) {
-            const whitespaceCount = eat_whitespaces(tokens, pos).length;
-            pos += whitespaceCount;
 
+        const args: TexNode[] = [];
+        if(['array', 'subarray'].includes(envName)) {
             if (pos >= tokens.length || !tokens[pos].eq(LEFT_CURLY_BRACKET)) {
-                break;
+                throw new LatexParserError(`Missing arg for \\begin{${envName}}`);
             }
             const [arg, newPos] = this.parseNextArg(tokens, pos);
             args.push(arg);
             pos = newPos;
         }
 
+        pos += eat_whitespaces(tokens, pos).length; // ignore whitespaces and '\n' after \begin{envName}
+
+
         const exprInsideStart = pos;
 
         const endIdx = find_closing_end_command(tokens, start);
@@ -440,7 +441,7 @@ export class LatexParser {
         pos = endIdx + 1;
 
         assert(tokens[pos].eq(LEFT_CURLY_BRACKET));
-        assert(tokens[pos + 1].type === TexTokenType.TEXT);
+        assert(tokens[pos + 1].type === TexTokenType.LITERAL);
         assert(tokens[pos + 2].eq(RIGHT_CURLY_BRACKET));
         if (tokens[pos + 1].value !== envName) {
             throw new LatexParserError('Mismatched \\begin and \\end environments');

package/src/tex-tokenizer.ts

@@ -1,5 +1,4 @@
 import { TexToken, TexTokenType } from "./types";
-import { assert } from "./util";
 import { JSLex, Scanner } from "./jslex";
 
 export const TEX_UNARY_COMMANDS = [
@@ -59,15 +58,28 @@ function unescape(str: string): string {
 }
 
 const rules_map = new Map<string, (a: Scanner<TexToken>) => TexToken | TexToken[]>([
+    // math `\begin{array}{cc}`
     [
-        String.raw`\\(text|operatorname|begin|end|hspace){.+?}`, (s) => {
-            const text = s.text()!;
-            const command = text.substring(0, text.indexOf('{'));
-            const text_inside = text.substring(text.indexOf('{') + 1, text.lastIndexOf('}'));
+        String.raw`\\begin{(array|subarry)}{(.+?)}`, (s) => {
+            const match = s.reMatchArray()!;
             return [
-                new TexToken(TexTokenType.COMMAND, command),
+                new TexToken(TexTokenType.COMMAND, '\\begin'),
+                new TexToken(TexTokenType.CONTROL, '{'),
+                new TexToken(TexTokenType.LITERAL, match[1]),
+                new TexToken(TexTokenType.CONTROL, '}'),
+                new TexToken(TexTokenType.CONTROL, '{'),
+                new TexToken(TexTokenType.LITERAL, match[2]),
+                new TexToken(TexTokenType.CONTROL, '}'),
+            ]
+        }
+    ],
+    [
+        String.raw`\\(text|operatorname|begin|end|hspace|array){(.+?)}`, (s) => {
+            const match = s.reMatchArray()!;
+            return [
+                new TexToken(TexTokenType.COMMAND, '\\' + match[1]),
                 new TexToken(TexTokenType.CONTROL, '{'),
-                new TexToken(TexTokenType.TEXT, unescape(text_inside)),
+                new TexToken(TexTokenType.LITERAL, unescape(match[2])),
                 new TexToken(TexTokenType.CONTROL, '}')
             ]
         }
@@ -80,14 +92,11 @@ const rules_map = new Map<string, (a: Scanner<TexToken>) => TexToken | TexToken[
     [String.raw`\\[{}%$&#_|]`, (s) => new TexToken(TexTokenType.ELEMENT, s.text()!)],
     // e.g. match `\frac13`, `\frac1 b`, `\frac a b`
     [String.raw`(\\[a-zA-Z]+)(\s*\d|\s+[a-zA-Z])\s*([0-9a-zA-Z])`, (s) => {
-        const text = s.text()!;
-        const regex = RegExp(String.raw`(\\[a-zA-Z]+)(\s*\d|\s+[a-zA-Z])\s*([0-9a-zA-Z])`);
-        const match = text.match(regex);
-        assert(match !== null);
+        const match = s.reMatchArray()!;
         const command = match![1];
         if (TEX_BINARY_COMMANDS.includes(command.substring(1))) {
-            const arg1 = match![2].trimStart();
-            const arg2 = match![3];
+            const arg1 = match[2].trimStart();
+            const arg2 = match[3];
             return [
                 new TexToken(TexTokenType.COMMAND, command),
                 new TexToken(TexTokenType.ELEMENT, arg1),
@@ -100,13 +109,10 @@ const rules_map = new Map<string, (a: Scanner<TexToken>) => TexToken | TexToken[
     }],
     // e.g. match `\sqrt3`, `\sqrt a`
     [String.raw`(\\[a-zA-Z]+)(\s*\d|\s+[a-zA-Z])`, (s) => {
-        const text = s.text()!;
-        const regex = RegExp(String.raw`(\\[a-zA-Z]+)(\s*\d|\s+[a-zA-Z])`);
-        const match = text.match(regex);
-        assert(match !== null);
-        const command = match![1];
+        const match = s.reMatchArray()!;
+        const command = match[1];
         if (TEX_UNARY_COMMANDS.includes(command.substring(1))) {
-            const arg1 = match![2].trimStart();
+            const arg1 = match[2].trimStart();
             return [
                 new TexToken(TexTokenType.COMMAND, command),
                 new TexToken(TexTokenType.ELEMENT, arg1),
@@ -116,10 +122,7 @@ const rules_map = new Map<string, (a: Scanner<TexToken>) => TexToken | TexToken[
             return [];
         }
     }],
-    [String.raw`\\[a-zA-Z]+`, (s) => {
-        const command = s.text()!;
-        return [ new TexToken(TexTokenType.COMMAND, command), ];
-    }],
+    [String.raw`\\[a-zA-Z]+`, (s) => new TexToken(TexTokenType.COMMAND, s.text()!)],
     // Numbers like "123", "3.14"
     [String.raw`[0-9]+(\.[0-9]+)?`, (s) => new TexToken(TexTokenType.ELEMENT, s.text()!)],
     [String.raw`[a-zA-Z]`, (s) => new TexToken(TexTokenType.ELEMENT, s.text()!)],

package/src/tex-writer.ts

@@ -1,5 +1,5 @@
 import { array_includes, array_split } from "./generic";
-import { TexNode, TexToken, TexSupsubData, TexTokenType } from "./types";
+import { TexNode, TexToken, TexTokenType } from "./types";
 
 
 export class TexWriter {

package/src/types.ts

@@ -3,7 +3,7 @@ import { array_includes } from "./generic";
 export enum TexTokenType {
     ELEMENT,
     COMMAND,
-    TEXT,
+    LITERAL,
     COMMENT,
     SPACE,
     NEWLINE,
@@ -26,10 +26,8 @@ export class TexToken {
 
     public toString(): string {
         switch (this.type) {
-            case TexTokenType.TEXT:
-                return `\\text{${this.value}}`;
             case TexTokenType.COMMENT:
-                return `%${this.value}`;
+                return "%" + this.value;
             default:
                 return this.value;
         }
@@ -56,7 +54,7 @@ export type TexArrayData = TexNode[][];
  * empty: special type when something is empty. e.g. the base of _{a} or ^{a}
  * whitespace: space, tab, newline
  */
-type TexNodeType = 'element' | 'text' | 'comment' | 'whitespace' | 'control' | 'ordgroup' | 'supsub'
+type TexNodeType = 'element' | 'text'  | 'literal' | 'comment' | 'whitespace' | 'control' | 'ordgroup' | 'supsub'
              | 'unaryFunc' | 'binaryFunc' | 'leftright' | 'beginend' | 'symbol' | 'empty' | 'unknownMacro';
 
 
@@ -89,16 +87,6 @@ export class TexNode {
         return this.type === other.type && this.content === other.content;
     }
 
-    public toString(): string {
-        switch (this.type) {
-            case 'text':
-                return `\\text{${this.content}}`;
-            default:
-                throw new Error(`toString() is not implemented for type ${this.type}`);
-        }
-    }
-
-
     public serialize(): TexToken[] {
         switch (this.type) {
             case 'empty':
@@ -110,8 +98,15 @@ export class TexNode {
             }
             case 'symbol':
                 return [new TexToken(TexTokenType.COMMAND, this.content)];
+            case 'literal':
+                return [new TexToken(TexTokenType.LITERAL, this.content)];
             case 'text':
-                return [new TexToken(TexTokenType.TEXT, this.content)];
+                return [
+                    new TexToken(TexTokenType.COMMAND, '\\text'),
+                    new TexToken(TexTokenType.ELEMENT, '{'),
+                    new TexToken(TexTokenType.LITERAL, this.content),
+                    new TexToken(TexTokenType.ELEMENT, '}'),
+                ];
             case 'comment':
                 return [new TexToken(TexTokenType.COMMENT, this.content)];
             case 'whitespace': {
@@ -243,6 +238,7 @@ export enum TypstTokenType {
     NONE,
     SYMBOL,
     ELEMENT,
+    LITERAL,
     TEXT,
     COMMENT,
     SPACE,
@@ -271,6 +267,8 @@ export class TypstToken {
         switch(this.type) {
             case TypstTokenType.NONE:
                 return new TypstNode('none', '#none');
+            case TypstTokenType.LITERAL:
+                return new TypstNode('literal', this.value);
             case TypstTokenType.TEXT:
                 return new TypstNode('text', this.value);
             case TypstTokenType.COMMENT:
@@ -326,11 +324,10 @@ export interface TypstLrData {
     rightDelim: string | null;
 }
 
-type TypstNodeType = 'atom' | 'symbol' | 'text' | 'control' | 'comment' | 'whitespace'
+type TypstNodeType = 'atom' | 'symbol' | 'text' | 'literal' | 'control' | 'comment' | 'whitespace'
             | 'none' | 'group' | 'supsub' | 'funcCall' | 'fraction' | 'align' | 'matrix' | 'cases' | 'unknown';
 
-export type TypstPrimitiveValue = string | boolean | TypstNode;
-export type TypstNamedParams = { [key: string]: TypstPrimitiveValue };
+export type TypstNamedParams = { [key: string]: TypstNode };
 
 
 export class TypstNode {
@@ -382,12 +379,21 @@ export class TypstNode {
                 return false;
         }
     }
+
+    public toString(): string {
+        switch (this.type) {
+            case 'text':
+                return `"${this.content}"`;
+            case 'comment':
+                return `//${this.content}`;
+            default:
+                return this.content;
+        }
+    }
 }
 
 // #none
 export const TYPST_NONE = new TypstNode('none', '#none');
-export const TYPST_TRUE: TypstPrimitiveValue = true;
-export const TYPST_FALSE: TypstPrimitiveValue = false;
 
 /**
  * ATTENTION:

package/src/typst-parser.ts

@@ -413,22 +413,7 @@ export class TypstParser {
                             if(g.args!.length !== 3) {
                                 throw new TypstParserError('Invalid number of arguments for delim');
                             }
-                             switch (g.args![pos_colon + 1].type) {
-                                case 'text': {
-                                    np['delim'] = g.args![pos_colon + 1].content;
-                                    break;
-                                }
-                                case 'none': {
-                                   np['delim'] = TYPST_NONE;
-                                    break;
-                                }
-                                case 'symbol': {
-                                    np['delim'] = g.args![pos_colon + 1];
-                                    break;
-                                }
-                                default:
-                                    throw new TypstParserError('Not implemented for other types of delim');
-                            }
+                            np['delim'] = g.args![pos_colon + 1];
                         } else {
                             throw new TypstParserError('Not implemented for other named parameters');
                         }

package/src/typst-tokenizer.ts

@@ -59,6 +59,15 @@ const rules_map = new Map<string, (a: Scanner<TypstToken>) => TypstToken | Typst
     ],
     [String.raw`[0-9]+(\.[0-9]+)?`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
     [String.raw`[+\-*/=\'<>!.,;?()\[\]|]`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
+    [String.raw`#h\((.+?)\)`, (s) => {
+        const match = s.reMatchArray()!;
+        return [
+            new TypstToken(TypstTokenType.SYMBOL, "#h"),
+            new TypstToken(TypstTokenType.ELEMENT, "("),
+            new TypstToken(TypstTokenType.LITERAL, match[1]),
+            new TypstToken(TypstTokenType.ELEMENT, ")"),
+        ];
+    }],
     [String.raw`[a-zA-Z\.]+`, (s) => {
         return new TypstToken(s.text()!.length === 1? TypstTokenType.ELEMENT: TypstTokenType.SYMBOL, s.text()!);
     }],

package/src/typst-writer.ts

@@ -1,6 +1,5 @@
-import { TexNode, TypstNode, TypstPrimitiveValue, TypstSupsubData, TypstToken, TypstTokenType } from "./types";
+import { TexNode, TypstNode, TypstSupsubData, TypstToken, TypstTokenType } from "./types";
 import { shorthandMap } from "./typst-shorthands";
-import { assert } from "./util";
 
 function is_delimiter(c: TypstNode): boolean {
     return c.type === 'atom' && ['(', ')', '[', ']', '{', '}', '|', '⌊', '⌋', '⌈', '⌉'].includes(c.content);
@@ -13,20 +12,6 @@ const TYPST_NEWLINE: TypstToken = new TypstToken(TypstTokenType.SYMBOL, '\n');
 
 const SOFT_SPACE = new TypstToken(TypstTokenType.CONTROL, ' ');
 
-function typst_primitive_to_string(value: TypstPrimitiveValue) {
-    switch (typeof value) {
-        case 'string':
-            return `"${value}"`;
-        case 'number':
-            return (value as number).toString();
-        case 'boolean':
-            return (value as boolean) ? '#true' : '#false';
-        default:
-            assert(value instanceof TypstNode, 'Not a valid primitive value');
-            return (value as TypstNode).content;
-    }
-}
-
 export class TypstWriterError extends Error {
     node: TexNode | TypstNode | TypstToken;
 
@@ -132,6 +117,9 @@ export class TypstWriter {
                 this.queue.push(new TypstToken(TypstTokenType.SYMBOL, content));
                 break;
             }
+            case 'literal':
+                this.queue.push(new TypstToken(TypstTokenType.LITERAL, node.content));
+                break;
             case 'text':
                 this.queue.push(new TypstToken(TypstTokenType.TEXT, node.content));
                 break;
@@ -198,8 +186,7 @@ export class TypstWriter {
                 }
                 if (node.options) {
                     for (const [key, value] of Object.entries(node.options)) {
-                        const value_str = typst_primitive_to_string(value);
-                        this.queue.push(new TypstToken(TypstTokenType.SYMBOL, `, ${key}: ${value_str}`));
+                        this.queue.push(new TypstToken(TypstTokenType.LITERAL, `, ${key}: ${value.toString()}`));
                     }
                 }
                 this.queue.push(TYPST_RIGHT_PARENTHESIS);
@@ -246,8 +233,7 @@ export class TypstWriter {
                 this.queue.push(TYPST_LEFT_PARENTHESIS);
                 if (node.options) {
                     for (const [key, value] of Object.entries(node.options)) {
-                        const value_str = typst_primitive_to_string(value);
-                        this.queue.push(new TypstToken(TypstTokenType.SYMBOL, `${key}: ${value_str}, `));
+                        this.queue.push(new TypstToken(TypstTokenType.LITERAL, `${key}: ${value.toString()}, `));
                     }
                 }
                 matrix.forEach((row, i) => {
@@ -282,8 +268,7 @@ export class TypstWriter {
                 this.queue.push(TYPST_LEFT_PARENTHESIS);
                 if (node.options) {
                     for (const [key, value] of Object.entries(node.options)) {
-                        const value_str = typst_primitive_to_string(value);
-                        this.queue.push(new TypstToken(TypstTokenType.SYMBOL, `${key}: ${value_str}, `));
+                        this.queue.push(new TypstToken(TypstTokenType.LITERAL, `${key}: ${value.toString()}, `));
                     }
                 }
                 cases.forEach((row, i) => {
@@ -316,7 +301,7 @@ export class TypstWriter {
     }
 
     private appendWithBracketsIfNeeded(node: TypstNode): boolean {
-        let need_to_wrap = ['group', 'supsub', 'fraction','empty'].includes(node.type);
+        let need_to_wrap = ['group', 'supsub', 'align', 'fraction','empty'].includes(node.type);
 
         if (node.type === 'group') {
             if (node.args!.length === 0) {