tex2typst 0.3.24 → 0.3.26

package/src/tex-tokenizer.ts

@@ -46,6 +46,7 @@ export const TEX_BINARY_COMMANDS = [
     'tbinom',
     'overset',
     'underset',
+    'textcolor',
 ]
 
 
@@ -58,7 +59,7 @@ function unescape(str: string): string {
 }
 
 const rules_map = new Map<string, (a: Scanner<TexToken>) => TexToken | TexToken[]>([
-    // math `\begin{array}{cc}`
+    // match `\begin{array}{cc}`
     [
         String.raw`\\begin{(array|subarry)}{(.+?)}`, (s) => {
             const match = s.reMatchArray()!;
@@ -74,7 +75,7 @@ const rules_map = new Map<string, (a: Scanner<TexToken>) => TexToken | TexToken[
         }
     ],
     [
-        String.raw`\\(text|operatorname|begin|end|hspace|array){(.+?)}`, (s) => {
+        String.raw`\\(text|operatorname|textcolor|begin|end|hspace|array){(.+?)}`, (s) => {
             const match = s.reMatchArray()!;
             return [
                 new TexToken(TexTokenType.COMMAND, '\\' + match[1]),

package/src/tex-types.ts

@@ -18,7 +18,7 @@ export enum TexTokenType {
 }
 
 export class TexToken {
-    type: TexTokenType;
+    readonly type: TexTokenType;
     value: string;
 
     constructor(type: TexTokenType, value: string) {
@@ -55,6 +55,7 @@ export interface TexSupsubData {
 
 // \left. or \right. will be represented as null.
 export interface TexLeftRightData {
+    body: TexNode;
     left: TexToken | null;
     right: TexToken | null;
 }
@@ -68,14 +69,12 @@ type TexNodeType = 'terminal' | 'text' | 'ordgroup' | 'supsub'
 
 
 export abstract class TexNode {
-    type: TexNodeType;
+    readonly type: TexNodeType;
     head: TexToken;
-    args?: TexNode[];
 
-    constructor(type: TexNodeType, head: TexToken | null, args?: TexNode[]) {
+    constructor(type: TexNodeType, head: TexToken | null) {
         this.type = type;
         this.head = head ? head : TexToken.EMPTY;
-        this.args = args;
     }
 
     // Note that this is only shallow equality.
@@ -148,12 +147,14 @@ export class TexText extends TexNode {
 }
 
 export class TexGroup extends TexNode {
-    constructor(args: TexNode[]) {
-        super('ordgroup', TexToken.EMPTY, args);
+    public items: TexNode[];
+    constructor(items: TexNode[]) {
+        super('ordgroup', TexToken.EMPTY);
+        this.items = items;
     }
 
     public serialize(): TexToken[] {
-        return this.args!.map((n) => n.serialize()).flat();
+        return this.items.map((n) => n.serialize()).flat();
     }
 }
 
@@ -163,7 +164,7 @@ export class TexSupSub extends TexNode {
     public sub: TexNode | null;
 
     constructor(data: TexSupsubData) {
-        super('supsub', TexToken.EMPTY, []);
+        super('supsub', TexToken.EMPTY);
         this.base = data.base;
         this.sup = data.sup;
         this.sub = data.sub;
@@ -211,11 +212,14 @@ export class TexSupSub extends TexNode {
 }
 
 export class TexFuncCall extends TexNode {
+    public args: TexNode[];
+
     // For type="sqrt", it's additional argument wrapped square bracket. e.g. 3 in \sqrt[3]{x}
     public data: TexNode | null;
 
     constructor(head: TexToken, args: TexNode[], data: TexNode | null = null) {
-        super('funcCall', head, args);
+        super('funcCall', head);
+        this.args = args;
         this.data = data;
     }
 
@@ -230,7 +234,7 @@ export class TexFuncCall extends TexNode {
             tokens.push(new TexToken(TexTokenType.ELEMENT, ']'));
         }
 
-        for (const arg of this.args!) {
+        for (const arg of this.args) {
             tokens.push(new TexToken(TexTokenType.ELEMENT, '{'));
             tokens = tokens.concat(arg.serialize());
             tokens.push(new TexToken(TexTokenType.ELEMENT, '}'));
@@ -241,11 +245,13 @@ export class TexFuncCall extends TexNode {
 }
 
 export class TexLeftRight extends TexNode {
+    public body: TexNode;
     public left: TexToken | null;
     public right: TexToken | null;
 
-    constructor(args: TexNode[], data: TexLeftRightData) {
-        super('leftright', TexToken.EMPTY, args);
+    constructor(data: TexLeftRightData) {
+        super('leftright', TexToken.EMPTY);
+        this.body = data.body;
         this.left = data.left;
         this.right = data.right;
     }
@@ -254,7 +260,7 @@ export class TexLeftRight extends TexNode {
         let tokens: TexToken[] = [];
         tokens.push(new TexToken(TexTokenType.COMMAND, '\\left'));
         tokens.push(new TexToken(TexTokenType.ELEMENT, this.left? this.left.value: '.'));
-        tokens = tokens.concat(this.args!.map((n) => n.serialize()).flat());
+        tokens = tokens.concat(this.body.serialize());
         tokens.push(new TexToken(TexTokenType.COMMAND, '\\right'));
         tokens.push(new TexToken(TexTokenType.ELEMENT, this.right? this.right.value: '.'));
         return tokens;
@@ -263,11 +269,14 @@ export class TexLeftRight extends TexNode {
 
 export class TexBeginEnd extends TexNode {
     public matrix: TexNode[][];
+    // for environment="array" or "subarray", there's additional data like {c|c} right after \begin{env}
+    public data: TexNode | null;
 
-    constructor(head: TexToken, args: TexNode[], data: TexNode[][]) {
+    constructor(head: TexToken, matrix: TexNode[][], data: TexNode | null = null) {
         assert(head.type === TexTokenType.LITERAL);
-        super('beginend', head, args);
-        this.matrix = data;
+        super('beginend', head);
+        this.matrix = matrix;
+        this.data = data;
     }
 
     public serialize(): TexToken[] {
@@ -330,6 +339,8 @@ export function writeTexTokenBuffer(buffer: string, token: TexToken): string {
         no_need_space ||= /[\(\[{]\s*(-|\+)$/.test(buffer) || buffer === '-' || buffer === '+';
         // "&=" instead of "& ="
         no_need_space ||= buffer.endsWith('&') && str === '=';
+        // "2y" instead of "2 y"
+        no_need_space ||= /\d$/.test(buffer) && /^[a-zA-Z]$/.test(str);
     }
 
     if (!no_need_space) {

package/src/typst-parser.ts

@@ -1,6 +1,6 @@
 
 import { array_find } from "./generic";
-import { TypstCases, TypstFraction, TypstFuncCall, TypstGroup, TypstLeftright, TypstLeftRightData, TypstMatrix, TypstNode, TypstSupsub } from "./typst-types";
+import { TypstFraction, TypstFuncCall, TypstGroup, TypstLeftright, TypstLeftRightData, TypstMarkupFunc, TypstMatrixLike, TypstNode, TypstSupsub, TypstTerminal } from "./typst-types";
 import { TypstNamedParams } from "./typst-types";
 import { TypstSupsubData } from "./typst-types";
 import { TypstToken } from "./typst-types";
@@ -19,7 +19,6 @@ function eat_primes(tokens: TypstToken[], start: number): number {
     return pos - start;
 }
 
-
 function _find_closing_match(tokens: TypstToken[], start: number,
         leftBrackets: TypstToken[], rightBrackets: TypstToken[]): number {
     assert(tokens[start].isOneOf(leftBrackets));
@@ -168,10 +167,10 @@ function process_operators(nodes: TypstNode[], parenthesis = false): TypstNode {
                 let numerator = args.pop()!;
 
                 if(denominator.type === 'leftright') {
-                    denominator = new TypstGroup(denominator.args!);
+                    denominator = (denominator as TypstLeftright).body;
                 }
                 if(numerator.type === 'leftright') {
-                    numerator = new TypstGroup(numerator.args!);
+                    numerator = (numerator as TypstLeftright).body;
                 }
 
                 args.push(new TypstFraction([numerator, denominator]));
@@ -181,15 +180,24 @@ function process_operators(nodes: TypstNode[], parenthesis = false): TypstNode {
             }
         }
     }
+    const body = args.length === 1? args[0]: new TypstGroup(args);
     if(parenthesis) {
-        return new TypstLeftright(null, args, { left: LEFT_PARENTHESES, right: RIGHT_PARENTHESES } as TypstLeftRightData);
+        return new TypstLeftright(null, { body: body, left: LEFT_PARENTHESES, right: RIGHT_PARENTHESES } as TypstLeftRightData);
     } else {
-        if(args.length === 1) {
-            return args[0];
-        } else {
-            return new TypstGroup(args);
-        }
+        return body;
+    }
+}
+
+function parse_named_params(groups: TypstGroup[]): TypstNamedParams {
+    const COLON = new TypstToken(TypstTokenType.ELEMENT, ':').toNode();
+
+    const np: TypstNamedParams = {};
+    for (const group of groups) {
+        assert(group.items.length == 3);
+        assert(group.items[1].eq(COLON));
+        np[group.items[0].toString()] = new TypstTerminal(new TypstToken(TypstTokenType.LITERAL, group.items[2].toString()));
     }
+    return np;
 }
 
 export class TypstParserError extends Error {
@@ -322,19 +330,31 @@ export class TypstParser {
             if (start + 1 < tokens.length && tokens[start + 1].eq(LEFT_PARENTHESES)) {
                 if(firstToken.value === 'mat') {
                     const [matrix, named_params, newPos] = this.parseMatrix(tokens, start + 1, SEMICOLON, COMMA);
-                    const mat = new TypstMatrix(matrix);
+                    const mat = new TypstMatrixLike(firstToken, matrix);
                     mat.setOptions(named_params);
                     return [mat, newPos];
                 }
                 if(firstToken.value === 'cases') {
                     const [cases, named_params, newPos] = this.parseMatrix(tokens, start + 1, COMMA, CONTROL_AND);
-                    const casesNode = new TypstCases(cases);
+                    const casesNode = new TypstMatrixLike(firstToken, cases);
                     casesNode.setOptions(named_params);
                     return [casesNode, newPos];
                 }
                 if (firstToken.value === 'lr') {
                     return this.parseLrArguments(tokens, start + 1);
                 }
+                if (['#heading', '#text'].includes(firstToken.value)) {
+                    const [args, newPos] = this.parseArguments(tokens, start + 1);
+                    const named_params = parse_named_params(args as TypstGroup[]);
+                    assert(tokens[newPos].eq(LEFT_BRACKET));
+                    const DOLLAR = new TypstToken(TypstTokenType.ELEMENT, '$');
+                    const end = _find_closing_match(tokens, newPos + 1, [DOLLAR], [DOLLAR]);
+                    const [group, _] = this.parseGroup(tokens, newPos + 2, end);
+                    assert(tokens[end + 1].eq(RIGHT_BRACKET));
+                    const markup_func = new TypstMarkupFunc(firstToken, [group]);
+                    markup_func.setOptions(named_params);
+                    return [markup_func, end + 2];
+                }
                 const [args, newPos] = this.parseArguments(tokens, start + 1);
                 const func_call = new TypstFuncCall(firstToken, args);
                 return [func_call, newPos];
@@ -359,13 +379,13 @@ export class TypstParser {
             const inner_end = find_closing_delim(tokens, inner_start);
             const inner_args= this.parseArgumentsWithSeparator(tokens, inner_start + 1, inner_end, COMMA);
             return [
-                new TypstLeftright(lr_token, inner_args, {left: tokens[inner_start], right: tokens[inner_end]}),
+                new TypstLeftright(lr_token, { body: new TypstGroup(inner_args), left: tokens[inner_start], right: tokens[inner_end]}),
                 end + 1,
             ];
         } else {
             const [args, end] = this.parseArguments(tokens, start);
             return [
-                new TypstLeftright(lr_token, args, { left: null, right: null }),
+                new TypstLeftright(lr_token, { body: new TypstGroup(args), left: null, right: null }),
                 end,
             ];
         }
@@ -400,18 +420,18 @@ export class TypstParser {
                             continue;
                         }
 
-                        const g = arr[i];
-                        const pos_colon = array_find(g.args!, COLON);
+                        const g = arr[i] as TypstGroup;
+                        const pos_colon = array_find(g.items, COLON);
                         if(pos_colon === -1 || pos_colon === 0) {
                             continue;
                         }
                         to_delete.push(i);
-                        const param_name = g.args![pos_colon - 1];
+                        const param_name = g.items[pos_colon - 1];
                         if(param_name.eq(new TypstToken(TypstTokenType.SYMBOL, 'delim').toNode())) {
-                            if(g.args!.length !== 3) {
+                            if(g.items.length !== 3) {
                                 throw new TypstParserError('Invalid number of arguments for delim');
                             }
-                            np['delim'] = g.args![pos_colon + 1];
+                            np['delim'] = g.items[pos_colon + 1];
                         } else {
                             throw new TypstParserError('Not implemented for other named parameters');
                         }

package/src/typst-tokenizer.ts

@@ -69,10 +69,10 @@ const rules_map = new Map<string, (a: Scanner<TypstToken>) => TypstToken | Typst
             new TypstToken(TypstTokenType.ELEMENT, ")"),
         ];
     }],
-    [String.raw`[a-zA-Z\.]+`, (s) => {
+    [String.raw`#none`, (s) => new TypstToken(TypstTokenType.NONE, s.text()!)],
+    [String.raw`#?[a-zA-Z\.]+`, (s) => {
         return new TypstToken(s.text()!.length === 1? TypstTokenType.ELEMENT: TypstTokenType.SYMBOL, s.text()!);
     }],
-    [String.raw`#none`, (s) => new TypstToken(TypstTokenType.NONE, s.text()!)],
     [String.raw`.`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
 ]);
 

package/src/typst-types.ts

@@ -14,7 +14,7 @@ export enum TypstTokenType {
 
 
 export class TypstToken {
-    type: TypstTokenType;
+    readonly type: TypstTokenType;
     value: string;
 
     constructor(type: TypstTokenType, content: string) {
@@ -46,6 +46,7 @@ export class TypstToken {
     }
 
     public static readonly NONE = new TypstToken(TypstTokenType.NONE, '#none');
+    public static readonly EMPTY = new TypstToken(TypstTokenType.ELEMENT, '');
 }
 
 export interface TypstSupsubData {
@@ -56,6 +57,7 @@ export interface TypstSupsubData {
 
 
 export interface TypstLeftRightData {
+    body: TypstNode;
     left: TypstToken | null;
     right: TypstToken | null;
 }
@@ -64,22 +66,21 @@ export interface TypstLeftRightData {
  * fraction: `1/2`, `(x + y)/2`, `(1+x)/(1-x)`
  * group: `a + 1/3`
  * leftright: `(a + 1/3)`, `[a + 1/3)`, `lr(]sum_(x=1)^n])`
+ * markupFunc: `#heading(level: 2)[something]`, `#text(fill: red)[some text and math $x + y$]`
  */
-export type TypstNodeType = 'terminal' | 'group' | 'supsub' | 'funcCall' | 'fraction'| 'leftright' | 'align' | 'matrix' | 'cases';
+export type TypstNodeType = 'terminal' | 'group' | 'supsub' | 'funcCall' | 'fraction'| 'leftright' | 'matrixLike'| 'markupFunc';
 
 export type TypstNamedParams = { [key: string]: TypstNode; };
 
 export abstract class TypstNode {
     readonly type: TypstNodeType;
     head: TypstToken;
-    args?: TypstNode[];
     // Some Typst functions accept additional options. e.g. mat() has option "delim", op() has option "limits"
     options?: TypstNamedParams;
 
-    constructor(type: TypstNodeType, head: TypstToken | null, args?: TypstNode[]) {
+    constructor(type: TypstNodeType, head: TypstToken | null) {
         this.type = type;
         this.head = head ? head : TypstToken.NONE;
-        this.args = args;
     }
 
     // whether the node is over high so that if it's wrapped in braces, \left and \right should be used in its TeX form
@@ -115,12 +116,14 @@ export class TypstTerminal extends TypstNode {
 }
 
 export class TypstGroup extends TypstNode {
-    constructor(args: TypstNode[]) {
-        super('group', TypstToken.NONE, args);
+    public items: TypstNode[];
+    constructor(items: TypstNode[]) {
+        super('group', TypstToken.NONE);
+        this.items = items;
     }
 
     public isOverHigh(): boolean {
-        return this.args!.some((n) => n.isOverHigh());
+        return this.items.some((n) => n.isOverHigh());
     }
 }
 
@@ -130,7 +133,7 @@ export class TypstSupsub extends TypstNode {
     public sub: TypstNode | null;
 
     constructor(data: TypstSupsubData) {
-        super('supsub', TypstToken.NONE, []);
+        super('supsub', TypstToken.NONE);
         this.base = data.base;
         this.sup = data.sup;
         this.sub = data.sub;
@@ -142,21 +145,26 @@ export class TypstSupsub extends TypstNode {
 }
 
 export class TypstFuncCall extends TypstNode {
+    public args: TypstNode[];
     constructor(head: TypstToken, args: TypstNode[]) {
-        super('funcCall', head, args);
+        super('funcCall', head);
+        this.args = args;
     }
 
     public isOverHigh(): boolean {
         if (this.head.value === 'frac') {
             return true;
         }
-        return this.args!.some((n) => n.isOverHigh());
+        return this.args.some((n) => n.isOverHigh());
     }
 }
 
 export class TypstFraction extends TypstNode {
+    public args: TypstNode[];
+
     constructor(args: TypstNode[]) {
-        super('fraction', TypstToken.NONE, args);
+        super('fraction', TypstToken.NONE);
+        this.args = args;
     }
 
     public isOverHigh(): boolean {
@@ -166,56 +174,57 @@ export class TypstFraction extends TypstNode {
 
 
 export class TypstLeftright extends TypstNode {
+    public body: TypstNode;
     public left: TypstToken | null;
     public right: TypstToken | null;
 
-    constructor(head: TypstToken | null, args: TypstNode[], data: TypstLeftRightData) {
-        super('leftright', head, args);
+    // head is either null or 'lr'
+    constructor(head: TypstToken | null, data: TypstLeftRightData) {
+        super('leftright', head);
+        this.body = data.body;
         this.left = data.left;
         this.right = data.right;
     }
 
     public isOverHigh(): boolean {
-        return this.args!.some((n) => n.isOverHigh());
+        return this.body.isOverHigh();
     }
 }
 
-export class TypstAlign extends TypstNode {
-    public matrix: TypstNode[][];
 
-    constructor(data: TypstNode[][]) {
-        super('align', TypstToken.NONE, []);
-        this.matrix = data;
-    }
-
-    public isOverHigh(): boolean {
-        return true;
-    }
-}
-
-export class TypstMatrix extends TypstNode {
+export class TypstMatrixLike extends TypstNode {
     public matrix: TypstNode[][];
 
-    constructor(data: TypstNode[][]) {
-        super('matrix', TypstToken.NONE, []);
+    // head is 'mat', 'cases' or null
+    constructor(head: TypstToken | null, data: TypstNode[][]) {
+        super('matrixLike', head);
         this.matrix = data;
     }
 
     public isOverHigh(): boolean {
         return true;
     }
+
+    static readonly MAT = new TypstToken(TypstTokenType.SYMBOL, 'mat');
+    static readonly CASES = new TypstToken(TypstTokenType.SYMBOL, 'cases');
 }
 
-export class TypstCases extends TypstNode {
-    public matrix: TypstNode[][];
+export class TypstMarkupFunc extends TypstNode {
+    /*
+    In idealized situations, for `#heading([some text and math $x + y$ example])`,
+    fragments would be [TypstMarkup{"some text and math "}, TypstNode{"x + y"}, TypstMarkup{" example"}]
+    At present, we haven't implemented anything about TypstMarkup.
+    So only pattens like `#heading(level: 2)[$x+y$]`, `#text(fill: red)[$x + y$]` are supported.
+    Therefore, fragments is always a list containing exactly 1 TypstNode in well-working situations.
+    */
+    public fragments: TypstNode[];
 
-    constructor(data: TypstNode[][]) {
-        super('cases', TypstToken.NONE, []);
-        this.matrix = data;
+    constructor(head: TypstToken, fragments: TypstNode[]) {
+        super('markupFunc', head);
+        this.fragments = fragments;
     }
 
     public isOverHigh(): boolean {
-        return true;
+        return this.fragments.some((n) => n.isOverHigh());
     }
 }
-