tex2typst 0.3.10 → 0.3.12

package/src/convert.ts

@@ -1,4 +1,4 @@
-import { TexNode, TypstNode, TexSupsubData, TypstSupsubData, TexSqrtData, Tex2TypstOptions, TYPST_NULL, TYPST_TRUE, TypstPrimitiveValue, TypstToken, TypstTokenType, TypstLrData, TexArrayData } from "./types";
+import { TexNode, TypstNode, TexSupsubData, TypstSupsubData, TexSqrtData, Tex2TypstOptions, TYPST_NONE, TYPST_TRUE, TypstPrimitiveValue, TypstToken, TypstTokenType, TypstLrData, TexArrayData } from "./types";
 import { TypstWriterError } from "./typst-writer";
 import { symbolMap, reverseSymbolMap } from "./map";
 import { array_join } from "./generic";
@@ -90,7 +90,7 @@ function convert_overset(node: TexNode, options: Tex2TypstOptions): TypstNode {
 export function convert_tex_node_to_typst(node: TexNode, options: Tex2TypstOptions = {}): TypstNode {
     switch (node.type) {
         case 'empty':
-            return new TypstNode('empty', '');
+            return TYPST_NONE;
         case 'whitespace':
             return new TypstNode('whitespace', node.content);
         case 'ordgroup':
@@ -103,8 +103,16 @@ export function convert_tex_node_to_typst(node: TexNode, options: Tex2TypstOptio
             return new TypstNode('atom', tex_token_to_typst(node.content));
         case 'symbol':
             return new TypstNode('symbol', tex_token_to_typst(node.content));
-        case 'text':
+        case 'text': {
+            if ((/[^\x00-\x7F]+/).test(node.content) && options.nonAsciiWrapper !== "") {
+                return new TypstNode(
+                    'funcCall',
+                    options.nonAsciiWrapper!,
+                    [new TypstNode('text', node.content)]
+                );
+            }
             return new TypstNode('text', node.content);
+        }
         case 'comment':
             return new TypstNode('comment', node.content);
         case 'supsub': {
@@ -128,8 +136,8 @@ export function convert_tex_node_to_typst(node: TexNode, options: Tex2TypstOptio
             const data: TypstSupsubData = {
                 base: convert_tex_node_to_typst(base, options),
             };
-            if (data.base.type === 'empty') {
-                data.base = new TypstNode('text', '');
+            if (data.base.type === 'none') {
+                data.base = new TypstNode('none', '');
             }
 
             if (sup) {
@@ -281,10 +289,10 @@ export function convert_tex_node_to_typst(node: TexNode, options: Tex2TypstOptio
                 return new TypstNode('cases', '', [], data);
             }
             if (node.content!.endsWith('matrix')) {
-                let delim: TypstPrimitiveValue = null;
+                let delim: TypstPrimitiveValue;
                 switch (node.content) {
                     case 'matrix':
-                        delim = TYPST_NULL;
+                        delim = TYPST_NONE;
                         break;
                     case 'pmatrix':
                         delim = '(';
@@ -299,7 +307,7 @@ export function convert_tex_node_to_typst(node: TexNode, options: Tex2TypstOptio
                         delim = '|';
                         break;
                     case 'Vmatrix': {
-                        delim = new TypstToken(TypstTokenType.SYMBOL, 'bar.v.double');
+                        delim = new TypstNode('symbol', 'bar.v.double');
                         break;
                     }
                     default:
@@ -388,7 +396,8 @@ export function convert_typst_node_to_tex(node: TypstNode): TexNode {
         ]);
     }
     switch (node.type) {
-        case 'empty':
+        case 'none':
+            // e.g. Typst `#none^2` is converted to TeX `^2`
             return new TexNode('empty', '');
         case 'whitespace':
             return new TexNode('whitespace', node.content);
@@ -513,29 +522,44 @@ export function convert_typst_node_to_tex(node: TypstNode): TexNode {
             let env_type = 'pmatrix';
             if (node.options) {
                 if ('delim' in node.options) {
-                    switch (node.options.delim) {
-                        case TYPST_NULL:
-                            env_type = 'matrix';
-                            break;
-                        case '[':
-                            env_type = 'bmatrix';
-                            break;
-                        case ']':
-                            env_type = 'bmatrix';
-                            break;
-                        case '{':
-                            env_type = 'Bmatrix';
-                            break;
-                        case '}':
-                            env_type = 'Bmatrix';
-                            break;
-                        case '|':
-                            env_type = 'vmatrix';
-                            break;
-                        case ')':
-                        case '(':
-                        default:
-                            env_type = 'pmatrix';
+                    const delim = node.options.delim;
+                    if (delim instanceof TypstNode) {
+                        switch (delim.content) {
+                            case '#none':
+                                env_type = 'matrix';
+                                break;
+                            case 'bar.v.double':
+                                env_type = 'Vmatrix';
+                                break;
+                            case 'bar':
+                            case 'bar.v':
+                                env_type = 'vmatrix';
+                                break;
+                            default:
+                                throw new Error(`Unexpected delimiter ${delim.content}`);
+                        }
+                    } else {
+                        switch (delim) {
+                            case '[':
+                                env_type = 'bmatrix';
+                                break;
+                            case ']':
+                                env_type = 'bmatrix';
+                                break;
+                            case '{':
+                                env_type = 'Bmatrix';
+                                break;
+                            case '}':
+                                env_type = 'Bmatrix';
+                                break;
+                            case '|':
+                                env_type = 'vmatrix';
+                                break;
+                            case ')':
+                            case '(':
+                            default:
+                                env_type = 'pmatrix';
+                        }
                     }
                 }
             }

package/src/index.ts

@@ -16,6 +16,7 @@ export function tex2typst(tex: string, options?: Tex2TypstOptions): string {
         keepSpaces: false,
         fracToSlash: true,
         inftyToOo: false,
+        nonAsciiWrapper: "",
         customTexMacros: {}
     };
 

package/src/tex-parser.ts

@@ -209,9 +209,12 @@ const rules_map = new Map<string, (a: Scanner<TexToken>) => TexToken | TexToken[
         const command = s.text()!;
         return [ new TexToken(TexTokenType.COMMAND, command), ];
     }],
-    [String.raw`[0-9]+`, (s) => new TexToken(TexTokenType.ELEMENT, s.text()!)],
+    // Numbers like "123", "3.14"
+    [String.raw`[0-9]+(\.[0-9]+)?`, (s) => new TexToken(TexTokenType.ELEMENT, s.text()!)],
     [String.raw`[a-zA-Z]`, (s) => new TexToken(TexTokenType.ELEMENT, s.text()!)],
     [String.raw`[+\-*/='<>!.,;:?()\[\]|]`, (s) => new TexToken(TexTokenType.ELEMENT, s.text()!)],
+    // non-ASCII characters
+    [String.raw`[^\x00-\x7F]`, (s) => new TexToken(TexTokenType.ELEMENT, s.text()!)],
     [String.raw`.`, (s) => new TexToken(TexTokenType.UNKNOWN, s.text()!)],
 ]);
 
@@ -273,9 +276,7 @@ export class LatexParser {
         }
 
         let node: TexNode;
-        if (results.length === 0) {
-            node = EMPTY_NODE;
-        } else if (results.length === 1) {
+        if (results.length === 1) {
             node = results[0];
         } else {
             node = new TexNode('ordgroup', '', results);

package/src/types.ts

@@ -233,12 +233,12 @@ export class TexNode {
 }
 
 export enum TypstTokenType {
+    NONE,
     SYMBOL,
     ELEMENT,
     TEXT,
     COMMENT,
     SPACE,
-    SOFT_SPACE,
     CONTROL,
     NEWLINE,
 }
@@ -262,6 +262,8 @@ export class TypstToken {
 
     public toNode(): TypstNode {
         switch(this.type) {
+            case TypstTokenType.NONE:
+                return new TypstNode('none', '#none');
             case TypstTokenType.TEXT:
                 return new TypstNode('text', this.value);
             case TypstTokenType.COMMENT:
@@ -279,7 +281,7 @@ export class TypstToken {
                     case '':
                     case '_':
                     case '^':
-                        return new TypstNode('empty', '');
+                        throw new Error(`Should not convert ${controlChar} to a node`);
                     case '&':
                         return new TypstNode('control', '&');
                     case '\\':
@@ -318,15 +320,11 @@ export interface TypstLrData {
 }
 
 type TypstNodeType = 'atom' | 'symbol' | 'text' | 'control' | 'comment' | 'whitespace'
-            | 'empty' | 'group' | 'supsub' | 'funcCall' | 'fraction' | 'align' | 'matrix' | 'cases' | 'unknown';
+            | 'none' | 'group' | 'supsub' | 'funcCall' | 'fraction' | 'align' | 'matrix' | 'cases' | 'unknown';
 
-export type TypstPrimitiveValue = string | boolean | null | TypstToken;
+export type TypstPrimitiveValue = string | boolean | TypstNode;
 export type TypstNamedParams = { [key: string]: TypstPrimitiveValue };
 
-// #none
-export const TYPST_NULL: TypstPrimitiveValue = null;
-export const TYPST_TRUE: TypstPrimitiveValue = true;
-export const TYPST_FALSE: TypstPrimitiveValue = false;
 
 export class TypstNode {
     type: TypstNodeType;
@@ -379,6 +377,11 @@ export class TypstNode {
     }
 }
 
+// #none
+export const TYPST_NONE = new TypstNode('none', '#none');
+export const TYPST_TRUE: TypstPrimitiveValue = true;
+export const TYPST_FALSE: TypstPrimitiveValue = false;
+
 export interface Tex2TypstOptions {
     nonStrict?: boolean; // default is true
     preferTypstIntrinsic?: boolean; // default is true,
@@ -386,6 +389,7 @@ export interface Tex2TypstOptions {
     keepSpaces?: boolean; // default is false
     fracToSlash?: boolean; // default is true
     inftyToOo?: boolean; // default is false
+    nonAsciiWrapper?: string; // default is ""
     customTexMacros?: { [key: string]: string };
     // TODO: custom typst functions
 }

package/src/typst-parser.ts

@@ -1,13 +1,10 @@
 
 import { array_find } from "./generic";
-import { TYPST_NULL, TypstLrData, TypstNamedParams, TypstNode, TypstSupsubData, TypstToken, TypstTokenType } from "./types";
+import { TYPST_NONE, TypstLrData, TypstNamedParams, TypstNode, TypstSupsubData, TypstToken, TypstTokenType } from "./types";
 import { assert, isalpha } from "./util";
 import { reverseShorthandMap } from "./typst-shorthands";
 import { JSLex, Scanner } from "./jslex";
 
-
-const TYPST_EMPTY_NODE = new TypstNode('empty', '');
-
 const TYPST_SHORTHANDS = Array.from(reverseShorthandMap.keys());
 
 // TODO: In Typst, y' ' is not the same as y''.
@@ -78,6 +75,7 @@ const rules_map = new Map<string, (a: Scanner<TypstToken>) => TypstToken | Typst
     [String.raw`[a-zA-Z\.]+`, (s) => {
         return new TypstToken(s.text()!.length === 1? TypstTokenType.ELEMENT: TypstTokenType.SYMBOL, s.text()!);
     }],
+    [String.raw`#none`, (s) => new TypstToken(TypstTokenType.NONE, s.text()!)],
     [String.raw`.`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
 ]);
 
@@ -168,12 +166,12 @@ const DIV = new TypstNode('atom', '/');
 
 
 
-function next_non_whitespace(nodes: TypstNode[], start: number): TypstNode {
+function next_non_whitespace(nodes: TypstNode[], start: number): TypstNode | null {
     let pos = start;
     while (pos < nodes.length && nodes[pos].type === 'whitespace') {
         pos++;
     }
-    return pos === nodes.length ? TYPST_EMPTY_NODE : nodes[pos];
+    return pos === nodes.length ? null : nodes[pos];
 }
 
 function trim_whitespace_around_operators(nodes: TypstNode[]): TypstNode[] {
@@ -185,7 +183,7 @@ function trim_whitespace_around_operators(nodes: TypstNode[]): TypstNode[] {
             if(after_operator) {
                 continue;
             }
-            if(next_non_whitespace(nodes, i + 1).eq(DIV)) {
+            if(next_non_whitespace(nodes, i + 1)?.eq(DIV)) {
                 continue;
             }
         }
@@ -253,9 +251,7 @@ function process_operators(nodes: TypstNode[], parenthesis = false): TypstNode {
     if(parenthesis) {
         return new TypstNode('group', 'parenthesis', args);
     } else {
-        if(args.length === 0) {
-            return TYPST_EMPTY_NODE;
-        } else if(args.length === 1) {
+        if(args.length === 1) {
             return args[0];
         } else {
             return new TypstNode('group', '', args);
@@ -322,9 +318,7 @@ export class TypstParser {
         if(parentheses) {
             node = process_operators(results, true);
         } else {
-            if (results.length === 0) {
-                node = TYPST_EMPTY_NODE;
-            } else if (results.length === 1) {
+            if (results.length === 1) {
                 node = results[0];
             } else {
                 node = process_operators(results);
@@ -489,19 +483,24 @@ export class TypstParser {
                         to_delete.push(i);
                         const param_name = g.args![pos_colon - 1];
                         if(param_name.eq(new TypstNode('symbol', 'delim'))) {
-                            if(g.args![pos_colon + 1].type === 'text') {
-                                np['delim'] = g.args![pos_colon + 1].content;
-                                if(g.args!.length !== 3) {
-                                    throw new TypstParserError('Invalid number of arguments for delim');
+                            if(g.args!.length !== 3) {
+                                throw new TypstParserError('Invalid number of arguments for delim');
+                            }
+                             switch (g.args![pos_colon + 1].type) {
+                                case 'text': {
+                                    np['delim'] = g.args![pos_colon + 1].content;
+                                    break;
+                                }
+                                case 'none': {
+                                   np['delim'] = TYPST_NONE;
+                                    break;
                                 }
-                            } else if(g.args![pos_colon + 1].eq(new TypstNode('atom', '#'))) {
-                                // TODO: should parse #none properly
-                                if(g.args!.length !== 4 || !g.args![pos_colon + 2].eq(new TypstNode('symbol', 'none'))) { 
-                                    throw new TypstParserError('Invalid number of arguments for delim');
+                                case 'symbol': {
+                                    np['delim'] = g.args![pos_colon + 1];
+                                    break;
                                 }
-                                np['delim'] = TYPST_NULL;
-                            } else {
-                                throw new TypstParserError('Not implemented for other types of delim');
+                                default:
+                                    throw new TypstParserError('Not implemented for other types of delim');
                             }
                         } else {
                             throw new TypstParserError('Not implemented for other named parameters');
@@ -543,9 +542,7 @@ export class TypstParser {
             }
 
             let arg: TypstNode;
-            if (nodes.length === 0) {
-                arg = TYPST_EMPTY_NODE;
-            } else if (nodes.length === 1) {
+            if (nodes.length === 1) {
                 arg = nodes[0];
             } else {
                 arg = process_operators(nodes);

package/src/typst-writer.ts

@@ -1,5 +1,6 @@
 import { TexNode, TypstNode, TypstPrimitiveValue, TypstSupsubData, TypstToken, TypstTokenType } from "./types";
 import { shorthandMap } from "./typst-shorthands";
+import { assert } from "./util";
 
 function is_delimiter(c: TypstNode): boolean {
     return c.type === 'atom' && ['(', ')', '[', ']', '{', '}', '|', '⌊', '⌋', '⌈', '⌉'].includes(c.content);
@@ -19,12 +20,8 @@ function typst_primitive_to_string(value: TypstPrimitiveValue) {
         case 'boolean':
             return (value as boolean) ? '#true' : '#false';
         default:
-            if (value === null) {
-                return '#none';
-            } else if (value instanceof TypstToken) {
-                return value.toString();
-            }
-            throw new TypstWriterError(`Invalid primitive value: ${value}`, value);
+            assert(value instanceof TypstNode, 'Not a valid primitive value');
+            return (value as TypstNode).content;
     }
 }
 
@@ -108,7 +105,8 @@ export class TypstWriter {
     // Serialize a tree of TypstNode into a list of TypstToken
     public serialize(node: TypstNode) {
         switch (node.type) {
-            case 'empty':
+            case 'none':
+                this.queue.push(new TypstToken(TypstTokenType.NONE, '#none'));
                 break;
             case 'atom': {
                 if (node.content === ',' && this.insideFunctionDepth > 0) {
@@ -163,7 +161,7 @@ export class TypstWriter {
                 const has_prime = (sup && sup.type === 'atom' && sup.content === '\'');
                 if (has_prime) {
                     // Put prime symbol before '_'. Because $y_1'$ is not displayed properly in Typst (so far)
-                    // e.g. 
+                    // e.g.
                     // y_1' -> y'_1
                     // y_{a_1}' -> y'_{a_1}
                     this.queue.push(new TypstToken(TypstTokenType.ELEMENT, '\''));
@@ -185,7 +183,9 @@ export class TypstWriter {
             case 'funcCall': {
                 const func_symbol: TypstToken = new TypstToken(TypstTokenType.SYMBOL, node.content);
                 this.queue.push(func_symbol);
-                this.insideFunctionDepth++;
+                if (node.content !== 'lr') {
+                    this.insideFunctionDepth++;
+                }
                 this.queue.push(TYPST_LEFT_PARENTHESIS);
                 for (let i = 0; i < node.args!.length; i++) {
                     this.serialize(node.args![i]);
@@ -200,7 +200,9 @@ export class TypstWriter {
                     }
                 }
                 this.queue.push(TYPST_RIGHT_PARENTHESIS);
-                this.insideFunctionDepth--;
+                if (node.content !== 'lr') {
+                    this.insideFunctionDepth--;
+                }
                 break;
             }
             case 'fraction': {
@@ -317,10 +319,15 @@ export class TypstWriter {
         let need_to_wrap = ['group', 'supsub', 'empty'].includes(node.type);
 
         if (node.type === 'group') {
-            const first = node.args![0];
-            const last = node.args![node.args!.length - 1];
-            if (is_delimiter(first) && is_delimiter(last)) {
-                need_to_wrap = false;
+            if (node.args!.length === 0) {
+                // e.g. TeX `P_{}` converts to Typst `P_()`
+                need_to_wrap = true;
+            } else {
+                const first = node.args![0];
+                const last = node.args![node.args!.length - 1];
+                if (is_delimiter(first) && is_delimiter(last)) {
+                    need_to_wrap = false;
+                }
             }
         }