tex2typst 0.3.17 → 0.3.19

package/src/types.ts

@@ -125,6 +125,13 @@ export class TexNode {
             case 'ordgroup': {
                 return this.args!.map((n) => n.serialize()).flat();
             }
+            case 'leftright': {
+                let tokens = this.args!.map((n) => n.serialize()).flat();
+                tokens.splice(0, 0, new TexToken(TexTokenType.COMMAND, '\\left'));
+                tokens.splice(tokens.length - 1, 0, new TexToken(TexTokenType.COMMAND, '\\right'));
+
+                return tokens;
+            }
             case 'unaryFunc': {
                 let tokens: TexToken[] = [];
                 tokens.push(new TexToken(TexTokenType.COMMAND, this.content));
@@ -382,14 +389,20 @@ export const TYPST_NONE = new TypstNode('none', '#none');
 export const TYPST_TRUE: TypstPrimitiveValue = true;
 export const TYPST_FALSE: TypstPrimitiveValue = false;
 
+/**
+ * ATTENTION:
+ * Don't use any options except those explicitly documented in
+ *   https://github.com/qwinsi/tex2typst/blob/main/docs/api-reference.md
+ * Any undocumented options may be not working at present or break in the future!
+ */
 export interface Tex2TypstOptions {
-    nonStrict?: boolean; // default is true
-    preferTypstIntrinsic?: boolean; // default is true,
-    preferShorthands?: boolean; // default is true
-    keepSpaces?: boolean; // default is false
-    fracToSlash?: boolean; // default is true
-    inftyToOo?: boolean; // default is false
-    nonAsciiWrapper?: string; // default is ""
+    nonStrict?: boolean; /** default is true */
+    preferShorthands?: boolean; /** default is true */
+    keepSpaces?: boolean; /** default is false */
+    fracToSlash?: boolean; /** default is true */
+    inftyToOo?: boolean; /** default is false */
+    optimize?: boolean; /** default is true */
+    nonAsciiWrapper?: string; /** default is "" */
     customTexMacros?: { [key: string]: string };
     // TODO: custom typst functions
 }

package/src/typst-parser.ts

@@ -1,11 +1,9 @@
 
 import { array_find } from "./generic";
 import { TYPST_NONE, TypstLrData, TypstNamedParams, TypstNode, TypstSupsubData, TypstToken, TypstTokenType } from "./types";
+import { tokenize_typst } from "./typst-tokenizer";
 import { assert, isalpha } from "./util";
-import { reverseShorthandMap } from "./typst-shorthands";
-import { JSLex, Scanner } from "./jslex";
 
-const TYPST_SHORTHANDS = Array.from(reverseShorthandMap.keys());
 
 // TODO: In Typst, y' ' is not the same as y''.
 // The parser should be able to parse the former correctly.
@@ -18,77 +16,6 @@ function eat_primes(tokens: TypstToken[], start: number): number {
 }
 
 
-function generate_regex_for_shorthands(): string {
-    const regex_list = TYPST_SHORTHANDS.map((s) => {
-        s = s.replaceAll('|', '\\|');
-        s = s.replaceAll('.', '\\.');
-        s = s.replaceAll('[', '\\[');
-        s = s.replaceAll(']', '\\]');
-        return s;
-    });
-    return `(${regex_list.join('|')})`;
-}
-
-
-const REGEX_SHORTHANDS = generate_regex_for_shorthands();
-
-const rules_map = new Map<string, (a: Scanner<TypstToken>) => TypstToken | TypstToken[]>([
-    [String.raw`//[^\n]*`, (s) => new TypstToken(TypstTokenType.COMMENT, s.text()!.substring(2))],
-    [String.raw`/`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
-    [String.raw`[_^&]`, (s) => new TypstToken(TypstTokenType.CONTROL, s.text()!)],
-    [String.raw`\r?\n`, (_s) => new TypstToken(TypstTokenType.NEWLINE, "\n")],
-    [String.raw`\s+`, (s) => new TypstToken(TypstTokenType.SPACE, s.text()!)],
-    [String.raw`\\[$&#_]`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
-    [String.raw`\\\n`, (s) => {
-        return [
-            new TypstToken(TypstTokenType.CONTROL, "\\"),
-            new TypstToken(TypstTokenType.NEWLINE, "\n"),
-        ]
-    }],
-    [String.raw`\\\s`, (s) => {
-        return [
-            new TypstToken(TypstTokenType.CONTROL, "\\"),
-            new TypstToken(TypstTokenType.SPACE, " "),
-        ]
-    }],
-    // this backslash is dummy and will be ignored in later stages
-    [String.raw`\\\S`, (_s) => new TypstToken(TypstTokenType.CONTROL, "")],
-    [
-        String.raw`"([^"]|(\\"))*"`,
-        (s) => {
-            const text = s.text()!.substring(1, s.text()!.length - 1);
-            // replace all escape characters with their actual characters
-            text.replaceAll('\\"', '"');
-            return new TypstToken(TypstTokenType.TEXT, text);
-        }
-    ],
-    [
-        REGEX_SHORTHANDS,
-        (s) => {
-            const shorthand = s.text()!;
-            const symbol = reverseShorthandMap.get(shorthand)!;
-            return new TypstToken(TypstTokenType.SYMBOL, symbol);
-        }
-    ],
-    [String.raw`[0-9]+(\.[0-9]+)?`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
-    [String.raw`[+\-*/=\'<>!.,;?()\[\]|]`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
-    [String.raw`[a-zA-Z\.]+`, (s) => {
-        return new TypstToken(s.text()!.length === 1? TypstTokenType.ELEMENT: TypstTokenType.SYMBOL, s.text()!);
-    }],
-    [String.raw`#none`, (s) => new TypstToken(TypstTokenType.NONE, s.text()!)],
-    [String.raw`.`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
-]);
-
-const spec = {
-    "start": rules_map
-};
-
-export function tokenize_typst(input: string): TypstToken[] {
-    const lexer = new JSLex<TypstToken>(spec);
-    return lexer.collect(input);
-}
-
-
 function _find_closing_match(tokens: TypstToken[], start: number,
         leftBrackets: TypstToken[], rightBrackets: TypstToken[]): number {
     assert(tokens[start].isOneOf(leftBrackets));

package/src/typst-tokenizer.ts

@@ -0,0 +1,76 @@
+import { TypstToken, TypstTokenType } from "./types";
+import { reverseShorthandMap } from "./typst-shorthands";
+import { JSLex, Scanner } from "./jslex";
+
+const TYPST_SHORTHANDS = Array.from(reverseShorthandMap.keys());
+
+
+function generate_regex_for_shorthands(): string {
+    const regex_list = TYPST_SHORTHANDS.map((s) => {
+        s = s.replaceAll('|', '\\|');
+        s = s.replaceAll('.', '\\.');
+        s = s.replaceAll('[', '\\[');
+        s = s.replaceAll(']', '\\]');
+        return s;
+    });
+    return `(${regex_list.join('|')})`;
+}
+
+
+const REGEX_SHORTHANDS = generate_regex_for_shorthands();
+
+const rules_map = new Map<string, (a: Scanner<TypstToken>) => TypstToken | TypstToken[]>([
+    [String.raw`//[^\n]*`, (s) => new TypstToken(TypstTokenType.COMMENT, s.text()!.substring(2))],
+    [String.raw`/`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
+    [String.raw`[_^&]`, (s) => new TypstToken(TypstTokenType.CONTROL, s.text()!)],
+    [String.raw`\r?\n`, (_s) => new TypstToken(TypstTokenType.NEWLINE, "\n")],
+    [String.raw`\s+`, (s) => new TypstToken(TypstTokenType.SPACE, s.text()!)],
+    [String.raw`\\[$&#_]`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
+    [String.raw`\\\n`, (s) => {
+        return [
+            new TypstToken(TypstTokenType.CONTROL, "\\"),
+            new TypstToken(TypstTokenType.NEWLINE, "\n"),
+        ]
+    }],
+    [String.raw`\\\s`, (s) => {
+        return [
+            new TypstToken(TypstTokenType.CONTROL, "\\"),
+            new TypstToken(TypstTokenType.SPACE, " "),
+        ]
+    }],
+    // this backslash is dummy and will be ignored in later stages
+    [String.raw`\\\S`, (_s) => new TypstToken(TypstTokenType.CONTROL, "")],
+    [
+        String.raw`"([^"]|(\\"))*"`,
+        (s) => {
+            const text = s.text()!.substring(1, s.text()!.length - 1);
+            // replace all escape characters with their actual characters
+            text.replaceAll('\\"', '"');
+            return new TypstToken(TypstTokenType.TEXT, text);
+        }
+    ],
+    [
+        REGEX_SHORTHANDS,
+        (s) => {
+            const shorthand = s.text()!;
+            const symbol = reverseShorthandMap.get(shorthand)!;
+            return new TypstToken(TypstTokenType.SYMBOL, symbol);
+        }
+    ],
+    [String.raw`[0-9]+(\.[0-9]+)?`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
+    [String.raw`[+\-*/=\'<>!.,;?()\[\]|]`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
+    [String.raw`[a-zA-Z\.]+`, (s) => {
+        return new TypstToken(s.text()!.length === 1? TypstTokenType.ELEMENT: TypstTokenType.SYMBOL, s.text()!);
+    }],
+    [String.raw`#none`, (s) => new TypstToken(TypstTokenType.NONE, s.text()!)],
+    [String.raw`.`, (s) => new TypstToken(TypstTokenType.ELEMENT, s.text()!)],
+]);
+
+const spec = {
+    "start": rules_map
+};
+
+export function tokenize_typst(input: string): TypstToken[] {
+    const lexer = new JSLex<TypstToken>(spec);
+    return lexer.collect(input);
+}

package/src/typst-writer.ts

@@ -11,6 +11,8 @@ const TYPST_RIGHT_PARENTHESIS: TypstToken = new TypstToken(TypstTokenType.ELEMEN
 const TYPST_COMMA: TypstToken = new TypstToken(TypstTokenType.ELEMENT, ',');
 const TYPST_NEWLINE: TypstToken = new TypstToken(TypstTokenType.SYMBOL, '\n');
 
+const SOFT_SPACE = new TypstToken(TypstTokenType.CONTROL, ' ');
+
 function typst_primitive_to_string(value: TypstPrimitiveValue) {
     switch (typeof value) {
         case 'string':
@@ -40,6 +42,7 @@ export interface TypstWriterOptions {
     preferShorthands: boolean;
     keepSpaces: boolean;
     inftyToOo: boolean;
+    optimize: boolean;
 }
 
 export class TypstWriter {
@@ -47,17 +50,19 @@ export class TypstWriter {
     private preferShorthands: boolean;
     private keepSpaces: boolean;
     private inftyToOo: boolean;
+    private optimize: boolean;
 
     protected buffer: string = "";
     protected queue: TypstToken[] = [];
 
     private insideFunctionDepth = 0;
 
-    constructor(opt: TypstWriterOptions) {
-        this.nonStrict = opt.nonStrict;
-        this.preferShorthands = opt.preferShorthands;
-        this.keepSpaces = opt.keepSpaces;
-        this.inftyToOo = opt.inftyToOo;
+    constructor(options: TypstWriterOptions) {
+        this.nonStrict = options.nonStrict;
+        this.preferShorthands = options.preferShorthands;
+        this.keepSpaces = options.keepSpaces;
+        this.inftyToOo = options.inftyToOo;
+        this.optimize = options.optimize;
     }
 
 
@@ -68,8 +73,6 @@ export class TypstWriter {
             return;
         }
 
-        // TODO: "C \frac{xy}{z}" should translate to "C (x y)/z" instead of "C(x y)/z"
-
         let no_need_space = false;
         // putting the first token in clause
         no_need_space ||= /[\(\[\|]$/.test(this.buffer) && /^\w/.test(str);
@@ -163,7 +166,7 @@ export class TypstWriter {
                     // Put prime symbol before '_'. Because $y_1'$ is not displayed properly in Typst (so far)
                     // e.g.
                     // y_1' -> y'_1
-                    // y_{a_1}' -> y'_{a_1}
+                    // y_{a_1}' -> y'_(a_1)
                     this.queue.push(new TypstToken(TypstTokenType.ELEMENT, '\''));
                     trailing_space_needed = false;
                 }
@@ -176,7 +179,7 @@ export class TypstWriter {
                     trailing_space_needed = this.appendWithBracketsIfNeeded(sup);
                 }
                 if (trailing_space_needed) {
-                    this.queue.push(new TypstToken(TypstTokenType.CONTROL, ' '));
+                    this.queue.push(SOFT_SPACE);
                 }
                 break;
             }
@@ -207,7 +210,16 @@ export class TypstWriter {
             }
             case 'fraction': {
                 const [numerator, denominator] = node.args!;
-                this.appendWithBracketsIfNeeded(numerator);
+                const pos = this.queue.length;
+                const no_wrap = this.appendWithBracketsIfNeeded(numerator);
+
+                // This is a dirty hack to force `C \frac{xy}{z}`to translate to `C (x y)/z` instead of `C(x y)/z`
+                // To solve this properly, we should implement a Typst formatter
+                const wrapped = !no_wrap;
+                if (wrapped) {
+                    this.queue.splice(pos, 0, SOFT_SPACE);
+                }
+
                 this.queue.push(new TypstToken(TypstTokenType.ELEMENT, '/'));
                 this.appendWithBracketsIfNeeded(denominator);
                 break;
@@ -331,20 +343,24 @@ export class TypstWriter {
     }
 
     protected flushQueue() {
-        const SOFT_SPACE = new TypstToken(TypstTokenType.CONTROL, ' ');
+        const dummy_token = new TypstToken(TypstTokenType.SYMBOL, '');
 
         // delete soft spaces if they are not needed
         for(let i = 0; i < this.queue.length; i++) {
             let token = this.queue[i];
             if (token.eq(SOFT_SPACE)) {
-                if (i === this.queue.length - 1) {
-                    this.queue[i].value = '';
-                } else if (this.queue[i + 1].isOneOf([TYPST_RIGHT_PARENTHESIS, TYPST_COMMA, TYPST_NEWLINE])) {
-                    this.queue[i].value = '';
+                const to_delete = (i === 0)
+                                || (i === this.queue.length - 1)
+                                || this.queue[i - 1].isOneOf([TYPST_NEWLINE])
+                                || this.queue[i + 1].isOneOf([TYPST_RIGHT_PARENTHESIS, TYPST_COMMA, TYPST_NEWLINE]);
+                if (to_delete) {
+                    this.queue[i] = dummy_token;
                 }
             }
         }
 
+        this.queue = this.queue.filter((token) => !token.eq(dummy_token));
+
         this.queue.forEach((token) => {
             this.writeBuffer(token)
         });
@@ -375,9 +391,11 @@ export class TypstWriter {
             res = res.replace(/round\(\)/g, 'round("")');
             return res;
         }
-        const all_passes = [smartFloorPass, smartCeilPass, smartRoundPass];
-        for (const pass of all_passes) {
-            this.buffer = pass(this.buffer);
+        if (this.optimize) {
+            const all_passes = [smartFloorPass, smartCeilPass, smartRoundPass];
+            for (const pass of all_passes) {
+                this.buffer = pass(this.buffer);
+            }
         }
         return this.buffer;
     }

package/TODO.md

@@ -1 +0,0 @@
-- Typst math `limits(Y)^X` to TeX `\overset{X}{Y}`

package/docs/api-reference.md

@@ -1,64 +0,0 @@
-# API Reference of tex2typst.js
-
-## Basic usage
-
-```javascript
-import { tex2typst, typst2tex } from 'tex2typst';
-
-let tex = "e \\overset{\\text{def}}{=} \\lim_{{n \\to \\infty}} \left(1 + \\frac{1}{n}\\right)^n";
-let typst = tex2typst(tex);
-console.log(typst);
-// e eq.def lim_(n -> infinity)(1 + 1/n)^n
-
-let tex_recovered = typst2tex(typst);
-console.log(tex_recovered);
-// e \overset{\text{def}}{=} \lim_{n \rightarrow \infty} \left(1 + \frac{1}{n} \right)^n
-```
-
-## Advanced options
-
-`tex2typst` function accepts an optional second argument, which is an object containing options to customize the conversion.
-
-```typescript
-interface Tex2TypstOptions {
-    preferShorthands?: boolean;
-    fracToSlash?: boolean;
-    inftyToOo?: boolean;
-}
-```
-
-- `preferShorthands`: If set to `true`, the function will prefer using shorthands in Typst (e.g., `->` instead of `arrow.r`, `<<` instead of `lt.double`) when converting TeX to Typst. Default is `ture`.
-
-```javascript
-let tex = "a \\rightarrow b \\ll c";
-let typst1 = tex2typst(tex, { preferShorthands: false });
-console.log(typst1);
-// a arrow.r b lt.double c
-let typst2 = tex2typst(tex, { preferShorthands: true });
-console.log(typst2);
-// a -> b << c
-```
-
-- `fracToSlash`: If set to `true`, the Typst result will use the slash notation for fractions. Default is `true`.
-
-```javascript
-let tex = "\\frac{a}{b}";
-let tpyst1 = tex2typst(tex, { fracToSlash: false });
-console.log(typst1);
-// frac(a, b)
-let typst2 = tex2typst(tex, { fracToSlash: true });
-console.log(typst2);
-// a / b
-```
-
-- `inftyToOo`: If set to `true`, `\infty` converts to `oo` instead of `infinity`. Default is `false`.
-
-```javascript
-let tex = "\\infty";
-let typst1 = tex2typst(tex, { inftyToOo: false });
-console.log(typst1);
-// infinity
-let typst2 = tex2typst(tex, { inftyToOo: true });
-console.log(typst2);
-// oo
-```

package/tools/make-shorthand-map.py

@@ -1,33 +0,0 @@
-import urllib.request
-import html
-from bs4 import BeautifulSoup
-
-
-if __name__ == '__main__':
-    shorthand_map = []
-
-
-    url = "https://typst.app/docs/reference/symbols/"
-    with urllib.request.urlopen(url) as response:
-        html_text = response.read().decode('utf-8')
-
-    soup = BeautifulSoup(html_text, 'html.parser')
-
-    # <ul class="symbol-grid">
-    ul_list = soup.find_all('ul', class_='symbol-grid')
-    # ul_shorthands_markup = ul_list[0]
-    ul_shorthands_math = ul_list[1]
-
-    li_list = ul_shorthands_math.find_all('li')
-    for li in li_list:
-        # e.g. <li id="symbol-arrow.r" data-math-shorthand="-&gt;"><button>...</button></li>
-        # ==> typst = "arrow.r"
-        # ==> shorthand = "->"
-        typst = li['id'][7:]
-        shorthand = html.unescape(li['data-math-shorthand'])
-        shorthand_map.append((typst, shorthand))
-
-    # Sort by length of shorthand, order from longest to shortest
-    shorthand_map.sort(key=lambda x: len(x[1]), reverse=True)
-    for typst, shorthand in shorthand_map:
-        print(f"['{typst}', '{shorthand}'],")

package/tools/make-symbol-map.py

@@ -1,35 +0,0 @@
-import urllib.request
-from bs4 import BeautifulSoup
-
-if __name__ == '__main__':
-    symbol_map = {}
-
-    url = "https://typst.app/docs/reference/symbols/sym/"
-    with urllib.request.urlopen(url) as response:
-        html_text = response.read().decode('utf-8')
-
-    soup = BeautifulSoup(html_text, 'html.parser')
-    # <ul class="symbol-grid">
-    ul = soup.find('ul', class_='symbol-grid')
-    li_list = ul.find_all('li')
-    for li in li_list:
-        # e.g. <li id="symbol-brace.r.double" data-latex-name="\rBrace" data-codepoint="10628"><button>...</button></li>
-        # ==> latex = rBrace
-        # ==> typst = brace.r.double
-        # ==> unicode = 10628 = \u2984
-        latex = li.get('data-latex-name', None)
-        typst = li['id'][7:]
-        unicode = int(li['data-codepoint'])
-        if latex is not None:
-            # some latex macro can be associated with multiple typst
-            # e.g. \equiv can be mapped to equal or equiv.triple
-            # We only keep the first one
-            if latex not in symbol_map:
-                symbol_map[latex] = typst
-
-    # sort the pairs with alphabetical order of latex
-    sorted_keys = sorted(list(symbol_map.keys()), key=str.lower)
-    sorted_symbol_map = [(key, symbol_map[key]) for key in sorted_keys]
-    for latex, typst in sorted_symbol_map:
-        print(f"    ['{latex[1:]}', '{typst}'],")
-        # print(f'{latex[1:]} = "{typst}"')