tex2typst 0.3.2 → 0.3.3

package/README.md

@@ -35,7 +35,7 @@ import { tex2typst, typst2tex } from 'tex2typst';
 let tex = "e \overset{\text{def}}{=} \lim_{{n \to \infty}} \left(1 + \frac{1}{n}\right)^n";
 let typst = tex2typst(tex);
 console.log(typst);
-// e eq.def lim_(n arrow.r infinity)(1 + 1/n)^n
+// e eq.def lim_(n -> infinity)(1 + 1/n)^n
 
 let tex_recovered = typst2tex(typst);
 console.log(tex_recovered);

package/dist/index.js

@@ -1382,15 +1382,248 @@ var TypstNode = class {
 function isalpha(char) {
   return "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ".includes(char);
 }
-function isdigit(char) {
-  return "0123456789".includes(char);
-}
 function assert(condition, message = "") {
   if (!condition) {
     throw new Error(message);
   }
 }
 
+// src/jslex.ts
+var EOF = {};
+function matchcompare(m1, m2) {
+  if (m2.len !== m1.len) {
+    return m2.len - m1.len;
+  } else {
+    return m1.index - m2.index;
+  }
+}
+var Scanner = class {
+  constructor(input, lexer) {
+    // position within input stream
+    this._pos = 0;
+    // current line number
+    this._line = 0;
+    // current column number
+    this._col = 0;
+    this._offset = 0;
+    this._less = null;
+    this._go = false;
+    this._newstate = null;
+    this._text = null;
+    this._leng = null;
+    this._input = input;
+    this._lexer = lexer;
+    this._state = lexer.states[0];
+  }
+  /**
+   * Analogous to yytext and yyleng in lex - will be set during scan.
+   */
+  text() {
+    return this._text;
+  }
+  leng() {
+    return this._leng;
+  }
+  /**
+   * Position of in stream, line number and column number of match.
+   */
+  pos() {
+    return this._pos;
+  }
+  line() {
+    return this._line;
+  }
+  column() {
+    return this._col;
+  }
+  /**
+   * Analogous to input() in lex.
+   * @return {string} The next character in the stream.
+   */
+  input() {
+    return this._input.charAt(this._pos + this._leng + this._offset++);
+  }
+  /**
+   * Similar to unput() in lex, but does not allow modifying the stream.
+   * @return {int} The offset position after the operation.
+   */
+  unput() {
+    return this._offset = this._offset > 0 ? this._offset-- : 0;
+  }
+  /**
+   * Analogous to yyless(n) in lex - retains the first n characters from this pattern, and returns
+   * the rest to the input stream, such that they will be used in the next pattern-matching operation.
+   * @param {int} n Number of characters to retain.
+   * @return {int} Length of the stream after the operation has completed.
+   */
+  less(n) {
+    this._less = n;
+    this._offset = 0;
+    this._text = this._text.substring(0, n);
+    return this._leng = this._text.length;
+  }
+  /**
+   * Like less(), but instead of retaining the first n characters, it chops off the last n.
+   * @param {int} n Number of characters to chop.
+   * @return {int} Length of the stream after the operation has completed.
+   */
+  pushback(n) {
+    return this.less(this._leng - n);
+  }
+  /**
+   * Similar to REJECT in lex, except it doesn't break the current execution context.
+   * TIP: reject() should be the last instruction in a spec callback.
+   */
+  reject() {
+    this._go = true;
+  }
+  /**
+   * Analogous to BEGIN in lex - sets the named state (start condition).
+   * @param {string|int} state Name of state to switch to, or ordinal number (0 is first, etc).
+   * @return {string} The new state on successful switch, throws exception on failure.
+   */
+  begin(state) {
+    if (this._lexer.specification[state]) {
+      return this._newstate = state;
+    }
+    const s = this._lexer.states[parseInt(state)];
+    if (s) {
+      return this._newstate = s;
+    }
+    throw "Unknown state '" + state + "' requested";
+  }
+  /**
+   * Simple accessor for reading in the current state.
+   * @return {string} The current state.
+   */
+  state() {
+    return this._state;
+  }
+  /**
+   * Scan method to be returned to caller - grabs the next token and fires appropriate calback.
+   * @return {T} The next token extracted from the stream.
+   */
+  scan() {
+    if (this._pos >= this._input.length) {
+      return EOF;
+    }
+    const str = this._input.substring(this._pos);
+    const rules = this._lexer.specification[this._state];
+    const matches = [];
+    for (let i = 0; i < rules.length; i++) {
+      const rule = rules[i];
+      const mt = str.match(rule.re);
+      if (mt !== null && mt[0].length > 0) {
+        matches.push({
+          index: i,
+          text: mt[0],
+          len: mt[0].length,
+          rule
+        });
+      }
+    }
+    if (matches.length === 0) {
+      throw new Error("No match found for input '" + str + "'");
+    }
+    matches.sort(matchcompare);
+    this._go = true;
+    let result;
+    let m;
+    for (let j = 0, n = matches.length; j < n && this._go; j++) {
+      this._offset = 0;
+      this._less = null;
+      this._go = false;
+      this._newstate = null;
+      m = matches[j];
+      this._text = m.text;
+      this._leng = m.len;
+      result = m.rule.action(this);
+      if (this._newstate && this._newstate != this._state) {
+        this._state = this._newstate;
+        break;
+      }
+    }
+    const text = this._less === null ? m.text : m.text.substring(0, this._less);
+    const len = text.length;
+    this._pos += len + this._offset;
+    const nlm = text.match(/\n/g);
+    if (nlm !== null) {
+      this._line += nlm.length;
+      this._col = len - text.lastIndexOf("\n") - 1;
+    } else {
+      this._col += len;
+    }
+    return result;
+  }
+};
+var JSLex = class {
+  constructor(spec3) {
+    this.states = Object.keys(spec3);
+    this.specification = {};
+    for (const s of this.states) {
+      const rule_map = spec3[s];
+      if (s in this.specification) {
+        throw "Duplicate state declaration encountered for state '" + s + "'";
+      }
+      this.specification[s] = [];
+      for (const [k, v] of rule_map.entries()) {
+        let re;
+        try {
+          re = new RegExp("^" + k);
+        } catch (err) {
+          throw "Invalid regexp '" + k + "' in state '" + s + "' (" + err.message + ")";
+        }
+        this.specification[s].push({
+          re,
+          action: v
+        });
+      }
+    }
+  }
+  /**
+   * Scanner function - makes a new scanner object which is used to get tokens one at a time.
+   * @param {string} input Input text to tokenize.
+   * @return {function} Scanner function.
+   */
+  scanner(input) {
+    return new Scanner(input, this);
+  }
+  /**
+   * Similar to lex's yylex() function, consumes all input, calling calback for each token.
+   * @param {string} input Text to lex.
+   * @param {function} callback Function to execute for each token.
+   */
+  lex(input, callback) {
+    const scanner = this.scanner(input);
+    while (true) {
+      const token = scanner.scan();
+      if (token === EOF) {
+        return;
+      }
+      if (token !== void 0) {
+        callback(token);
+      }
+    }
+  }
+  /**
+   * Consumes all input, collecting tokens along the way.
+   * @param {string} input Text to lex.
+   * @return {array} List of tokens, may contain an Error at the end.
+   */
+  collect(input) {
+    const tokens = [];
+    const callback = function(item) {
+      if (Array.isArray(item)) {
+        tokens.push(...item);
+      } else {
+        tokens.push(item);
+      }
+    };
+    this.lex(input, callback);
+    return tokens;
+  }
+};
+
 // src/tex-parser.ts
 var UNARY_COMMANDS = [
   "sqrt",
@@ -1420,7 +1653,9 @@ var UNARY_COMMANDS = [
   "underline",
   "vec",
   "widehat",
-  "widetilde"
+  "widetilde",
+  "overleftarrow",
+  "overrightarrow"
 ];
 var BINARY_COMMANDS = [
   "frac",
@@ -1469,13 +1704,6 @@ function eat_primes(tokens, start) {
   }
   return pos - start;
 }
-function eat_command_name(latex, start) {
-  let pos = start;
-  while (pos < latex.length && isalpha(latex[pos])) {
-    pos += 1;
-  }
-  return latex.substring(start, pos);
-}
 function find_closing_match(tokens, start, leftToken, rightToken) {
   assert(tokens[start].eq(leftToken));
   let count = 1;
@@ -1503,126 +1731,46 @@ var END_COMMAND = new TexToken(1 /* COMMAND */, "\\end");
 function find_closing_end_command(tokens, start) {
   return find_closing_match(tokens, start, BEGIN_COMMAND, END_COMMAND);
 }
-function find_closing_curly_bracket_char(latex, start) {
-  assert(latex[start] === "{");
-  let count = 1;
-  let pos = start + 1;
-  while (count > 0) {
-    if (pos >= latex.length) {
-      throw new LatexParserError("Unmatched curly brackets");
-    }
-    if (pos + 1 < latex.length && ["\\{", "\\}"].includes(latex.substring(pos, pos + 2))) {
-      pos += 2;
-      continue;
-    }
-    if (latex[pos] === "{") {
-      count += 1;
-    } else if (latex[pos] === "}") {
-      count -= 1;
-    }
-    pos += 1;
+function unescape(str) {
+  const chars = ["{", "}", "\\", "$", "&", "#", "_", "%"];
+  for (const char of chars) {
+    str = str.replaceAll("\\" + char, char);
   }
-  return pos - 1;
+  return str;
 }
-function tokenize(latex) {
-  const tokens = [];
-  let pos = 0;
-  while (pos < latex.length) {
-    const firstChar = latex[pos];
-    let token;
-    switch (firstChar) {
-      case "%": {
-        let newPos = pos + 1;
-        while (newPos < latex.length && latex[newPos] !== "\n") {
-          newPos += 1;
-        }
-        token = new TexToken(3 /* COMMENT */, latex.slice(pos + 1, newPos));
-        pos = newPos;
-        break;
-      }
-      case "{":
-      case "}":
-      case "_":
-      case "^":
-      case "&":
-        token = new TexToken(6 /* CONTROL */, firstChar);
-        pos++;
-        break;
-      case "\n":
-        token = new TexToken(5 /* NEWLINE */, firstChar);
-        pos++;
-        break;
-      case "\r": {
-        if (pos + 1 < latex.length && latex[pos + 1] === "\n") {
-          token = new TexToken(5 /* NEWLINE */, "\n");
-          pos += 2;
-        } else {
-          token = new TexToken(5 /* NEWLINE */, "\n");
-          pos++;
-        }
-        break;
-      }
-      case " ": {
-        let newPos = pos;
-        while (newPos < latex.length && latex[newPos] === " ") {
-          newPos += 1;
-        }
-        token = new TexToken(4 /* SPACE */, latex.slice(pos, newPos));
-        pos = newPos;
-        break;
-      }
-      case "\\": {
-        if (pos + 1 >= latex.length) {
-          throw new LatexParserError("Expecting command name after \\");
-        }
-        const firstTwoChars = latex.slice(pos, pos + 2);
-        if (["\\\\", "\\,"].includes(firstTwoChars)) {
-          token = new TexToken(6 /* CONTROL */, firstTwoChars);
-        } else if (["\\{", "\\}", "\\%", "\\$", "\\&", "\\#", "\\_", "\\|"].includes(firstTwoChars)) {
-          token = new TexToken(0 /* ELEMENT */, firstTwoChars);
-        } else {
-          const command = eat_command_name(latex, pos + 1);
-          token = new TexToken(1 /* COMMAND */, "\\" + command);
-        }
-        pos += token.value.length;
-        break;
-      }
-      default: {
-        if (isdigit(firstChar)) {
-          let newPos = pos;
-          while (newPos < latex.length && isdigit(latex[newPos])) {
-            newPos += 1;
-          }
-          token = new TexToken(0 /* ELEMENT */, latex.slice(pos, newPos));
-        } else if (isalpha(firstChar)) {
-          token = new TexToken(0 /* ELEMENT */, firstChar);
-        } else if ("+-*/='<>!.,;:?()[]|".includes(firstChar)) {
-          token = new TexToken(0 /* ELEMENT */, firstChar);
-        } else {
-          token = new TexToken(7 /* UNKNOWN */, firstChar);
-        }
-        pos += token.value.length;
-      }
-    }
-    tokens.push(token);
-    if (token.type === 1 /* COMMAND */ && ["\\text", "\\operatorname", "\\begin", "\\end"].includes(token.value)) {
-      if (pos >= latex.length || latex[pos] !== "{") {
-        throw new LatexParserError(`No content for ${token.value} command`);
-      }
-      tokens.push(new TexToken(6 /* CONTROL */, "{"));
-      const posClosingBracket = find_closing_curly_bracket_char(latex, pos);
-      pos++;
-      let textInside = latex.slice(pos, posClosingBracket);
-      const chars = ["{", "}", "\\", "$", "&", "#", "_", "%"];
-      for (const char of chars) {
-        textInside = textInside.replaceAll("\\" + char, char);
-      }
-      tokens.push(new TexToken(2 /* TEXT */, textInside));
-      tokens.push(new TexToken(6 /* CONTROL */, "}"));
-      pos = posClosingBracket + 1;
-    }
-  }
-  return tokens;
+var rules_map = /* @__PURE__ */ new Map([
+  [
+    String.raw`\\(text|operatorname|begin|end){.+?}`,
+    (s) => {
+      const text = s.text();
+      const command = text.substring(0, text.indexOf("{"));
+      const text_inside = text.substring(text.indexOf("{") + 1, text.lastIndexOf("}"));
+      return [
+        new TexToken(1 /* COMMAND */, command),
+        new TexToken(6 /* CONTROL */, "{"),
+        new TexToken(2 /* TEXT */, unescape(text_inside)),
+        new TexToken(6 /* CONTROL */, "}")
+      ];
+    }
+  ],
+  [String.raw`%[^\n]*`, (s) => new TexToken(3 /* COMMENT */, s.text().substring(1))],
+  [String.raw`[{}_^&]`, (s) => new TexToken(6 /* CONTROL */, s.text())],
+  [String.raw`\r?\n`, (_s) => new TexToken(5 /* NEWLINE */, "\n")],
+  [String.raw`\s+`, (s) => new TexToken(4 /* SPACE */, s.text())],
+  [String.raw`\\[\\,]`, (s) => new TexToken(6 /* CONTROL */, s.text())],
+  [String.raw`\\[{}%$&#_|]`, (s) => new TexToken(0 /* ELEMENT */, s.text())],
+  [String.raw`\\[a-zA-Z]+`, (s) => new TexToken(1 /* COMMAND */, s.text())],
+  [String.raw`[0-9]+`, (s) => new TexToken(0 /* ELEMENT */, s.text())],
+  [String.raw`[a-zA-Z]`, (s) => new TexToken(0 /* ELEMENT */, s.text())],
+  [String.raw`[+\-*/='<>!.,;:?()\[\]|]`, (s) => new TexToken(0 /* ELEMENT */, s.text())],
+  [String.raw`.`, (s) => new TexToken(7 /* UNKNOWN */, s.text())]
+]);
+var spec = {
+  "start": rules_map
+};
+function tokenize_tex(input) {
+  const lexer = new JSLex(spec);
+  return lexer.collect(input);
 }
 var LatexParserError = class extends Error {
   constructor(message) {
@@ -1938,7 +2086,7 @@ function passExpandCustomTexMacros(tokens, customTexMacros) {
   let out_tokens = [];
   for (const token of tokens) {
     if (token.type === 1 /* COMMAND */ && customTexMacros[token.value]) {
-      const expanded_tokens = tokenize(customTexMacros[token.value]);
+      const expanded_tokens = tokenize_tex(customTexMacros[token.value]);
       out_tokens = out_tokens.concat(expanded_tokens);
     } else {
       out_tokens.push(token);
@@ -1948,7 +2096,7 @@ function passExpandCustomTexMacros(tokens, customTexMacros) {
 }
 function parseTex(tex, customTexMacros) {
   const parser = new LatexParser();
-  let tokens = tokenize(tex);
+  let tokens = tokenize_tex(tex);
   tokens = passIgnoreWhitespaceBeforeScriptMark(tokens);
   tokens = passExpandCustomTexMacros(tokens, customTexMacros);
   return parser.parse(tokens);
@@ -2486,6 +2634,20 @@ function convert_tex_node_to_typst(node, options = {}) {
       if (node.content === "\\mathbb" && arg0.type === "atom" && /^[A-Z]$/.test(arg0.content)) {
         return new TypstNode("symbol", arg0.content + arg0.content);
       }
+      if (node.content === "\\overrightarrow") {
+        return new TypstNode(
+          "funcCall",
+          "arrow",
+          [arg0]
+        );
+      }
+      if (node.content === "\\overleftarrow") {
+        return new TypstNode(
+          "funcCall",
+          "accent",
+          [arg0, new TypstNode("symbol", "arrow.l")]
+        );
+      }
       if (node.content === "\\operatorname") {
         const body = node.args;
         if (body.length !== 1 || body[0].type !== "text") {
@@ -2772,139 +2934,67 @@ function eat_primes2(tokens, start) {
   }
   return pos - start;
 }
-function eat_identifier_name(typst, start) {
-  let pos = start;
-  while (pos < typst.length && (isalpha(typst[pos]) || typst[pos] === ".")) {
-    pos += 1;
-  }
-  return typst.substring(start, pos);
-}
-function try_eat_shorthand(typst, start) {
-  for (const shorthand of TYPST_SHORTHANDS) {
-    if (typst.startsWith(shorthand, start)) {
-      return shorthand;
-    }
-  }
-  return null;
+function generate_regex_for_shorthands() {
+  const regex_list = TYPST_SHORTHANDS.map((s) => {
+    s = s.replaceAll("|", "\\|");
+    s = s.replaceAll(".", "\\.");
+    s = s.replaceAll("[", "\\[");
+    s = s.replaceAll("]", "\\]");
+    return s;
+  });
+  return `(${regex_list.join("|")})`;
 }
-function tokenize_typst(typst) {
-  const tokens = [];
-  let pos = 0;
-  while (pos < typst.length) {
-    const firstChar = typst[pos];
-    let token;
-    switch (firstChar) {
-      case "_":
-      case "^":
-      case "&":
-        token = new TypstToken(6 /* CONTROL */, firstChar);
-        pos++;
-        break;
-      case "\n":
-        token = new TypstToken(7 /* NEWLINE */, firstChar);
-        pos++;
-        break;
-      case "\r": {
-        if (pos + 1 < typst.length && typst[pos + 1] === "\n") {
-          token = new TypstToken(7 /* NEWLINE */, "\n");
-          pos += 2;
-        } else {
-          token = new TypstToken(7 /* NEWLINE */, "\n");
-          pos++;
-        }
-        break;
-      }
-      case " ": {
-        let newPos = pos;
-        while (newPos < typst.length && typst[newPos] === " ") {
-          newPos++;
-        }
-        token = new TypstToken(4 /* SPACE */, typst.substring(pos, newPos));
-        pos = newPos;
-        break;
-      }
-      case "/": {
-        if (pos < typst.length && typst[pos + 1] === "/") {
-          let newPos = pos + 2;
-          while (newPos < typst.length && typst[newPos] !== "\n") {
-            newPos++;
-          }
-          token = new TypstToken(3 /* COMMENT */, typst.slice(pos + 2, newPos));
-          pos = newPos;
-        } else {
-          token = new TypstToken(1 /* ELEMENT */, "/");
-          pos++;
-        }
-        break;
-      }
-      case "\\": {
-        if (pos + 1 >= typst.length) {
-          throw new Error("Expecting a character after \\");
-        }
-        const firstTwoChars = typst.substring(pos, pos + 2);
-        if (["\\$", "\\&", "\\#", "\\_"].includes(firstTwoChars)) {
-          token = new TypstToken(1 /* ELEMENT */, firstTwoChars);
-          pos += 2;
-        } else if (["\\\n", "\\ "].includes(firstTwoChars)) {
-          token = new TypstToken(6 /* CONTROL */, "\\");
-          pos += 1;
-        } else {
-          token = new TypstToken(6 /* CONTROL */, "");
-          pos++;
-        }
-        break;
-      }
-      case '"': {
-        let newPos = pos + 1;
-        while (newPos < typst.length) {
-          if (typst[newPos] === '"' && typst[newPos - 1] !== "\\") {
-            break;
-          }
-          newPos++;
-        }
-        let text = typst.substring(pos + 1, newPos);
-        const chars = ['"', "\\"];
-        for (const char of chars) {
-          text = text.replaceAll("\\" + char, char);
-        }
-        token = new TypstToken(2 /* TEXT */, text);
-        pos = newPos + 1;
-        break;
-      }
-      default: {
-        const shorthand = try_eat_shorthand(typst, pos);
-        if (shorthand !== null) {
-          token = new TypstToken(0 /* SYMBOL */, reverseShorthandMap.get(shorthand));
-          pos += shorthand.length;
-          break;
-        }
-        if (isdigit(firstChar)) {
-          let newPos = pos;
-          while (newPos < typst.length && isdigit(typst[newPos])) {
-            newPos += 1;
-          }
-          if (newPos < typst.length && typst[newPos] === ".") {
-            newPos += 1;
-            while (newPos < typst.length && isdigit(typst[newPos])) {
-              newPos += 1;
-            }
-          }
-          token = new TypstToken(1 /* ELEMENT */, typst.slice(pos, newPos));
-        } else if ("+-*/='<>!.,;?()[]|".includes(firstChar)) {
-          token = new TypstToken(1 /* ELEMENT */, firstChar);
-        } else if (isalpha(firstChar)) {
-          const identifier = eat_identifier_name(typst, pos);
-          const _type = identifier.length === 1 ? 1 /* ELEMENT */ : 0 /* SYMBOL */;
-          token = new TypstToken(_type, identifier);
-        } else {
-          token = new TypstToken(1 /* ELEMENT */, firstChar);
-        }
-        pos += token.value.length;
-      }
-    }
-    tokens.push(token);
-  }
-  return tokens;
+var REGEX_SHORTHANDS = generate_regex_for_shorthands();
+var rules_map2 = /* @__PURE__ */ new Map([
+  [String.raw`//[^\n]*`, (s) => new TypstToken(3 /* COMMENT */, s.text().substring(2))],
+  [String.raw`/`, (s) => new TypstToken(1 /* ELEMENT */, s.text())],
+  [String.raw`[_^&]`, (s) => new TypstToken(6 /* CONTROL */, s.text())],
+  [String.raw`\r?\n`, (_s) => new TypstToken(7 /* NEWLINE */, "\n")],
+  [String.raw`\s+`, (s) => new TypstToken(4 /* SPACE */, s.text())],
+  [String.raw`\\[$&#_]`, (s) => new TypstToken(1 /* ELEMENT */, s.text())],
+  [String.raw`\\\n`, (s) => {
+    return [
+      new TypstToken(6 /* CONTROL */, "\\"),
+      new TypstToken(7 /* NEWLINE */, "\n")
+    ];
+  }],
+  [String.raw`\\\s`, (s) => {
+    return [
+      new TypstToken(6 /* CONTROL */, "\\"),
+      new TypstToken(4 /* SPACE */, " ")
+    ];
+  }],
+  // this backslash is dummy and will be ignored in later stages
+  [String.raw`\\\S`, (_s) => new TypstToken(6 /* CONTROL */, "")],
+  [
+    String.raw`"([^"]|(\\"))*"`,
+    (s) => {
+      const text = s.text().substring(1, s.text().length - 1);
+      text.replaceAll('\\"', '"');
+      return new TypstToken(2 /* TEXT */, text);
+    }
+  ],
+  [
+    REGEX_SHORTHANDS,
+    (s) => {
+      const shorthand = s.text();
+      const symbol = reverseShorthandMap.get(shorthand);
+      return new TypstToken(0 /* SYMBOL */, symbol);
+    }
+  ],
+  [String.raw`[0-9]+(\.[0-9]+)?`, (s) => new TypstToken(1 /* ELEMENT */, s.text())],
+  [String.raw`[+\-*/=\'<>!.,;?()\[\]|]`, (s) => new TypstToken(1 /* ELEMENT */, s.text())],
+  [String.raw`[a-zA-Z\.]+`, (s) => {
+    return new TypstToken(s.text().length === 1 ? 1 /* ELEMENT */ : 0 /* SYMBOL */, s.text());
+  }],
+  [String.raw`.`, (s) => new TypstToken(1 /* ELEMENT */, s.text())]
+]);
+var spec2 = {
+  "start": rules_map2
+};
+function tokenize_typst(input) {
+  const lexer = new JSLex(spec2);
+  return lexer.collect(input);
 }
 function find_closing_match2(tokens, start) {
   assert(tokens[start].isOneOf([LEFT_PARENTHESES, LEFT_BRACKET, LEFT_CURLY_BRACKET2]));