tex2typst 0.6.1 → 0.6.2

package/dist/index.js

@@ -324,7 +324,7 @@ function array_intersperse(array, sep) {
 }
 
 // src/lex.ts
-var EOF = {};
+var EOF = null;
 function matchcompare(m1, m2) {
   const m1_len = m1.reMatchArray[0].length;
   const m2_len = m2.reMatchArray[0].length;
@@ -334,114 +334,31 @@ function matchcompare(m1, m2) {
     return m1.index - m2.index;
   }
 }
+var ScanResult = class _ScanResult {
+  result;
+  status;
+  constructor(status, result) {
+    this.result = result;
+    this.status = status;
+  }
+  static Accepted(result) {
+    return new _ScanResult(0 /* ACCEPTED */, result);
+  }
+  static Rejected() {
+    return new _ScanResult(1 /* REJECTED */, []);
+  }
+  static Error(message) {
+    return new _ScanResult(2 /* ERROR */, []);
+  }
+};
 var Scanner = class {
   _input;
-  _lexer;
+  rules;
   // position within input stream
   _pos = 0;
-  // current line number
-  _line = 0;
-  // current column number
-  _col = 0;
-  _offset = 0;
-  _less = null;
-  _go = false;
-  _newstate = null;
-  _state;
-  _text = null;
-  _leng = null;
-  _reMatchArray = null;
-  constructor(input, lexer) {
+  constructor(input, rules) {
     this._input = input;
-    this._lexer = lexer;
-    this._state = lexer.states[0];
-  }
-  /**
-   * Analogous to yytext and yyleng in lex - will be set during scan.
-   */
-  text() {
-    return this._text;
-  }
-  leng() {
-    return this._leng;
-  }
-  reMatchArray() {
-    return this._reMatchArray;
-  }
-  /**
-   * Position of in stream, line number and column number of match.
-   */
-  pos() {
-    return this._pos;
-  }
-  line() {
-    return this._line;
-  }
-  column() {
-    return this._col;
-  }
-  /**
-   * Analogous to input() in lex.
-   * @return {string} The next character in the stream.
-   */
-  input() {
-    return this._input.charAt(this._pos + this._leng + this._offset++);
-  }
-  /**
-   * Similar to unput() in lex, but does not allow modifying the stream.
-   * @return {int} The offset position after the operation.
-   */
-  unput() {
-    return this._offset = this._offset > 0 ? this._offset-- : 0;
-  }
-  /**
-   * Analogous to yyless(n) in lex - retains the first n characters from this pattern, and returns
-   * the rest to the input stream, such that they will be used in the next pattern-matching operation.
-   * @param {int} n Number of characters to retain.
-   * @return {int} Length of the stream after the operation has completed.
-   */
-  less(n) {
-    this._less = n;
-    this._offset = 0;
-    this._text = this._text.substring(0, n);
-    return this._leng = this._text.length;
-  }
-  /**
-   * Like less(), but instead of retaining the first n characters, it chops off the last n.
-   * @param {int} n Number of characters to chop.
-   * @return {int} Length of the stream after the operation has completed.
-   */
-  pushback(n) {
-    return this.less(this._leng - n);
-  }
-  /**
-   * Similar to REJECT in lex, except it doesn't break the current execution context.
-   * TIP: reject() should be the last instruction in a spec callback.
-   */
-  reject() {
-    this._go = true;
-  }
-  /**
-   * Analogous to BEGIN in lex - sets the named state (start condition).
-   * @param {string|int} state Name of state to switch to, or ordinal number (0 is first, etc).
-   * @return {string} The new state on successful switch, throws exception on failure.
-   */
-  begin(state) {
-    if (this._lexer.specification[state]) {
-      return this._newstate = state;
-    }
-    const s = this._lexer.states[parseInt(state)];
-    if (s) {
-      return this._newstate = s;
-    }
-    throw "Unknown state '" + state + "' requested";
-  }
-  /**
-   * Simple accessor for reading in the current state.
-   * @return {string} The current state.
-   */
-  state() {
-    return this._state;
+    this.rules = rules;
   }
   /**
    * Scan method to be returned to caller - grabs the next token and fires appropriate calback.
@@ -452,7 +369,7 @@ var Scanner = class {
       return EOF;
     }
     const str = this._input.substring(this._pos);
-    const rules = this._lexer.specification[this._state];
+    const rules = this.rules;
     const matches = [];
     for (let i = 0; i < rules.length; i++) {
       const rule = rules[i];
@@ -469,87 +386,50 @@ var Scanner = class {
       throw new Error("No match found for input '" + str + "'");
     }
     matches.sort(matchcompare);
-    this._go = true;
-    let result;
-    let matched_text;
-    for (let j = 0, n = matches.length; j < n && this._go; j++) {
-      this._offset = 0;
-      this._less = null;
-      this._go = false;
-      this._newstate = null;
-      const m = matches[j];
-      matched_text = m.reMatchArray[0];
-      this._text = matched_text;
-      this._leng = matched_text.length;
-      this._reMatchArray = m.reMatchArray;
-      result = m.rule.action(this);
-      if (this._newstate && this._newstate != this._state) {
-        this._state = this._newstate;
-        break;
+    for (const m of matches) {
+      const matched_text = m.reMatchArray[0];
+      const result = m.rule.action({
+        pos: this._pos,
+        text: matched_text,
+        reMatchArray: m.reMatchArray
+      });
+      if (result.status === 0 /* ACCEPTED */) {
+        this._pos += matched_text.length;
+        return result.result;
       }
     }
-    const text = this._less === null ? matched_text : matched_text.substring(0, this._less);
-    const len = text.length;
-    this._pos += len + this._offset;
-    const nlm = text.match(/\n/g);
-    if (nlm !== null) {
-      this._line += nlm.length;
-      this._col = len - text.lastIndexOf("\n") - 1;
-    } else {
-      this._col += len;
-    }
-    return result;
+    throw new Error("No match found for input '" + str + "'");
   }
 };
 var JSLex = class {
-  states;
-  specification;
-  constructor(spec3) {
-    this.states = Object.keys(spec3);
-    this.specification = {};
-    for (const s of this.states) {
-      const rule_map = spec3[s];
-      if (s in this.specification) {
-        throw "Duplicate state declaration encountered for state '" + s + "'";
-      }
-      this.specification[s] = [];
-      for (const [k, v] of rule_map.entries()) {
-        let re;
-        try {
-          re = new RegExp("^" + k);
-        } catch (err) {
-          throw "Invalid regexp '" + k + "' in state '" + s + "' (" + err.message + ")";
-        }
-        this.specification[s].push({
-          re,
-          action: v
-        });
+  rules = [];
+  constructor(ruleMap) {
+    for (const [k, v] of ruleMap.entries()) {
+      let re;
+      try {
+        re = new RegExp("^" + k);
+      } catch (err) {
+        throw "Invalid regexp '" + k + "' (" + err.message + ")";
       }
+      this.rules.push({
+        re,
+        action: v
+      });
     }
   }
-  /**
-   * Scanner function - makes a new scanner object which is used to get tokens one at a time.
-   * @param {string} input Input text to tokenize.
-   * @return {function} Scanner function.
-   */
-  scanner(input) {
-    return new Scanner(input, this);
-  }
   /**
    * Similar to lex's yylex() function, consumes all input, calling calback for each token.
    * @param {string} input Text to lex.
    * @param {function} callback Function to execute for each token.
    */
   lex(input, callback) {
-    const scanner = this.scanner(input);
+    const scanner = new Scanner(input, this.rules);
     while (true) {
       const token = scanner.scan();
       if (token === EOF) {
-        return;
-      }
-      if (token !== void 0) {
-        callback(token);
+        break;
       }
+      callback(token);
     }
   }
   /**
@@ -647,8 +527,8 @@ var rules_map = /* @__PURE__ */ new Map([
   [
     String.raw`\\begin{(array|subarry)}{(.+?)}`,
     (s) => {
-      const match = s.reMatchArray();
-      return [
+      const match = s.reMatchArray;
+      return ScanResult.Accepted([
         new TexToken(2 /* COMMAND */, "\\begin"),
         new TexToken(7 /* CONTROL */, "{"),
         new TexToken(3 /* LITERAL */, match[1]),
@@ -656,74 +536,69 @@ var rules_map = /* @__PURE__ */ new Map([
         new TexToken(7 /* CONTROL */, "{"),
         new TexToken(3 /* LITERAL */, match[2]),
         new TexToken(7 /* CONTROL */, "}")
-      ];
+      ]);
     }
   ],
   [
     String.raw`\\(text|operatorname\*?|textcolor|begin|end|hspace|array)\s*{(.+?)}`,
     (s) => {
-      const match = s.reMatchArray();
-      return [
+      const match = s.reMatchArray;
+      return ScanResult.Accepted([
         new TexToken(2 /* COMMAND */, "\\" + match[1]),
         new TexToken(7 /* CONTROL */, "{"),
         new TexToken(3 /* LITERAL */, unescape(match[2])),
         new TexToken(7 /* CONTROL */, "}")
-      ];
+      ]);
     }
   ],
-  [String.raw`%[^\n]*`, (s) => new TexToken(4 /* COMMENT */, s.text().substring(1))],
-  [String.raw`[{}_^&]`, (s) => new TexToken(7 /* CONTROL */, s.text())],
-  [String.raw`\\[\\,:;!> ]`, (s) => new TexToken(7 /* CONTROL */, s.text())],
-  [String.raw`~`, (s) => new TexToken(7 /* CONTROL */, s.text())],
-  [String.raw`\r?\n`, (_s) => new TexToken(6 /* NEWLINE */, "\n")],
-  [String.raw`\s+`, (s) => new TexToken(5 /* SPACE */, s.text())],
-  [String.raw`\\[{}%$&#_|]`, (s) => new TexToken(1 /* ELEMENT */, s.text())],
+  [String.raw`%[^\n]*`, (s) => ScanResult.Accepted(new TexToken(4 /* COMMENT */, s.text.substring(1)))],
+  [String.raw`[{}_^&]`, (s) => ScanResult.Accepted([new TexToken(7 /* CONTROL */, s.text)])],
+  [String.raw`\\[\\,:;!> ]`, (s) => ScanResult.Accepted([new TexToken(7 /* CONTROL */, s.text)])],
+  [String.raw`~`, (s) => ScanResult.Accepted([new TexToken(7 /* CONTROL */, s.text)])],
+  [String.raw`\r?\n`, (_s) => ScanResult.Accepted([new TexToken(6 /* NEWLINE */, "\n")])],
+  [String.raw`\s+`, (s) => ScanResult.Accepted([new TexToken(5 /* SPACE */, s.text)])],
+  [String.raw`\\[{}%$&#_|]`, (s) => ScanResult.Accepted([new TexToken(1 /* ELEMENT */, s.text)])],
   // e.g. match `\frac13`, `\frac1 b`, `\frac a b`
   [String.raw`(\\[a-zA-Z]+)(\s*\d|\s+[a-zA-Z])\s*([0-9a-zA-Z])`, (s) => {
-    const match = s.reMatchArray();
+    const match = s.reMatchArray;
     const command = match[1];
     if (TEX_BINARY_COMMANDS.includes(command.substring(1))) {
       const arg1 = match[2].trimStart();
       const arg2 = match[3];
-      return [
+      return ScanResult.Accepted([
         new TexToken(2 /* COMMAND */, command),
         new TexToken(1 /* ELEMENT */, arg1),
         new TexToken(1 /* ELEMENT */, arg2)
-      ];
+      ]);
     } else {
-      s.reject();
-      return [];
+      return ScanResult.Rejected();
     }
   }],
   // e.g. match `\sqrt3`, `\sqrt a`
   [String.raw`(\\[a-zA-Z]+)(\s*\d|\s+[a-zA-Z])`, (s) => {
-    const match = s.reMatchArray();
+    const match = s.reMatchArray;
     const command = match[1];
     if (TEX_UNARY_COMMANDS.includes(command.substring(1))) {
       const arg1 = match[2].trimStart();
-      return [
+      return ScanResult.Accepted([
         new TexToken(2 /* COMMAND */, command),
         new TexToken(1 /* ELEMENT */, arg1)
-      ];
+      ]);
     } else {
-      s.reject();
-      return [];
+      return ScanResult.Rejected();
     }
   }],
-  [String.raw`\\[a-zA-Z]+`, (s) => new TexToken(2 /* COMMAND */, s.text())],
+  [String.raw`\\[a-zA-Z]+`, (s) => ScanResult.Accepted(new TexToken(2 /* COMMAND */, s.text))],
   // Numbers like "123", "3.14"
-  [String.raw`[0-9]+(\.[0-9]+)?`, (s) => new TexToken(1 /* ELEMENT */, s.text())],
-  [String.raw`[a-zA-Z]`, (s) => new TexToken(1 /* ELEMENT */, s.text())],
-  [String.raw`[+\-*/='<>!.,;:?()\[\]|]`, (s) => new TexToken(1 /* ELEMENT */, s.text())],
+  [String.raw`[0-9]+(\.[0-9]+)?`, (s) => ScanResult.Accepted(new TexToken(1 /* ELEMENT */, s.text))],
+  [String.raw`[a-zA-Z]`, (s) => ScanResult.Accepted(new TexToken(1 /* ELEMENT */, s.text))],
+  [String.raw`[+\-*/='<>!.,;:?()\[\]|]`, (s) => ScanResult.Accepted(new TexToken(1 /* ELEMENT */, s.text))],
   // non-ASCII characters
-  [String.raw`[^\x00-\x7F]`, (s) => new TexToken(1 /* ELEMENT */, s.text())],
-  [String.raw`.`, (s) => new TexToken(8 /* UNKNOWN */, s.text())]
+  [String.raw`[^\x00-\x7F]`, (s) => ScanResult.Accepted(new TexToken(1 /* ELEMENT */, s.text))],
+  [String.raw`.`, (s) => ScanResult.Accepted([new TexToken(8 /* UNKNOWN */, s.text)])]
 ]);
-var spec = {
-  "start": rules_map
-};
 function tokenize_tex(input) {
-  const lexer = new JSLex(spec);
+  const lexer = new JSLex(rules_map);
   return lexer.collect(input);
 }
 
@@ -3065,8 +2940,6 @@ function tex_token_to_typst(token, options) {
     case 7 /* CONTROL */: {
       if (token.value === "\\\\") {
         return new TypstToken(7 /* CONTROL */, "\\");
-      } else if (token.value === "\\!") {
-        return new TypstToken(1 /* SYMBOL */, "#h(-math.thin.amount)");
       } else if (token.value === "~") {
         const typst_symbol = symbolMap.get("~");
         return new TypstToken(1 /* SYMBOL */, typst_symbol);
@@ -3166,6 +3039,12 @@ function convert_tex_node_to_typst(abstractNode, options) {
   switch (abstractNode.type) {
     case "terminal": {
       const node2 = abstractNode;
+      if (node2.head.eq(new TexToken(7 /* CONTROL */, "\\!"))) {
+        return new TypstFuncCall(
+          new TypstToken(1 /* SYMBOL */, "#h"),
+          [new TypstToken(3 /* LITERAL */, "-math.thin.amount").toNode()]
+        );
+      }
       return tex_token_to_typst(node2.head, options).toNode();
     }
     case "text": {
@@ -3880,64 +3759,69 @@ function generate_regex_for_shorthands() {
 }
 var REGEX_SHORTHANDS = generate_regex_for_shorthands();
 var rules_map2 = /* @__PURE__ */ new Map([
-  [String.raw`//[^\n]*`, (s) => new TypstToken(5 /* COMMENT */, s.text().substring(2))],
-  [String.raw`/`, (s) => new TypstToken(2 /* ELEMENT */, s.text())],
-  [String.raw`[_^&]`, (s) => new TypstToken(7 /* CONTROL */, s.text())],
-  [String.raw`\r?\n`, (_s) => new TypstToken(8 /* NEWLINE */, "\n")],
-  [String.raw`\s+`, (s) => new TypstToken(6 /* SPACE */, s.text())],
-  [String.raw`\\[$&#_]`, (s) => new TypstToken(2 /* ELEMENT */, s.text())],
-  [String.raw`\\\n`, (s) => {
-    return [
+  [String.raw`//[^\n]*`, (s) => ScanResult.Accepted(new TypstToken(5 /* COMMENT */, s.text.substring(2)))],
+  [String.raw`/`, (s) => ScanResult.Accepted(new TypstToken(2 /* ELEMENT */, s.text))],
+  [String.raw`[_^&]`, (s) => ScanResult.Accepted(new TypstToken(7 /* CONTROL */, s.text))],
+  [String.raw`\r?\n`, (_s) => ScanResult.Accepted(new TypstToken(8 /* NEWLINE */, "\n"))],
+  [String.raw`\s+`, (s) => ScanResult.Accepted(new TypstToken(6 /* SPACE */, s.text))],
+  [String.raw`\\[$&#_]`, (s) => ScanResult.Accepted(new TypstToken(2 /* ELEMENT */, s.text))],
+  [
+    String.raw`\\\n`,
+    (s) => ScanResult.Accepted([
       new TypstToken(7 /* CONTROL */, "\\"),
       new TypstToken(8 /* NEWLINE */, "\n")
-    ];
-  }],
+    ])
+  ],
   [String.raw`\\\s`, (s) => {
-    return [
+    return ScanResult.Accepted([
       new TypstToken(7 /* CONTROL */, "\\"),
       new TypstToken(6 /* SPACE */, " ")
-    ];
+    ]);
   }],
   // this backslash is dummy and will be ignored in later stages
-  [String.raw`\\\S`, (_s) => new TypstToken(7 /* CONTROL */, "")],
+  [String.raw`\\\S`, (_s) => ScanResult.Accepted(new TypstToken(7 /* CONTROL */, ""))],
   [
     String.raw`"([^"]|(\\"))*"`,
     (s) => {
-      const text = s.text().substring(1, s.text().length - 1);
+      const text = s.text.substring(1, s.text.length - 1);
       text.replaceAll('\\"', '"');
-      return new TypstToken(4 /* TEXT */, text);
+      return ScanResult.Accepted(new TypstToken(4 /* TEXT */, text));
     }
   ],
   [
     REGEX_SHORTHANDS,
     (s) => {
-      const shorthand = s.text();
+      const shorthand = s.text;
       const symbol = reverseShorthandMap.get(shorthand);
-      return new TypstToken(1 /* SYMBOL */, symbol);
+      return ScanResult.Accepted(new TypstToken(1 /* SYMBOL */, symbol));
     }
   ],
-  [String.raw`[0-9]+(\.[0-9]+)?`, (s) => new TypstToken(2 /* ELEMENT */, s.text())],
-  [String.raw`[+\-*/=\'<>!.,;?()\[\]|]`, (s) => new TypstToken(2 /* ELEMENT */, s.text())],
-  [String.raw`#h\((.+?)\)`, (s) => {
-    const match = s.reMatchArray();
-    return [
-      new TypstToken(1 /* SYMBOL */, "#h"),
-      new TypstToken(2 /* ELEMENT */, "("),
-      new TypstToken(3 /* LITERAL */, match[1]),
-      new TypstToken(2 /* ELEMENT */, ")")
-    ];
-  }],
-  [String.raw`#none`, (s) => new TypstToken(0 /* NONE */, s.text())],
+  [String.raw`[0-9]+(\.[0-9]+)?`, (s) => ScanResult.Accepted(new TypstToken(2 /* ELEMENT */, s.text))],
+  [String.raw`[+\-*/=\'<>!.,;?()\[\]|]`, (s) => ScanResult.Accepted(new TypstToken(2 /* ELEMENT */, s.text))],
+  [
+    String.raw`#h\((.+?)\)`,
+    (s) => {
+      const match = s.reMatchArray;
+      return ScanResult.Accepted([
+        new TypstToken(1 /* SYMBOL */, "#h"),
+        new TypstToken(2 /* ELEMENT */, "("),
+        new TypstToken(3 /* LITERAL */, match[1]),
+        new TypstToken(2 /* ELEMENT */, ")")
+      ]);
+    }
+  ],
+  [String.raw`#none`, (s) => ScanResult.Accepted(new TypstToken(0 /* NONE */, s.text))],
+  [
+    String.raw`#none`,
+    (s) => ScanResult.Accepted(new TypstToken(0 /* NONE */, s.text))
+  ],
   [String.raw`#?[a-zA-Z\.]+`, (s) => {
-    return new TypstToken(s.text().length === 1 ? 2 /* ELEMENT */ : 1 /* SYMBOL */, s.text());
+    return ScanResult.Accepted(new TypstToken(s.text.length === 1 ? 2 /* ELEMENT */ : 1 /* SYMBOL */, s.text));
   }],
-  [String.raw`.`, (s) => new TypstToken(2 /* ELEMENT */, s.text())]
+  [String.raw`.`, (s) => ScanResult.Accepted(new TypstToken(2 /* ELEMENT */, s.text))]
 ]);
-var spec2 = {
-  "start": rules_map2
-};
 function tokenize_typst(input) {
-  const lexer = new JSLex(spec2);
+  const lexer = new JSLex(rules_map2);
   return lexer.collect(input);
 }