tova 0.3.4 → 0.3.6

package/src/lexer/lexer.js

@@ -3,6 +3,16 @@ import { TokenType, Keywords, Token } from './tokens.js';
 export class Lexer {
   static MAX_INTERPOLATION_DEPTH = 64;
 
+  // Pre-compiled regex constants (avoid re-compilation in hot loops)
+  static UNICODE_LETTER_RE = /\p{Letter}/u;
+  static UNICODE_ALPHANUM_RE = /[\p{Letter}\p{Number}\p{Mark}]/u;
+  static HEX_DIGIT_RE = /[0-9a-fA-F_]/;
+  static BINARY_DIGIT_RE = /[01_]/;
+  static OCTAL_DIGIT_RE = /[0-7_]/;
+  static REGEX_FLAG_RE = /[gimsuydv]/;
+  static REGEX_START_RE = /[\s\/*=]/;
+  static JSX_CF_KEYWORDS = new Set(['if', 'for', 'elif', 'else', 'match']);
+
   constructor(source, filename = '<stdin>', lineOffset = 0, columnOffset = 0, _depth = 0) {
     this.source = source;
     this.filename = filename;
@@ -20,6 +30,22 @@ export class Lexer {
     this._jsxExprDepth = 0;       // brace depth for {expr} inside JSX
     this._jsxCF = null;           // null | { paren: 0, brace: 0, keyword? } — control flow state
     this._matchBlockDepth = 0;    // brace depth for match body inside JSX
+    this._subLexer = null;        // reusable sub-lexer for string interpolation
+  }
+
+  reset(source, lineOffset, columnOffset) {
+    this.source = source;
+    this.tokens = [];
+    this.pos = 0;
+    this.line = 1 + lineOffset;
+    this.column = 1 + columnOffset;
+    this.length = source.length;
+    this._jsxStack = [];
+    this._jsxTagMode = null;
+    this._jsxSelfClosing = false;
+    this._jsxExprDepth = 0;
+    this._jsxCF = null;
+    this._matchBlockDepth = 0;
   }
 
   error(message) {
@@ -62,14 +88,14 @@ export class Lexer {
   isAlpha(ch) {
     if ((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch === '_') return true;
     // Unicode letter support
-    if (ch > '\x7f') return /\p{Letter}/u.test(ch);
+    if (ch > '\x7f') return Lexer.UNICODE_LETTER_RE.test(ch);
     return false;
   }
 
   isAlphaNumeric(ch) {
     if (this.isAlpha(ch) || this.isDigit(ch)) return true;
     // Unicode continue characters (combining marks, etc.)
-    if (ch > '\x7f') return /[\p{Letter}\p{Number}\p{Mark}]/u.test(ch);
+    if (ch > '\x7f') return Lexer.UNICODE_ALPHANUM_RE.test(ch);
     return false;
   }
 
@@ -77,16 +103,27 @@ export class Lexer {
     return ch === ' ' || ch === '\t' || ch === '\r';
   }
 
+  _processEscape(esc) {
+    switch (esc) {
+      case 'n': return '\n';
+      case 't': return '\t';
+      case 'r': return '\r';
+      case '\\': return '\\';
+      case '"': return '"';
+      case "'": return "'";
+      case '{': return '{';
+      case '}': return '}';
+      default: return '\\' + esc;
+    }
+  }
+
   _isJSXStart() {
     const nextCh = this.peek();
     if (!this.isAlpha(nextCh)) return false;
     // Check the token BEFORE < (LESS was already pushed, so it's at length-2)
     const prev = this.tokens.length > 1 ? this.tokens[this.tokens.length - 2] : null;
     if (!prev) return true;
-    const valueTypes = [TokenType.IDENTIFIER, TokenType.NUMBER, TokenType.STRING,
-      TokenType.STRING_TEMPLATE, TokenType.RPAREN, TokenType.RBRACKET, TokenType.RBRACE,
-      TokenType.TRUE, TokenType.FALSE, TokenType.NIL];
-    return !valueTypes.includes(prev.type);
+    return !Lexer.VALUE_TOKEN_TYPES.has(prev.type);
   }
 
   tokenize() {
@@ -153,17 +190,12 @@ export class Lexer {
       // Negative list: if previous token ends an expression (produces a value),
       // then / is division. Otherwise, / starts a regex.
       // This is simpler and more robust — new token types default to regex context.
-      const divisionContextTokens = [
-        TokenType.IDENTIFIER, TokenType.NUMBER, TokenType.STRING, TokenType.STRING_TEMPLATE,
-        TokenType.TRUE, TokenType.FALSE, TokenType.NIL,
-        TokenType.RPAREN, TokenType.RBRACKET, TokenType.RBRACE,
-      ];
-      if (prev && !divisionContextTokens.includes(prev.type)) {
+      if (prev && !Lexer.VALUE_TOKEN_TYPES.has(prev.type)) {
         this.scanRegex();
         return;
       }
       // At start of file (no prev token), treat / as regex if followed by a non-space, non-special char
-      if (!prev && this.pos + 1 < this.length && !/[\s\/*=]/.test(this.peek(1))) {
+      if (!prev && this.pos + 1 < this.length && !Lexer.REGEX_START_RE.test(this.peek(1))) {
         this.scanRegex();
         return;
       }
@@ -224,11 +256,10 @@ export class Lexer {
       }
       // Check if next non-ws starts a keyword (if/for/elif/else)
       if (this.isAlpha(nextNonWs)) {
-        let word = '', wp = pp;
-        while (wp < this.length && this.isAlphaNumeric(this.source[wp])) {
-          word += this.source[wp]; wp++;
-        }
-        if (['if', 'for', 'elif', 'else', 'match'].includes(word)) {
+        let wp = pp;
+        while (wp < this.length && this.isAlphaNumeric(this.source[wp])) wp++;
+        const word = this.source.substring(pp, wp);
+        if (Lexer.JSX_CF_KEYWORDS.has(word)) {
           while (this.pos < pp) this.advance();
           return;
         }
@@ -258,11 +289,10 @@ export class Lexer {
 
     // Check for JSX control flow keywords: if, for, elif, else, match
     if (this.isAlpha(ch)) {
-      let word = '', peekPos = this.pos;
-      while (peekPos < this.length && this.isAlphaNumeric(this.source[peekPos])) {
-        word += this.source[peekPos]; peekPos++;
-      }
-      if (['if', 'for', 'elif', 'else', 'match'].includes(word)) {
+      let peekPos = this.pos;
+      while (peekPos < this.length && this.isAlphaNumeric(this.source[peekPos])) peekPos++;
+      const word = this.source.substring(this.pos, peekPos);
+      if (Lexer.JSX_CF_KEYWORDS.has(word)) {
         this.scanIdentifier();
         // After keyword, enter control flow mode for normal scanning
         this._jsxCF = { paren: 0, brace: 0, keyword: word };
@@ -281,12 +311,14 @@ export class Lexer {
       const ch = this.peek();
       if (ch === '<' || ch === '{' || ch === '"' || ch === "'") break;
       // Stop at keywords if, for, elif, else preceded by whitespace
-      if (this.isAlpha(ch) && text.length > 0 && /\s$/.test(text)) {
-        let word = '', pp = this.pos;
-        while (pp < this.length && this.isAlphaNumeric(this.source[pp])) {
-          word += this.source[pp]; pp++;
+      if (this.isAlpha(ch) && text.length > 0) {
+        const lastCh = text[text.length - 1];
+        if (lastCh === ' ' || lastCh === '\t' || lastCh === '\n' || lastCh === '\r') {
+          let pp = this.pos;
+          while (pp < this.length && this.isAlphaNumeric(this.source[pp])) pp++;
+          const word = this.source.substring(this.pos, pp);
+          if (Lexer.JSX_CF_KEYWORDS.has(word)) break;
         }
-        if (['if', 'for', 'elif', 'else', 'match'].includes(word)) break;
       }
       text += this.advance();
     }
@@ -351,7 +383,7 @@ export class Lexer {
       if (next === 'x' || next === 'X') {
         this.advance(); // 0
         this.advance(); // x
-        while (this.pos < this.length && /[0-9a-fA-F_]/.test(this.peek())) {
+        while (this.pos < this.length && Lexer.HEX_DIGIT_RE.test(this.peek())) {
           const ch = this.advance();
           if (ch !== '_') value += ch;
         }
@@ -362,7 +394,7 @@ export class Lexer {
       if (next === 'b' || next === 'B') {
         this.advance(); // 0
         this.advance(); // b
-        while (this.pos < this.length && /[01_]/.test(this.peek())) {
+        while (this.pos < this.length && Lexer.BINARY_DIGIT_RE.test(this.peek())) {
           const ch = this.advance();
           if (ch !== '_') value += ch;
         }
@@ -373,7 +405,7 @@ export class Lexer {
       if (next === 'o' || next === 'O') {
         this.advance(); // 0
         this.advance(); // o
-        while (this.pos < this.length && /[0-7_]/.test(this.peek())) {
+        while (this.pos < this.length && Lexer.OCTAL_DIGIT_RE.test(this.peek())) {
           const ch = this.advance();
           if (ch !== '_') value += ch;
         }
@@ -383,41 +415,56 @@ export class Lexer {
       }
     }
 
-    // Decimal
-    while (this.pos < this.length && (this.isDigit(this.peek()) || this.peek() === '_')) {
-      const ch = this.advance();
-      if (ch !== '_') value += ch;
+    // Fast path: scan decimal number using index advancement (no string concat)
+    // Handles digits, underscores, decimal point, and exponent
+    const numStart = this.pos;
+    let hasUnderscore = false;
+    while (this.pos < this.length) {
+      const ch = this.source[this.pos];
+      if (ch >= '0' && ch <= '9') { this.pos++; this.column++; }
+      else if (ch === '_') { hasUnderscore = true; this.pos++; this.column++; }
+      else break;
     }
 
     // Decimal point — only consume if followed by a digit or underscore (not e.g. 15.minutes)
-    if (this.peek() === '.' && this.peek(1) !== '.' && (this.isDigit(this.peek(1)) || this.peek(1) === '_')) {
-      value += this.advance(); // .
-      while (this.pos < this.length && (this.isDigit(this.peek()) || this.peek() === '_')) {
-        const ch = this.advance();
-        if (ch !== '_') value += ch;
+    if (this.pos < this.length && this.source[this.pos] === '.') {
+      const next = this.pos + 1 < this.length ? this.source[this.pos + 1] : '';
+      if (next !== '.' && ((next >= '0' && next <= '9') || next === '_')) {
+        this.pos++; this.column++; // .
+        while (this.pos < this.length) {
+          const ch = this.source[this.pos];
+          if (ch >= '0' && ch <= '9') { this.pos++; this.column++; }
+          else if (ch === '_') { hasUnderscore = true; this.pos++; this.column++; }
+          else break;
+        }
       }
     }
 
     // Exponent
-    if (this.peek() === 'e' || this.peek() === 'E') {
-      const savedPos = this.pos;
-      const savedCol = this.column;
-      let expPart = this.advance(); // consume 'e'/'E'
-      if (this.peek() === '+' || this.peek() === '-') {
-        expPart += this.advance();
-      }
-      if (this.pos < this.length && this.isDigit(this.peek())) {
-        value += expPart;
-        while (this.pos < this.length && this.isDigit(this.peek())) {
-          value += this.advance();
+    if (this.pos < this.length) {
+      const ech = this.source[this.pos];
+      if (ech === 'e' || ech === 'E') {
+        const savedPos = this.pos;
+        const savedCol = this.column;
+        this.pos++; this.column++;
+        if (this.pos < this.length && (this.source[this.pos] === '+' || this.source[this.pos] === '-')) {
+          this.pos++; this.column++;
+        }
+        if (this.pos < this.length && this.source[this.pos] >= '0' && this.source[this.pos] <= '9') {
+          while (this.pos < this.length && this.source[this.pos] >= '0' && this.source[this.pos] <= '9') {
+            this.pos++; this.column++;
+          }
+        } else {
+          // No digits after exponent — backtrack
+          this.pos = savedPos;
+          this.column = savedCol;
         }
-      } else {
-        // No digits after exponent — backtrack, treat 'e' as separate token
-        this.pos = savedPos;
-        this.column = savedCol;
       }
     }
 
+    let numStr = this.source.substring(numStart, this.pos);
+    if (hasUnderscore) numStr = numStr.replace(/_/g, '');
+    value = numStr;
     this.tokens.push(new Token(TokenType.NUMBER, parseFloat(value), startLine, startCol));
   }
 
@@ -436,17 +483,7 @@ export class Lexer {
         if (this.pos >= this.length) {
           this.error('Unterminated string');
         }
-        const esc = this.advance();
-        switch (esc) {
-          case 'n': current += '\n'; break;
-          case 't': current += '\t'; break;
-          case 'r': current += '\r'; break;
-          case '\\': current += '\\'; break;
-          case '"': current += '"'; break;
-          case '{': current += '{'; break;
-          case '}': current += '}'; break;
-          default: current += '\\' + esc;
-        }
+        current += this._processEscape(this.advance());
         continue;
       }
 
@@ -508,8 +545,12 @@ export class Lexer {
         if (this._depth + 1 > Lexer.MAX_INTERPOLATION_DEPTH) {
           this.error('String interpolation nested too deeply (max ' + Lexer.MAX_INTERPOLATION_DEPTH + ' levels)');
         }
-        const subLexer = new Lexer(exprSource, this.filename, exprStartLine, exprStartCol, this._depth + 1);
-        const exprTokens = subLexer.tokenize();
+        if (!this._subLexer) {
+          this._subLexer = new Lexer(exprSource, this.filename, exprStartLine, exprStartCol, this._depth + 1);
+        } else {
+          this._subLexer.reset(exprSource, exprStartLine, exprStartCol);
+        }
+        const exprTokens = this._subLexer.tokenize();
         // Remove the EOF token
         exprTokens.pop();
 
@@ -566,17 +607,7 @@ export class Lexer {
         if (this.pos >= this.length) {
           this.error('Unterminated multiline string');
         }
-        const esc = this.advance();
-        switch (esc) {
-          case 'n': current += '\n'; break;
-          case 't': current += '\t'; break;
-          case 'r': current += '\r'; break;
-          case '\\': current += '\\'; break;
-          case '"': current += '"'; break;
-          case '{': current += '{'; break;
-          case '}': current += '}'; break;
-          default: current += '\\' + esc;
-        }
+        current += this._processEscape(this.advance());
         continue;
       }
 
@@ -591,30 +622,30 @@ export class Lexer {
         const exprStartLine = this.line - 1;
         const exprStartCol = this.column - 1;
         let depth = 1;
-        let exprSource = '';
+        const exprParts = [];
         while (this.pos < this.length && depth > 0) {
           const ch = this.peek();
           if (ch === '"' || ch === "'" || ch === '`') {
             const quote = ch;
-            exprSource += this.advance();
+            exprParts.push(this.advance());
             let strDepth = 0;
             while (this.pos < this.length) {
               if (this.peek() === '\\') {
-                exprSource += this.advance();
-                if (this.pos < this.length) exprSource += this.advance();
+                exprParts.push(this.advance());
+                if (this.pos < this.length) exprParts.push(this.advance());
               } else if (quote === '"' && this.peek() === '{') {
                 strDepth++;
-                exprSource += this.advance();
+                exprParts.push(this.advance());
               } else if (quote === '"' && this.peek() === '}' && strDepth > 0) {
                 strDepth--;
-                exprSource += this.advance();
+                exprParts.push(this.advance());
               } else if (this.peek() === quote && strDepth === 0) {
                 break;
               } else {
-                exprSource += this.advance();
+                exprParts.push(this.advance());
               }
             }
-            if (this.pos < this.length) exprSource += this.advance();
+            if (this.pos < this.length) exprParts.push(this.advance());
             continue;
           }
           if (ch === '{') depth++;
@@ -622,8 +653,9 @@ export class Lexer {
             depth--;
             if (depth === 0) break;
           }
-          exprSource += this.advance();
+          exprParts.push(this.advance());
         }
+        const exprSource = exprParts.join('');
 
         if (this.peek() !== '}') {
           this.error('Unterminated string interpolation in multiline string');
@@ -633,8 +665,12 @@ export class Lexer {
         if (this._depth + 1 > Lexer.MAX_INTERPOLATION_DEPTH) {
           this.error('String interpolation nested too deeply (max ' + Lexer.MAX_INTERPOLATION_DEPTH + ' levels)');
         }
-        const subLexer = new Lexer(exprSource, this.filename, exprStartLine, exprStartCol, this._depth + 1);
-        const exprTokens = subLexer.tokenize();
+        if (!this._subLexer) {
+          this._subLexer = new Lexer(exprSource, this.filename, exprStartLine, exprStartCol, this._depth + 1);
+        } else {
+          this._subLexer.reset(exprSource, exprStartLine, exprStartCol);
+        }
+        const exprTokens = this._subLexer.tokenize();
         exprTokens.pop();
 
         parts.push({ type: 'expr', tokens: exprTokens, source: exprSource });
@@ -760,15 +796,7 @@ export class Lexer {
         if (this.pos >= this.length) {
           this.error('Unterminated string');
         }
-        const esc = this.advance();
-        switch (esc) {
-          case 'n': value += '\n'; break;
-          case 't': value += '\t'; break;
-          case 'r': value += '\r'; break;
-          case '\\': value += '\\'; break;
-          case "'": value += "'"; break;
-          default: value += '\\' + esc;
-        }
+        value += this._processEscape(this.advance());
       } else {
         value += this.advance();
       }
@@ -821,7 +849,7 @@ export class Lexer {
 
     // Read flags
     let flags = '';
-    while (this.pos < this.length && /[gimsuydv]/.test(this.peek())) {
+    while (this.pos < this.length && Lexer.REGEX_FLAG_RE.test(this.peek())) {
       flags += this.advance();
     }
 
@@ -831,19 +859,31 @@ export class Lexer {
   scanIdentifier() {
     const startLine = this.line;
     const startCol = this.column;
-    let value = '';
+    const startPos = this.pos;
 
-    while (this.pos < this.length && this.isAlphaNumeric(this.peek())) {
-      value += this.advance();
+    // Fast path: scan ASCII identifier using index advancement (no string concat)
+    while (this.pos < this.length) {
+      const ch = this.source[this.pos];
+      if ((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch === '_' || (ch >= '0' && ch <= '9')) {
+        this.pos++;
+        this.column++;
+      } else if (ch > '\x7f' && Lexer.UNICODE_ALPHANUM_RE.test(ch)) {
+        this.pos++;
+        this.column++;
+      } else {
+        break;
+      }
     }
+    const value = this.source.substring(startPos, this.pos);
 
     // Raw string: r"no\escapes"
     if (value === 'r' && this.pos < this.length && this.peek() === '"') {
       this.advance(); // opening "
-      let raw = '';
+      const rawParts = [];
       while (this.pos < this.length && this.peek() !== '"') {
-        raw += this.advance();
+        rawParts.push(this.advance());
       }
+      const raw = rawParts.join('');
       if (this.pos >= this.length) {
         this.error('Unterminated raw string');
       }
@@ -1049,10 +1089,7 @@ export class Lexer {
                 this._jsxTagMode = 'open';
               } else {
                 const prev = this.tokens.length > 1 ? this.tokens[this.tokens.length - 2] : null;
-                const valueTypes = [TokenType.IDENTIFIER, TokenType.NUMBER, TokenType.STRING,
-                  TokenType.STRING_TEMPLATE, TokenType.RPAREN, TokenType.RBRACKET, TokenType.RBRACE,
-                  TokenType.TRUE, TokenType.FALSE, TokenType.NIL];
-                if (!prev || !valueTypes.includes(prev.type)) {
+                if (!prev || !Lexer.VALUE_TOKEN_TYPES.has(prev.type)) {
                   this._jsxTagMode = 'open';
                 }
               }
@@ -1139,8 +1176,19 @@ export class Lexer {
         }
         break;
 
+      case '@':
+        this.tokens.push(new Token(TokenType.AT, '@', startLine, startCol));
+        break;
+
       default:
         this.error(`Unexpected character: '${ch}'`);
     }
   }
 }
+
+// Initialize static Set after class definition (depends on TokenType)
+Lexer.VALUE_TOKEN_TYPES = new Set([
+  TokenType.IDENTIFIER, TokenType.NUMBER, TokenType.STRING,
+  TokenType.STRING_TEMPLATE, TokenType.RPAREN, TokenType.RBRACKET, TokenType.RBRACE,
+  TokenType.TRUE, TokenType.FALSE, TokenType.NIL
+]);

package/src/lexer/tokens.js

@@ -157,6 +157,9 @@ export const TokenType = {
   // Regex
   REGEX: 'REGEX',             // /pattern/flags
 
+  // Decorators
+  AT: 'AT',                    // @
+
   // Special
   EOF: 'EOF',
   DOCSTRING: 'DOCSTRING',     // /// comment