stone-lang 0.1.0

package/frontend/parsing/tokenizer.js

@@ -0,0 +1,692 @@
+/**
+ * Stone Language Lexer (Tokenizer)
+ *
+ * Converts Stone source code into a stream of tokens.
+ * This is the first phase of parsing.
+ */
+
+// Token types
+export const TokenType = {
+  // Literals
+  NUMBER: "NUMBER",
+  IMAGINARY: "IMAGINARY", // 4i or 4j (imaginary part of complex)
+  STRING: "STRING",
+  TRUE: "TRUE",
+  FALSE: "FALSE",
+  NONE: "NONE",
+
+  // Identifiers and keywords
+  IDENTIFIER: "IDENTIFIER",
+
+  // Keywords
+  FN: "FN",
+  RETURN: "RETURN",
+  IF: "IF",
+  ELIF: "ELIF",
+  ELSE: "ELSE",
+  FOR: "FOR",
+  WHILE: "WHILE",
+  IN: "IN",
+  BY: "BY",
+  BREAK: "BREAK",
+  CONTINUE: "CONTINUE",
+  STATE: "STATE",
+  STATIC: "STATIC",
+  AS: "AS",
+
+  // Module keywords
+  IMPORT: "IMPORT",
+  EXPORT: "EXPORT",
+  FROM: "FROM",
+
+  // Type keywords
+  TYPE: "TYPE",
+
+  // Extension keyword
+  EXTENSION: "EXTENSION",
+
+  // Operators
+  PLUS: "PLUS", // +
+  MINUS: "MINUS", // -
+  STAR: "STAR", // *
+  SLASH: "SLASH", // /
+  PERCENT: "PERCENT", // %
+  CARET: "CARET", // ^ (power)
+  AT: "AT", // @ (matrix multiplication)
+
+  // Element-wise operators (for arrays)
+  DOT_PLUS: "DOT_PLUS", // .+
+  DOT_MINUS: "DOT_MINUS", // .-
+  DOT_STAR: "DOT_STAR", // .*
+  DOT_SLASH: "DOT_SLASH", // ./
+  DOT_CARET: "DOT_CARET", // .^
+
+  // Logical operators
+  AND: "AND", // &&
+  OR: "OR", // ||
+  NOT: "NOT", // !
+  PIPE: "PIPE", // | (union type)
+
+  // Comparison
+  LT: "LT", // <
+  GT: "GT", // >
+  LTE: "LTE", // <=
+  GTE: "GTE", // >=
+  EQ: "EQ", // ==
+  NEQ: "NEQ", // !=
+
+  // Assignment
+  ASSIGN: "ASSIGN", // =
+  COLON_ASSIGN: "COLON_ASSIGN", // :=
+  LEFT_ARROW: "LEFT_ARROW", // <- (state transition operator)
+  PLUS_ASSIGN: "PLUS_ASSIGN", // +=
+  MINUS_ASSIGN: "MINUS_ASSIGN", // -=
+  STAR_ASSIGN: "STAR_ASSIGN", // *=
+  SLASH_ASSIGN: "SLASH_ASSIGN", // /=
+
+  // Increment/Decrement
+  INCREMENT: "INCREMENT", // ++
+  DECREMENT: "DECREMENT", // --
+
+  // Range operators
+  RANGE: "RANGE", // ..
+  RANGE_EXCLUSIVE: "RANGE_EXCLUSIVE", // ..<
+  RANGE_INCLUSIVE: "RANGE_INCLUSIVE", // ..=
+
+  // Delimiters
+  LPAREN: "LPAREN", // (
+  RPAREN: "RPAREN", // )
+  LBRACE: "LBRACE", // {
+  RBRACE: "RBRACE", // }
+  LBRACKET: "LBRACKET", // [
+  RBRACKET: "RBRACKET", // ]
+
+  // Punctuation
+  COMMA: "COMMA", // ,
+  COLON: "COLON", // :
+  DOUBLE_COLON: "DOUBLE_COLON", // ::
+  SEMICOLON: "SEMICOLON", // ;
+  DOT: "DOT", // .
+  ARROW: "ARROW", // ->
+  FAT_ARROW: "FAT_ARROW", // =>
+  QUESTION: "QUESTION", // ?
+  TILDE: "TILDE", // ~
+
+  // Special
+  NEWLINE: "NEWLINE",
+  EOF: "EOF",
+  COMMENT: "COMMENT",
+
+  // String interpolation
+  F_STRING_START: "F_STRING_START",
+  INTERPOLATION_START: "INTERPOLATION_START",
+  INTERPOLATION_END: "INTERPOLATION_END",
+};
+
+// Reserved keywords
+const KEYWORDS = {
+  fn: TokenType.FN,
+  return: TokenType.RETURN,
+  if: TokenType.IF,
+  elif: TokenType.ELIF,
+  else: TokenType.ELSE,
+  for: TokenType.FOR,
+  while: TokenType.WHILE,
+  in: TokenType.IN,
+  by: TokenType.BY,
+  break: TokenType.BREAK,
+  continue: TokenType.CONTINUE,
+  true: TokenType.TRUE,
+  false: TokenType.FALSE,
+  none: TokenType.NONE,
+  // Note: 'state' is NOT a keyword - it can be used as a regular identifier
+  static: TokenType.STATIC,
+  // 'as' is NOT a keyword - it's a contextual identifier recognized only in alias positions
+  // This allows: as = [1, 2, 3] while still supporting: x as alias := obj
+  // Module keywords
+  import: TokenType.IMPORT,
+  export: TokenType.EXPORT,
+  from: TokenType.FROM,
+  // Type keywords
+  type: TokenType.TYPE,
+  // Extension keyword
+  ext: TokenType.EXTENSION,
+};
+
+/**
+ * Token class
+ */
+export class Token {
+  constructor(type, value, line, column) {
+    this.type = type;
+    this.value = value;
+    this.line = line;
+    this.column = column;
+  }
+
+  toString() {
+    try {
+      let valueStr;
+      if (typeof this.value === "string" && this.value.length > 100) {
+        valueStr = `"${this.value.substring(0, 100)}..."`;
+      } else if (typeof this.value === "object") {
+        valueStr = "[Object]";
+      } else {
+        valueStr = JSON.stringify(this.value);
+      }
+      return `Token(${this.type}, ${valueStr}, ${this.line}:${this.column})`;
+    } catch (error) {
+      return `Token(${this.type}, [Error stringifying value], ${this.line}:${this.column})`;
+    }
+  }
+}
+
+/**
+ * Lexer class
+ */
+export class Lexer {
+  constructor(source, filename = "<stdin>") {
+    this.source = source;
+    this.filename = filename;
+    this.pos = 0;
+    this.line = 1;
+    this.column = 1;
+    this.tokens = [];
+  }
+
+  /**
+   * Tokenize the entire source
+   */
+  tokenize() {
+    while (this.pos < this.source.length) {
+      this.skipWhitespace();
+
+      if (this.pos >= this.source.length) break;
+
+      // Skip comments
+      if (this.peek() === "/" && this.peek(1) === "/") {
+        this.skipComment();
+        continue;
+      }
+
+      const token = this.nextToken();
+      if (token) {
+        this.tokens.push(token);
+      }
+    }
+
+    // Add EOF token
+    const eofToken = new Token(TokenType.EOF, null, this.line, this.column);
+    this.tokens.push(eofToken);
+
+    return this.tokens;
+  }
+
+  /**
+   * Get next token
+   */
+  nextToken() {
+    const char = this.peek();
+    const startLine = this.line;
+    const startColumn = this.column;
+
+    // Numbers
+    if (this.isDigit(char)) {
+      return this.readNumber(startLine, startColumn);
+    }
+
+    // Strings
+    if (char === '"' || char === "'") {
+      return this.readString(char, startLine, startColumn);
+    }
+
+    // F-strings (string interpolation)
+    if (char === "f" && (this.peek(1) === '"' || this.peek(1) === "'")) {
+      return this.readFString(startLine, startColumn);
+    }
+
+    // Identifiers and keywords
+    if (this.isAlpha(char) || char === "_") {
+      return this.readIdentifier(startLine, startColumn);
+    }
+
+    // Three-character operators (none currently)
+
+    // Two-character operators
+    const twoChar = this.peek() + this.peek(1);
+    if (twoChar === "&&") {
+      this.advance(2);
+      return new Token(TokenType.AND, "&&", startLine, startColumn);
+    }
+    if (twoChar === "||") {
+      this.advance(2);
+      return new Token(TokenType.OR, "||", startLine, startColumn);
+    }
+    if (twoChar === ":=") {
+      this.advance(2);
+      return new Token(TokenType.COLON_ASSIGN, ":=", startLine, startColumn);
+    }
+    if (twoChar === "::") {
+      this.advance(2);
+      return new Token(TokenType.DOUBLE_COLON, "::", startLine, startColumn);
+    }
+    // Element-wise operators: .+, .-, .*, ./, .^
+    if (twoChar === ".+") {
+      this.advance(2);
+      return new Token(TokenType.DOT_PLUS, ".+", startLine, startColumn);
+    }
+    if (twoChar === ".-") {
+      this.advance(2);
+      return new Token(TokenType.DOT_MINUS, ".-", startLine, startColumn);
+    }
+    if (twoChar === ".*") {
+      this.advance(2);
+      return new Token(TokenType.DOT_STAR, ".*", startLine, startColumn);
+    }
+    if (twoChar === "./") {
+      this.advance(2);
+      return new Token(TokenType.DOT_SLASH, "./", startLine, startColumn);
+    }
+    if (twoChar === ".^") {
+      this.advance(2);
+      return new Token(TokenType.DOT_CARET, ".^", startLine, startColumn);
+    }
+    if (twoChar === "..") {
+      this.advance(2);
+      if (this.peek() === "<") {
+        this.advance();
+        return new Token(
+          TokenType.RANGE_EXCLUSIVE,
+          "..<",
+          startLine,
+          startColumn
+        );
+      }
+      if (this.peek() === "=") {
+        this.advance();
+        return new Token(
+          TokenType.RANGE_INCLUSIVE,
+          "..=",
+          startLine,
+          startColumn
+        );
+      }
+      // Default range (inclusive)
+      return new Token(TokenType.RANGE, "..", startLine, startColumn);
+    }
+    if (twoChar === "<-") {
+      this.advance(2);
+      return new Token(TokenType.LEFT_ARROW, "<-", startLine, startColumn);
+    }
+    if (twoChar === "->") {
+      this.advance(2);
+      return new Token(TokenType.ARROW, "->", startLine, startColumn);
+    }
+    if (twoChar === "=>") {
+      this.advance(2);
+      return new Token(TokenType.FAT_ARROW, "=>", startLine, startColumn);
+    }
+    if (twoChar === "<=") {
+      this.advance(2);
+      return new Token(TokenType.LTE, "<=", startLine, startColumn);
+    }
+    if (twoChar === ">=") {
+      this.advance(2);
+      return new Token(TokenType.GTE, ">=", startLine, startColumn);
+    }
+    if (twoChar === "==") {
+      this.advance(2);
+      return new Token(TokenType.EQ, "==", startLine, startColumn);
+    }
+    if (twoChar === "!=") {
+      this.advance(2);
+      return new Token(TokenType.NEQ, "!=", startLine, startColumn);
+    }
+    if (twoChar === "++") {
+      this.advance(2);
+      return new Token(TokenType.INCREMENT, "++", startLine, startColumn);
+    }
+    if (twoChar === "--") {
+      this.advance(2);
+      return new Token(TokenType.DECREMENT, "--", startLine, startColumn);
+    }
+    if (twoChar === "+=") {
+      this.advance(2);
+      return new Token(TokenType.PLUS_ASSIGN, "+=", startLine, startColumn);
+    }
+    if (twoChar === "-=") {
+      this.advance(2);
+      return new Token(TokenType.MINUS_ASSIGN, "-=", startLine, startColumn);
+    }
+    if (twoChar === "*=") {
+      this.advance(2);
+      return new Token(TokenType.STAR_ASSIGN, "*=", startLine, startColumn);
+    }
+    if (twoChar === "/=") {
+      this.advance(2);
+      return new Token(TokenType.SLASH_ASSIGN, "/=", startLine, startColumn);
+    }
+
+    // Single-character tokens
+    this.advance();
+    switch (char) {
+      case "+":
+        return new Token(TokenType.PLUS, "+", startLine, startColumn);
+      case "-":
+        return new Token(TokenType.MINUS, "-", startLine, startColumn);
+      case "*":
+        return new Token(TokenType.STAR, "*", startLine, startColumn);
+      case "/":
+        return new Token(TokenType.SLASH, "/", startLine, startColumn);
+      case "%":
+        return new Token(TokenType.PERCENT, "%", startLine, startColumn);
+      case "^":
+        return new Token(TokenType.CARET, "^", startLine, startColumn);
+      case "@":
+        return new Token(TokenType.AT, "@", startLine, startColumn);
+      case "|":
+        return new Token(TokenType.PIPE, "|", startLine, startColumn);
+      case "<":
+        return new Token(TokenType.LT, "<", startLine, startColumn);
+      case ">":
+        return new Token(TokenType.GT, ">", startLine, startColumn);
+      case "=":
+        return new Token(TokenType.ASSIGN, "=", startLine, startColumn);
+      case "!":
+        return new Token(TokenType.NOT, "!", startLine, startColumn);
+      case "(":
+        return new Token(TokenType.LPAREN, "(", startLine, startColumn);
+      case ")":
+        return new Token(TokenType.RPAREN, ")", startLine, startColumn);
+      case "{":
+        return new Token(TokenType.LBRACE, "{", startLine, startColumn);
+      case "}":
+        return new Token(TokenType.RBRACE, "}", startLine, startColumn);
+      case "[":
+        return new Token(TokenType.LBRACKET, "[", startLine, startColumn);
+      case "]":
+        return new Token(TokenType.RBRACKET, "]", startLine, startColumn);
+      case ",":
+        return new Token(TokenType.COMMA, ",", startLine, startColumn);
+      case ":":
+        return new Token(TokenType.COLON, ":", startLine, startColumn);
+      case ";":
+        return new Token(TokenType.SEMICOLON, ";", startLine, startColumn);
+      case ".":
+        return new Token(TokenType.DOT, ".", startLine, startColumn);
+      case "?":
+        return new Token(TokenType.QUESTION, "?", startLine, startColumn);
+      case "~":
+        return new Token(TokenType.TILDE, "~", startLine, startColumn);
+      default:
+        throw new Error(
+          `Unexpected character '${char}' at ${startLine}:${startColumn}`
+        );
+    }
+  }
+
+  /**
+   * Read a number (int or float)
+   * Also handles imaginary literals: 4i or 4j
+   */
+  readNumber(line, column) {
+    let num = "";
+    let isFloat = false;
+
+    while (this.isDigit(this.peek()) || this.peek() === ".") {
+      if (this.peek() === ".") {
+        // Check if it's a range operator
+        if (
+          this.peek(1) === "." ||
+          this.peek(1) === "<" ||
+          this.peek(1) === "="
+        ) {
+          break;
+        }
+        if (isFloat) {
+          throw new Error(
+            `Invalid number: multiple decimal points at ${line}:${column}`
+          );
+        }
+        isFloat = true;
+      }
+      num += this.peek();
+      this.advance();
+    }
+
+    // Check for scientific notation (e.g., 1e-12, 3.14e+5, 2.5E10)
+    if (this.peek() === "e" || this.peek() === "E") {
+      num += this.peek();
+      this.advance();
+      isFloat = true;
+
+      // Optional sign after e/E
+      if (this.peek() === "+" || this.peek() === "-") {
+        num += this.peek();
+        this.advance();
+      }
+
+      // Must have at least one digit after e/E
+      if (!this.isDigit(this.peek())) {
+        throw new Error(
+          `Invalid number: expected digit after exponent at ${line}:${column}`
+        );
+      }
+
+      while (this.isDigit(this.peek())) {
+        num += this.peek();
+        this.advance();
+      }
+    }
+
+    const value = isFloat ? parseFloat(num) : parseInt(num, 10);
+
+    // Check for imaginary suffix (i or j)
+    if (this.peek() === "i" || this.peek() === "j") {
+      this.advance(); // consume the i/j
+      return new Token(TokenType.IMAGINARY, value, line, column);
+    }
+
+    return new Token(TokenType.NUMBER, value, line, column);
+  }
+
+  /**
+   * Read a string literal
+   */
+  readString(quote, line, column) {
+    this.advance(); // Skip opening quote
+    let str = "";
+
+    while (this.peek() !== quote && this.pos < this.source.length) {
+      if (this.peek() === "\\") {
+        this.advance();
+        const escaped = this.peek();
+        switch (escaped) {
+          case "n":
+            str += "\n";
+            break;
+          case "t":
+            str += "\t";
+            break;
+          case "r":
+            str += "\r";
+            break;
+          case "\\":
+            str += "\\";
+            break;
+          case quote:
+            str += quote;
+            break;
+          default:
+            str += escaped;
+        }
+        this.advance();
+      } else {
+        str += this.peek();
+        this.advance();
+      }
+    }
+
+    if (this.peek() !== quote) {
+      throw new Error(`Unterminated string at ${line}:${column}`);
+    }
+
+    this.advance(); // Skip closing quote
+    return new Token(TokenType.STRING, str, line, column);
+  }
+
+  /**
+   * Read an f-string (string interpolation)
+   */
+  readFString(line, column) {
+    this.advance(); // Skip 'f'
+    const quote = this.peek();
+    this.advance(); // Skip opening quote
+
+    let parts = [];
+    let currentPart = "";
+
+    while (this.peek() !== quote && this.pos < this.source.length) {
+      if (this.peek() === "{") {
+        if (currentPart) {
+          parts.push({ type: "literal", value: currentPart });
+          currentPart = "";
+        }
+
+        this.advance(); // Skip {
+        let expr = "";
+        let braceCount = 1;
+
+        while (braceCount > 0 && this.pos < this.source.length) {
+          if (this.peek() === "{") braceCount++;
+          if (this.peek() === "}") braceCount--;
+          if (braceCount > 0) {
+            expr += this.peek();
+          }
+          this.advance();
+        }
+
+        parts.push({ type: "interpolation", value: expr });
+      } else {
+        currentPart += this.peek();
+        this.advance();
+      }
+    }
+
+    if (currentPart) {
+      parts.push({ type: "literal", value: currentPart });
+    }
+
+    if (this.peek() !== quote) {
+      throw new Error(`Unterminated f-string at ${line}:${column}`);
+    }
+
+    this.advance(); // Skip closing quote
+
+    return new Token(TokenType.F_STRING_START, parts, line, column);
+  }
+
+  /**
+   * Read an identifier or keyword
+   * Supports prime notation: identifier followed by ' (apostrophe)
+   * Examples: sum', count', x'
+   */
+  readIdentifier(line, column) {
+    let id = "";
+
+    while (this.isAlphaNumeric(this.peek()) || this.peek() === "_") {
+      id += this.peek();
+      this.advance();
+    }
+
+    // Check for prime notation (apostrophe after identifier)
+    // This is used for loop next-state assignments: sum' = sum + x
+    // Multiple primes are allowed: x', x'', x''' etc.
+    while (this.peek() === "'") {
+      id += "'";
+      this.advance();
+    }
+
+    const type = KEYWORDS[id] || TokenType.IDENTIFIER;
+    return new Token(type, id, line, column);
+  }
+
+  /**
+   * Skip whitespace (but not newlines - those might be significant)
+   */
+  skipWhitespace() {
+    while (this.pos < this.source.length) {
+      const char = this.peek();
+      if (char === " " || char === "\t" || char === "\r" || char === "\n") {
+        if (char === "\n") {
+          this.line++;
+          this.column = 1;
+        } else {
+          this.column++;
+        }
+        this.pos++;
+      } else {
+        break;
+      }
+    }
+  }
+
+  /**
+   * Skip comment (// to end of line)
+   */
+  skipComment() {
+    while (this.peek() !== "\n" && this.pos < this.source.length) {
+      this.advance();
+    }
+  }
+
+  /**
+   * Peek at character at current position + offset
+   */
+  peek(offset = 0) {
+    const pos = this.pos + offset;
+    return pos < this.source.length ? this.source[pos] : null;
+  }
+
+  /**
+   * Advance position by n characters
+   */
+  advance(n = 1) {
+    for (let i = 0; i < n; i++) {
+      if (this.pos < this.source.length) {
+        if (this.source[this.pos] === "\n") {
+          this.line++;
+          this.column = 1;
+        } else {
+          this.column++;
+        }
+        this.pos++;
+      }
+    }
+  }
+
+  isDigit(char) {
+    if (char === null || char === undefined) return false;
+    return char >= "0" && char <= "9";
+  }
+
+  isAlpha(char) {
+    if (char === null || char === undefined) return false;
+    return (char >= "a" && char <= "z") || (char >= "A" && char <= "Z");
+  }
+
+  isAlphaNumeric(char) {
+    if (char === null || char === undefined) return false;
+    return this.isAlpha(char) || this.isDigit(char);
+  }
+}
+
+/**
+ * Helper function to tokenize source code
+ */
+export function tokenizeSource(source, filename) {
+  const lexer = new Lexer(source, filename);
+  return lexer.tokenize();
+}
+
+export const tokenize = tokenizeSource;