roll-parser 3.0.0-alpha.0 → 3.0.0

package/src/lexer/lexer.ts

@@ -4,25 +4,73 @@
  * @module lexer/lexer
  */
 
-import type { RollParserErrorCode } from '../errors';
-import { RollParserError } from '../errors';
-import { type Token, TokenType } from './tokens';
+import type { RollParserErrorCode } from '../errors.js';
+import { describeValue, RollParserError } from '../errors.js';
+import { type Token, TokenType } from './tokens.js';
 
 /**
  * Error thrown when the lexer encounters an invalid character.
+ *
+ * `position` is a zero-based UTF-16 offset into the input. It is deliberately
+ * absent from `message` — read it from the field, or uniformly across all
+ * roll-parser errors via `getErrorSpan`.
+ *
+ * Codes: `UNEXPECTED_CHARACTER` for a character that cannot start any token,
+ * `UNEXPECTED_IDENTIFIER` for a word that is not a known keyword.
+ *
+ * @example
+ * ```typescript
+ * import { LexerError, roll } from 'roll-parser';
+ *
+ * try {
+ *   roll('2d6+&');
+ * } catch (error) {
+ *   const typed = error as LexerError;
+ *   typed.code; // 'UNEXPECTED_CHARACTER'
+ *   typed.character; // '&'
+ *   typed.position; // 4
+ * }
+ * ```
+ *
+ * @category Errors
  */
 export class LexerError extends RollParserError {
+  /** Zero-based UTF-16 offset of the offending character in the input. */
   readonly position: number;
+  /**
+   * The offending text — a single character for `UNEXPECTED_CHARACTER` (the
+   * whole code point, so astral symbols are not split into surrogates), or
+   * the unrecognized word for `UNEXPECTED_IDENTIFIER`.
+   */
   readonly character: string;
 
-  constructor(message: string, code: RollParserErrorCode, position: number, character: string) {
-    super(`${message} at position ${position}: '${character}'`, code);
+  constructor(
+    message: string,
+    code: RollParserErrorCode,
+    position: number,
+    character: string,
+    options?: ErrorOptions,
+  ) {
+    super(`${message}: '${character}'`, code, options);
     this.name = 'LexerError';
     this.position = position;
     this.character = character;
   }
 }
 
+//
+// * Character codes
+//
+
+// Range tests compare code units: `char.toLowerCase()` per character allocated
+// a string on the lexer's hottest loop.
+const CHAR_DIGIT_0 = 48;
+const CHAR_DIGIT_9 = 57;
+const CHAR_UPPER_A = 65;
+const CHAR_UPPER_Z = 90;
+const CHAR_LOWER_A = 97;
+const CHAR_LOWER_Z = 122;
+
 /** Known identifier keywords mapped to their token types. */
 const IDENTIFIER_KEYWORDS: Record<string, TokenType> = {
   kh: TokenType.KEEP_HIGH,
@@ -39,10 +87,33 @@ const IDENTIFIER_KEYWORDS: Record<string, TokenType> = {
   ceil: TokenType.FUNCTION,
   round: TokenType.FUNCTION,
   abs: TokenType.FUNCTION,
+  sqrt: TokenType.FUNCTION,
+  pow: TokenType.FUNCTION,
   max: TokenType.FUNCTION,
   min: TokenType.FUNCTION,
+  s: TokenType.SORT_ASC,
+  sa: TokenType.SORT_ASC,
+  sd: TokenType.SORT_DESC,
+  cs: TokenType.CRIT_SUCCESS,
+  cf: TokenType.CRIT_FAIL,
 };
 
+/**
+ * Builds a hint for identifiers that start with a known keyword. Maximal
+ * munch merges adjacent modifiers when the first has no count — `4d6khs`
+ * lexes as one identifier `khs` instead of `kh` + `s`. Point the user at the
+ * explicit-count (or whitespace) split.
+ */
+function buildIdentifierHint(identifier: string): string {
+  for (let length = identifier.length - 1; length >= 1; length--) {
+    const prefix = identifier.slice(0, length);
+    if (IDENTIFIER_KEYWORDS[prefix] == null) continue;
+    const rest = identifier.slice(length);
+    return ` (did you mean '${prefix}' followed by '${rest}'? separate modifiers with a count or space, e.g. '${prefix}1${rest}')`;
+  }
+  return '';
+}
+
 /**
  * Lexer for dice notation.
  *
@@ -85,17 +156,18 @@ export class Lexer {
     const startPos = this.pos;
     const char = this.peek();
 
-    // * Numbers
     if (this.isDigit(char)) {
       return this.scanNumber();
     }
 
-    // * Identifiers (d, kh, kl, dh, dl, r, ro, f, vs, floor, ceil, ...)
     if (this.isAlpha(char)) {
       return this.scanIdentifier();
     }
 
-    // * Operators and punctuation
+    if (char === '@') {
+      return this.scanAt();
+    }
+
     this.advance();
 
     switch (char) {
@@ -120,6 +192,10 @@ export class Lexer {
         return this.createTokenAt(TokenType.RPAREN, char, startPos);
       case ',':
         return this.createTokenAt(TokenType.COMMA, char, startPos);
+      case '{':
+        return this.createTokenAt(TokenType.LBRACE, char, startPos);
+      case '}':
+        return this.createTokenAt(TokenType.RBRACE, char, startPos);
       case '>':
         if (this.match('=')) {
           return this.createTokenAt(TokenType.GREATER_EQUAL, '>=', startPos);
@@ -136,17 +212,23 @@ export class Lexer {
         if (this.match('!')) {
           return this.createTokenAt(TokenType.EXPLODE_COMPOUND, '!!', startPos);
         }
-        if (!this.isAtEnd() && this.peek().toLowerCase() === 'p') {
-          this.advance();
+        if (this.match('p') || this.match('P')) {
           return this.createTokenAt(TokenType.EXPLODE_PENETRATING, '!p', startPos);
         }
         return this.createTokenAt(TokenType.EXPLODE, char, startPos);
-      default:
-        throw new LexerError('Unexpected character', 'UNEXPECTED_CHARACTER', startPos, char);
+      default: {
+        // Surrogate pairs (emoji, astral symbols) span two code units —
+        // report the full code point instead of a lone surrogate ('�').
+        const codePoint = this.input.codePointAt(startPos);
+        const display = codePoint == null ? char : String.fromCodePoint(codePoint);
+        throw new LexerError('Unexpected character', 'UNEXPECTED_CHARACTER', startPos, display);
+      }
     }
   }
 
+  //
   // * Private helpers
+  //
 
   private skipWhitespace(): void {
     while (!this.isAtEnd() && this.isWhitespace(this.peek())) {
@@ -154,24 +236,23 @@ export class Lexer {
     }
   }
 
+  // Scanners slice once from a recorded start offset rather than accumulating
+  // `value += this.advance()` — one string per token instead of one per character.
   private scanNumber(): Token {
     const startPos = this.pos;
-    let value = '';
 
-    // Integer part
     while (!this.isAtEnd() && this.isDigit(this.peek())) {
-      value += this.advance();
+      this.pos++;
     }
 
-    // Decimal part
     if (!this.isAtEnd() && this.peek() === '.' && this.isDigit(this.peekNext())) {
-      value += this.advance(); // consume '.'
+      this.pos++;
       while (!this.isAtEnd() && this.isDigit(this.peek())) {
-        value += this.advance();
+        this.pos++;
       }
     }
 
-    return this.createTokenAt(TokenType.NUMBER, value, startPos);
+    return this.createTokenAt(TokenType.NUMBER, this.input.slice(startPos, this.pos), startPos);
   }
 
   /**
@@ -192,18 +273,15 @@ export class Lexer {
     const first = this.peek();
     const second = this.peekNext();
     if ((first === 'd' || first === 'D') && (second === 'f' || second === 'F')) {
-      this.advance();
-      this.advance();
+      this.pos += 2;
       return this.createTokenAt(TokenType.DICE_FATE, 'df', startPos);
     }
 
-    let value = '';
-
     while (!this.isAtEnd() && this.isAlpha(this.peek())) {
-      value += this.advance();
+      this.pos++;
     }
 
-    const lower = value.toLowerCase();
+    const lower = this.input.slice(startPos, this.pos).toLowerCase();
 
     if (lower === 'd' && !this.isAtEnd() && this.peek() === '%') {
       this.advance();
@@ -215,7 +293,55 @@ export class Lexer {
       return this.createTokenAt(tokenType, lower, startPos);
     }
 
-    throw new LexerError('Unexpected identifier', 'UNEXPECTED_IDENTIFIER', startPos, lower);
+    throw new LexerError(
+      `Unexpected identifier${buildIdentifierHint(lower)}`,
+      'UNEXPECTED_IDENTIFIER',
+      startPos,
+      lower,
+    );
+  }
+
+  /**
+   * Scans a variable reference introduced by `@`.
+   *
+   * Two forms:
+   * - Bare: `@name` where `name` matches `[A-Za-z_][A-Za-z0-9_]*` (case preserved).
+   * - Braced: `@{name}` where `name` is any run of printable characters except
+   *   `}` and newline (permits spaces, hyphens, digits).
+   *
+   * Case is preserved — distinct from `scanIdentifier`, which lowercases the
+   * captured value. The emitted token's `value` is the variable name without
+   * the leading `@` or the surrounding braces.
+   */
+  private scanAt(): Token {
+    const startPos = this.pos;
+    this.advance();
+
+    let name: string;
+    if (!this.isAtEnd() && this.peek() === '{') {
+      this.advance();
+      const nameStart = this.pos;
+      while (!this.isAtEnd() && this.peek() !== '}' && this.peek() !== '\n') {
+        this.advance();
+      }
+      if (this.isAtEnd() || this.peek() !== '}') {
+        throw new LexerError('Unterminated @{...} variable', 'UNEXPECTED_CHARACTER', startPos, '@');
+      }
+      name = this.input.slice(nameStart, this.pos);
+      this.advance();
+    } else {
+      const nameStart = this.pos;
+      if (this.isAtEnd() || !this.isIdentifierStart(this.peek())) {
+        throw new LexerError('Empty @ variable name', 'UNEXPECTED_CHARACTER', startPos, '@');
+      }
+      this.advance();
+      while (!this.isAtEnd() && this.isIdentifierPart(this.peek())) {
+        this.advance();
+      }
+      name = this.input.slice(nameStart, this.pos);
+    }
+
+    return this.createTokenAt(TokenType.AT, name, startPos);
   }
 
   private peek(): string {
@@ -241,41 +367,81 @@ export class Lexer {
     return this.pos >= this.input.length;
   }
 
+  // `NaN` from an empty `peek()` fails every comparison, so end-of-input
+  // still reads as "not a digit / not alpha" without an extra guard.
   private isDigit(char: string): boolean {
-    return char >= '0' && char <= '9';
+    const code = char.charCodeAt(0);
+    return code >= CHAR_DIGIT_0 && code <= CHAR_DIGIT_9;
   }
 
   private isAlpha(char: string): boolean {
-    const c = char.toLowerCase();
-    return c >= 'a' && c <= 'z';
+    const code = char.charCodeAt(0);
+    return (
+      (code >= CHAR_LOWER_A && code <= CHAR_LOWER_Z) ||
+      (code >= CHAR_UPPER_A && code <= CHAR_UPPER_Z)
+    );
+  }
+
+  private isIdentifierStart(char: string): boolean {
+    return this.isAlpha(char) || char === '_';
+  }
+
+  private isIdentifierPart(char: string): boolean {
+    return this.isAlpha(char) || this.isDigit(char) || char === '_';
   }
 
   private isWhitespace(char: string): boolean {
     return char === ' ' || char === '\t' || char === '\n' || char === '\r';
   }
 
+  // Both factories run after the token's characters were consumed, so
+  // `this.pos` is the exclusive end offset.
   private createToken(type: TokenType, value: string): Token {
-    return { type, value, position: this.pos };
+    return { type, value, position: this.pos, end: this.pos };
   }
 
   private createTokenAt(type: TokenType, value: string, position: number): Token {
-    return { type, value, position };
+    return { type, value, position, end: this.pos };
   }
 }
 
 /**
- * Tokenize a dice notation string.
+ * Tokenizes a dice notation string. The first stage of the pipeline —
+ * {@link parse} calls it for you; reach for `lex` directly only to build a
+ * syntax highlighter or an editor integration.
+ *
+ * Notation is case-insensitive and whitespace-tolerant: `2D20 + 5` and
+ * `2d20+5` produce the same tokens, and identifier tokens carry a lowercased
+ * `value`. The one exception is `@name`, whose case is preserved.
  *
  * @param input - The dice notation to tokenize
- * @returns Array of tokens including EOF
- * @throws {LexerError} If an invalid character is encountered
+ * @returns Every token in source order, always ending with one
+ *   `TokenType.EOF` token
+ * @throws {LexerError} If an invalid character or unknown identifier is found
+ * @throws {RollParserError} `INVALID_NOTATION_TYPE` when `input` is not a
+ *   string — raised before scanning, so it carries no position
  *
  * @example
  * ```typescript
+ * import { lex, TokenType } from 'roll-parser';
+ *
  * const tokens = lex('2d20+5');
- * // [NUMBER(2), DICE, NUMBER(20), PLUS, NUMBER(5), EOF]
+ * tokens.length; // 6 — NUMBER DICE NUMBER PLUS NUMBER EOF
+ * tokens[0]; // { type: TokenType.NUMBER, value: '2', position: 0, end: 1 }
+ * tokens[1].type === TokenType.DICE; // true
+ * tokens.at(-1)?.type === TokenType.EOF; // true
  * ```
+ *
+ * @category Core
  */
 export function lex(input: string): Token[] {
+  // ! The pipeline's only notation type guard — `parse` and `roll` both funnel through here.
+  if (typeof input !== 'string') {
+    throw new RollParserError(
+      `Notation must be a string, received ${describeValue(input)}`,
+      'INVALID_NOTATION_TYPE',
+    );
+  }
+
   return new Lexer(input).tokenize();
 }

package/src/lexer/tokens.ts

@@ -9,6 +9,8 @@
  *
  * Grouped semantically. Numeric values are stable identifiers — the specific
  * numbers don't matter, but they must be unique.
+ *
+ * @category AST
  */
 export enum TokenType {
   //
@@ -116,7 +118,11 @@ export enum TokenType {
   // * Functions
   //
 
-  /** Math function: 'floor', 'ceil', 'round', 'abs', 'max', 'min' */
+  /**
+   * Math function: 'floor', 'ceil', 'round', 'abs', 'sqrt', 'pow', 'max',
+   * 'min'. In postfix position, 'min'/'max' double as per-die clamp
+   * modifiers (`4d6min2`) — the parser decides by position, not the lexer.
+   */
   FUNCTION = 28,
 
   //
@@ -126,22 +132,81 @@ export enum TokenType {
   /** Versus operator: 'vs' */
   VS = 29,
 
+  //
+  // * Group boundaries
+  //
+
+  /** Left brace: '{' */
+  LBRACE = 30,
+  /** Right brace: '}' */
+  RBRACE = 31,
+
+  //
+  // * Variables
+  //
+
+  /** Variable reference prefix: '@' */
+  AT = 32,
+
+  //
+  // * Sort modifiers
+  //
+
+  /** Ascending sort: 's' or 'sa' */
+  SORT_ASC = 33,
+  /** Descending sort: 'sd' */
+  SORT_DESC = 34,
+
+  //
+  // * Crit thresholds
+  //
+
+  /** Critical success threshold: 'cs' */
+  CRIT_SUCCESS = 35,
+  /** Critical failure threshold: 'cf' */
+  CRIT_FAIL = 36,
+
   //
   // * End of input
   //
 
   /** End of input marker */
-  EOF = 30,
+  EOF = 37,
 }
 
 /**
- * A token produced by the lexer.
+ * A token produced by {@link lex}: what it is, the text it came from, and
+ * where in the input that text sits.
+ *
+ * `position`/`end` are a half-open range, so `input.slice(position, end)`
+ * recovers the original source text — useful for syntax highlighting, where
+ * `value` alone is lossy.
+ *
+ * @example
+ * ```typescript
+ * import { lex, TokenType } from 'roll-parser';
+ *
+ * const input = '4D6KH3';
+ * const tokens = lex(input);
+ * tokens[1]; // { type: TokenType.DICE, value: 'd', position: 1, end: 2 }
+ * tokens[3].type === TokenType.KEEP_HIGH; // true
+ * input.slice(tokens[3].position, tokens[3].end); // 'KH' — original casing
+ * tokens[3].value; // 'kh' — normalized
+ * ```
+ *
+ * @category AST
  */
 export type Token = {
   /** The type of this token */
-  type: TokenType;
+  readonly type: TokenType;
   /** The raw string value from input (lowercased for identifiers) */
-  value: string;
-  /** Zero-based position in the input string */
-  position: number;
+  readonly value: string;
+  /** Zero-based start offset in the input string (UTF-16 code units) */
+  readonly position: number;
+  /**
+   * Zero-based end offset (exclusive). Not always `position + value.length` —
+   * braced variables (`@{name}` stores only `name`) and case-normalized
+   * identifiers consume more input than their `value` retains.
+   */
+  readonly end: number;
 };