solve-engine 1.0.0-beta.0 → 1.0.0-beta.2

package/dist/Lexer-D9l4Yrk2.d.ts

@@ -0,0 +1,531 @@
+import { T as Token } from './Token-0jpvUdPY.js';
+
+/** Trie node for multi-word phrase matching. */
+interface PhraseNode {
+    /** Complete phrase token type (null = intermediate node) */
+    type?: string;
+    children: Map<string, PhraseNode>;
+}
+/**
+ * The optimized lookup structure built by TokenClassRegistry.build().
+ * Consumed by the Lexer for O(1) keyword → token type lookups and
+ * O(word-count) phrase matching.
+ */
+interface TokenLookup {
+    /** Lowercase keyword → token type. O(1) Map lookup. */
+    keywordToType: Map<string, string>;
+    /** Phrase trie for multi-word matching. Root node with children maps.
+     *  Null if no phrases registered. */
+    phraseTrie: PhraseNode | null;
+    /** Set of lowercase first-words of all registered phrases.
+     *  Used by the lexer to emit IDENT (not a phrase keyword) for words
+     *  that start multi-word phrases, deferring to the PhraseMatcher.
+     *
+     *  Example: "to" is in phraseStartWords because "to the power of" is a phrase.
+     *  When the lexer sees "to", it emits IDENT and lets the phrase matcher
+     *  combine "to the power of" into a single CARET token.
+     *
+     *  This prevents plugins from accidentally overriding phrase-start words.
+     */
+    phraseStartWords: Set<string>;
+    /** Case-sensitive unit names for UNIT fallback after keyword lookup fails. */
+    unitNames: ReadonlySet<string>;
+}
+
+/** What a line is, structurally, before anything tries to evaluate it. */
+type MarkdownLineType = 'expression' | 'prose' | 'heading' | 'blockquote' | 'list' | 'code_fence' | 'math_fence' | 'table' | 'table_separator' | 'hr' | 'wikilink' | 'comment' | 'empty';
+/**
+ * What a line is and whether it holds anything to evaluate.
+ *
+ * Produced by a character-level scan that never consults the keyword, unit or
+ * operator tables, so any lexer gives the same answer for the same line. See
+ * `__tests__/lexer/LineClassificationIsVocabularyIndependent.spec.ts`.
+ */
+interface LineClassification {
+    /** The type of this markdown line */
+    type: MarkdownLineType;
+    /** Whether this line should be skipped (no expression evaluation) */
+    skip: boolean;
+    /** Whether the line contains inline solve markers (`s`...``) */
+    hasInlineSolve: boolean;
+}
+/** Inline solve position with precise coordinates */
+interface InlineSolveSpan {
+    /** Character offset of the `s`` marker */
+    start: number;
+    /** Character offset past the closing `` ` `` */
+    end: number;
+    /** The expression text between the backticks */
+    expression: string;
+    /** 1-based column of the `s`` marker */
+    columnNumber: number;
+    /** Token index of INLINE_SOLVE_START in the line's token array (set during tokenization). */
+    startTokenIndex?: number;
+    /** Token index of closing BACKTICK_OPEN in the line's token array (set during tokenization). */
+    endTokenIndex?: number;
+}
+/**
+ * Result from a single line processed by scanDocument().
+ * Combines line classification, tokenized tokens, and inline solve spans
+ * into a single structure, eliminating the need for separate classifyLine()
+ * findInlineSolves(), and per-line Lexer.reset() calls.
+ */
+interface ScanLineResult {
+    /** The raw line text (without trailing newline). */
+    text: string;
+    /** 1-based line number within the document. */
+    lineNumber: number;
+    /** Character offset of the line start within the document. */
+    startOffset: number;
+    /** Character offset of the line end (before newline). */
+    endOffset: number;
+    /** The line classification. */
+    classification: LineClassification;
+    /** Tokenized tokens (empty array if line is skipped). */
+    tokens: Token[];
+    /** Inline solve spans found in this line (empty if none). */
+    inlineSolves: InlineSolveSpan[];
+}
+/**
+ * A token, as the lexer produces it.
+ *
+ * A class rather than an object literal because tokens are created on every
+ * keystroke and a shared hidden class keeps that path predictable for the
+ * engine running it.
+ */
+declare class LexerToken implements Token {
+    type: string;
+    typeId: number;
+    value: string;
+    text: string;
+    offset: number;
+    lineBreaks: number;
+    line: number;
+    col: number;
+    constructor(type: string, typeId: number, value: string, text: string, offset: number, lineBreaks: number, line: number, col: number);
+}
+/**
+ * Plugin interface for extending the ExpressionLexer with custom tokens.
+ *
+ * Plugins can register:
+ * - `keywords`: Map identifier strings to custom token types (checked after locale keywords).
+ * - `operators`: Map multi-character operator sequences to custom token types.
+ * - `units`: Register additional unit identifiers (checked alongside built-in units).
+ *
+ * Multi-word phrase matching has been moved to the TokenNormalizer post-lexer
+ * stage. To register phrase patterns, use `IEnginePackage.normalizerRules` instead.
+ *
+ * All registrations are additive, built-in patterns still work.
+ */
+interface LexerVocabulary {
+    /**
+     * Keyword → tokenType mappings. Each key is a lowercase identifier that,
+     * when encountered, will emit the specified token type instead of IDENT.
+     * These are checked AFTER the locale's built-in keywordMap, so locale
+     * keywords take priority.
+     */
+    keywords?: Record<string, string>;
+    /**
+     * Multi-character operator → tokenType mappings. Each key is the exact
+     * character sequence (e.g., "::", "->", "=>") and the value is the token
+     * type to emit. Two-character operators take priority during matching.
+     * Built-in operators (==, !=, >=, <=, <<, >>) always take priority.
+     */
+    operators?: Record<string, string>;
+    /**
+     * Additional unit identifiers to recognize (e.g., "gp", "osrs", "tile").
+     * These are checked alongside the built-in `knownUnits` set.
+     */
+    units?: string[];
+    /**
+     * Whole-line patterns matched against the RAW line text, BEFORE any
+     * per-character tokenization begins.
+     *
+     * Every other extension point in this file (`keywords`/`operators`/
+     * `units`, plus `IEnginePackage.phrases`/`normalizerRules`) transforms
+     * a token STREAM -- they all assume the line is, at some granularity,
+     * valid Solve syntax. This hook exists for the one shape that isn't:
+     * a package whose grammar captures arbitrary free-form text terminated
+     * by a fixed marker (e.g. a natural-language query ending in `= ?`),
+     * where the text itself ("distance to the moon") would never tokenize
+     * or parse as a normal expression and must be captured verbatim
+     * instead -- see `packages/knowledge/` for the reference use.
+     *
+     * Each entry's `pattern` is tested (via `RegExp.exec`) against the
+     * full, untrimmed line text. If it matches AND capture group 1 is
+     * non-empty after trimming, the ENTIRE line becomes a single
+     * synthetic token of `tokenType` whose `value`/`text` is the trimmed
+     * capture group -- the character-by-character scanner never runs for
+     * that line. Patterns are tried in registration order; the first
+     * match wins. A package registering a rule here still needs a
+     * `prefixParselets` entry for `tokenType` to actually consume the
+     * resulting token.
+     *
+     * Because this bypasses tokenization entirely, a matching line can
+     * contain characters that would otherwise be lexer errors (unmatched
+     * quotes, stray symbols, ...) -- by design, since the whole point is
+     * to hand the package raw text the normal pipeline was never meant to
+     * parse.
+     */
+    rawLinePatterns?: Array<{
+        pattern: RegExp;
+        tokenType: string;
+    }>;
+}
+/**
+ * Character-by-character tokenizer for expression text.
+ *
+ * Scans a raw line/expression string into a stream of typed tokens
+ * (numbers, identifiers, operators, units, keywords, ...), handling
+ * markdown-line classification (`classifyLine`), inline `` s`...` `` solve
+ * spans, and package-contributed vocabulary (registered via
+ * {@link registerVocabulary}/{@link unregisterVocabulary}, keywords
+ * operators, and units a package wants recognized as their own token
+ * types rather than falling through to generic identifiers).
+ *
+ * Most consumers should use the higher-level {@link Lexer} wrapper, which
+ * adds streaming `next()`/`peek()` access over this class's scan results.
+ */
+declare class ExpressionLexer {
+    private static readonly CHAR_CLASS;
+    /**
+     * Configured TokenLookup from TokenClassRegistry. When set, replaces
+     * the internal keyword map and unit set with registry-built equivalents.
+     * Enables data-driven keyword/unit registration across locale keywords,
+     * provider keywords, and plugins.
+     *
+     * Set at construction time via the constructor parameter. Plugin-registered
+     * keywords/units (via registerVocabulary()) are checked alongside
+     * the configuredLookup, neither source is bypassed.
+     */ private configuredLookup;
+    private input;
+    private pos;
+    private len;
+    private line;
+    private lineStartPos;
+    private keywordMap;
+    private mergedKeywords;
+    private mergedUnits;
+    private pluginKeywordMap;
+    private pluginOperators;
+    private pluginUnits;
+    private hasPluginOps;
+    private localeCode;
+    private locale;
+    /**
+     * Inline solve spans collected during the most recent tokenization pass.
+     * Populated by [Symbol.iterator]() and consumed by scanDocument().
+     */
+    _inlineSolveSpans: InlineSolveSpan[];
+    private pluginRawLinePatterns;
+    /**
+     * If a `rawLinePatterns` rule matches the FULL text most recently passed
+     * to {@link reset}, this holds the single synthetic token that
+     * {@link tokenizeAll} should return instead of running the
+     * character-by-character scanner. Cleared (re-evaluated) on every
+     * {@link reset} call. `null` when no plugin registered any raw-line
+     * patterns, or none matched, the overwhelmingly common case, checked
+     * with a `length === 0` guard before ever touching this field so a
+     * plugin-free lexer pays zero cost for the feature.
+     */
+    private pendingRawLineToken;
+    /**
+     * Test `text` against every registered `rawLinePatterns` rule, in
+     * registration order. Returns a synthetic token for the first rule
+     * whose `pattern` matches AND whose capture group 1 is non-empty after
+     * trimming; returns `null` if no rule matches (the normal
+     * character-by-character scanner should run instead).
+     */
+    private matchRawLine;
+    /** Rebuild merged keyword and unit collections after plugin registration. */
+    private rebuildMergedCollections;
+    constructor(localeCode?: string, lookup?: TokenLookup);
+    /**
+     * Register a plugin to extend the lexer with custom tokens.
+     *
+     * All registrations are additive, built-in patterns still work.
+     * Keywords, operators, and units from the plugin are merged
+     * with existing ones. Calling multiple times adds more entries.
+     *
+     * Note: multi-word phrases are now handled by the TokenNormalizer
+     * (see `IEnginePackage.normalizerRules`), not the lexer.
+     *
+     * Built-in tokens CANNOT be overridden. Throws a EngineError if the
+     * plugin attempts to register a keyword, operator, or unit
+     * that conflicts with a built-in one.
+     */
+    registerVocabulary(plugin: LexerVocabulary): void;
+    /**
+     * Unregister a plugin, removing its custom tokens from the lexer.
+     *
+     * This is the inverse of registerVocabulary(). All keywords, operators,
+     * and units registered by the plugin are removed. After
+     * unregistration, those tokens will revert to their default behavior
+     * (e.g., keywords become IDENT, operators become ERROR).
+     *
+     * Calling unregisterVocabulary with a plugin that was never registered
+     * is safe, it simply has no effect.
+     */
+    unregisterVocabulary(plugin: LexerVocabulary): void;
+    reset(input: string): void;
+    /**
+     * Scan a full document text in a single pass, classifying each line and
+     * tokenizing non-skipped lines.
+     *
+     * Replaces the separate classifyLine() + findInlineSolves() + per-line
+     * reset() + tokenizeAll() pattern with a single character-by-character
+     * walk through the entire document. Key benefits:
+     *
+     * - **Single reset()**: `this.pos`, `this.len`, `this.line`, and
+     *   `this.lineStartPos` are set once for the whole document, not per-line.
+     * - **Single classification**: classifyLine() runs once per line inline;
+     *   skipped lines are jumped over without tokenization.
+     * - **Shared tokenization**: Non-skipped lines are tokenized using the
+     *   existing state machine, yielding Token[] without per-line reset().
+     * - **Inline solve detection**: findInlineSolves() is called only for
+     *   lines that classifyLine() marks as having inline solves.
+     *
+     * Tokenization is scoped to each line by temporarily restricting
+     * `this.len` to the line end position, so the [Symbol.iterator]
+     * generator naturally stops at the line boundary. After tokenization,
+     * `this.len` is restored and `this.pos` advances past the newline.
+     *
+     * @param text The full document text (with newlines).
+     * @returns Array of ScanLineResult, one per line, in document order.
+     */
+    scanDocument(text: string): ScanLineResult[];
+    /**
+     * Tokenize an expression string into an array of Tokens.
+     *
+     * Delegates to the lazy [Symbol.iterator]() generator and collects all
+     * yielded tokens via Array.from(). For memory-sensitive use cases, prefer
+     * iterating the lexer directly with for...of to avoid array allocation.
+     *
+     * Optimizations:
+     *  - CHAR_CLASS jump table (Uint8Array) → switch on small integers
+     *  - Direct character-code dispatch (c0 cached pattern)
+     *  - Mathematical digit parsing (integer math, not slice+parseFloat)
+     *  - Inline operator tokenizer with two-char peek-ahead
+     *  - Whitespace eliminated in-lexer (never emitted)
+     *  - 0-char and 1-char fast paths
+     */
+    tokenizeAll(): Token[];
+    /**
+     * Lazy token-by-token generator. Yields each token without allocating an
+     * intermediate Token[] array. Supports for...of and spread usage.
+     *
+     * Usage:
+     *   for (const t of lexer) { ... }  // lazy, no array allocation
+     *   const tokens = [...lexer];       // materializes via spread
+     *   const tokens = lexer.tokenizeAll(); // materializes via Array.from()
+     *
+     * IMPORTANT: This generator captures `this.len` ONCE at creation time
+     * (const len = this.len). `scanDocument()` relies on this behavior to
+     * scope tokenization to a single line by temporarily restricting
+     * `this.len` to the line end position before creating the iterator.
+     * Do NOT refactor to re-read `this.len` mid-loop without also updating
+     * `scanDocument()`.
+     */
+    [Symbol.iterator](): Generator<Token, void, undefined>;
+    /**
+     * Character-by-character number parsing.
+     *
+     * Supports: integers, floats, scientific notation (1.5e10, 1.5e-10),
+     * hex (0xFF), binary (0b1010), octal (0o17), BigInt suffix (123n),
+     * thousands separators (1,234 or 1.234.567).
+     *
+     * Returns a LexerToken and advances `this.pos` past the number.
+     */
+    private tokenizeNumber;
+    /**
+     * Reads [a-zA-Z_][a-zA-Z0-9_]* and resolves to:
+     *   - A unit type (via knownUnits, case-sensitive)
+     *   - A keyword type (via locale keywordMap, case-insensitive)
+     *   - IDENT if none of the above
+     *
+     * Multi-word phrases (e.g., "to the power of") are handled by the
+     * TokenNormalizer post-lexer pass, not the lexer.
+     */
+    private tokenizeIdentifier;
+    /**
+     * Peek past in-expression whitespace (space, tab) from `pos` to check
+     * if the next significant character is '('.
+     */
+    private isFollowedByLParen;
+    /**
+     * Reads an operator/punctuation token.
+     * Handles two-char operators (==, !=, >=, <=, **) and the special
+     * cases << (LSHIFT) and >> (RSHIFT).
+     */
+    private tokenizeOperator;
+    /**
+     * Reads a double-quoted string literal. Supports backslash escapes.
+     */
+    private tokenizeString;
+    /**
+     * L1 expression gating: quickly determine if a line contains any
+     * characters that indicate an expression (digits, operators, currency,
+     * backticks, parentheses, etc.).
+     *
+     * Pure prose lines (e.g., "The quick brown fox jumps over the lazy dog")
+     * return false and can be skipped without full tokenization (L2).
+     *
+     * This is a fast character-by-character scan that stops at the first
+     * expression indicator. Called once per line in classifyFromPositions().
+     */
+    static hasExpressionIndicators(input: string, start: number, end: number): boolean;
+    /**
+     * Classify a line by its character positions within this.input.
+     * Reads directly from this.input using start/end boundaries.
+     * DOES NOT modify this.pos, purely a read-only classifier.
+     */
+    private classifyFromPositions;
+    /**
+     * Classify a single line of markdown text.
+     */
+    classifyLine(lineText: string): LineClassification;
+    /**
+     * Every keyword this lexer currently recognizes, locale keywords
+     * (`pi`, `sqrt`, `convert`, ...) merged with any plugin-contributed ones
+     * from `registerVocabulary()` (e.g. a package's custom keywords), mapped to
+     * the token type they lex to. A snapshot copy, not a live reference
+     * mutating the return value has no effect on the lexer.
+     */
+    getKeywords(): Record<string, string>;
+    /**
+     * Find all inline solve markers in a line with precise coordinate mapping.
+     */
+    findInlineSolves(lineText: string): InlineSolveSpan[];
+    /**
+     * Reads a comment: # to end of line, or // to end of line.
+     */
+    private tokenizeComment;
+}
+
+/**
+ * Lexer state machine modes.
+ * - Main: document-level scanning with markdown classification
+ * - Inline: expression embedded in markdown inline solve (`s\`...\``)
+ * - String: inside a double-quoted string literal
+ */
+declare enum LexerState {
+    Main = "main",
+    Inline = "inline",
+    String = "string"
+}
+
+/**
+ * Editor-agnostic semantic classification for a lexer token.
+ *
+ * This is the entire contract solve-js's language service exposes to any
+ * editor integration (CodeMirror, VS Code, ...): a category, nothing about
+ * how it should be rendered. CSS class names, VS Code semantic-token
+ * indices, etc. are all downstream adapter concerns. See
+ * `language/adapters/`.
+ *
+ * The built-in categories cover every core (non-plugin) token type. Solve-js
+ * packages may contribute additional, open-ended category strings for their
+ * own custom token types (see `registerTokenCategory` in TokenCategoryMap.ts)
+ *, the `(string & {})` union member keeps those assignable without widening
+ * the whole type to a bare `string` and losing autocomplete for the built-ins.
+ */
+type TokenCategory = "number" | "string" | "keyword" | "operator" | "comparison" | "bitwise" | "function" | "variable" | "unit" | "datetime" | "vector" | "punctuation" | "error" | (string & {});
+
+/**
+ * Public tokenizer wrapper around {@link ExpressionLexer}.
+ *
+ * `ExpressionLexer` does the actual character-by-character scanning;
+ * `Lexer` adds a materialized-token-array streaming interface
+ * (`next()`/`peek()`) plus line-classification state (`reset()`) so
+ * callers can iterate a line's tokens without re-scanning on each peek.
+ *
+ * Each `ExpressionEngine` instance owns its own `Lexer`, and packages
+ * extend it via {@link registerVocabulary} (keywords, operators, units)
+ * see `IEnginePackage.lexerVocabulary`.
+ */
+declare class Lexer {
+    /** Expression-mode lexer (Phase A: V8-optimized, replaces moo) */
+    private expressionLexer;
+    private currentState;
+    private peekedToken;
+    private hasPeeked;
+    private tokens;
+    private tokenIdx;
+    /**
+     * @param localeCode - Locale code (e.g., "en", "de"). Defaults to "en".
+     * @param tokenLookup - Optional TokenLookup from TokenClassRegistry.
+     *   When provided, configures ExpressionLexer to use registry-built
+     *   keyword/unit/phrase lookups instead of internal instance maps.
+     */
+    constructor(localeCode?: string, tokenLookup?: TokenLookup);
+    reset(input: string, state?: LexerState): void;
+    /**
+     * Classify a single line of markdown text (Phase B).
+     * Delegates to the ExpressionLexer's character-by-character scanner.
+     */
+    classifyLine(lineText: string): LineClassification;
+    /**
+     * Find all inline solve markers in a line (Phase B).
+     * Delegates to the ExpressionLexer's character-by-character scanner.
+     */
+    findInlineSolves(lineText: string): InlineSolveSpan[];
+    /**
+     * Every keyword this lexer currently recognizes (locale + plugin-contributed),
+     * mapped to the token type it lexes to. Delegates to the ExpressionLexer.
+     */
+    getKeywords(): Record<string, string>;
+    next(): Token | undefined;
+    peek(): Token | undefined;
+    [Symbol.iterator](): Iterator<Token>;
+    /**
+     * Register a plugin to extend the lexer with custom tokens.
+     * Delegates to the underlying ExpressionLexer.
+     *
+     * @see LexerVocabulary for the supported extension points.
+     */
+    registerVocabulary(plugin: LexerVocabulary): void;
+    /**
+     * Unregister a plugin, removing its custom tokens from the lexer.
+     * Delegates to the underlying ExpressionLexer.
+     */
+    unregisterVocabulary(plugin: LexerVocabulary): void;
+    /**
+     * Reset the lexer for expression-only text, skips the classifyLine()
+     * overhead in reset() for callers that already know the input is an
+     * evaluable expression (e.g., after isEmptyLine() confirmed non-skip).
+     */
+    resetExpression(input: string): void;
+    /**
+     * Scan a full document in one pass, classifying each line and
+     * tokenizing non-skipped lines. Delegates to ExpressionLexer.
+     *
+     * @returns ScanLineResult[], one per line, with classification + tokens.
+     */
+    scanDocument(text: string): ScanLineResult[];
+    getState(): LexerState;
+    setState(state: LexerState): void;
+    getHighlightTokens(lineText: string): {
+        type: string;
+        value: string;
+        offset: number;
+        col: number;
+        length: number;
+        category: TokenCategory | undefined;
+    }[];
+    private collectHighlightTokens;
+}
+/**
+ * A lexer for operations that do not depend on registered vocabulary.
+ *
+ * Line classification and inline-solve detection read characters looking for
+ * headings, comment markers, fences and backtick spans, and never consult the
+ * keyword, unit or operator tables. Every lexer therefore returns the same
+ * answer, so the callers that have no engine to ask can use this one. Checked
+ * by `__tests__/lexer/LineClassificationIsVocabularyIndependent.spec.ts`.
+ *
+ * Do not tokenize with this. An engine's own lexer carries the vocabulary its
+ * packages registered; this one carries none.
+ */
+declare const sharedLexer: Lexer;
+
+export { ExpressionLexer as E, type InlineSolveSpan as I, Lexer as L, type MarkdownLineType as M, type ScanLineResult as S, type TokenLookup as T, LexerState as a, LexerToken as b, type LexerVocabulary as c, type LineClassification as d, type TokenCategory as e, sharedLexer as s };

package/dist/NormalizerRule-BrVoVjmP.d.cts

@@ -0,0 +1,163 @@
+import { T as Token } from './Token-0jpvUdPY.cjs';
+
+/**
+ * NormalizerRule, pluggable token normalization rule for the
+ * TokenNormalizer post-lexer pass.
+ *
+ * ## Purpose
+ * After the ExpressionLexer produces raw tokens (numbers, identifiers,
+ * operators, etc.), the TokenNormalizer applies domain-specific rules
+ * to transform the token stream before parsing. This keeps the lexer
+ * focused on single-token production and moves multi-token pattern
+ * matching into a dedicated normalization layer.
+ *
+ * ## How rules work
+ * Rules are applied in priority order (highest first). At each token
+ * position, the normalizer tries every rule in priority order until one
+ * matches. Matched tokens are consumed and replaced; unmatched tokens
+ * pass through unchanged.
+ *
+ * ## What rules can do
+ * - **Phrase fusion**: Merge consecutive words into compound tokens
+ *   (e.g., `"to" "the" "power" "of"` → `CARET`)
+ * - **Implicit operators**: Insert missing operators between tokens
+ *   (e.g., `NUMBER IDENT` → `NUMBER STAR IDENT`)
+ * - **Domain transformations**: Coalesce item names, currency pairs, etc.
+ *
+ * ## Why a separate file?
+ * This is a duplicate-free copy of the interface defined in
+ * TokenNormalizer.ts. Storing it in a separate file avoids circular
+ * imports, TokenNormalizer imports NormalizerRule, and rule factories
+ * import TokenNormalizer's `createFusedToken`.
+ *
+ * @module NormalizerRule
+ */
+
+/**
+ * Result of a successful rule match attempt against the token stream.
+ *
+ * When a {@link NormalizerRule.match} function finds a pattern at the
+ * current position, it returns a NormalizerMatch describing how many
+ * tokens to consume and what to replace them with.
+ *
+ * @example
+ * ```ts
+ * // The phrase "to the power of" (5 tokens) becomes a single CARET token
+ * const match: NormalizerMatch = {
+ *   consumed: 5,
+ *   replacement: [caretToken],
+ * };
+ * ```
+ */
+interface NormalizerMatch {
+    /**
+     * Number of tokens consumed from the stream at the match position.
+     * Must be ≥ 1, a match always advances the cursor.
+     */
+    consumed: number;
+    /**
+     * Replacement tokens to insert at the match position.
+     * May be empty (deletion), a single token (fusion), or multiple
+     * tokens (expansion/splitting).
+     */
+    replacement: Token[];
+    /**
+     * Human-readable rule name for diagnostic fusion tracking.
+     * When set, the normalizer uses this instead of the rule's `name`
+     * in {@link TokenFusion} records. Used by {@link PhraseTrie} to
+     * report which specific phrase matched (e.g., "phrase:to the power of").
+     */
+    ruleName?: string;
+}
+/**
+ * A pluggable normalization rule registered with the TokenNormalizer.
+ *
+ * Each rule has a {@link name}, {@link priority}, and {@link match} function.
+ * The match function receives the current token stream and a position,
+ * and returns a {@link NormalizerMatch} on success or `null` on failure.
+ *
+ * ## Priority ordering
+ * Higher priority rules are tried first at each position. This allows
+ * long phrases (priority 100, e.g. "to the power of") to match before
+ * shorter fragments (priority 80, e.g. "power of").
+ *
+ * ## Match contract
+ * - Must be pure (no side effects, no mutation of input tokens)
+ * - Must return `null` for any position that doesn't match
+ * - Consumed tokens must be consecutive starting at `pos`
+ * - Replacement tokens must be valid for downstream parsing
+ *
+ * @example
+ * ```ts
+ * // A phrase fusion rule that converts "to the power of" into CARET
+ * const phraseRule: NormalizerRule = {
+ *   name: 'phrase:to the power of',
+ *   priority: 100,
+ *   match: (tokens, pos) => {
+ *     if (pos + 4 > tokens.length) return null;
+ *     const phrase = tokens.slice(pos, pos + 5)
+ *       .map(t => t.value.toLowerCase()).join(' ');
+ *     if (phrase === 'to the power of') {
+ *       return {
+ *         consumed: 5,
+ *         replacement: [createFusedToken('CARET', 'to the power of', tokens.slice(pos, pos + 5))],
+ *       };
+ *     }
+ *     return null;
+ *   },
+ * };
+ * ```
+ */
+interface NormalizerRule {
+    /**
+     * Human-readable name for debugging and diagnostic display.
+     * Convention: `"category:description"`, e.g. `"phrase:to the power of"`.
+     */
+    readonly name: string;
+    /**
+     * Priority for ordering rules. Higher values are tried first.
+     * Recommended ranges:
+     * - 100: Long multi-word phrase fusion (e.g., "to the power of")
+     * - 80:  Short phrase fusion (e.g., "power of", "times by")
+     * - 50:  Implicit operator insertion (e.g., implicit multiply)
+     * - 20:  Domain-specific transformations
+     */
+    readonly priority: number;
+    /**
+     * Attempt to match a pattern starting at position `pos` in the token stream.
+     *
+     * @param tokens - The current token stream (may be partially normalized from prior passes)
+     * @param pos    - The current position to attempt matching from
+     * @returns A {@link NormalizerMatch} if the pattern is found, or `null` if no match
+     */
+    match(tokens: Token[], pos: number): NormalizerMatch | null;
+}
+/**
+ * Record of a token fusion event performed by the normalizer.
+ *
+ * When a rule merges multiple source tokens into fewer replacement tokens,
+ * the normalizer fires a {@link NormalizerOptions.onFusion | fusion callback}
+ * with this record. The playground uses these records to render the
+ * fusion detail table showing exactly which tokens were merged and by
+ * which rule.
+ *
+ * @example
+ * ```ts
+ * // "to" "the" "power" "of" fused into CARET "^"
+ * const fusion: TokenFusion = {
+ *   rule: "phrase:to the power of",
+ *   sourceTokens: [toToken, theToken, powerToken, ofToken],
+ *   fusedToken: caretToken,
+ * };
+ * ```
+ */
+interface TokenFusion {
+    /** The name of the rule that triggered this fusion (e.g., "phrase:to the power of") */
+    rule: string;
+    /** The original tokens before fusion, always ≥ 2 tokens */
+    sourceTokens: Token[];
+    /** The resulting fused token with its new type and combined value */
+    fusedToken: Token;
+}
+
+export type { NormalizerRule as N, TokenFusion as T, NormalizerMatch as a };