solve-engine 1.0.0-beta.0 → 1.0.0-beta.2

package/dist/Lexer-BJdhlnej.d.cts

@@ -0,0 +1,531 @@
+import { T as Token } from './Token-0jpvUdPY.cjs';
+
+/** Trie node for multi-word phrase matching. */
+interface PhraseNode {
+    /** Complete phrase token type (null = intermediate node) */
+    type?: string;
+    children: Map<string, PhraseNode>;
+}
+/**
+ * The optimized lookup structure built by TokenClassRegistry.build().
+ * Consumed by the Lexer for O(1) keyword → token type lookups and
+ * O(word-count) phrase matching.
+ */
+interface TokenLookup {
+    /** Lowercase keyword → token type. O(1) Map lookup. */
+    keywordToType: Map<string, string>;
+    /** Phrase trie for multi-word matching. Root node with children maps.
+     *  Null if no phrases registered. */
+    phraseTrie: PhraseNode | null;
+    /** Set of lowercase first-words of all registered phrases.
+     *  Used by the lexer to emit IDENT (not a phrase keyword) for words
+     *  that start multi-word phrases, deferring to the PhraseMatcher.
+     *
+     *  Example: "to" is in phraseStartWords because "to the power of" is a phrase.
+     *  When the lexer sees "to", it emits IDENT and lets the phrase matcher
+     *  combine "to the power of" into a single CARET token.
+     *
+     *  This prevents plugins from accidentally overriding phrase-start words.
+     */
+    phraseStartWords: Set<string>;
+    /** Case-sensitive unit names for UNIT fallback after keyword lookup fails. */
+    unitNames: ReadonlySet<string>;
+}
+
+/** What a line is, structurally, before anything tries to evaluate it. */
+type MarkdownLineType = 'expression' | 'prose' | 'heading' | 'blockquote' | 'list' | 'code_fence' | 'math_fence' | 'table' | 'table_separator' | 'hr' | 'wikilink' | 'comment' | 'empty';
+/**
+ * What a line is and whether it holds anything to evaluate.
+ *
+ * Produced by a character-level scan that never consults the keyword, unit or
+ * operator tables, so any lexer gives the same answer for the same line. See
+ * `__tests__/lexer/LineClassificationIsVocabularyIndependent.spec.ts`.
+ */
+interface LineClassification {
+    /** The type of this markdown line */
+    type: MarkdownLineType;
+    /** Whether this line should be skipped (no expression evaluation) */
+    skip: boolean;
+    /** Whether the line contains inline solve markers (`s`...``) */
+    hasInlineSolve: boolean;
+}
+/** Inline solve position with precise coordinates */
+interface InlineSolveSpan {
+    /** Character offset of the `s`` marker */
+    start: number;
+    /** Character offset past the closing `` ` `` */
+    end: number;
+    /** The expression text between the backticks */
+    expression: string;
+    /** 1-based column of the `s`` marker */
+    columnNumber: number;
+    /** Token index of INLINE_SOLVE_START in the line's token array (set during tokenization). */
+    startTokenIndex?: number;
+    /** Token index of closing BACKTICK_OPEN in the line's token array (set during tokenization). */
+    endTokenIndex?: number;
+}
+/**
+ * Result from a single line processed by scanDocument().
+ * Combines line classification, tokenized tokens, and inline solve spans
+ * into a single structure, eliminating the need for separate classifyLine()
+ * findInlineSolves(), and per-line Lexer.reset() calls.
+ */
+interface ScanLineResult {
+    /** The raw line text (without trailing newline). */
+    text: string;
+    /** 1-based line number within the document. */
+    lineNumber: number;
+    /** Character offset of the line start within the document. */
+    startOffset: number;
+    /** Character offset of the line end (before newline). */
+    endOffset: number;
+    /** The line classification. */
+    classification: LineClassification;
+    /** Tokenized tokens (empty array if line is skipped). */
+    tokens: Token[];
+    /** Inline solve spans found in this line (empty if none). */
+    inlineSolves: InlineSolveSpan[];
+}
+/**
+ * A token, as the lexer produces it.
+ *
+ * A class rather than an object literal because tokens are created on every
+ * keystroke and a shared hidden class keeps that path predictable for the
+ * engine running it.
+ */
+declare class LexerToken implements Token {
+    type: string;
+    typeId: number;
+    value: string;
+    text: string;
+    offset: number;
+    lineBreaks: number;
+    line: number;
+    col: number;
+    constructor(type: string, typeId: number, value: string, text: string, offset: number, lineBreaks: number, line: number, col: number);
+}
+/**
+ * Plugin interface for extending the ExpressionLexer with custom tokens.
+ *
+ * Plugins can register:
+ * - `keywords`: Map identifier strings to custom token types (checked after locale keywords).
+ * - `operators`: Map multi-character operator sequences to custom token types.
+ * - `units`: Register additional unit identifiers (checked alongside built-in units).
+ *
+ * Multi-word phrase matching has been moved to the TokenNormalizer post-lexer
+ * stage. To register phrase patterns, use `IEnginePackage.normalizerRules` instead.
+ *
+ * All registrations are additive, built-in patterns still work.
+ */
+interface LexerVocabulary {
+    /**
+     * Keyword → tokenType mappings. Each key is a lowercase identifier that,
+     * when encountered, will emit the specified token type instead of IDENT.
+     * These are checked AFTER the locale's built-in keywordMap, so locale
+     * keywords take priority.
+     */
+    keywords?: Record<string, string>;
+    /**
+     * Multi-character operator → tokenType mappings. Each key is the exact
+     * character sequence (e.g., "::", "->", "=>") and the value is the token
+     * type to emit. Two-character operators take priority during matching.
+     * Built-in operators (==, !=, >=, <=, <<, >>) always take priority.
+     */
+    operators?: Record<string, string>;
+    /**
+     * Additional unit identifiers to recognize (e.g., "gp", "osrs", "tile").
+     * These are checked alongside the built-in `knownUnits` set.
+     */
+    units?: string[];
+    /**
+     * Whole-line patterns matched against the RAW line text, BEFORE any
+     * per-character tokenization begins.
+     *
+     * Every other extension point in this file (`keywords`/`operators`/
+     * `units`, plus `IEnginePackage.phrases`/`normalizerRules`) transforms
+     * a token STREAM -- they all assume the line is, at some granularity,
+     * valid Solve syntax. This hook exists for the one shape that isn't:
+     * a package whose grammar captures arbitrary free-form text terminated
+     * by a fixed marker (e.g. a natural-language query ending in `= ?`),
+     * where the text itself ("distance to the moon") would never tokenize
+     * or parse as a normal expression and must be captured verbatim
+     * instead -- see `packages/knowledge/` for the reference use.
+     *
+     * Each entry's `pattern` is tested (via `RegExp.exec`) against the
+     * full, untrimmed line text. If it matches AND capture group 1 is
+     * non-empty after trimming, the ENTIRE line becomes a single
+     * synthetic token of `tokenType` whose `value`/`text` is the trimmed
+     * capture group -- the character-by-character scanner never runs for
+     * that line. Patterns are tried in registration order; the first
+     * match wins. A package registering a rule here still needs a
+     * `prefixParselets` entry for `tokenType` to actually consume the
+     * resulting token.
+     *
+     * Because this bypasses tokenization entirely, a matching line can
+     * contain characters that would otherwise be lexer errors (unmatched
+     * quotes, stray symbols, ...) -- by design, since the whole point is
+     * to hand the package raw text the normal pipeline was never meant to
+     * parse.
+     */
+    rawLinePatterns?: Array<{
+        pattern: RegExp;
+        tokenType: string;
+    }>;
+}
+/**
+ * Character-by-character tokenizer for expression text.
+ *
+ * Scans a raw line/expression string into a stream of typed tokens
+ * (numbers, identifiers, operators, units, keywords, ...), handling
+ * markdown-line classification (`classifyLine`), inline `` s`...` `` solve
+ * spans, and package-contributed vocabulary (registered via
+ * {@link registerVocabulary}/{@link unregisterVocabulary}, keywords
+ * operators, and units a package wants recognized as their own token
+ * types rather than falling through to generic identifiers).
+ *
+ * Most consumers should use the higher-level {@link Lexer} wrapper, which
+ * adds streaming `next()`/`peek()` access over this class's scan results.
+ */
+declare class ExpressionLexer {
+    private static readonly CHAR_CLASS;
+    /**
+     * Configured TokenLookup from TokenClassRegistry. When set, replaces
+     * the internal keyword map and unit set with registry-built equivalents.
+     * Enables data-driven keyword/unit registration across locale keywords,
+     * provider keywords, and plugins.
+     *
+     * Set at construction time via the constructor parameter. Plugin-registered
+     * keywords/units (via registerVocabulary()) are checked alongside
+     * the configuredLookup, neither source is bypassed.
+     */ private configuredLookup;
+    private input;
+    private pos;
+    private len;
+    private line;
+    private lineStartPos;
+    private keywordMap;
+    private mergedKeywords;
+    private mergedUnits;
+    private pluginKeywordMap;
+    private pluginOperators;
+    private pluginUnits;
+    private hasPluginOps;
+    private localeCode;
+    private locale;
+    /**
+     * Inline solve spans collected during the most recent tokenization pass.
+     * Populated by [Symbol.iterator]() and consumed by scanDocument().
+     */
+    _inlineSolveSpans: InlineSolveSpan[];
+    private pluginRawLinePatterns;
+    /**
+     * If a `rawLinePatterns` rule matches the FULL text most recently passed
+     * to {@link reset}, this holds the single synthetic token that
+     * {@link tokenizeAll} should return instead of running the
+     * character-by-character scanner. Cleared (re-evaluated) on every
+     * {@link reset} call. `null` when no plugin registered any raw-line
+     * patterns, or none matched, the overwhelmingly common case, checked
+     * with a `length === 0` guard before ever touching this field so a
+     * plugin-free lexer pays zero cost for the feature.
+     */
+    private pendingRawLineToken;
+    /**
+     * Test `text` against every registered `rawLinePatterns` rule, in
+     * registration order. Returns a synthetic token for the first rule
+     * whose `pattern` matches AND whose capture group 1 is non-empty after
+     * trimming; returns `null` if no rule matches (the normal
+     * character-by-character scanner should run instead).
+     */
+    private matchRawLine;
+    /** Rebuild merged keyword and unit collections after plugin registration. */
+    private rebuildMergedCollections;
+    constructor(localeCode?: string, lookup?: TokenLookup);
+    /**
+     * Register a plugin to extend the lexer with custom tokens.
+     *
+     * All registrations are additive, built-in patterns still work.
+     * Keywords, operators, and units from the plugin are merged
+     * with existing ones. Calling multiple times adds more entries.
+     *
+     * Note: multi-word phrases are now handled by the TokenNormalizer
+     * (see `IEnginePackage.normalizerRules`), not the lexer.
+     *
+     * Built-in tokens CANNOT be overridden. Throws a EngineError if the
+     * plugin attempts to register a keyword, operator, or unit
+     * that conflicts with a built-in one.
+     */
+    registerVocabulary(plugin: LexerVocabulary): void;
+    /**
+     * Unregister a plugin, removing its custom tokens from the lexer.
+     *
+     * This is the inverse of registerVocabulary(). All keywords, operators,
+     * and units registered by the plugin are removed. After
+     * unregistration, those tokens will revert to their default behavior
+     * (e.g., keywords become IDENT, operators become ERROR).
+     *
+     * Calling unregisterVocabulary with a plugin that was never registered
+     * is safe, it simply has no effect.
+     */
+    unregisterVocabulary(plugin: LexerVocabulary): void;
+    reset(input: string): void;
+    /**
+     * Scan a full document text in a single pass, classifying each line and
+     * tokenizing non-skipped lines.
+     *
+     * Replaces the separate classifyLine() + findInlineSolves() + per-line
+     * reset() + tokenizeAll() pattern with a single character-by-character
+     * walk through the entire document. Key benefits:
+     *
+     * - **Single reset()**: `this.pos`, `this.len`, `this.line`, and
+     *   `this.lineStartPos` are set once for the whole document, not per-line.
+     * - **Single classification**: classifyLine() runs once per line inline;
+     *   skipped lines are jumped over without tokenization.
+     * - **Shared tokenization**: Non-skipped lines are tokenized using the
+     *   existing state machine, yielding Token[] without per-line reset().
+     * - **Inline solve detection**: findInlineSolves() is called only for
+     *   lines that classifyLine() marks as having inline solves.
+     *
+     * Tokenization is scoped to each line by temporarily restricting
+     * `this.len` to the line end position, so the [Symbol.iterator]
+     * generator naturally stops at the line boundary. After tokenization,
+     * `this.len` is restored and `this.pos` advances past the newline.
+     *
+     * @param text The full document text (with newlines).
+     * @returns Array of ScanLineResult, one per line, in document order.
+     */
+    scanDocument(text: string): ScanLineResult[];
+    /**
+     * Tokenize an expression string into an array of Tokens.
+     *
+     * Delegates to the lazy [Symbol.iterator]() generator and collects all
+     * yielded tokens via Array.from(). For memory-sensitive use cases, prefer
+     * iterating the lexer directly with for...of to avoid array allocation.
+     *
+     * Optimizations:
+     *  - CHAR_CLASS jump table (Uint8Array) → switch on small integers
+     *  - Direct character-code dispatch (c0 cached pattern)
+     *  - Mathematical digit parsing (integer math, not slice+parseFloat)
+     *  - Inline operator tokenizer with two-char peek-ahead
+     *  - Whitespace eliminated in-lexer (never emitted)
+     *  - 0-char and 1-char fast paths
+     */
+    tokenizeAll(): Token[];
+    /**
+     * Lazy token-by-token generator. Yields each token without allocating an
+     * intermediate Token[] array. Supports for...of and spread usage.
+     *
+     * Usage:
+     *   for (const t of lexer) { ... }  // lazy, no array allocation
+     *   const tokens = [...lexer];       // materializes via spread
+     *   const tokens = lexer.tokenizeAll(); // materializes via Array.from()
+     *
+     * IMPORTANT: This generator captures `this.len` ONCE at creation time
+     * (const len = this.len). `scanDocument()` relies on this behavior to
+     * scope tokenization to a single line by temporarily restricting
+     * `this.len` to the line end position before creating the iterator.
+     * Do NOT refactor to re-read `this.len` mid-loop without also updating
+     * `scanDocument()`.
+     */
+    [Symbol.iterator](): Generator<Token, void, undefined>;
+    /**
+     * Character-by-character number parsing.
+     *
+     * Supports: integers, floats, scientific notation (1.5e10, 1.5e-10),
+     * hex (0xFF), binary (0b1010), octal (0o17), BigInt suffix (123n),
+     * thousands separators (1,234 or 1.234.567).
+     *
+     * Returns a LexerToken and advances `this.pos` past the number.
+     */
+    private tokenizeNumber;
+    /**
+     * Reads [a-zA-Z_][a-zA-Z0-9_]* and resolves to:
+     *   - A unit type (via knownUnits, case-sensitive)
+     *   - A keyword type (via locale keywordMap, case-insensitive)
+     *   - IDENT if none of the above
+     *
+     * Multi-word phrases (e.g., "to the power of") are handled by the
+     * TokenNormalizer post-lexer pass, not the lexer.
+     */
+    private tokenizeIdentifier;
+    /**
+     * Peek past in-expression whitespace (space, tab) from `pos` to check
+     * if the next significant character is '('.
+     */
+    private isFollowedByLParen;
+    /**
+     * Reads an operator/punctuation token.
+     * Handles two-char operators (==, !=, >=, <=, **) and the special
+     * cases << (LSHIFT) and >> (RSHIFT).
+     */
+    private tokenizeOperator;
+    /**
+     * Reads a double-quoted string literal. Supports backslash escapes.
+     */
+    private tokenizeString;
+    /**
+     * L1 expression gating: quickly determine if a line contains any
+     * characters that indicate an expression (digits, operators, currency,
+     * backticks, parentheses, etc.).
+     *
+     * Pure prose lines (e.g., "The quick brown fox jumps over the lazy dog")
+     * return false and can be skipped without full tokenization (L2).
+     *
+     * This is a fast character-by-character scan that stops at the first
+     * expression indicator. Called once per line in classifyFromPositions().
+     */
+    static hasExpressionIndicators(input: string, start: number, end: number): boolean;
+    /**
+     * Classify a line by its character positions within this.input.
+     * Reads directly from this.input using start/end boundaries.
+     * DOES NOT modify this.pos, purely a read-only classifier.
+     */
+    private classifyFromPositions;
+    /**
+     * Classify a single line of markdown text.
+     */
+    classifyLine(lineText: string): LineClassification;
+    /**
+     * Every keyword this lexer currently recognizes, locale keywords
+     * (`pi`, `sqrt`, `convert`, ...) merged with any plugin-contributed ones
+     * from `registerVocabulary()` (e.g. a package's custom keywords), mapped to
+     * the token type they lex to. A snapshot copy, not a live reference
+     * mutating the return value has no effect on the lexer.
+     */
+    getKeywords(): Record<string, string>;
+    /**
+     * Find all inline solve markers in a line with precise coordinate mapping.
+     */
+    findInlineSolves(lineText: string): InlineSolveSpan[];
+    /**
+     * Reads a comment: # to end of line, or // to end of line.
+     */
+    private tokenizeComment;
+}
+
+/**
+ * Lexer state machine modes.
+ * - Main: document-level scanning with markdown classification
+ * - Inline: expression embedded in markdown inline solve (`s\`...\``)
+ * - String: inside a double-quoted string literal
+ */
+declare enum LexerState {
+    Main = "main",
+    Inline = "inline",
+    String = "string"
+}
+
+/**
+ * Editor-agnostic semantic classification for a lexer token.
+ *
+ * This is the entire contract solve-js's language service exposes to any
+ * editor integration (CodeMirror, VS Code, ...): a category, nothing about
+ * how it should be rendered. CSS class names, VS Code semantic-token
+ * indices, etc. are all downstream adapter concerns. See
+ * `language/adapters/`.
+ *
+ * The built-in categories cover every core (non-plugin) token type. Solve-js
+ * packages may contribute additional, open-ended category strings for their
+ * own custom token types (see `registerTokenCategory` in TokenCategoryMap.ts)
+ *, the `(string & {})` union member keeps those assignable without widening
+ * the whole type to a bare `string` and losing autocomplete for the built-ins.
+ */
+type TokenCategory = "number" | "string" | "keyword" | "operator" | "comparison" | "bitwise" | "function" | "variable" | "unit" | "datetime" | "vector" | "punctuation" | "error" | (string & {});
+
+/**
+ * Public tokenizer wrapper around {@link ExpressionLexer}.
+ *
+ * `ExpressionLexer` does the actual character-by-character scanning;
+ * `Lexer` adds a materialized-token-array streaming interface
+ * (`next()`/`peek()`) plus line-classification state (`reset()`) so
+ * callers can iterate a line's tokens without re-scanning on each peek.
+ *
+ * Each `ExpressionEngine` instance owns its own `Lexer`, and packages
+ * extend it via {@link registerVocabulary} (keywords, operators, units)
+ * see `IEnginePackage.lexerVocabulary`.
+ */
+declare class Lexer {
+    /** Expression-mode lexer (Phase A: V8-optimized, replaces moo) */
+    private expressionLexer;
+    private currentState;
+    private peekedToken;
+    private hasPeeked;
+    private tokens;
+    private tokenIdx;
+    /**
+     * @param localeCode - Locale code (e.g., "en", "de"). Defaults to "en".
+     * @param tokenLookup - Optional TokenLookup from TokenClassRegistry.
+     *   When provided, configures ExpressionLexer to use registry-built
+     *   keyword/unit/phrase lookups instead of internal instance maps.
+     */
+    constructor(localeCode?: string, tokenLookup?: TokenLookup);
+    reset(input: string, state?: LexerState): void;
+    /**
+     * Classify a single line of markdown text (Phase B).
+     * Delegates to the ExpressionLexer's character-by-character scanner.
+     */
+    classifyLine(lineText: string): LineClassification;
+    /**
+     * Find all inline solve markers in a line (Phase B).
+     * Delegates to the ExpressionLexer's character-by-character scanner.
+     */
+    findInlineSolves(lineText: string): InlineSolveSpan[];
+    /**
+     * Every keyword this lexer currently recognizes (locale + plugin-contributed),
+     * mapped to the token type it lexes to. Delegates to the ExpressionLexer.
+     */
+    getKeywords(): Record<string, string>;
+    next(): Token | undefined;
+    peek(): Token | undefined;
+    [Symbol.iterator](): Iterator<Token>;
+    /**
+     * Register a plugin to extend the lexer with custom tokens.
+     * Delegates to the underlying ExpressionLexer.
+     *
+     * @see LexerVocabulary for the supported extension points.
+     */
+    registerVocabulary(plugin: LexerVocabulary): void;
+    /**
+     * Unregister a plugin, removing its custom tokens from the lexer.
+     * Delegates to the underlying ExpressionLexer.
+     */
+    unregisterVocabulary(plugin: LexerVocabulary): void;
+    /**
+     * Reset the lexer for expression-only text, skips the classifyLine()
+     * overhead in reset() for callers that already know the input is an
+     * evaluable expression (e.g., after isEmptyLine() confirmed non-skip).
+     */
+    resetExpression(input: string): void;
+    /**
+     * Scan a full document in one pass, classifying each line and
+     * tokenizing non-skipped lines. Delegates to ExpressionLexer.
+     *
+     * @returns ScanLineResult[], one per line, with classification + tokens.
+     */
+    scanDocument(text: string): ScanLineResult[];
+    getState(): LexerState;
+    setState(state: LexerState): void;
+    getHighlightTokens(lineText: string): {
+        type: string;
+        value: string;
+        offset: number;
+        col: number;
+        length: number;
+        category: TokenCategory | undefined;
+    }[];
+    private collectHighlightTokens;
+}
+/**
+ * A lexer for operations that do not depend on registered vocabulary.
+ *
+ * Line classification and inline-solve detection read characters looking for
+ * headings, comment markers, fences and backtick spans, and never consult the
+ * keyword, unit or operator tables. Every lexer therefore returns the same
+ * answer, so the callers that have no engine to ask can use this one. Checked
+ * by `__tests__/lexer/LineClassificationIsVocabularyIndependent.spec.ts`.
+ *
+ * Do not tokenize with this. An engine's own lexer carries the vocabulary its
+ * packages registered; this one carries none.
+ */
+declare const sharedLexer: Lexer;
+
+export { ExpressionLexer as E, type InlineSolveSpan as I, Lexer as L, type MarkdownLineType as M, type ScanLineResult as S, type TokenLookup as T, LexerState as a, LexerToken as b, type LexerVocabulary as c, type LineClassification as d, type TokenCategory as e, sharedLexer as s };