solve-engine 1.0.0-beta.2 → 1.0.0-beta.3

package/dist/TokenNormalizer-B8zmhO5X.d.ts

@@ -0,0 +1,377 @@
+import { T as Token } from './Token-DnU2hwFY.js';
+
+/**
+ * NormalizerRule, pluggable token normalization rule for the
+ * TokenNormalizer post-lexer pass.
+ *
+ * ## Purpose
+ * After the ExpressionLexer produces raw tokens (numbers, identifiers,
+ * operators, etc.), the TokenNormalizer applies domain-specific rules
+ * to transform the token stream before parsing. This keeps the lexer
+ * focused on single-token production and moves multi-token pattern
+ * matching into a dedicated normalization layer.
+ *
+ * ## How rules work
+ * Rules are applied in priority order (highest first). At each token
+ * position, the normalizer tries every rule in priority order until one
+ * matches. Matched tokens are consumed and replaced; unmatched tokens
+ * pass through unchanged.
+ *
+ * ## What rules can do
+ * - **Phrase fusion**: Merge consecutive words into compound tokens
+ *   (e.g., `"to" "the" "power" "of"` → `CARET`)
+ * - **Implicit operators**: Insert missing operators between tokens
+ *   (e.g., `NUMBER IDENT` → `NUMBER STAR IDENT`)
+ * - **Domain transformations**: Coalesce item names, currency pairs, etc.
+ *
+ * ## Why a separate file?
+ * This is a duplicate-free copy of the interface defined in
+ * TokenNormalizer.ts. Storing it in a separate file avoids circular
+ * imports, TokenNormalizer imports NormalizerRule, and rule factories
+ * import TokenNormalizer's `createFusedToken`.
+ *
+ * @module NormalizerRule
+ */
+
+/**
+ * Result of a successful rule match attempt against the token stream.
+ *
+ * When a {@link NormalizerRule.match} function finds a pattern at the
+ * current position, it returns a NormalizerMatch describing how many
+ * tokens to consume and what to replace them with.
+ *
+ * @example
+ * ```ts
+ * // The phrase "to the power of" (5 tokens) becomes a single CARET token
+ * const match: NormalizerMatch = {
+ *   consumed: 5,
+ *   replacement: [caretToken],
+ * };
+ * ```
+ */
+interface NormalizerMatch {
+    /**
+     * Number of tokens consumed from the stream at the match position.
+     * Must be ≥ 1, a match always advances the cursor.
+     */
+    consumed: number;
+    /**
+     * Replacement tokens to insert at the match position.
+     * May be empty (deletion), a single token (fusion), or multiple
+     * tokens (expansion/splitting).
+     */
+    replacement: Token[];
+    /**
+     * Human-readable rule name for diagnostic fusion tracking.
+     * When set, the normalizer uses this instead of the rule's `name`
+     * in {@link TokenFusion} records. Used by {@link PhraseTrie} to
+     * report which specific phrase matched (e.g., "phrase:to the power of").
+     */
+    ruleName?: string;
+}
+/**
+ * A pluggable normalization rule registered with the TokenNormalizer.
+ *
+ * Each rule has a {@link name}, {@link priority}, and {@link match} function.
+ * The match function receives the current token stream and a position,
+ * and returns a {@link NormalizerMatch} on success or `null` on failure.
+ *
+ * ## Priority ordering
+ * Higher priority rules are tried first at each position. This allows
+ * long phrases (priority 100, e.g. "to the power of") to match before
+ * shorter fragments (priority 80, e.g. "power of").
+ *
+ * ## Match contract
+ * - Must be pure (no side effects, no mutation of input tokens)
+ * - Must return `null` for any position that doesn't match
+ * - Consumed tokens must be consecutive starting at `pos`
+ * - Replacement tokens must be valid for downstream parsing
+ *
+ * @example
+ * ```ts
+ * // A phrase fusion rule that converts "to the power of" into CARET
+ * const phraseRule: NormalizerRule = {
+ *   name: 'phrase:to the power of',
+ *   priority: 100,
+ *   match: (tokens, pos) => {
+ *     if (pos + 4 > tokens.length) return null;
+ *     const phrase = tokens.slice(pos, pos + 5)
+ *       .map(t => t.value.toLowerCase()).join(' ');
+ *     if (phrase === 'to the power of') {
+ *       return {
+ *         consumed: 5,
+ *         replacement: [createFusedToken('CARET', 'to the power of', tokens.slice(pos, pos + 5))],
+ *       };
+ *     }
+ *     return null;
+ *   },
+ * };
+ * ```
+ */
+interface NormalizerRule {
+    /**
+     * Human-readable name for debugging and diagnostic display.
+     * Convention: `"category:description"`, e.g. `"phrase:to the power of"`.
+     */
+    readonly name: string;
+    /**
+     * Priority for ordering rules. Higher values are tried first.
+     * Recommended ranges:
+     * - 100: Long multi-word phrase fusion (e.g., "to the power of")
+     * - 80:  Short phrase fusion (e.g., "power of", "times by")
+     * - 50:  Implicit operator insertion (e.g., implicit multiply)
+     * - 20:  Domain-specific transformations
+     */
+    readonly priority: number;
+    /**
+     * Attempt to match a pattern starting at position `pos` in the token stream.
+     *
+     * @param tokens - The current token stream (may be partially normalized from prior passes)
+     * @param pos    - The current position to attempt matching from
+     * @returns A {@link NormalizerMatch} if the pattern is found, or `null` if no match
+     */
+    match(tokens: Token[], pos: number): NormalizerMatch | null;
+}
+/**
+ * Record of a token fusion event performed by the normalizer.
+ *
+ * When a rule merges multiple source tokens into fewer replacement tokens,
+ * the normalizer fires a {@link NormalizerOptions.onFusion | fusion callback}
+ * with this record. The playground uses these records to render the
+ * fusion detail table showing exactly which tokens were merged and by
+ * which rule.
+ *
+ * @example
+ * ```ts
+ * // "to" "the" "power" "of" fused into CARET "^"
+ * const fusion: TokenFusion = {
+ *   rule: "phrase:to the power of",
+ *   sourceTokens: [toToken, theToken, powerToken, ofToken],
+ *   fusedToken: caretToken,
+ * };
+ * ```
+ */
+interface TokenFusion {
+    /** The name of the rule that triggered this fusion (e.g., "phrase:to the power of") */
+    rule: string;
+    /** The original tokens before fusion, always ≥ 2 tokens */
+    sourceTokens: Token[];
+    /** The resulting fused token with its new type and combined value */
+    fusedToken: Token;
+}
+
+/**
+ * TokenNormalizer, post-lexer token normalization pass.
+ *
+ * ## Purpose
+ * Applies domain-specific {@link NormalizerRule | NormalizerRules} to the raw
+ * token stream produced by the {@link ExpressionLexer}. This keeps the lexer
+ * slim and focused on single-token production, while multi-token pattern
+ * matching (phrases, implicit operators, domain merges) lives here.
+ *
+ * ## What rules can do
+ * - **Phrase fusion**: Merge consecutive words into compound tokens
+ *   (e.g., `IDENT + ... + IDENT` → `CARET`)
+ * - **Implicit operator insertion**: Insert missing operators between tokens
+ *   (e.g., `NUMBER IDENT` → `NUMBER STAR IDENT`)
+ * - **Domain-specific transformations**: Coalesce item names, currency pairs,
+ *   percentage syntax, etc.
+ *
+ * ## Architecture
+ * Providers register NormalizerRules alongside Parselets and OpCode handlers
+ * via {@link IEnginePackage.normalizerRules}. The normalizer applies them
+ * greedily left-to-right in multiple passes with safety limits.
+ *
+ * @module TokenNormalizer
+ */
+
+/**
+ * Configuration options for the normalization pass.
+ *
+ * These control safety limits and diagnostic callbacks. The defaults
+ * are chosen to be generous enough for any realistic expression while
+ * preventing runaway token expansion from recursive rules.
+ */
+interface NormalizerOptions {
+    /**
+     * Maximum number of full passes over the token stream before bailing out.
+     * Prevents infinite loops from recursive rule chains.
+     * @default 100
+     */
+    maxPasses?: number;
+    /**
+     * Maximum number of tokens allowed after normalization.
+     * If exceeded, an Error is thrown rather than passing a bloated stream
+     * to the parser.
+     * @default 10000
+     */
+    maxTokens?: number;
+    /**
+     * Callback invoked for each fusion event during normalization.
+     * Used by diagnostic mode to populate {@link NormalizerOutput.fusions}.
+     * When `undefined`, fusions are still tracked internally but no callbacks fire.
+     */
+    onFusion?: (fusion: TokenFusion) => void;
+}
+/**
+ * Creates a new normalized token from fused source tokens.
+ *
+ * The fused token inherits position information (offset, line, column)
+ * from the first source token, which preserves source-map accuracy
+ * for error messages and diagnostic highlighting.
+ *
+ * It also records where the source text ENDS, on `sourceEnd`. The start alone
+ * is not enough to describe the span a fusion covers, because `text` is the
+ * replacement rather than the original: `10 frames` fuses into a FRAME_COUNT
+ * whose text is `10`, and a timecode fuses into a token whose text is a
+ * comma-separated tuple that appears nowhere in the line. Anything painting the
+ * line needs both ends, and only this function is in a position to know them.
+ *
+ * @param type         - The new token type (e.g., "CARET", "TIMES_BY")
+ * @param text         - The combined text representation (e.g., "to the power of")
+ * @param sourceTokens - The original tokens being fused (at least 2)
+ * @returns A new {@link LexerToken} with the fused type and combined text
+ */
+declare function createFusedToken(type: string, text: string, sourceTokens: Token[]): Token;
+/**
+ * Token normalizer: applies {@link NormalizerRule | NormalizerRules} to a token stream.
+ *
+ * ## Lifecycle
+ * 1. **Registration**: Rules are added via {@link register} and sorted by priority
+ * 2. **Normalization**: {@link normalize} applies rules greedily left-to-right
+ * 3. **Cleanup**: {@link clear} or {@link unregister} removes rules
+ *
+ * ## Normalization algorithm
+ * The normalizer uses a greedy left-to-right multi-pass algorithm:
+ * - At each token position, rules are tried in priority order (highest first)
+ * - When a rule matches, matched tokens are consumed and replaced
+ * - Processing continues from the replacement position
+ * - Multiple passes handle cascading matches (one rule's output triggers another)
+ * - Safety limits ({@link NormalizerOptions.maxPasses}) prevent infinite loops
+ *
+ * @example
+ * ```ts
+ * const normalizer = new TokenNormalizer();
+ * normalizer.register(phraseRule);       // "to the power of" → CARET
+ * normalizer.register(implicitMultRule); // "2 x" → "2 * x"
+ * const normalized = normalizer.normalize(rawTokens);
+ * ```
+ */
+declare class TokenNormalizer {
+    /** Registered rules, unsorted, the source of truth. */
+    private rules;
+    /**
+     * Priority-sorted copy of {@link rules}, rebuilt lazily on the next
+     * {@link normalize} call after a mutation. Rules are registered once at
+     * engine/package-registration time and essentially never change during a
+     * session, but normalize() runs on every keystroke-driven evaluation, an
+     * earlier version re-sorted a fresh copy of `rules` on every single call,
+     * which meant every keystroke paid for an allocation + sort of a list that
+     * had usually not changed since the last one. `null` means "stale, rebuild
+     * on next use"; {@link register}/{@link unregister}/{@link clear} all
+     * invalidate it.
+     */
+    private sortedRulesCache;
+    /**
+     * Phrase trie for single-pass multi-word phrase fusion.
+     * Tried at each token position BEFORE other rules, the trie walk
+     * is O(depth) vs O(R × W) for separate rule matching.
+     */
+    private phraseTrie;
+    /** Merged options with defaults applied. */
+    private options;
+    /**
+     * @param options - Configuration overrides for safety limits and diagnostic callbacks
+     */
+    constructor(options?: NormalizerOptions);
+    /**
+     * Register a normalization rule.
+     *
+     * Rules are sorted by priority (descending) on each {@link normalize} call.
+     * Multiple rules can share the same priority, they are tried in registration
+     * order when priorities are equal.
+     *
+     * @param rule - The rule to register
+     */
+    register(rule: NormalizerRule): void;
+    /**
+     * Unregister a normalization rule by its {@link NormalizerRule.name | name}.
+     *
+     * If multiple rules share the same name, all are removed. This is safe to
+     * call with a name that doesn't match any rule, it simply has no effect.
+     *
+     * @param ruleName - The name of the rule to remove
+     */
+    unregister(ruleName: string): void;
+    /**
+     * Remove all registered rules, resetting the normalizer to its initial state.
+     * Also clears the phrase trie.
+     */
+    clear(): void;
+    /**
+     * Priority-sorted view of {@link rules} (descending priority; registration
+     * order preserved for ties, since {@link Array.prototype.sort} is stable).
+     * Cached until the next mutation. See {@link sortedRulesCache}.
+     */
+    private getSortedRules;
+    /**
+     * Get the number of currently registered rules (excludes phrase trie entries).
+     */
+    get ruleCount(): number;
+    /**
+     * Register a multi-word phrase for fusion into a single compound token.
+     *
+     * This is the preferred way to add phrase patterns. It inserts into the
+     * internal {@link PhraseTrie}, which collapses all phrase rules into a
+     * single O(depth) trie walk per position, no separate rule scanning.
+     *
+     * @param phrase    - Multi-word phrase (e.g., "to the power of", "abyssal whip")
+     * @param tokenType - Target token type after fusion (e.g., "CARET", "ITEM")
+     */
+    addPhrase(phrase: string, tokenType: string): void;
+    /**
+     * Check whether a word can start any registered phrase.
+     *
+     * Used by {@link implicitMultiplyRule} to suppress `*` insertion
+     * before phrase-starting identifiers (e.g., "2 power of 3" → `2 ^ 3`,
+     * not `2 * power of 3`). Delegates to {@link PhraseTrie.canStart}.
+     */
+    /**
+     * Get all registered phrases and their target token types.
+     *
+     * Exposes the full phrase trie structure for diagnostic rendering
+     * in the playground's NormalizerTab. Returns ALL registered phrases,
+     * not just the ones that matched in the last evaluation.
+     */
+    getPhrases(): Record<string, string>;
+    canStartPhrase(word: string): boolean;
+    /**
+     * Normalize a token stream by applying all registered rules.
+     *
+     * ## Algorithm
+     * Applies rules greedily left-to-right in multiple passes:
+     * 1. Sort rules by priority (descending)
+     * 2. Walk the token stream left to right
+     * 3. At each position, try rules in priority order
+     * 4. On match: consume matched tokens, insert replacements, restart from insert point
+     * 5. On no match: pass token through unchanged
+     * 6. Repeat until a full pass produces no changes, or maxPasses is reached
+     *
+     * ## Fusion tracking
+     * When a rule consumes more tokens than it produces, the normalizer calls
+     * `onFusion` with a {@link TokenFusion} record for diagnostic collection.
+     * This populates {@link NormalizerOutput.fusions} in the playground pipeline view.
+     *
+     * ## Safety
+     * If the normalized token count exceeds {@link NormalizerOptions.maxTokens},
+     * an Error is thrown to prevent memory exhaustion from runaway rule expansion.
+     *
+     * @param tokens   - Raw tokens from the lexer
+     * @param onFusion - Optional fusion callback (overrides {@link NormalizerOptions.onFusion})
+     * @returns Normalized tokens ready for parsing
+     * @throws {Error} If the normalized token count exceeds maxTokens
+     */
+    normalize(tokens: Token[], onFusion?: (fusion: TokenFusion) => void): Token[];
+}
+
+export { type NormalizerMatch as N, type TokenFusion as T, type NormalizerRule as a, type NormalizerOptions as b, TokenNormalizer as c, createFusedToken as d };