solve-engine 1.0.0-beta.2 → 1.0.0-beta.4

package/dist/TokenNormalizer-t_GotBxr.d.ts

@@ -0,0 +1,377 @@
+import { T as Token } from './Token-BzG5G4ja.js';
+
+/**
+ * NormalizerRule, pluggable token normalization rule for the
+ * TokenNormalizer post-lexer pass.
+ *
+ * ## Purpose
+ * After the ExpressionLexer produces raw tokens (numbers, identifiers,
+ * operators, etc.), the TokenNormalizer applies domain-specific rules
+ * to transform the token stream before parsing. This keeps the lexer
+ * focused on single-token production and moves multi-token pattern
+ * matching into a dedicated normalization layer.
+ *
+ * ## How rules work
+ * Rules are applied in priority order (highest first). At each token
+ * position, the normalizer tries every rule in priority order until one
+ * matches. Matched tokens are consumed and replaced; unmatched tokens
+ * pass through unchanged.
+ *
+ * ## What rules can do
+ * - **Phrase fusion**: Merge consecutive words into compound tokens
+ *   (e.g., `"to" "the" "power" "of"` → `CARET`)
+ * - **Implicit operators**: Insert missing operators between tokens
+ *   (e.g., `NUMBER IDENT` → `NUMBER STAR IDENT`)
+ * - **Domain transformations**: Coalesce item names, currency pairs, etc.
+ *
+ * ## Why a separate file?
+ * This is a duplicate-free copy of the interface defined in
+ * TokenNormalizer.ts. Storing it in a separate file avoids circular
+ * imports, TokenNormalizer imports NormalizerRule, and rule factories
+ * import TokenNormalizer's `createFusedToken`.
+ *
+ * @module NormalizerRule
+ */
+
+/**
+ * Result of a successful rule match attempt against the token stream.
+ *
+ * When a {@link NormalizerRule.match} function finds a pattern at the
+ * current position, it returns a NormalizerMatch describing how many
+ * tokens to consume and what to replace them with.
+ *
+ * @example
+ * ```ts
+ * // The phrase "to the power of" (5 tokens) becomes a single CARET token
+ * const match: NormalizerMatch = {
+ *   consumed: 5,
+ *   replacement: [caretToken],
+ * };
+ * ```
+ */
+interface NormalizerMatch {
+    /**
+     * Number of tokens consumed from the stream at the match position.
+     * Must be ≥ 1, a match always advances the cursor.
+     */
+    consumed: number;
+    /**
+     * Replacement tokens to insert at the match position.
+     * May be empty (deletion), a single token (fusion), or multiple
+     * tokens (expansion/splitting).
+     */
+    replacement: Token[];
+    /**
+     * Human-readable rule name for diagnostic fusion tracking.
+     * When set, the normalizer uses this instead of the rule's `name`
+     * in {@link TokenFusion} records. Used by {@link PhraseTrie} to
+     * report which specific phrase matched (e.g., "phrase:to the power of").
+     */
+    ruleName?: string;
+}
+/**
+ * A pluggable normalization rule registered with the TokenNormalizer.
+ *
+ * Each rule has a {@link name}, {@link priority}, and {@link match} function.
+ * The match function receives the current token stream and a position,
+ * and returns a {@link NormalizerMatch} on success or `null` on failure.
+ *
+ * ## Priority ordering
+ * Higher priority rules are tried first at each position. This allows
+ * long phrases (priority 100, e.g. "to the power of") to match before
+ * shorter fragments (priority 80, e.g. "power of").
+ *
+ * ## Match contract
+ * - Must be pure (no side effects, no mutation of input tokens)
+ * - Must return `null` for any position that doesn't match
+ * - Consumed tokens must be consecutive starting at `pos`
+ * - Replacement tokens must be valid for downstream parsing
+ *
+ * @example
+ * ```ts
+ * // A phrase fusion rule that converts "to the power of" into CARET
+ * const phraseRule: NormalizerRule = {
+ *   name: 'phrase:to the power of',
+ *   priority: 100,
+ *   match: (tokens, pos) => {
+ *     if (pos + 4 > tokens.length) return null;
+ *     const phrase = tokens.slice(pos, pos + 5)
+ *       .map(t => t.value.toLowerCase()).join(' ');
+ *     if (phrase === 'to the power of') {
+ *       return {
+ *         consumed: 5,
+ *         replacement: [createFusedToken('CARET', 'to the power of', tokens.slice(pos, pos + 5))],
+ *       };
+ *     }
+ *     return null;
+ *   },
+ * };
+ * ```
+ */
+interface NormalizerRule {
+    /**
+     * Human-readable name for debugging and diagnostic display.
+     * Convention: `"category:description"`, e.g. `"phrase:to the power of"`.
+     */
+    readonly name: string;
+    /**
+     * Priority for ordering rules. Higher values are tried first.
+     * Recommended ranges:
+     * - 100: Long multi-word phrase fusion (e.g., "to the power of")
+     * - 80:  Short phrase fusion (e.g., "power of", "times by")
+     * - 50:  Implicit operator insertion (e.g., implicit multiply)
+     * - 20:  Domain-specific transformations
+     */
+    readonly priority: number;
+    /**
+     * Attempt to match a pattern starting at position `pos` in the token stream.
+     *
+     * @param tokens - The current token stream (may be partially normalized from prior passes)
+     * @param pos    - The current position to attempt matching from
+     * @returns A {@link NormalizerMatch} if the pattern is found, or `null` if no match
+     */
+    match(tokens: Token[], pos: number): NormalizerMatch | null;
+}
+/**
+ * Record of a token fusion event performed by the normalizer.
+ *
+ * When a rule merges multiple source tokens into fewer replacement tokens,
+ * the normalizer fires a {@link NormalizerOptions.onFusion | fusion callback}
+ * with this record. The playground uses these records to render the
+ * fusion detail table showing exactly which tokens were merged and by
+ * which rule.
+ *
+ * @example
+ * ```ts
+ * // "to" "the" "power" "of" fused into CARET "^"
+ * const fusion: TokenFusion = {
+ *   rule: "phrase:to the power of",
+ *   sourceTokens: [toToken, theToken, powerToken, ofToken],
+ *   fusedToken: caretToken,
+ * };
+ * ```
+ */
+interface TokenFusion {
+    /** The name of the rule that triggered this fusion (e.g., "phrase:to the power of") */
+    rule: string;
+    /** The original tokens before fusion, always ≥ 2 tokens */
+    sourceTokens: Token[];
+    /** The resulting fused token with its new type and combined value */
+    fusedToken: Token;
+}
+
+/**
+ * TokenNormalizer, post-lexer token normalization pass.
+ *
+ * ## Purpose
+ * Applies domain-specific {@link NormalizerRule | NormalizerRules} to the raw
+ * token stream produced by the {@link ExpressionLexer}. This keeps the lexer
+ * slim and focused on single-token production, while multi-token pattern
+ * matching (phrases, implicit operators, domain merges) lives here.
+ *
+ * ## What rules can do
+ * - **Phrase fusion**: Merge consecutive words into compound tokens
+ *   (e.g., `IDENT + ... + IDENT` → `CARET`)
+ * - **Implicit operator insertion**: Insert missing operators between tokens
+ *   (e.g., `NUMBER IDENT` → `NUMBER STAR IDENT`)
+ * - **Domain-specific transformations**: Coalesce item names, currency pairs,
+ *   percentage syntax, etc.
+ *
+ * ## Architecture
+ * Providers register NormalizerRules alongside Parselets and OpCode handlers
+ * via {@link IEnginePackage.normalizerRules}. The normalizer applies them
+ * greedily left-to-right in multiple passes with safety limits.
+ *
+ * @module TokenNormalizer
+ */
+
+/**
+ * Configuration options for the normalization pass.
+ *
+ * These control safety limits and diagnostic callbacks. The defaults
+ * are chosen to be generous enough for any realistic expression while
+ * preventing runaway token expansion from recursive rules.
+ */
+interface NormalizerOptions {
+    /**
+     * Maximum number of full passes over the token stream before bailing out.
+     * Prevents infinite loops from recursive rule chains.
+     * @default 100
+     */
+    maxPasses?: number;
+    /**
+     * Maximum number of tokens allowed after normalization.
+     * If exceeded, an Error is thrown rather than passing a bloated stream
+     * to the parser.
+     * @default 10000
+     */
+    maxTokens?: number;
+    /**
+     * Callback invoked for each fusion event during normalization.
+     * Used by diagnostic mode to populate {@link NormalizerOutput.fusions}.
+     * When `undefined`, fusions are still tracked internally but no callbacks fire.
+     */
+    onFusion?: (fusion: TokenFusion) => void;
+}
+/**
+ * Creates a new normalized token from fused source tokens.
+ *
+ * The fused token inherits position information (offset, line, column)
+ * from the first source token, which preserves source-map accuracy
+ * for error messages and diagnostic highlighting.
+ *
+ * It also records where the source text ENDS, on `sourceEnd`. The start alone
+ * is not enough to describe the span a fusion covers, because `text` is the
+ * replacement rather than the original: `10 frames` fuses into a FRAME_COUNT
+ * whose text is `10`, and a timecode fuses into a token whose text is a
+ * comma-separated tuple that appears nowhere in the line. Anything painting the
+ * line needs both ends, and only this function is in a position to know them.
+ *
+ * @param type         - The new token type (e.g., "CARET", "TIMES_BY")
+ * @param text         - The combined text representation (e.g., "to the power of")
+ * @param sourceTokens - The original tokens being fused (at least 2)
+ * @returns A new {@link LexerToken} with the fused type and combined text
+ */
+declare function createFusedToken(type: string, text: string, sourceTokens: Token[]): Token;
+/**
+ * Token normalizer: applies {@link NormalizerRule | NormalizerRules} to a token stream.
+ *
+ * ## Lifecycle
+ * 1. **Registration**: Rules are added via {@link register} and sorted by priority
+ * 2. **Normalization**: {@link normalize} applies rules greedily left-to-right
+ * 3. **Cleanup**: {@link clear} or {@link unregister} removes rules
+ *
+ * ## Normalization algorithm
+ * The normalizer uses a greedy left-to-right multi-pass algorithm:
+ * - At each token position, rules are tried in priority order (highest first)
+ * - When a rule matches, matched tokens are consumed and replaced
+ * - Processing continues from the replacement position
+ * - Multiple passes handle cascading matches (one rule's output triggers another)
+ * - Safety limits ({@link NormalizerOptions.maxPasses}) prevent infinite loops
+ *
+ * @example
+ * ```ts
+ * const normalizer = new TokenNormalizer();
+ * normalizer.register(phraseRule);       // "to the power of" → CARET
+ * normalizer.register(implicitMultRule); // "2 x" → "2 * x"
+ * const normalized = normalizer.normalize(rawTokens);
+ * ```
+ */
+declare class TokenNormalizer {
+    /** Registered rules, unsorted, the source of truth. */
+    private rules;
+    /**
+     * Priority-sorted copy of {@link rules}, rebuilt lazily on the next
+     * {@link normalize} call after a mutation. Rules are registered once at
+     * engine/package-registration time and essentially never change during a
+     * session, but normalize() runs on every keystroke-driven evaluation, an
+     * earlier version re-sorted a fresh copy of `rules` on every single call,
+     * which meant every keystroke paid for an allocation + sort of a list that
+     * had usually not changed since the last one. `null` means "stale, rebuild
+     * on next use"; {@link register}/{@link unregister}/{@link clear} all
+     * invalidate it.
+     */
+    private sortedRulesCache;
+    /**
+     * Phrase trie for single-pass multi-word phrase fusion.
+     * Tried at each token position BEFORE other rules, the trie walk
+     * is O(depth) vs O(R × W) for separate rule matching.
+     */
+    private phraseTrie;
+    /** Merged options with defaults applied. */
+    private options;
+    /**
+     * @param options - Configuration overrides for safety limits and diagnostic callbacks
+     */
+    constructor(options?: NormalizerOptions);
+    /**
+     * Register a normalization rule.
+     *
+     * Rules are sorted by priority (descending) on each {@link normalize} call.
+     * Multiple rules can share the same priority, they are tried in registration
+     * order when priorities are equal.
+     *
+     * @param rule - The rule to register
+     */
+    register(rule: NormalizerRule): void;
+    /**
+     * Unregister a normalization rule by its {@link NormalizerRule.name | name}.
+     *
+     * If multiple rules share the same name, all are removed. This is safe to
+     * call with a name that doesn't match any rule, it simply has no effect.
+     *
+     * @param ruleName - The name of the rule to remove
+     */
+    unregister(ruleName: string): void;
+    /**
+     * Remove all registered rules, resetting the normalizer to its initial state.
+     * Also clears the phrase trie.
+     */
+    clear(): void;
+    /**
+     * Priority-sorted view of {@link rules} (descending priority; registration
+     * order preserved for ties, since {@link Array.prototype.sort} is stable).
+     * Cached until the next mutation. See {@link sortedRulesCache}.
+     */
+    private getSortedRules;
+    /**
+     * Get the number of currently registered rules (excludes phrase trie entries).
+     */
+    get ruleCount(): number;
+    /**
+     * Register a multi-word phrase for fusion into a single compound token.
+     *
+     * This is the preferred way to add phrase patterns. It inserts into the
+     * internal {@link PhraseTrie}, which collapses all phrase rules into a
+     * single O(depth) trie walk per position, no separate rule scanning.
+     *
+     * @param phrase    - Multi-word phrase (e.g., "to the power of", "abyssal whip")
+     * @param tokenType - Target token type after fusion (e.g., "CARET", "ITEM")
+     */
+    addPhrase(phrase: string, tokenType: string): void;
+    /**
+     * Check whether a word can start any registered phrase.
+     *
+     * Used by {@link implicitMultiplyRule} to suppress `*` insertion
+     * before phrase-starting identifiers (e.g., "2 power of 3" → `2 ^ 3`,
+     * not `2 * power of 3`). Delegates to {@link PhraseTrie.canStart}.
+     */
+    /**
+     * Get all registered phrases and their target token types.
+     *
+     * Exposes the full phrase trie structure for diagnostic rendering
+     * in the playground's NormalizerTab. Returns ALL registered phrases,
+     * not just the ones that matched in the last evaluation.
+     */
+    getPhrases(): Record<string, string>;
+    canStartPhrase(word: string): boolean;
+    /**
+     * Normalize a token stream by applying all registered rules.
+     *
+     * ## Algorithm
+     * Applies rules greedily left-to-right in multiple passes:
+     * 1. Sort rules by priority (descending)
+     * 2. Walk the token stream left to right
+     * 3. At each position, try rules in priority order
+     * 4. On match: consume matched tokens, insert replacements, restart from insert point
+     * 5. On no match: pass token through unchanged
+     * 6. Repeat until a full pass produces no changes, or maxPasses is reached
+     *
+     * ## Fusion tracking
+     * When a rule consumes more tokens than it produces, the normalizer calls
+     * `onFusion` with a {@link TokenFusion} record for diagnostic collection.
+     * This populates {@link NormalizerOutput.fusions} in the playground pipeline view.
+     *
+     * ## Safety
+     * If the normalized token count exceeds {@link NormalizerOptions.maxTokens},
+     * an Error is thrown to prevent memory exhaustion from runaway rule expansion.
+     *
+     * @param tokens   - Raw tokens from the lexer
+     * @param onFusion - Optional fusion callback (overrides {@link NormalizerOptions.onFusion})
+     * @returns Normalized tokens ready for parsing
+     * @throws {Error} If the normalized token count exceeds maxTokens
+     */
+    normalize(tokens: Token[], onFusion?: (fusion: TokenFusion) => void): Token[];
+}
+
+export { type NormalizerMatch as N, type TokenFusion as T, type NormalizerRule as a, type NormalizerOptions as b, TokenNormalizer as c, createFusedToken as d };

package/dist/{chunk-HWSZJQCI.js → chunk-2CS6OMZK.js}

@@ -1,7 +1,7 @@
-import { getTokenCategory } from './chunk-KVILKGMS.js';
-import { ExpressionLexer } from './chunk-XVWCOTR6.js';
-import { getLocale } from './chunk-GOLDJNMZ.js';
-import { knownUnits } from './chunk-LIPPNDBE.js';
+import { getTokenCategory } from './chunk-AJA6LUI7.js';
+import { ExpressionLexer } from './chunk-SFQWJMKT.js';
+import { getLocale } from './chunk-XBTEO4OB.js';
+import { knownUnits } from './chunk-M5E34VG5.js';
 
 // src/lexer/LexerState.ts
 var LexerState = /* @__PURE__ */ ((LexerState2) => {
@@ -143,24 +143,45 @@ var Lexer = class {
     }
     return this.collectHighlightTokens(lineText);
   }
-  collectHighlightTokens(lineText) {
+  /**
+   * The same tokens {@link getHighlightTokens} reduces, before reduction.
+   *
+   * Exists because normalization operates on tokens, not on the flattened
+   * shape, and a consumer that wants phrase-fused highlighting has to run the
+   * normalizer between the two. See `LanguageService.getSemanticTokens`.
+   *
+   * @param lineText - One line of source.
+   * @returns Every token on the line that is worth painting, unreduced.
+   */
+  getHighlightTokenObjects(lineText) {
+    const classification = this.expressionLexer.classifyLine(lineText);
+    if (classification.skip && lineText.startsWith("> ")) {
+      return this.collectTokenObjects(lineText.slice(2));
+    }
+    if (classification.skip) return [];
+    return this.collectTokenObjects(lineText);
+  }
+  collectTokenObjects(lineText) {
     this.resetExpression(lineText);
     const result = [];
     for (const token of this) {
       if (token.type === "WS" || token.type === "NEWLINE") continue;
       if (token.type.startsWith("MD_")) continue;
       if (token.type === "INLINE_SOLVE_START" || token.type === "BACKTICK_CLOSE") continue;
-      result.push({
-        type: token.type,
-        value: token.value,
-        offset: token.offset,
-        col: token.col,
-        length: token.value.length,
-        category: getTokenCategory(token.type)
-      });
+      result.push(token);
     }
     return result;
   }
+  collectHighlightTokens(lineText) {
+    return this.collectTokenObjects(lineText).map((token) => ({
+      type: token.type,
+      value: token.value,
+      offset: token.offset,
+      col: token.col,
+      length: token.value.length,
+      category: getTokenCategory(token.type)
+    }));
+  }
 };
 var sharedLexer = new Lexer("en", void 0);
 
@@ -296,6 +317,7 @@ var BUILTIN_PHRASES = {
   "decrease by": "DECREASE_BY",
   "times by": "TIMES_BY",
   "multiply by": "MULTIPLY_BY",
+  "multiplied by": "MULTIPLY_BY",
   "divide by": "DIVIDE_BY"
 };
 function buildTokenLookup(localeCode = "en") {
@@ -307,5 +329,5 @@ function buildTokenLookup(localeCode = "en") {
 }
 
 export { Lexer, LexerState, buildTokenLookup, sharedLexer };
-//# sourceMappingURL=chunk-HWSZJQCI.js.map
-//# sourceMappingURL=chunk-HWSZJQCI.js.map
+//# sourceMappingURL=chunk-2CS6OMZK.js.map
+//# sourceMappingURL=chunk-2CS6OMZK.js.map

package/dist/chunk-2CS6OMZK.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/lexer/LexerState.ts","../src/lexer/Lexer.ts","../src/lexer/TokenClassRegistry.ts","../src/lexer/tokenRegistration.ts"],"names":["LexerState"],"mappings":";;;;;;AAMO,IAAK,UAAA,qBAAAA,WAAAA,KAAL;AACN,EAAAA,YAAA,MAAA,CAAA,GAAO,MAAA;AACP,EAAAA,YAAA,QAAA,CAAA,GAAS,QAAA;AACT,EAAAA,YAAA,QAAA,CAAA,GAAS,QAAA;AAHE,EAAA,OAAAA,WAAAA;AAAA,CAAA,EAAA,UAAA,IAAA,EAAA;;;ACaL,IAAM,QAAN,MAAY;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAkBjB,WAAA,CAAY,UAAA,GAAa,IAAA,EAAM,WAAA,EAA2B;AAf1D,IAAA,IAAA,CAAQ,YAAA,GAAA,MAAA;AAER,IAAA,IAAA,CAAQ,SAAA,GAAY,KAAA;AAIpB;AAAA;AAAA,IAAA,IAAA,CAAQ,SAAkB,EAAC;AAC3B,IAAA,IAAA,CAAQ,QAAA,GAAmB,CAAA;AAYzB,IAAA,IAAA,CAAK,eAAA,GAAkB,IAAI,eAAA,CAAgB,UAAA,EAAY,WAAW,CAAA;AAAA,EACpE;AAAA,EAEA,KAAA,CAAM,OAAe,KAAA,EAA0B;AAC7C,IAAA,MAAM,QAAA,GAAW,KAAA,IAAA,MAAA;AACjB,IAAA,IAAA,CAAK,YAAA,GAAe,QAAA;AACpB,IAAA,IAAA,CAAK,SAAA,GAAY,KAAA;AACjB,IAAA,IAAA,CAAK,WAAA,GAAc,MAAA;AAKnB,IAAA,IAAI,QAAA,KAAA,MAAA,aAA8B;AAChC,MAAA,MAAM,cAAA,GAAiB,IAAA,CAAK,eAAA,CAAgB,YAAA,CAAa,KAAK,CAAA;AAC9D,MAAA,IAAI,eAAe,IAAA,EAAM;AACvB,QAAA,IAAA,CAAK,SAAS,EAAC;AACf,QAAA,IAAA,CAAK,QAAA,GAAW,CAAA;AAChB,QAAA;AAAA,MACF;AAEA,MAAA,IAAA,CAAK,eAAA,CAAgB,MAAM,KAAK,CAAA;AAChC,MAAA,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,eAAA,CAAgB,WAAA,EAAY;AAC/C,MAAA,IAAA,CAAK,QAAA,GAAW,CAAA;AAAA,IAClB,CAAA,MAAO;AAEL,MAAA,IAAA,CAAK,eAAA,CAAgB,MAAM,KAAK,CAAA;AAChC,MAAA,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,eAAA,CAAgB,WAAA,EAAY;AAC/C,MAAA,IAAA,CAAK,QAAA,GAAW,CAAA;AAAA,IAClB;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,aAAa,QAAA,EAAsC;AACjD,IAAA,OAAO,IAAA,CAAK,eAAA,CAAgB,YAAA,CAAa,QAAQ,CAAA;AAAA,EACnD;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,iBAAiB,QAAA,EAAkB;AACjC,IAAA,OAAO,IAAA,CAAK,eAAA,CAAgB,gBAAA,CAAiB,QAAQ,CAAA;AAAA,EACvD;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,WAAA,GAAsC;AACpC,IAAA,OAAO,IAAA,CAAK,gBAAgB,WAAA,EAAY;AAAA,EAC1C;AAAA,EAEA,IAAA,GAA0B;AACxB,IAAA,IAAI,KAAK,SAAA,EAAW;AAClB,MAAA,IAAA,CAAK,SAAA,GAAY,KAAA;AACjB,MAAA,OAAO,IAAA,CAAK,WAAA;AAAA,IACd;AAEA,IAAA,IAAI,IAAA,CAAK,QAAA,GAAW,IAAA,CAAK,MAAA,CAAO,MAAA,EAAQ;AACtC,MAAA,OAAO,IAAA,CAAK,MAAA,CAAO,IAAA,CAAK,QAAA,EAAU,CAAA;AAAA,IACpC;AACA,IAAA,OAAO,MAAA;AAAA,EACT;AAAA,EAEA,IAAA,GAA0B;AACxB,IAAA,IAAI,IAAA,CAAK,SAAA,EAAW,OAAO,IAAA,CAAK,WAAA;AAChC,IAAA,IAAA,CAAK,WAAA,GAAc,KAAK,IAAA,EAAK;AAC7B,IAAA,IAAA,CAAK,SAAA,GAAY,IAAA;AACjB,IAAA,OAAO,IAAA,CAAK,WAAA;AAAA,EACd;AAAA,EAEA,CAAC,MAAA,CAAO,QAAQ,CAAA,GAAqB;AACnC,IAAA,OAAO,IAAA,CAAK,MAAA,CAAO,MAAA,CAAO,QAAQ,CAAA,EAAE;AAAA,EACtC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,mBAAmB,MAAA,EAA+B;AAChD,IAAA,IAAA,CAAK,eAAA,CAAgB,mBAAmB,MAAM,CAAA;AAAA,EAChD;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,qBAAqB,MAAA,EAA+B;AAClD,IAAA,IAAA,CAAK,eAAA,CAAgB,qBAAqB,MAAM,CAAA;AAAA,EAClD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,gBAAgB,KAAA,EAAqB;AACnC,IAAA,IAAA,CAAK,YAAA,GAAA,MAAA;AACL,IAAA,IAAA,CAAK,SAAA,GAAY,KAAA;AACjB,IAAA,IAAA,CAAK,WAAA,GAAc,MAAA;AACnB,IAAA,IAAA,CAAK,eAAA,CAAgB,MAAM,KAAK,CAAA;AAChC,IAAA,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,eAAA,CAAgB,WAAA,EAAY;AAC/C,IAAA,IAAA,CAAK,QAAA,GAAW,CAAA;AAAA,EAClB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,aAAa,IAAA,EAAgC;AAC3C,IAAA,OAAO,IAAA,CAAK,eAAA,CAAgB,YAAA,CAAa,IAAI,CAAA;AAAA,EAC/C;AAAA,EAEA,QAAA,GAAuB;AACrB,IAAA,OAAO,IAAA,CAAK,YAAA;AAAA,EACd;AAAA,EAEA,SAAS,KAAA,EAAyB;AAChC,IAAA,IAAA,CAAK,YAAA,GAAe,KAAA;AAAA,EACtB;AAAA,EAEA,mBAAmB,QAAA,EAAqI;AACtJ,IAAA,MAAM,cAAA,GAAiB,IAAA,CAAK,eAAA,CAAgB,YAAA,CAAa,QAAQ,CAAA;AAKjE,IAAA,IAAI,cAAA,CAAe,IAAA,IAAQ,QAAA,CAAS,UAAA,CAAW,IAAI,CAAA,EAAG;AACpD,MAAA,OAAO,IAAA,CAAK,sBAAA,CAAuB,QAAA,CAAS,KAAA,CAAM,CAAC,CAAC,CAAA;AAAA,IACtD;AAEA,IAAA,IAAI,eAAe,IAAA,EAAM;AACvB,MAAA,OAAO,EAAC;AAAA,IACV;AAEA,IAAA,OAAO,IAAA,CAAK,uBAAuB,QAAQ,CAAA;AAAA,EAC7C;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,yBAAyB,QAAA,EAA2B;AAClD,IAAA,MAAM,cAAA,GAAiB,IAAA,CAAK,eAAA,CAAgB,YAAA,CAAa,QAAQ,CAAA;AACjE,IAAA,IAAI,cAAA,CAAe,IAAA,IAAQ,QAAA,CAAS,UAAA,CAAW,IAAI,CAAA,EAAG;AACpD,MAAA,OAAO,IAAA,CAAK,mBAAA,CAAoB,QAAA,CAAS,KAAA,CAAM,CAAC,CAAC,CAAA;AAAA,IACnD;AACA,IAAA,IAAI,cAAA,CAAe,IAAA,EAAM,OAAO,EAAC;AACjC,IAAA,OAAO,IAAA,CAAK,oBAAoB,QAAQ,CAAA;AAAA,EAC1C;AAAA,EAEQ,oBAAoB,QAAA,EAA2B;AACrD,IAAA,IAAA,CAAK,gBAAgB,QAAQ,CAAA;AAC7B,IAAA,MAAM,SAAkB,EAAC;AACzB,IAAA,KAAA,MAAW,SAAS,IAAA,EAAM;AACxB,MAAA,IAAI,KAAA,CAAM,IAAA,KAAS,IAAA,IAAQ,KAAA,CAAM,SAAS,SAAA,EAAW;AACrD,MAAA,IAAI,KAAA,CAAM,IAAA,CAAK,UAAA,CAAW,KAAK,CAAA,EAAG;AAClC,MAAA,IAAI,KAAA,CAAM,IAAA,KAAS,oBAAA,IAAwB,KAAA,CAAM,SAAS,gBAAA,EAAkB;AAC5E,MAAA,MAAA,CAAO,KAAK,KAAK,CAAA;AAAA,IACnB;AACA,IAAA,OAAO,MAAA;AAAA,EACT;AAAA,EAEQ,uBAAuB,QAAA,EAAqI;AAClK,IAAA,OAAO,IAAA,CAAK,mBAAA,CAAoB,QAAQ,CAAA,CAAE,IAAI,CAAA,KAAA,MAAU;AAAA,MACtD,MAAM,KAAA,CAAM,IAAA;AAAA,MACZ,OAAO,KAAA,CAAM,KAAA;AAAA,MACb,QAAQ,KAAA,CAAM,MAAA;AAAA,MACd,KAAK,KAAA,CAAM,GAAA;AAAA,MACX,MAAA,EAAQ,MAAM,KAAA,CAAM,MAAA;AAAA,MACpB,QAAA,EAAU,gBAAA,CAAiB,KAAA,CAAM,IAAI;AAAA,KACvC,CAAE,CAAA;AAAA,EACJ;AACF;AAcO,IAAM,WAAA,GAAc,IAAI,KAAA,CAAM,IAAA,EAAM,MAAS;;;AC/I7C,IAAM,qBAAN,MAAyB;AAAA,EAAzB,WAAA,GAAA;AACL,IAAA,IAAA,CAAQ,UAAwB,EAAC;AACjC,IAAA,IAAA,CAAQ,gBAAA,GAAkD,IAAA;AAC1D,IAAA,IAAA,CAAQ,eAAA,GAAiD,IAAA;AACzD,IAAA,IAAA,CAAQ,SAAA,GAAwC,IAAA;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUhD,SAAS,UAAA,EAA8B;AACrC,IAAA,IAAA,CAAK,OAAA,CAAQ,KAAK,UAAU,CAAA;AAAA,EAC9B;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,WAAW,SAAA,EAAyB;AAClC,IAAA,IAAA,CAAK,UAAU,IAAA,CAAK,OAAA,CAAQ,OAAO,CAAA,CAAA,KAAK,CAAA,CAAE,cAAc,SAAS,CAAA;AAAA,EACnE;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,SAAA,CAAU,YAAoC,SAAA,EAA0C;AACtF,IAAA,IAAA,CAAK,gBAAA,GAAmB,UAAA;AACxB,IAAA,IAAA,CAAK,kBAAkB,SAAA,IAAa,IAAA;AAAA,EACtC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,SAAS,SAAA,EAAsC;AAC7C,IAAA,IAAA,CAAK,SAAA,GAAY,SAAA;AAAA,EACnB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,KAAA,GAAqB;AACnB,IAAA,MAAM,aAAA,uBAAoB,GAAA,EAAoB;AAG9C,IAAA,IAAI,KAAK,gBAAA,EAAkB;AACzB,MAAA,KAAA,MAAW,CAAC,SAAS,SAAS,CAAA,IAAK,OAAO,OAAA,CAAQ,IAAA,CAAK,gBAAgB,CAAA,EAAG;AACxE,QAAA,aAAA,CAAc,GAAA,CAAI,OAAA,CAAQ,WAAA,EAAY,EAAG,SAAS,CAAA;AAAA,MACpD;AAAA,IACF;AAGA,IAAA,MAAM,MAAA,GAAS,CAAC,GAAG,IAAA,CAAK,OAAO,CAAA,CAAE,IAAA,CAAK,CAAC,CAAA,EAAG,OAAO,CAAA,CAAE,QAAA,IAAY,CAAA,KAAM,CAAA,CAAE,YAAY,CAAA,CAAE,CAAA;AACrF,IAAA,KAAA,MAAW,MAAM,MAAA,EAAQ;AACvB,MAAA,KAAA,MAAW,OAAA,IAAW,MAAA,CAAO,IAAA,CAAK,EAAA,CAAG,QAAQ,CAAA,EAAG;AAC9C,QAAA,aAAA,CAAc,GAAA,CAAI,OAAA,CAAQ,WAAA,EAAY,EAAG,GAAG,SAAS,CAAA;AAAA,MACvD;AAAA,IACF;AAGA,IAAA,MAAM,UAAA,GAAa,KAAK,eAAA,EAAgB;AAGxC,IAAA,MAAM,gBAAA,GAAmB,KAAK,qBAAA,EAAsB;AAEpD,IAAA,OAAO;AAAA,MACL,aAAA;AAAA,MACA,UAAA;AAAA,MACA,gBAAA;AAAA,MACA,SAAA,EAAW,IAAA,CAAK,SAAA,oBAAa,IAAI,GAAA;AAAI,KACvC;AAAA,EACF;AAAA;AAAA,EAIQ,eAAA,GAAqC;AAC3C,IAAA,MAAM,IAAA,GAAmB,EAAE,QAAA,kBAAU,IAAI,KAAI,EAAE;AAG/C,IAAA,IAAI,KAAK,eAAA,EAAiB;AACxB,MAAA,KAAA,MAAW,CAAC,QAAQ,SAAS,CAAA,IAAK,OAAO,OAAA,CAAQ,IAAA,CAAK,eAAe,CAAA,EAAG;AACtE,QAAA,IAAA,CAAK,YAAA,CAAa,IAAA,EAAM,MAAA,CAAO,WAAA,IAAe,SAAS,CAAA;AAAA,MACzD;AAAA,IACF;AAGA,IAAA,KAAA,MAAW,EAAA,IAAM,KAAK,OAAA,EAAS;AAC7B,MAAA,IAAI,CAAC,GAAG,OAAA,EAAS;AACjB,MAAA,KAAA,MAAW,MAAA,IAAU,MAAA,CAAO,IAAA,CAAK,EAAA,CAAG,OAAO,CAAA,EAAG;AAC5C,QAAA,IAAA,CAAK,aAAa,IAAA,EAAM,MAAA,CAAO,WAAA,EAAY,EAAG,GAAG,SAAS,CAAA;AAAA,MAC5D;AAAA,IACF;AAEA,IAAA,OAAO,IAAA,CAAK,QAAA,CAAS,IAAA,GAAO,CAAA,GAAI,IAAA,GAAO,IAAA;AAAA,EACzC;AAAA,EAEQ,YAAA,CAAa,IAAA,EAAkB,MAAA,EAAgB,SAAA,EAAyB;AAC9E,IAAA,MAAM,KAAA,GAAQ,MAAA,CAAO,KAAA,CAAM,GAAG,CAAA;AAC9B,IAAA,IAAI,IAAA,GAAO,IAAA;AACX,IAAA,KAAA,MAAW,QAAQ,KAAA,EAAO;AACxB,MAAA,IAAI,CAAC,IAAA,CAAK,QAAA,CAAS,GAAA,CAAI,IAAI,CAAA,EAAG;AAC5B,QAAA,IAAA,CAAK,QAAA,CAAS,IAAI,IAAA,EAAM,EAAE,0BAAU,IAAI,GAAA,IAAO,CAAA;AAAA,MACjD;AACA,MAAA,IAAA,GAAO,IAAA,CAAK,QAAA,CAAS,GAAA,CAAI,IAAI,CAAA;AAAA,IAC/B;AAEA,IAAA,IAAI,CAAC,KAAK,IAAA,EAAM;AACd,MAAA,IAAA,CAAK,IAAA,GAAO,SAAA;AAAA,IACd;AAAA,EACF;AAAA,EAEQ,qBAAA,GAAqC;AAC3C,IAAA,MAAM,UAAA,uBAAiB,GAAA,EAAY;AAGnC,IAAA,IAAI,KAAK,eAAA,EAAiB;AACxB,MAAA,KAAA,MAAW,MAAA,IAAU,MAAA,CAAO,IAAA,CAAK,IAAA,CAAK,eAAe,CAAA,EAAG;AACtD,QAAA,MAAM,QAAQ,MAAA,CAAO,KAAA,CAAM,GAAG,CAAA,CAAE,CAAC,EAAE,WAAA,EAAY;AAC/C,QAAA,UAAA,CAAW,IAAI,KAAK,CAAA;AAAA,MACtB;AAAA,IACF;AAGA,IAAA,KAAA,MAAW,EAAA,IAAM,KAAK,OAAA,EAAS;AAC7B,MAAA,IAAI,CAAC,GAAG,OAAA,EAAS;AACjB,MAAA,KAAA,MAAW,MAAA,IAAU,MAAA,CAAO,IAAA,CAAK,EAAA,CAAG,OAAO,CAAA,EAAG;AAC5C,QAAA,MAAM,QAAQ,MAAA,CAAO,KAAA,CAAM,GAAG,CAAA,CAAE,CAAC,EAAE,WAAA,EAAY;AAC/C,QAAA,UAAA,CAAW,IAAI,KAAK,CAAA;AAAA,MACtB;AAAA,IACF;AAEA,IAAA,OAAO,UAAA;AAAA,EACT;AACF,CAAA;;;AClOA,IAAM,eAAA,GAA0C;AAAA,EAC9C,iBAAA,EAAmB,OAAA;AAAA,EACnB,UAAA,EAAY,OAAA;AAAA,EACZ,aAAA,EAAe,aAAA;AAAA,EACf,aAAA,EAAe,aAAA;AAAA,EACf,UAAA,EAAY,UAAA;AAAA,EACZ,aAAA,EAAe,aAAA;AAAA,EACf,eAAA,EAAiB,aAAA;AAAA,EACjB,WAAA,EAAa;AACf,CAAA;AAiBO,SAAS,gBAAA,CAAiB,aAAa,IAAA,EAAmB;AAC/D,EAAA,MAAM,MAAA,GAAkB,UAAU,UAAU,CAAA;AAC5C,EAAA,MAAM,QAAA,GAAW,IAAI,kBAAA,EAAmB;AAGxC,EAAA,QAAA,CAAS,SAAA,CAAU,MAAA,CAAO,UAAA,EAAY,eAAe,CAAA;AAGrD,EAAA,QAAA,CAAS,SAAS,UAAU,CAAA;AAG5B,EAAA,OAAO,SAAS,KAAA,EAAM;AACxB","file":"chunk-2CS6OMZK.js","sourcesContent":["/**\n * Lexer state machine modes.\n * - Main: document-level scanning with markdown classification\n * - Inline: expression embedded in markdown inline solve (`s\\`...\\``)\n * - String: inside a double-quoted string literal\n */\nexport enum LexerState {\n\tMain = \"main\",\n\tInline = \"inline\",\n\tString = \"string\",\n}\n","import { ExpressionLexer, LineClassification, LexerVocabulary, type ScanLineResult } from \"./ExpressionLexer\";\nimport { Token } from \"@solve-js/lexer/Token\";\nimport { LexerState } from \"@solve-js/lexer/LexerState\";\nimport { getTokenCategory } from \"@solve-js/language/TokenCategoryMap\";\nimport type { TokenCategory } from \"@solve-js/language/TokenCategory\";\nimport type { TokenLookup } from \"@solve-js/lexer/TokenClassRegistry\";\n\n/**\n * Public tokenizer wrapper around {@link ExpressionLexer}.\n *\n * `ExpressionLexer` does the actual character-by-character scanning;\n * `Lexer` adds a materialized-token-array streaming interface\n * (`next()`/`peek()`) plus line-classification state (`reset()`) so\n * callers can iterate a line's tokens without re-scanning on each peek.\n *\n * Each `ExpressionEngine` instance owns its own `Lexer`, and packages\n * extend it via {@link registerVocabulary} (keywords, operators, units)\n * see `IEnginePackage.lexerVocabulary`.\n */\nexport class Lexer {\n  /** Expression-mode lexer (Phase A: V8-optimized, replaces moo) */\n  private expressionLexer: ExpressionLexer;\n  private currentState: LexerState = LexerState.Main;\n  private peekedToken: Token | undefined;\n  private hasPeeked = false;\n\n  // Materialized token array from the last reset() call, used for\n  // next()/peek() streaming access.\n  private tokens: Token[] = [];\n  private tokenIdx: number = 0;\n\n  /**\n   * @param localeCode - Locale code (e.g., \"en\", \"de\"). Defaults to \"en\".\n   * @param tokenLookup - Optional TokenLookup from TokenClassRegistry.\n   *   When provided, configures ExpressionLexer to use registry-built\n   *   keyword/unit/phrase lookups instead of internal instance maps.\n   */\n  constructor(localeCode = \"en\", tokenLookup?: TokenLookup) {\n    // Pass the lookup directly to ExpressionLexer's constructor, it's an\n    // instance field now, not a static. Each Lexer instance gets its own\n    // isolated lookup, preventing cross-instance corruption.\n    this.expressionLexer = new ExpressionLexer(localeCode, tokenLookup);\n  }\n\n  reset(input: string, state?: LexerState): void {\n    const newState = state ?? LexerState.Main;\n    this.currentState = newState;\n    this.hasPeeked = false;\n    this.peekedToken = undefined;\n\n    // Phase B: Main state classifies the line with the markdown scanner.\n    // Skip lines (headings, fences, HRs, etc.) produce empty token arrays.\n    // Expression lines and lines with inline solves are tokenized normally.\n    if (newState === LexerState.Main) {\n      const classification = this.expressionLexer.classifyLine(input);\n      if (classification.skip) {\n        this.tokens = [];\n        this.tokenIdx = 0;\n        return;\n      }\n      // Expression line or markdown line with inline solves, tokenize.\n      this.expressionLexer.reset(input);\n      this.tokens = this.expressionLexer.tokenizeAll();\n      this.tokenIdx = 0;\n    } else {\n      // Non-main states (Inline, String), expression tokenization.\n      this.expressionLexer.reset(input);\n      this.tokens = this.expressionLexer.tokenizeAll();\n      this.tokenIdx = 0;\n    }\n  }\n\n  /**\n   * Classify a single line of markdown text (Phase B).\n   * Delegates to the ExpressionLexer's character-by-character scanner.\n   */\n  classifyLine(lineText: string): LineClassification {\n    return this.expressionLexer.classifyLine(lineText);\n  }\n\n  /**\n   * Find all inline solve markers in a line (Phase B).\n   * Delegates to the ExpressionLexer's character-by-character scanner.\n   */\n  findInlineSolves(lineText: string) {\n    return this.expressionLexer.findInlineSolves(lineText);\n  }\n\n  /**\n   * Every keyword this lexer currently recognizes (locale + plugin-contributed),\n   * mapped to the token type it lexes to. Delegates to the ExpressionLexer.\n   */\n  getKeywords(): Record<string, string> {\n    return this.expressionLexer.getKeywords();\n  }\n\n  next(): Token | undefined {\n    if (this.hasPeeked) {\n      this.hasPeeked = false;\n      return this.peekedToken;\n    }\n    // Materialized token array (ExpressionLexer path).\n    if (this.tokenIdx < this.tokens.length) {\n      return this.tokens[this.tokenIdx++];\n    }\n    return undefined;\n  }\n\n  peek(): Token | undefined {\n    if (this.hasPeeked) return this.peekedToken;\n    this.peekedToken = this.next();\n    this.hasPeeked = true;\n    return this.peekedToken;\n  }\n\n  [Symbol.iterator](): Iterator<Token> {\n    return this.tokens[Symbol.iterator]();\n  }\n\n  /**\n   * Register a plugin to extend the lexer with custom tokens.\n   * Delegates to the underlying ExpressionLexer.\n   *\n   * @see LexerVocabulary for the supported extension points.\n   */\n  registerVocabulary(plugin: LexerVocabulary): void {\n    this.expressionLexer.registerVocabulary(plugin);\n  }\n\n  /**\n   * Unregister a plugin, removing its custom tokens from the lexer.\n   * Delegates to the underlying ExpressionLexer.\n   */\n  unregisterVocabulary(plugin: LexerVocabulary): void {\n    this.expressionLexer.unregisterVocabulary(plugin);\n  }\n\n  /**\n   * Reset the lexer for expression-only text, skips the classifyLine()\n   * overhead in reset() for callers that already know the input is an\n   * evaluable expression (e.g., after isEmptyLine() confirmed non-skip).\n   */\n  resetExpression(input: string): void {\n    this.currentState = LexerState.Main;\n    this.hasPeeked = false;\n    this.peekedToken = undefined;\n    this.expressionLexer.reset(input);\n    this.tokens = this.expressionLexer.tokenizeAll();\n    this.tokenIdx = 0;\n  }\n\n  /**\n   * Scan a full document in one pass, classifying each line and\n   * tokenizing non-skipped lines. Delegates to ExpressionLexer.\n   *\n   * @returns ScanLineResult[], one per line, with classification + tokens.\n   */\n  scanDocument(text: string): ScanLineResult[] {\n    return this.expressionLexer.scanDocument(text);\n  }\n\n  getState(): LexerState {\n    return this.currentState;\n  }\n\n  setState(state: LexerState): void {\n    this.currentState = state;\n  }\n\n  getHighlightTokens(lineText: string): {type: string; value: string; offset: number; col: number; length: number; category: TokenCategory | undefined}[] {\n    const classification = this.expressionLexer.classifyLine(lineText);\n\n    // For blockquote lines, strip the \"> \" prefix and tokenize the expression content.\n    // This lets expressions inside blockquotes (e.g., \"> 1 + 2\") get syntax highlighted\n    // while pure structural lines (headings, code fences) remain unhighlighted.\n    if (classification.skip && lineText.startsWith(\"> \")) {\n      return this.collectHighlightTokens(lineText.slice(2));\n    }\n\n    if (classification.skip) {\n      return [];\n    }\n\n    return this.collectHighlightTokens(lineText);\n  }\n\n  /**\n   * The same tokens {@link getHighlightTokens} reduces, before reduction.\n   *\n   * Exists because normalization operates on tokens, not on the flattened\n   * shape, and a consumer that wants phrase-fused highlighting has to run the\n   * normalizer between the two. See `LanguageService.getSemanticTokens`.\n   *\n   * @param lineText - One line of source.\n   * @returns Every token on the line that is worth painting, unreduced.\n   */\n  getHighlightTokenObjects(lineText: string): Token[] {\n    const classification = this.expressionLexer.classifyLine(lineText);\n    if (classification.skip && lineText.startsWith(\"> \")) {\n      return this.collectTokenObjects(lineText.slice(2));\n    }\n    if (classification.skip) return [];\n    return this.collectTokenObjects(lineText);\n  }\n\n  private collectTokenObjects(lineText: string): Token[] {\n    this.resetExpression(lineText);\n    const result: Token[] = [];\n    for (const token of this) {\n      if (token.type === \"WS\" || token.type === \"NEWLINE\") continue;\n      if (token.type.startsWith(\"MD_\")) continue;\n      if (token.type === \"INLINE_SOLVE_START\" || token.type === \"BACKTICK_CLOSE\") continue;\n      result.push(token);\n    }\n    return result;\n  }\n\n  private collectHighlightTokens(lineText: string): {type: string; value: string; offset: number; col: number; length: number; category: TokenCategory | undefined}[] {\n    return this.collectTokenObjects(lineText).map(token => ({\n      type: token.type,\n      value: token.value,\n      offset: token.offset,\n      col: token.col,\n      length: token.value.length,\n      category: getTokenCategory(token.type),\n    }));\n  }\n}\n\n/**\n * A lexer for operations that do not depend on registered vocabulary.\n *\n * Line classification and inline-solve detection read characters looking for\n * headings, comment markers, fences and backtick spans, and never consult the\n * keyword, unit or operator tables. Every lexer therefore returns the same\n * answer, so the callers that have no engine to ask can use this one. Checked\n * by `__tests__/lexer/LineClassificationIsVocabularyIndependent.spec.ts`.\n *\n * Do not tokenize with this. An engine's own lexer carries the vocabulary its\n * packages registered; this one carries none.\n */\nexport const sharedLexer = new Lexer(\"en\", undefined);","/**\n * TokenClass, Plugin-extensible keyword registration for the Lexer.\n *\n * Providers call `registry.register(tokenClass)` to teach the lexer about\n * their keywords. The registry merges locale keywords, provider keywords,\n * phrase mappings, and unit names into an optimized TokenLookup structure\n * consumed by the Lexer.\n *\n * @example\n * registry.register({\n *   tokenType: 'CARET',\n *   keywords: {},\n *   phrases: { 'to the power of': true, 'power of': true },\n *   priority: 10,\n *   description: 'Exponentiation operators (x^y)',\n * });\n */\nexport interface TokenClass {\n  /** The token type string produced by the lexer (e.g., \"FUNC\", \"PI\", \"CARET\").\n   *  Must match a token type that a ParseletRegistry has a parselet for. */\n  tokenType: string;\n\n  /** Single-word keywords (case-insensitive). The lexer lowercases input\n   *  before lookup, so these should be lowercase. Example:\n   *  { sqrt: true, abs: true, sin: true, cos: true } for tokenType \"FUNC\" */\n  keywords: Record<string, boolean>;\n\n  /** Multi-word phrases (case-insensitive). Matched by the built-in PhraseMatcher\n   *  via the phrase trie. Example:\n   *  { \"to the power of\": true, \"power of\": true } for tokenType \"CARET\" */\n  phrases?: Record<string, boolean>;\n\n  /** Priority for conflict resolution. When two TokenClasses register\n   *  the same keyword, the higher-priority class wins. Locale keywords\n   *  have priority 0 (set via setLocale). Providers should use\n   *  priority >= 10 to override locale defaults. Default: 0 */\n  priority?: number;\n\n  /** Human-readable description for debugging and introspection */\n  description?: string;\n}\n\n// ── Phrase Trie ──────────────────────────────────────────────────────────────\n\n/** Trie node for multi-word phrase matching. */\nexport interface PhraseNode {\n  /** Complete phrase token type (null = intermediate node) */\n  type?: string;\n  children: Map<string, PhraseNode>;\n}\n\n// ── TokenLookup, Optimized lookup structure for the Lexer ──────────────────\n\n/**\n * The optimized lookup structure built by TokenClassRegistry.build().\n * Consumed by the Lexer for O(1) keyword → token type lookups and\n * O(word-count) phrase matching.\n */\nexport interface TokenLookup {\n  /** Lowercase keyword → token type. O(1) Map lookup. */\n  keywordToType: Map<string, string>;\n\n  /** Phrase trie for multi-word matching. Root node with children maps.\n   *  Null if no phrases registered. */\n  phraseTrie: PhraseNode | null;\n\n  /** Set of lowercase first-words of all registered phrases.\n   *  Used by the lexer to emit IDENT (not a phrase keyword) for words\n   *  that start multi-word phrases, deferring to the PhraseMatcher.\n   *\n   *  Example: \"to\" is in phraseStartWords because \"to the power of\" is a phrase.\n   *  When the lexer sees \"to\", it emits IDENT and lets the phrase matcher\n   *  combine \"to the power of\" into a single CARET token.\n   *\n   *  This prevents plugins from accidentally overriding phrase-start words.\n   */\n  phraseStartWords: Set<string>;\n\n  /** Case-sensitive unit names for UNIT fallback after keyword lookup fails. */\n  unitNames: ReadonlySet<string>;\n}\n\n// ── TokenClassRegistry ───────────────────────────────────────────────────────\n\n/**\n * Central registry for keyword→token-type mappings.\n *\n * Providers register TokenClasses; locales provide keyword maps;\n * units provide a name set. `build()` merges all sources into an\n * optimized TokenLookup consumed by the Lexer.\n *\n * Merge order (later overrides earlier):\n *   1. Locale keywords (priority 0)\n *   2. Provider keywords (sorted by priority ascending, higher priority wins)\n *\n * Unit names are stored separately (checked AFTER keyword lookup fails).\n * Phrases are stored in a trie for O(phrase-length) matching.\n */\nexport class TokenClassRegistry {\n  private classes: TokenClass[] = [];\n  private localeKeywordMap: Record<string, string> | null = null;\n  private localePhraseMap: Record<string, string> | null = null;\n  private unitNames: ReadonlySet<string> | null = null;\n\n  /**\n   * Register a provider's TokenClass. Must be called BEFORE build().\n   * Can be called multiple times to add more entries.\n   *\n   * Built-in token types CANNOT be overridden, throws a EngineError\n   * if the TokenClass attempts to register a keyword that conflicts\n   * with an already-registered token type.\n   */\n  register(tokenClass: TokenClass): void {\n    this.classes.push(tokenClass);\n  }\n\n  /**\n   * Unregister all TokenClasses for a given token type.\n   * Useful for plugin unload. Requires rebuild() to take effect.\n   */\n  unregister(tokenType: string): void {\n    this.classes = this.classes.filter(c => c.tokenType !== tokenType);\n  }\n\n  /**\n   * Set the locale's keyword→type map and optional phrase map.\n   * Called on locale change. Priority 0 (cannot override providers with higher priority).\n   */\n  setLocale(keywordMap: Record<string, string>, phraseMap?: Record<string, string>): void {\n    this.localeKeywordMap = keywordMap;\n    this.localePhraseMap = phraseMap ?? null;\n  }\n\n  /**\n   * Set the unit name set. Called when unit list changes.\n   * Units are stored separately (checked AFTER keyword lookup fails).\n   *\n   * Takes a ReadonlySet because the caller's set is derived from the\n   * conversion tables and must not be mutated; this class only ever reads it.\n   */\n  setUnits(unitNames: ReadonlySet<string>): void {\n    this.unitNames = unitNames;\n  }\n\n  /**\n   * Build the optimized TokenLookup from all registered sources.\n   *\n   * Merge order (later overrides earlier):\n   *   1. Locale keywords (priority 0)\n   *   2. Provider classes (sorted by priority ascending)\n   *\n   * Returns a frozen TokenLookup that the Lexer consumes.\n   * Call build() again after register()/setLocale()/setUnits() changes.\n   */\n  build(): TokenLookup {\n    const keywordToType = new Map<string, string>();\n\n    // Layer 1: Locale keywords (priority 0, lowest)\n    if (this.localeKeywordMap) {\n      for (const [keyword, tokenType] of Object.entries(this.localeKeywordMap)) {\n        keywordToType.set(keyword.toLowerCase(), tokenType);\n      }\n    }\n\n    // Layer 2: Provider keywords (sorted by priority ascending, higher wins)\n    const sorted = [...this.classes].sort((a, b) => (a.priority ?? 0) - (b.priority ?? 0));\n    for (const tc of sorted) {\n      for (const keyword of Object.keys(tc.keywords)) {\n        keywordToType.set(keyword.toLowerCase(), tc.tokenType);\n      }\n    }\n\n    // Build phrase trie from locale + provider phrases\n    const phraseTrie = this.buildPhraseTrie();\n\n    // Build phraseStartWords set (first word of every phrase)\n    const phraseStartWords = this.buildPhraseStartWords();\n\n    return {\n      keywordToType,\n      phraseTrie,\n      phraseStartWords,\n      unitNames: this.unitNames ?? new Set(),\n    };\n  }\n\n  // ── Private helpers ─────────────────────────────────────────────────────\n\n  private buildPhraseTrie(): PhraseNode | null {\n    const root: PhraseNode = { children: new Map() };\n\n    // Layer 1: Locale phrases\n    if (this.localePhraseMap) {\n      for (const [phrase, tokenType] of Object.entries(this.localePhraseMap)) {\n        this.insertPhrase(root, phrase.toLowerCase(), tokenType);\n      }\n    }\n\n    // Layer 2: Provider phrases\n    for (const tc of this.classes) {\n      if (!tc.phrases) continue;\n      for (const phrase of Object.keys(tc.phrases)) {\n        this.insertPhrase(root, phrase.toLowerCase(), tc.tokenType);\n      }\n    }\n\n    return root.children.size > 0 ? root : null;\n  }\n\n  private insertPhrase(root: PhraseNode, phrase: string, tokenType: string): void {\n    const words = phrase.split(' ');\n    let node = root;\n    for (const word of words) {\n      if (!node.children.has(word)) {\n        node.children.set(word, { children: new Map() });\n      }\n      node = node.children.get(word)!;\n    }\n    // Only set type if not already set, first-registered (locale) wins\n    if (!node.type) {\n      node.type = tokenType;\n    }\n  }\n\n  private buildPhraseStartWords(): Set<string> {\n    const startWords = new Set<string>();\n\n    // Locale phrase first words\n    if (this.localePhraseMap) {\n      for (const phrase of Object.keys(this.localePhraseMap)) {\n        const first = phrase.split(' ')[0].toLowerCase();\n        startWords.add(first);\n      }\n    }\n\n    // Provider phrase first words\n    for (const tc of this.classes) {\n      if (!tc.phrases) continue;\n      for (const phrase of Object.keys(tc.phrases)) {\n        const first = phrase.split(' ')[0].toLowerCase();\n        startWords.add(first);\n      }\n    }\n\n    return startWords;\n  }\n}","/**\n * tokenRegistration.ts, Bootstrap for building TokenLookup from locale, units, and phrases.\n *\n * This module centralizes the assembly of the TokenLookup consumed by ExpressionLexer.\n * It merges:\n *   1. Locale keywords (from ILocale.keywordMap)\n *   2. Built-in phrase patterns (to the power of, increase by, etc.)\n *   3. Known unit names (from units.ts)\n *\n * The resulting TokenLookup is frozen and passed to ExpressionLexer.configuredLookup\n * at construction time, enabling data-driven keyword/unit/phrase resolution.\n */\nimport { TokenClassRegistry } from '@solve-js/lexer/TokenClassRegistry';\nimport type { TokenLookup } from '@solve-js/lexer/TokenClassRegistry';\nimport { getLocale, type ILocale } from '@solve-js/constants/locales';\nimport { knownUnits } from '@solve-js/lexer/units';\n\n// ── Built-in phrase map ───────────────────────────────────────────────────\n// These are the multi-word expressions handled as compound tokens.\n// Matched by the PhraseMatcher (trie-based) in ExpressionLexer.tryMatchPhrase().\nconst BUILTIN_PHRASES: Record<string, string> = {\n  'to the power of': 'CARET',\n  'power of': 'CARET',\n  'increase by': 'INCREASE_BY',\n  'decrease by': 'DECREASE_BY',\n  'times by': 'TIMES_BY',\n  'multiply by': 'MULTIPLY_BY',\n  'multiplied by': 'MULTIPLY_BY',\n  'divide by': 'DIVIDE_BY',\n};\n\n/**\n * Build a TokenLookup from locale keywords, known units, and built-in phrases.\n *\n * Merge order (later overrides earlier):\n *   1. Locale keywords (priority 0, lowest, can be overridden by providers)\n *   2. Built-in phrases (via locale phraseMap)\n *   3. Known units (checked AFTER keyword lookup fails, via unitNames set)\n *\n * The resulting TokenLookup replaces the internal keyword map, unit set,\n * phrase trie, and phraseStartWords in ExpressionLexer when set via\n * ExpressionLexer.configuredLookup.\n *\n * @param localeCode - The locale code (e.g., \"en\", \"de\"). Defaults to \"en\".\n * @returns A frozen TokenLookup ready for consumption by the lexer.\n */\nexport function buildTokenLookup(localeCode = 'en'): TokenLookup {\n  const locale: ILocale = getLocale(localeCode);\n  const registry = new TokenClassRegistry();\n\n  // Layer 1: Locale keywords (priority 0, lowest)\n  registry.setLocale(locale.keywordMap, BUILTIN_PHRASES);\n\n  // Layer 2: Known units (checked after keyword lookup)\n  registry.setUnits(knownUnits);\n\n  // Build and return the lookup\n  return registry.build();\n}\n"]}