solve-engine 1.0.0-beta.2 → 1.0.0-beta.3

package/dist/{Lexer-BJdhlnej.d.cts → Lexer-Dkt7KGaU.d.ts}

@@ -1,4 +1,4 @@
-import { T as Token } from './Token-0jpvUdPY.cjs';
+import { T as Token } from './Token-DnU2hwFY.js';
 
 /** Trie node for multi-word phrase matching. */
 interface PhraseNode {
@@ -102,7 +102,11 @@ declare class LexerToken implements Token {
     lineBreaks: number;
     line: number;
     col: number;
-    constructor(type: string, typeId: number, value: string, text: string, offset: number, lineBreaks: number, line: number, col: number);
+    /** See {@link Token.sourceEnd}. Set only on normalizer-fused tokens. */
+    sourceEnd?: number | undefined;
+    constructor(type: string, typeId: number, value: string, text: string, offset: number, lineBreaks: number, line: number, col: number, 
+    /** See {@link Token.sourceEnd}. Set only on normalizer-fused tokens. */
+    sourceEnd?: number | undefined);
 }
 /**
  * Plugin interface for extending the ExpressionLexer with custom tokens.
@@ -512,6 +516,18 @@ declare class Lexer {
         length: number;
         category: TokenCategory | undefined;
     }[];
+    /**
+     * The same tokens {@link getHighlightTokens} reduces, before reduction.
+     *
+     * Exists because normalization operates on tokens, not on the flattened
+     * shape, and a consumer that wants phrase-fused highlighting has to run the
+     * normalizer between the two. See `LanguageService.getSemanticTokens`.
+     *
+     * @param lineText - One line of source.
+     * @returns Every token on the line that is worth painting, unreduced.
+     */
+    getHighlightTokenObjects(lineText: string): Token[];
+    private collectTokenObjects;
     private collectHighlightTokens;
 }
 /**

package/dist/{Lexer-D9l4Yrk2.d.ts → Lexer-y8uY6n6t.d.cts}

@@ -1,4 +1,4 @@
-import { T as Token } from './Token-0jpvUdPY.js';
+import { T as Token } from './Token-DnU2hwFY.cjs';
 
 /** Trie node for multi-word phrase matching. */
 interface PhraseNode {
@@ -102,7 +102,11 @@ declare class LexerToken implements Token {
     lineBreaks: number;
     line: number;
     col: number;
-    constructor(type: string, typeId: number, value: string, text: string, offset: number, lineBreaks: number, line: number, col: number);
+    /** See {@link Token.sourceEnd}. Set only on normalizer-fused tokens. */
+    sourceEnd?: number | undefined;
+    constructor(type: string, typeId: number, value: string, text: string, offset: number, lineBreaks: number, line: number, col: number, 
+    /** See {@link Token.sourceEnd}. Set only on normalizer-fused tokens. */
+    sourceEnd?: number | undefined);
 }
 /**
  * Plugin interface for extending the ExpressionLexer with custom tokens.
@@ -512,6 +516,18 @@ declare class Lexer {
         length: number;
         category: TokenCategory | undefined;
     }[];
+    /**
+     * The same tokens {@link getHighlightTokens} reduces, before reduction.
+     *
+     * Exists because normalization operates on tokens, not on the flattened
+     * shape, and a consumer that wants phrase-fused highlighting has to run the
+     * normalizer between the two. See `LanguageService.getSemanticTokens`.
+     *
+     * @param lineText - One line of source.
+     * @returns Every token on the line that is worth painting, unreduced.
+     */
+    getHighlightTokenObjects(lineText: string): Token[];
+    private collectTokenObjects;
     private collectHighlightTokens;
 }
 /**

package/dist/{PackageRegistry-D-Tv_7ca.d.cts → PackageRegistry-CblRmrHc.d.cts}

@@ -1,15 +1,15 @@
-import { a as PrecedenceParser, b as PrefixParselet, I as InfixParselet } from './Parselet-CK8cQu2a.cjs';
+import { a as PrecedenceParser, b as PrefixParselet, I as InfixParselet } from './Parselet-C974vhZG.cjs';
 import { V as Value } from './Value-KtAQNHur.cjs';
 import { IVariableSource } from './variables.cjs';
-import { M as MarkdownLineType, L as Lexer, e as TokenCategory, c as LexerVocabulary } from './Lexer-BJdhlnej.cjs';
+import { M as MarkdownLineType, L as Lexer, e as TokenCategory, c as LexerVocabulary } from './Lexer-y8uY6n6t.cjs';
 import { IAsyncResolver } from './resolvers.cjs';
-import { T as TokenFusion, N as NormalizerRule } from './NormalizerRule-BrVoVjmP.cjs';
+import { T as TokenFusion, c as TokenNormalizer, a as NormalizerRule } from './TokenNormalizer-CqBCrYkR.cjs';
 import { D as DependencyGraph, V as VM, a as DagSnapshot, b as EngineContext, S as ScopeManager, L as LineExecutionContext } from './ScopeManager-CKfQtl7l.cjs';
 import { a as BytecodeProgram } from './BytecodeBuilder-3egMyuDi.cjs';
 import { QueryClient } from '@tanstack/query-core';
 import { E as EngineError } from './EngineError-DvPSnhZi.cjs';
 import { a as DiagnosticReportJSON, D as DiagnosticPipeline } from './pipeline-DPUQt7yL.cjs';
-import { T as Token } from './Token-0jpvUdPY.cjs';
+import { T as Token } from './Token-DnU2hwFY.cjs';
 import { D as DEFAULT_CONFIG, E as EngineConfig } from './Configuration-fVpd-HC-.cjs';
 
 /**
@@ -1958,6 +1958,17 @@ declare class ExpressionEngine {
     getContext(): EngineContext;
     getScopeManager(): ScopeManager;
     getLexer(): Lexer;
+    /**
+     * This engine's token normalizer, with every phrase and rule its packages
+     * registered.
+     *
+     * Exposed for the highlighting path. `LanguageService` classifies tokens
+     * after the lexer but can only see phrase-fused types (a date literal, a
+     * timecode, a package's own fused token) by running the same normalizer
+     * evaluation runs. Sharing this instance rather than building a second one
+     * is what keeps the two from disagreeing.
+     */
+    getNormalizer(): TokenNormalizer;
     /**
      * Every completion candidate contributed by currently-registered
      * packages (`IEnginePackage.completionItems`), flattened across all of
@@ -2174,6 +2185,20 @@ interface LanguageServiceOptions {
      * evaluation engine's DAG snapshot instead.
      */
     variableNameSource?: () => Iterable<string>;
+    /**
+     * Run the normalizer on the highlighting path, so phrase-fused tokens are
+     * classified as the thing the parser will actually see.
+     *
+     * Off by default, and the default is a judgement rather than an oversight.
+     * Highlighting runs per keystroke, normalization is real work, and a host
+     * that is happy with lexer-level categories should not start paying for
+     * fusion because a new version shipped. Turn it on and `12/09/2026` is one
+     * `datetime` span instead of five number and operator spans; leave it off
+     * and nothing about this class changes.
+     *
+     * See `benchmarks/languageServiceBenchmarks.spec.ts` for what it costs.
+     */
+    normalizeForHighlighting?: boolean;
 }
 /**
  * Editor-agnostic "language server" for solve expressions: turns a line of
@@ -2229,11 +2254,42 @@ interface LanguageServiceOptions {
 declare class LanguageService {
     private engine;
     private variableNameSource;
+    private readonly normalizeForHighlighting;
     private cache;
     private staticCompletionCandidates;
     private staticCompletionIndex;
     constructor(engine?: ExpressionEngine | null, options?: LanguageServiceOptions);
     private defaultVariableNames;
+    /**
+     * Lex a line, normalize it, and place every resulting token back in the
+     * source text.
+     *
+     * ## Why placing them back is the hard part
+     *
+     * Normalization produces three kinds of token and only one of them can be
+     * highlighted the obvious way.
+     *
+     * A token the normalizer left alone still describes its own text, so its
+     * span is `offset` to `offset + value.length`, exactly as before.
+     *
+     * A FUSED token does not. `10 frames` becomes a FRAME_COUNT whose value is
+     * `10`, and a timecode becomes a token whose value is a comma-separated
+     * tuple appearing nowhere on the line. Those carry `sourceEnd`, set by
+     * `createFusedToken`, which is the only place that knows where the fusion
+     * ended.
+     *
+     * An INSERTED token has no text at all. Implicit multiplication puts a STAR
+     * at the following token's offset, so `5(3)` gains a `*` sitting exactly
+     * where the `(` is. Painting it would colour a character the reader never
+     * typed as an operator, and would overlap the token that really is there.
+     * These are dropped, detected by the one test that needs no cooperation
+     * from any rule: a token with no recorded fusion span must match the text
+     * at its own offset, and an inserted one does not.
+     *
+     * @param lineText - The raw line.
+     * @returns Spans in the same shape `Lexer.getHighlightTokens` returns.
+     */
+    private classifyNormalized;
     /**
      * Classify every recognized token on one line.
      *

package/dist/{PackageRegistry-DaH4aIKP.d.ts → PackageRegistry-wUQyVEyw.d.ts}

@@ -1,15 +1,15 @@
-import { a as PrecedenceParser, b as PrefixParselet, I as InfixParselet } from './Parselet-ConOIdRO.js';
+import { a as PrecedenceParser, b as PrefixParselet, I as InfixParselet } from './Parselet-CsmtM3It.js';
 import { V as Value } from './Value-KtAQNHur.js';
 import { IVariableSource } from './variables.js';
-import { M as MarkdownLineType, L as Lexer, e as TokenCategory, c as LexerVocabulary } from './Lexer-D9l4Yrk2.js';
+import { M as MarkdownLineType, L as Lexer, e as TokenCategory, c as LexerVocabulary } from './Lexer-Dkt7KGaU.js';
 import { IAsyncResolver } from './resolvers.js';
-import { T as TokenFusion, N as NormalizerRule } from './NormalizerRule-CEjf1FyD.js';
+import { T as TokenFusion, c as TokenNormalizer, a as NormalizerRule } from './TokenNormalizer-B8zmhO5X.js';
 import { D as DependencyGraph, V as VM, a as DagSnapshot, b as EngineContext, S as ScopeManager, L as LineExecutionContext } from './ScopeManager-B5k0nk05.js';
 import { a as BytecodeProgram } from './BytecodeBuilder-3egMyuDi.js';
 import { QueryClient } from '@tanstack/query-core';
 import { E as EngineError } from './EngineError-DvPSnhZi.js';
 import { a as DiagnosticReportJSON, D as DiagnosticPipeline } from './pipeline-CrsRPmsv.js';
-import { T as Token } from './Token-0jpvUdPY.js';
+import { T as Token } from './Token-DnU2hwFY.js';
 import { D as DEFAULT_CONFIG, E as EngineConfig } from './Configuration-fVpd-HC-.js';
 
 /**
@@ -1958,6 +1958,17 @@ declare class ExpressionEngine {
     getContext(): EngineContext;
     getScopeManager(): ScopeManager;
     getLexer(): Lexer;
+    /**
+     * This engine's token normalizer, with every phrase and rule its packages
+     * registered.
+     *
+     * Exposed for the highlighting path. `LanguageService` classifies tokens
+     * after the lexer but can only see phrase-fused types (a date literal, a
+     * timecode, a package's own fused token) by running the same normalizer
+     * evaluation runs. Sharing this instance rather than building a second one
+     * is what keeps the two from disagreeing.
+     */
+    getNormalizer(): TokenNormalizer;
     /**
      * Every completion candidate contributed by currently-registered
      * packages (`IEnginePackage.completionItems`), flattened across all of
@@ -2174,6 +2185,20 @@ interface LanguageServiceOptions {
      * evaluation engine's DAG snapshot instead.
      */
     variableNameSource?: () => Iterable<string>;
+    /**
+     * Run the normalizer on the highlighting path, so phrase-fused tokens are
+     * classified as the thing the parser will actually see.
+     *
+     * Off by default, and the default is a judgement rather than an oversight.
+     * Highlighting runs per keystroke, normalization is real work, and a host
+     * that is happy with lexer-level categories should not start paying for
+     * fusion because a new version shipped. Turn it on and `12/09/2026` is one
+     * `datetime` span instead of five number and operator spans; leave it off
+     * and nothing about this class changes.
+     *
+     * See `benchmarks/languageServiceBenchmarks.spec.ts` for what it costs.
+     */
+    normalizeForHighlighting?: boolean;
 }
 /**
  * Editor-agnostic "language server" for solve expressions: turns a line of
@@ -2229,11 +2254,42 @@ interface LanguageServiceOptions {
 declare class LanguageService {
     private engine;
     private variableNameSource;
+    private readonly normalizeForHighlighting;
     private cache;
     private staticCompletionCandidates;
     private staticCompletionIndex;
     constructor(engine?: ExpressionEngine | null, options?: LanguageServiceOptions);
     private defaultVariableNames;
+    /**
+     * Lex a line, normalize it, and place every resulting token back in the
+     * source text.
+     *
+     * ## Why placing them back is the hard part
+     *
+     * Normalization produces three kinds of token and only one of them can be
+     * highlighted the obvious way.
+     *
+     * A token the normalizer left alone still describes its own text, so its
+     * span is `offset` to `offset + value.length`, exactly as before.
+     *
+     * A FUSED token does not. `10 frames` becomes a FRAME_COUNT whose value is
+     * `10`, and a timecode becomes a token whose value is a comma-separated
+     * tuple appearing nowhere on the line. Those carry `sourceEnd`, set by
+     * `createFusedToken`, which is the only place that knows where the fusion
+     * ended.
+     *
+     * An INSERTED token has no text at all. Implicit multiplication puts a STAR
+     * at the following token's offset, so `5(3)` gains a `*` sitting exactly
+     * where the `(` is. Painting it would colour a character the reader never
+     * typed as an operator, and would overlap the token that really is there.
+     * These are dropped, detected by the one test that needs no cooperation
+     * from any rule: a token with no recorded fusion span must match the text
+     * at its own offset, and an inserted one does not.
+     *
+     * @param lineText - The raw line.
+     * @returns Spans in the same shape `Lexer.getHighlightTokens` returns.
+     */
+    private classifyNormalized;
     /**
      * Classify every recognized token on one line.
      *

package/dist/{Parselet-CK8cQu2a.d.cts → Parselet-C974vhZG.d.cts}

@@ -1,5 +1,5 @@
 import { B as BytecodeBuilder } from './BytecodeBuilder-3egMyuDi.cjs';
-import { T as Token } from './Token-0jpvUdPY.cjs';
+import { T as Token } from './Token-DnU2hwFY.cjs';
 import { D as DiagnosticPipeline } from './pipeline-DPUQt7yL.cjs';
 
 /**

package/dist/{Parselet-ConOIdRO.d.ts → Parselet-CsmtM3It.d.ts}

@@ -1,5 +1,5 @@
 import { B as BytecodeBuilder } from './BytecodeBuilder-3egMyuDi.js';
-import { T as Token } from './Token-0jpvUdPY.js';
+import { T as Token } from './Token-DnU2hwFY.js';
 import { D as DiagnosticPipeline } from './pipeline-CrsRPmsv.js';
 
 /**

package/dist/{Token-0jpvUdPY.d.cts → Token-DnU2hwFY.d.cts}

@@ -19,6 +19,20 @@ interface Token {
     lineBreaks: number;
     line: number;
     col: number;
+    /**
+     * End offset of the source text this token was fused from, exclusive.
+     *
+     * Only set on tokens the normalizer built by fusing several lexer tokens
+     * into one, and only there because those are the tokens whose `value` no
+     * longer describes the text they came from. `10 frames` fuses into a
+     * FRAME_COUNT whose value is `10`, so `offset + value.length` would cover
+     * two characters of a nine-character phrase.
+     *
+     * Evaluation never reads this. It exists so that a consumer painting a line
+     * (see `LanguageService.getSemanticTokens`) can highlight a fused phrase as
+     * the one thing the parser will see rather than as its parts.
+     */
+    sourceEnd?: number;
 }
 /**
  * Canonical token type name constants.

package/dist/{Token-0jpvUdPY.d.ts → Token-DnU2hwFY.d.ts}

@@ -19,6 +19,20 @@ interface Token {
     lineBreaks: number;
     line: number;
     col: number;
+    /**
+     * End offset of the source text this token was fused from, exclusive.
+     *
+     * Only set on tokens the normalizer built by fusing several lexer tokens
+     * into one, and only there because those are the tokens whose `value` no
+     * longer describes the text they came from. `10 frames` fuses into a
+     * FRAME_COUNT whose value is `10`, so `offset + value.length` would cover
+     * two characters of a nine-character phrase.
+     *
+     * Evaluation never reads this. It exists so that a consumer painting a line
+     * (see `LanguageService.getSemanticTokens`) can highlight a fused phrase as
+     * the one thing the parser will see rather than as its parts.
+     */
+    sourceEnd?: number;
 }
 /**
  * Canonical token type name constants.