parseman 0.47.1 → 0.48.1

package/dist/compiler/token-alphabet.d.ts

@@ -1,9 +1,19 @@
 /**
- * Derived tokenization, part 1: the terminal ALPHABET.
+ * Derived tokenization groundwork: primitive kernels and lexical-token families.
  *
- * Parseman knows every terminal in the composed grammar at macro time — every
- * `literal`, `keywords`, and `regex`. This module walks the combinator graph
- * and assigns each distinct terminal a small integer TOKEN ID.
+ * `collectAlphabet()` is the preserved primitive-terminal scanner groundwork:
+ * it knows every `literal`, `keywords`, and `regex` in the composed grammar and
+ * assigns each one a small integer KERNEL id. Those ids are implementation
+ * details, not parser tokens. In particular, unwrapping
+ * `token(sequence(identifier, optional(literal('('))))` must never publish an
+ * identifier id followed by a `(` id: `token()` defines ONE contiguous source
+ * token and ONE CST leaf for the complete range.
+ *
+ * `collectLexicalAlphabet()` is the production-facing boundary. It assigns ids
+ * to authored `token()` families, normalizes effect-free bodies into canonical
+ * lexical IR, interns equivalent recognizers, and records the downstream
+ * `dispatch()` outcome classifiers over the same range. Primitive terminals
+ * stay private recognizer machinery.
  *
  * Two things are deliberately separated, because conflating them is what makes
  * a naive derived scanner useless:
@@ -20,7 +30,7 @@
  * Nothing here decides HOW a token is recognized (see `token-scanner.ts`); this
  * module only decides WHAT the tokens are and WHICH ones each site may see.
  */
-import type { Combinator } from '../types.ts';
+import type { AutoNotCheck, Combinator, FirstSet, ParserDef } from '../types.ts';
 /** A terminal in the derived alphabet, with its globally assigned id. */
 export type TokenTerminal = {
     kind: 'literal';
@@ -51,6 +61,800 @@ export type Alphabet = {
     /** The combinator that contributed each id (first one wins), for candidate sets. */
     originOf: Map<number, Combinator<unknown>>;
 };
+/** Effect-free recognition IR underneath one authored `token()` boundary. */
+export type LexicalIr = {
+    readonly kind: 'literal';
+    readonly value: string;
+    readonly caseInsensitive: boolean;
+} | {
+    readonly kind: 'keywords';
+    readonly words: readonly string[];
+    readonly caseInsensitive: boolean;
+    readonly boundary: string | undefined;
+} | {
+    readonly kind: 'regex';
+    readonly source: string;
+    readonly flags: string;
+} | {
+    readonly kind: 'sequence';
+    readonly parts: readonly LexicalIr[];
+} | {
+    readonly kind: 'choice';
+    readonly arms: readonly LexicalIr[];
+} | {
+    readonly kind: 'repeat';
+    readonly body: LexicalIr;
+    readonly min: number;
+    readonly max: number | null;
+    readonly greedy: boolean;
+    readonly mode: 'possessive' | 'backtracking';
+} | {
+    readonly kind: 'assert';
+    readonly positive: boolean;
+    readonly body: LexicalIr;
+} | {
+    readonly kind: 'balanced';
+    readonly open: string;
+    readonly close: string;
+    readonly strict: boolean;
+    readonly raw: boolean;
+    /** Ordered explicit skippers; ambient trivia/scanSkip precede them unless raw. */
+    readonly skip: readonly LexicalIr[];
+};
+/** First complete executable lexical body. It is intentionally a semantic
+ * shape rather than an opcode: the encoder relocates the regex into the table
+ * pool only after TOKEN wins the compiler-only cost comparison. */
+export type OptionalSuffixLexBody = {
+    readonly kind: 'regex-optional-code-unit';
+    readonly source: string;
+    readonly flags: string;
+    readonly suffix: string;
+};
+export type ExecutableOptionalSuffixLexBody = OptionalSuffixLexBody & {
+    /** Match the two authored terminal line-publication points exactly. */
+    readonly baseCanMatchNewline: boolean;
+    readonly suffixCanMatchNewline: boolean;
+    /** Failure authority belongs to the resolved final regex, not the lazy
+     * construction thunk that originally named it. */
+    readonly expected: readonly string[];
+};
+/** A direct authored terminal under token(). The boundary shell still owns
+ * source slicing/CST; this body owns exactly the terminal recognizer and lines. */
+export type ExecutableTerminalLexBody = {
+    readonly kind: 'regex-terminal';
+    readonly source: string;
+    readonly flags: string;
+    readonly canMatchNewline: boolean;
+    readonly expected: readonly string[];
+};
+export type ExecutableOrdered4LexBody = {
+    readonly kind: 'ordered4-terminals';
+    readonly commonFirstSet: FirstSet;
+    readonly commonMissExpected: readonly string[];
+    readonly expected: readonly string[];
+    readonly arms: readonly [
+        ExecutableTerminalLexBody,
+        ExecutableTerminalLexBody,
+        ExecutableTerminalLexBody,
+        ExecutableTerminalLexBody
+    ];
+};
+export type ExecutableNot2LexBody = {
+    readonly kind: 'not2-ordered4-terminal';
+    readonly guard0Expected: readonly string[];
+    readonly guard1Expected: readonly string[];
+    readonly terminalExpected: readonly string[];
+    readonly guard0: ExecutableOrdered4LexBody['arms'];
+    readonly guard1: ExecutableTerminalLexBody;
+    readonly terminal: ExecutableTerminalLexBody;
+};
+export type ExecutableLexNode = {
+    readonly kind: 'terminal';
+    readonly terminal: ExecutableTerminalLexBody;
+} | {
+    readonly kind: 'sequence';
+    readonly parts: readonly ExecutableLexNode[];
+} | {
+    readonly kind: 'choice';
+    readonly arms: readonly ExecutableLexNode[];
+    readonly firstSets: readonly FirstSet[];
+    readonly armExpected: readonly (readonly string[])[];
+    readonly expected: readonly string[];
+} | {
+    readonly kind: 'not';
+    readonly body: ExecutableLexNode;
+    readonly expected: readonly string[];
+} | {
+    readonly kind: 'optional';
+    readonly body: ExecutableLexNode;
+};
+export type ExecutableFixedLexBody = {
+    readonly kind: 'fixed-tree';
+    readonly root: ExecutableLexNode;
+};
+/** A balanced token whose recognition remains owned by the canonical scanner
+ * pool. The selected shell removes only the losing token/parent topology; it
+ * never restates ambient scanSkip, recovery, or skipper semantics. */
+export type ExecutableBalancedLexBody = {
+    readonly kind: 'balanced-scan';
+    readonly open: string;
+    readonly close: string;
+    readonly raw: boolean;
+    readonly strict: boolean;
+    readonly ownSkip: readonly Combinator<unknown>[];
+};
+export type ExecutableLexBody = ExecutableOptionalSuffixLexBody | ExecutableTerminalLexBody | ExecutableOrdered4LexBody | ExecutableNot2LexBody | ExecutableFixedLexBody | ExecutableBalancedLexBody;
+export type LexBodyCandidate = {
+    readonly strategy: 'character';
+    readonly estimatedOps: number;
+} | {
+    readonly strategy: 'token';
+    readonly estimatedOps: number;
+    readonly body: OptionalSuffixLexBody;
+};
+/** Exact recognizer shape admitted by the first end-to-end Stage-C tranche. */
+export declare function optionalSuffixLexBody(ir: LexicalIr): OptionalSuffixLexBody | undefined;
+/** Compare two COMPLETE candidates. The CHARACTER cost is the existing
+ * token-boundary + sequence + regex + optional + literal body. The TOKEN cost
+ * is one selected body and one materialization boundary; neither candidate is
+ * serialized until this decision returns. */
+export declare function selectOptionalSuffixLexBody(ir: LexicalIr): LexBodyCandidate;
+/** Refuse wrapper/effect lookalikes even when normalization erases them to the
+ * same language. This first executable body owns only the authored token
+ * boundary around a direct sequence(regex, optional(literal-one-code-unit)). */
+export declare function directOptionalSuffixTokenBody(parser: Combinator<unknown>, resolve?: (name: string) => Combinator<unknown> | undefined): ExecutableOptionalSuffixLexBody | undefined;
+/** Exact direct token terminal bodies. `keywords()` is represented by the same
+ * canonical sticky RegExp its authored parser and OP_RX lowering use. Literal
+ * stays CHARACTER for now: case-insensitive literal() intentionally uses ASCII
+ * folding, which is not JavaScript RegExp `/i` semantics for non-ASCII text. */
+export declare function directTerminalTokenBody(parser: Combinator<unknown>, resolve?: (name: string) => Combinator<unknown> | undefined): ExecutableTerminalLexBody | undefined;
+/** First composite selected body: the hot Less at-keyword choice. Keep this a
+ * fixed template, not a recursive lexical interpreter. Equal finite first sets
+ * are load-bearing: at a matching lead all four arms really run in source
+ * order, while an excluded lead contributes only the choice's static opener
+ * set and no probe event. */
+export declare function directOrdered4TokenBody(parser: Combinator<unknown>, resolve?: (name: string) => Combinator<unknown> | undefined): ExecutableOrdered4LexBody | undefined;
+export declare function directNot2TokenBody(parser: Combinator<unknown>, resolve?: (name: string) => Combinator<unknown> | undefined): ExecutableNot2LexBody | undefined;
+export declare function directFixedTokenBody(parser: Combinator<unknown>, resolve?: (name: string) => Combinator<unknown> | undefined): ExecutableFixedLexBody | undefined;
+export declare function directBalancedTokenBody(parser: Combinator<unknown>, resolve?: (name: string) => Combinator<unknown> | undefined): ExecutableBalancedLexBody | undefined;
+export declare function directExecutableTokenBody(parser: Combinator<unknown>, resolve?: (name: string) => Combinator<unknown> | undefined): ExecutableLexBody | undefined;
+/** One canonical recognizer spec, shared by every family with equal lexical IR. */
+export type LexicalRecognizer = {
+    readonly id: number;
+    readonly key: string;
+    readonly ir: LexicalIr;
+    /** Crosswalk into the compiler-wide canonical recognition-family interner. */
+    readonly capabilityFamilyId: number;
+};
+/**
+ * Spelling-specific failure/probe contract. Families may share recognition IR
+ * without sharing where a structured spelling fails or what it expects.
+ */
+export type LexicalDiagnosticPlan = {
+    readonly id: number;
+    readonly body: Combinator<unknown>;
+};
+/** Pointer-free, compiler-only checkpoints inside one authored token body. */
+export type LexicalDiagnosticEvent = {
+    readonly op: 'FAIL';
+    readonly state: number;
+    readonly expectedId: number;
+} | {
+    readonly op: 'ASSERT';
+    readonly state: number;
+    readonly innerStart: number;
+    readonly innerEnd: number;
+    readonly positive: boolean;
+    readonly expectedId: number;
+    readonly snapshotPolicy: 'saveLookaheadMark';
+    readonly execution: 'deferred';
+} | {
+    readonly op: 'REQUIRE';
+    readonly state: number;
+    readonly childStart: number;
+    readonly childEnd: number;
+    readonly expectedId: number;
+    readonly when: 'mandatory-iteration-2-to-min';
+    readonly committedChild: 'propagate';
+    readonly probe: 'child-only';
+    readonly anchor: 'repeat-position';
+} | {
+    readonly op: 'BAL_CLOSE_STRICT';
+    readonly state: number;
+    readonly expectedId: number;
+    readonly probe: true;
+    readonly committed: false;
+    readonly when: 'close-miss-after-open-body-success';
+} | {
+    readonly op: 'BAL_CLOSE_RECOVER';
+    readonly state: number;
+    readonly expectedId: number;
+    readonly probe: true;
+    readonly committed: false;
+    readonly when: 'close-miss-after-open-body-success';
+    readonly result: 'parseError';
+    readonly errorSpan: 'close-position';
+    readonly lineAnnotation: 'when-active';
+    readonly errorSink: 'when-active';
+    readonly cstErrorCapture: 'when-active';
+};
+/**
+ * Stage-B2 diagnostic metadata. It is intentionally absent from TableProgram:
+ * assertion execution and every token-boundary effect remain capability GAPs.
+ */
+export type LexicalTransitionDiagnosticPlan = {
+    readonly id: number;
+    readonly stateCount: number;
+    readonly expected: readonly (readonly string[])[];
+    readonly events: readonly LexicalDiagnosticEvent[];
+};
+export type LexicalControlNode = {
+    readonly id: number;
+    readonly parentControlId?: number;
+    /** Authored parser kind, or the conditional/repeated balanced skipper role. */
+    readonly kind: ParserDef['tag'] | 'balanced' | 'balanced-skip';
+    readonly stateStart: number;
+    readonly stateEnd: number;
+};
+export type LexicalControlPlan = {
+    readonly id: number;
+    /** Pointer-free source-control index over the one normalization session. */
+    readonly controls: readonly LexicalControlNode[];
+};
+/** One globally interned lexical family/base-token id. */
+export type LexicalTokenFamily = {
+    readonly id: number;
+    readonly recognizerId: number;
+};
+/** One authored `token()` site, with spelling-specific diagnostics/effects. */
+export type LexicalTokenSite = {
+    readonly parser: Combinator<unknown>;
+    readonly body: Combinator<unknown>;
+    /** Absent on a refused site; refused metadata allocates no diagnostic plan. */
+    readonly diagnosticId?: number;
+    /** All three are absent when normalization declined. */
+    readonly familyId?: number;
+    readonly recognizerId?: number;
+    readonly refusal?: string;
+};
+export type LexicalCapabilityStatus = {
+    readonly kind: 'complete';
+} | {
+    readonly kind: 'gap';
+    readonly reason: string;
+} | {
+    readonly kind: 'impossible';
+    readonly proof: string;
+};
+export type LexicalCapabilityPhase = {
+    readonly representation: LexicalCapabilityStatus;
+    /** A complete body is constructible for every required reader/variant before selection. */
+    readonly executableLowering: LexicalCapabilityStatus;
+};
+export type LexicalCapabilityObligations = {
+    readonly recognition: LexicalCapabilityPhase;
+    readonly diagnostics: LexicalCapabilityPhase;
+    readonly boundaryPlan: LexicalCapabilityPhase;
+    readonly materializationPlan: LexicalCapabilityPhase;
+    readonly supportedVariants: LexicalCapabilityPhase;
+    readonly bindingAndReachability: LexicalCapabilityPhase;
+};
+/** Built-in authored token() context transaction. Compiler-only in this tranche. */
+export type LexicalBoundaryPlan = {
+    readonly id: 0;
+    readonly kind: 'token-context-transaction';
+};
+/** Built-in authored token() source-range value/CST materialization. */
+export type LexicalMaterializationPlan = {
+    readonly id: 0;
+    readonly kind: 'token-source-range';
+};
+/** Exact construction-time grammar wrapper policy, not merely its effective context. */
+export type LexicalGrammarWrapperSpec = {
+    readonly id: number;
+    readonly sourceOperationId: number;
+    readonly clearTrivia: boolean;
+    readonly triviaBindingId?: number;
+    readonly trackLines: 'on' | 'off' | 'inherit';
+    readonly captureTrivia: boolean;
+    readonly captureTriviaKindsId?: number;
+    readonly rootCapture: 'opaque' | 'inherit';
+    readonly clonePolicy: 'spread-existing-or-create-canonical';
+    readonly postChildPolicy: 'line-propagate-then-annotate-or-return';
+};
+/** Well-nested overlay over the sole LexicalIr/B2 control/state authority. */
+export type LexicalWrapperFrame = {
+    readonly id: number;
+    readonly parentFrameId?: number;
+    readonly controlId: number;
+    readonly stateStart: number;
+    readonly stateEnd: number;
+    readonly kind: 'token';
+    readonly boundaryPlanId: 0;
+    readonly materializationPlanId: 0;
+} | {
+    readonly id: number;
+    readonly parentFrameId?: number;
+    readonly controlId: number;
+    readonly stateStart: number;
+    readonly stateEnd: number;
+    readonly kind: 'grammar';
+    readonly wrapperSpecId: number;
+};
+export type LexicalBoundaryTopology = {
+    readonly id: number;
+    /** Source-order outer-before-inner frames; recognition/control is not duplicated. */
+    readonly frames: readonly LexicalWrapperFrame[];
+};
+export type LexicalCapabilityContext = {
+    readonly trivia: Combinator<unknown> | undefined;
+    readonly scanSkip: readonly Combinator<unknown>[];
+    readonly trackLines: boolean;
+    readonly captureTrivia: boolean;
+    readonly opaqueRootCapture: boolean;
+    readonly dynamicState: boolean;
+};
+/** Immutable compiler occurrence context. Live scanner callbacks are policy,
+ * never identity-bearing lexical facts or pending-range keys. */
+export type LexicalContextSnapshot = {
+    readonly id: number;
+    readonly key: string;
+    readonly hasTrivia: boolean;
+    readonly hasScanSkip: boolean;
+    readonly trackLines: boolean;
+    readonly captureTrivia: boolean;
+    readonly opaqueRootCapture: boolean;
+    readonly dynamicState: boolean;
+};
+export type LexicalOwnSkipEntry = {
+    readonly semanticKey?: string;
+    readonly ir?: LexicalIr;
+    readonly status: LexicalCapabilityStatus;
+};
+/** Construction-owned skipper children only. Ambient trivia/scanSkip never
+ * enter this pool because their object and `.parse` lookup are observable. */
+export type LexicalOwnSkipPlan = {
+    readonly id: number;
+    readonly entries: readonly LexicalOwnSkipEntry[];
+};
+export type LexicalScanConsumerPolicy = {
+    readonly id: number;
+    readonly siteId: number;
+    readonly path: string;
+    readonly contextSnapshotId: number;
+    readonly kind: 'scanTo';
+    readonly raw: boolean;
+    readonly ambient: 'enumerate-trivia-then-scanSkip-per-parse-attempt' | 'none-raw';
+    readonly lookup: 'dynamic-parse-property-every-skip-test';
+    readonly context: 'detached-per-attempt-state-errors-only';
+    readonly ownSkipPlanId?: number;
+    readonly pendingReuse: 'forbidden';
+    readonly status: LexicalCapabilityStatus;
+} | {
+    readonly id: number;
+    readonly siteId: number;
+    readonly path: string;
+    readonly contextSnapshotId: number;
+    readonly kind: 'balanced';
+    readonly raw: boolean;
+    readonly ambient: 'array-identity-interior-cache' | 'none-raw';
+    readonly cache: 'lookup-every-attempt-enumerate-on-miss';
+    readonly lookup: 'dynamic-parse-property-every-skip-test';
+    readonly context: 'token-cleared-original';
+    readonly ownSkipPlanId?: number;
+    readonly pendingReuse: 'forbidden';
+    readonly status: LexicalCapabilityStatus;
+};
+/**
+ * Compiler-only phase-A census record. These ids are deliberately not family
+ * ids: two authored sites can share one future language while remaining two
+ * independently auditable capability obligations.
+ */
+export type LexicalCapabilitySite = {
+    readonly id: number;
+    /** Final-graph compiler body identity. Never serialized as a parser pointer. */
+    readonly bodyId: number;
+    /** Interned recognition-language summary; never used as a completeness id. */
+    readonly languageId: number;
+    readonly path: string;
+    /** Effective lexical scope at this occurrence; compiler-only, never a family id. */
+    readonly contextKey: string;
+    readonly contextSnapshotId: number;
+    /** Context seen by recognition after token() clears trivia/capture sinks. */
+    readonly recognitionContextKey: string;
+    readonly context: LexicalCapabilityContext;
+    readonly recognitionContext: LexicalCapabilityContext;
+    readonly semanticKey: string;
+    readonly atom: 'terminal' | 'token' | 'choice' | 'dispatch';
+    readonly parser: Combinator<unknown>;
+    readonly obligations: LexicalCapabilityObligations;
+    /** Internal body metadata only; never claims the boundary effects obligation. */
+    readonly diagnosticPlanId?: number;
+    /** Compiler-only boundary overlay; absent when exact representation declined. */
+    readonly boundaryTopologyId?: number;
+    /** Control ancestry authority used by the boundary overlay. */
+    readonly controlPlanId?: number;
+    /** Scan consumers owned by this atomic occurrence, in structural order. */
+    readonly scanConsumerPolicyIds: readonly number[];
+    /** Derived from `obligations`; callers cannot independently set it. */
+    readonly status: LexicalCapabilityStatus;
+};
+export type LexicalCapabilityLanguage = {
+    readonly id: number;
+    readonly atom: LexicalCapabilitySite['atom'];
+    readonly semanticKey: string;
+};
+/**
+ * One compiler-wide canonical source-range language. IDs use the lexical
+ * family namespace even while phase B is disabled; decision plans and the
+ * legacy token collector cross-reference this one IR interner.
+ */
+export type LexicalDecisionFamily = {
+    readonly id: number;
+    readonly semanticKey: string;
+    readonly ir: LexicalIr;
+};
+/**
+ * A compatible view of one family range. `prefix` owns its own shorter end;
+ * consumers must never substitute the full family end (the `a | ab` rule).
+ */
+export type LexicalDecisionOutcomeView = {
+    readonly kind: 'whole';
+    readonly relation: 'equal';
+} | {
+    readonly kind: 'predicate';
+    readonly relation: 'equal';
+    readonly match: Exclude<LexicalOutcomeMatch, {
+        kind: 'otherwise';
+    }>;
+} | {
+    readonly kind: 'language';
+    readonly relation: 'equal' | 'prefix';
+    readonly ir: LexicalIr;
+};
+/** Global, family-qualified atomic outcome/view identity. */
+export type LexicalDecisionOutcome = {
+    readonly id: number;
+    readonly familyId: number;
+    readonly semanticKey: string;
+    readonly view: LexicalDecisionOutcomeView;
+};
+export type LexicalDecisionAcceptance = {
+    readonly kind: 'outcomes';
+    readonly outcomeIds: readonly number[];
+} | {
+    readonly kind: 'otherwise';
+    readonly excludingOutcomeIds: readonly number[];
+} | {
+    readonly kind: 'unrestricted';
+} | {
+    readonly kind: 'impossible';
+} | {
+    readonly kind: 'gap';
+    readonly reason: string;
+};
+/** One source-ordered arm/route for one candidate family at one occurrence. */
+export type LexicalDecisionArm = {
+    readonly armId: number;
+    readonly acceptance: LexicalDecisionAcceptance;
+    readonly usesRouted: boolean;
+    readonly dynamicGate: boolean;
+};
+export type LexicalDecisionFamilyPlan = {
+    readonly familyId: number;
+    readonly arms: readonly LexicalDecisionArm[];
+};
+/** Source-ordered decision regions within which a recognized range may remain
+ * live. A dynamic scanner is its own closed epoch: callbacks and ambient skip
+ * policy may mutate parse state, so no lexical fact may cross that arm in
+ * either direction. This is compiler evidence for one complete TOKEN body,
+ * never a request to replay the CHARACTER body. */
+export type LexicalDecisionReuseEpoch = {
+    readonly armIds: readonly number[];
+    readonly pendingReuse: 'forbidden' | 'unproved';
+    readonly boundary: 'none' | 'dynamic-scan';
+};
+/** Occurrence-local ordered decision proof; never serialized in this tranche. */
+export type LexicalDecisionSite = {
+    readonly siteId: number;
+    readonly atom: 'choice' | 'dispatch';
+    readonly path: string;
+    readonly contextKey: string;
+    readonly contextSnapshotId: number;
+    readonly families: readonly LexicalDecisionFamilyPlan[];
+    /** Dispatch-only source-order routed policy. This remains Stage-A authority;
+     * the effect projection references the site rather than copying route bits. */
+    readonly routeUsesRouted?: readonly boolean[];
+    /** Families not proven at this occurrence remain admitted through this
+     * conservative relation. It is an explicit TOKEN-body route, never a request
+     * to replay the character parser. */
+    readonly fallback: 'unrestricted';
+    /** Missing inclusion/partition proofs reduce pruning precision only. Every
+     * authored arm still runs its own exact replacement recognizer in PEG order. */
+    readonly precisionNotes: readonly string[];
+    /** Dynamic scanner callbacks are effectful and can never publish a reusable
+     * cross-arm lexical fact. All other sites remain unproved in this tranche. */
+    readonly pendingReuse: 'forbidden' | 'unproved';
+    readonly reuseEpochs: readonly LexicalDecisionReuseEpoch[];
+};
+/** Final winner-resolved fixed choice classes, shared with table encoding. */
+export type LexicalChoiceClassAuthority = {
+    readonly id: number;
+    readonly armIds: readonly number[];
+    readonly firstSets: readonly FirstSet[];
+};
+/** A non-authored execution order such as longest literal first. */
+export type LexicalChoiceExecutionOrder = {
+    readonly id: number;
+    readonly armIds: readonly number[];
+};
+/** Exact classified spelling to authored arm identity. */
+export type LexicalClassifySpellingMap = {
+    readonly id: number;
+    readonly entries: readonly {
+        readonly spelling: string;
+        readonly armId: number;
+    }[];
+};
+/** Transform callback identities applied to an already recognized spelling. */
+export type LexicalClassifyProjectionPlan = {
+    readonly id: number;
+    readonly entries: readonly {
+        readonly armId: number;
+        readonly transformIds: readonly number[];
+        readonly transformSourceIds: readonly (number | -1)[];
+        readonly transformCount: number;
+    }[];
+};
+export type LexicalExpectedAuthority = {
+    readonly id: number;
+    readonly values: readonly string[];
+};
+/** Compiler-local callback identity; never a TableProgram or public wire value. */
+export type LexicalDecisionCallbackAuthority = {
+    readonly id: number;
+    readonly callback: ((state: unknown) => boolean) | ((value: unknown, span: {
+        start: number;
+        end: number;
+    }) => unknown);
+};
+/** Named emitted projection for a callback; compiler-only until Stage C selects. */
+export type LexicalDecisionSourceAuthority = {
+    readonly id: number;
+    readonly source: string;
+};
+/** Exact post-success rejection checks, interned without re-deriving them. */
+export type LexicalAutoNotAuthority = {
+    readonly id: number;
+    readonly checks: readonly AutoNotCheck[];
+};
+export type LexicalFinalDecisionAuthority = {
+    readonly id: number;
+    readonly atom: 'choice';
+    readonly decisionSiteId: number;
+    readonly authoredArmIds: readonly number[];
+    readonly mode: {
+        readonly kind: 'ordered';
+    } | {
+        readonly kind: 'exclusive';
+        readonly classAuthorityId: number;
+    } | {
+        readonly kind: 'longest';
+        readonly orderAuthorityId: number;
+    } | {
+        readonly kind: 'classify';
+        readonly superArmId: number;
+        readonly spellingMapAuthorityId: number;
+        readonly projectionAuthorityId: number;
+    };
+    /** Static total-miss set only for strategies that never consult dynamic gates. */
+    readonly staticMissExpectedAuthorityId?: number;
+} | {
+    readonly id: number;
+    readonly atom: 'dispatch';
+    readonly decisionSiteId: number;
+    readonly selectorBindingEdgeId: number;
+    readonly classifierAuthorityId: number;
+    readonly orderedRouteIds: readonly number[];
+    readonly noRouteExpectedAuthorityId: number;
+};
+export type LexicalDecisionChildEffect = {
+    readonly id: number;
+    readonly decisionSiteId: number;
+    readonly armId: number;
+    readonly childBindingEdgeId: number;
+    readonly expectedAuthorityId: number;
+    readonly gateCallbackId?: number;
+    readonly gateSourceId?: number;
+    readonly autoNotAuthorityId?: number;
+};
+export type LexicalDecisionEffectProgram = {
+    readonly id: number;
+    readonly decisionSiteId: number;
+    readonly finalDecisionAuthorityId: number;
+    readonly phase: LexicalCapabilityPhase;
+    readonly childEffectIds: readonly number[];
+    /** Exact fixed-edge projections consumed by the three existing readers. */
+    readonly childBindingProjectionIds: readonly number[];
+    readonly referenceTemplateId: number;
+    readonly capturedTemplateId: number;
+    readonly namedTemplateId: number;
+    readonly readerMask: number;
+    readonly variantMask: number;
+    readonly semanticDigest: number;
+    readonly pendingRollbackPolicyId: 0;
+    readonly commitPolicyId: 0;
+};
+/** Compiler-only C2 pools. No member is a TableProgram or runtime wire field. */
+export type LexicalDecisionEffectInventory = {
+    readonly finalDecisionAuthorities: readonly LexicalFinalDecisionAuthority[];
+    readonly decisionEffects: readonly LexicalDecisionEffectProgram[];
+    readonly decisionChildEffects: readonly LexicalDecisionChildEffect[];
+    readonly choiceClassAuthorities: readonly LexicalChoiceClassAuthority[];
+    readonly choiceExecutionOrders: readonly LexicalChoiceExecutionOrder[];
+    readonly classifySpellingMaps: readonly LexicalClassifySpellingMap[];
+    readonly classifyProjectionPlans: readonly LexicalClassifyProjectionPlan[];
+    readonly decisionExpectedAuthorities: readonly LexicalExpectedAuthority[];
+    readonly decisionCallbackAuthorities: readonly LexicalDecisionCallbackAuthority[];
+    readonly decisionSourceAuthorities: readonly LexicalDecisionSourceAuthority[];
+    readonly decisionAutoNotAuthorities: readonly LexicalAutoNotAuthority[];
+};
+/** One fixed incoming/root edge whose direct linked-body candidates remain owed. */
+export type LexicalBindingEdge = {
+    readonly id: number;
+    readonly path: string;
+    readonly contextKey: string;
+    /** Effective context at the child occurrence, after the parent boundary. */
+    readonly childContextKey: string;
+    readonly contextSnapshotId: number;
+    readonly parentTag: ParserDef['tag'] | 'root';
+    readonly childTag: ParserDef['tag'];
+    readonly projectionId?: number;
+    readonly status: LexicalCapabilityStatus;
+};
+/** Compiler-owned fixed-edge lowering. These are numeric construction facts,
+ * not a parse-time micro-IR: each reader consumes only its own preselected
+ * template and the common digest proves all three describe the same edge. */
+export type LexicalBindingProjection = {
+    readonly id: number;
+    readonly edgeId: number;
+    readonly parentBodyId: number;
+    readonly childBodyId: number;
+    readonly childOrdinal: number;
+    readonly referenceOperand: number;
+    readonly capturedSlotId: number;
+    readonly capturedTemplateId: number;
+    readonly namedSymbolId: number;
+    readonly namedTemplateId: number;
+    /** bit 0 reference, bit 1 captured/CSP, bit 2 named/emitted. */
+    readonly readerMask: number;
+    /** Strict, recovery, tracking, CST, probe, and coverage construction modes. */
+    readonly variantMask: number;
+    readonly semanticDigest: number;
+};
+/** One executable primitive-terminal candidate plus every fixed incoming edge.
+ * This remains compiler-only until Stage C selects a complete semantic body. */
+export type LexicalTerminalProjection = {
+    readonly id: number;
+    readonly siteId: number;
+    readonly bodyId: number;
+    readonly languageId: number;
+    readonly referenceBodyId: number;
+    readonly capturedTemplateId: number;
+    readonly namedSymbolId: number;
+    readonly incomingBindingProjectionIds: readonly number[];
+    readonly readerMask: number;
+    readonly variantMask: number;
+    readonly semanticDigest: number;
+    readonly status: LexicalCapabilityStatus;
+};
+export type LexicalOutcomeMatch = {
+    readonly kind: 'exact';
+    readonly values: readonly string[];
+    readonly caseInsensitive: boolean;
+} | {
+    readonly kind: 'startsWith' | 'endsWith';
+    readonly value: string;
+    readonly caseInsensitive: boolean;
+} | {
+    readonly kind: 'matches';
+    readonly value: string;
+    readonly flags: string;
+    readonly caseInsensitive: boolean;
+} | {
+    readonly kind: 'otherwise';
+    readonly excluding: readonly Exclude<LexicalOutcomeMatch, {
+        kind: 'otherwise';
+    }>[];
+};
+/** A globally reusable classification id for one family/range predicate. */
+export type LexicalOutcomeSpec = {
+    readonly id: number;
+    readonly familyId: number;
+    readonly match: LexicalOutcomeMatch;
+};
+/** One compatible view over a family range at a particular dispatch site. */
+export type LexicalTokenOutcome = {
+    readonly id: number;
+    readonly match: LexicalOutcomeMatch;
+};
+/** One ordered dispatch arm. IDs classify; this route owns the branch/cut. */
+export type LexicalTokenRoute = {
+    readonly index: number;
+    readonly kind: 'exact' | 'matcher' | 'otherwise';
+    readonly acceptedIds: readonly number[];
+    readonly matches: readonly LexicalOutcomeMatch[];
+    readonly parser: Combinator<unknown>;
+    readonly usesRouted: boolean;
+};
+/** Dispatch outcomes stay site-local; ordered dispatch semantics are unchanged. */
+export type LexicalTokenClassifier = {
+    readonly dispatch: Combinator<unknown>;
+    readonly familyId: number;
+    /** True when reaching token() crossed a wrapper whose effects must still run. */
+    readonly selectorEffects: boolean;
+    /** Compatible range views, flattened and allowed to repeat an id by route. */
+    readonly outcomes: readonly LexicalTokenOutcome[];
+    /** Authoritative ordered route/cut identity. */
+    readonly routes: readonly LexicalTokenRoute[];
+};
+export type LexicalAlphabet = {
+    /**
+     * Compiler-only graph metadata. `diagnostics`, `sites`, `classifiers`, and
+     * `familyIdOf` contain live combinators/Maps and are NOT TableProgram data.
+     * A serializer must project the numeric/IR specs and relocate site references.
+     */
+    readonly recognizers: readonly LexicalRecognizer[];
+    readonly diagnostics: readonly LexicalDiagnosticPlan[];
+    readonly families: readonly LexicalTokenFamily[];
+    readonly sites: readonly LexicalTokenSite[];
+    readonly outcomes: readonly LexicalOutcomeSpec[];
+    readonly classifiers: readonly LexicalTokenClassifier[];
+    readonly familyIdOf: ReadonlyMap<Combinator<unknown>, number>;
+    /** Whole-final-grammar, ownership-aware phase-A capability census. */
+    readonly capabilities: readonly LexicalCapabilitySite[];
+    /** Interned language view only; completeness is always occurrence-based. */
+    readonly capabilityLanguages: readonly LexicalCapabilityLanguage[];
+    /** Every distinct fixed parent/root edge, independently GAP until linked. */
+    readonly bindingEdges: readonly LexicalBindingEdge[];
+    readonly bindingProjections: readonly LexicalBindingProjection[];
+    readonly terminalProjections: readonly LexicalTerminalProjection[];
+    /** Phase-A compatible range/arm algebra; compiler-only and occurrence-local. */
+    readonly decisionFamilies: readonly LexicalDecisionFamily[];
+    readonly decisionOutcomes: readonly LexicalDecisionOutcome[];
+    readonly decisions: readonly LexicalDecisionSite[];
+    readonly decisionEffectPlan: LexicalDecisionEffectInventory;
+    readonly transitionDiagnostics: readonly LexicalTransitionDiagnosticPlan[];
+    readonly boundaryPlans: readonly LexicalBoundaryPlan[];
+    readonly materializationPlans: readonly LexicalMaterializationPlan[];
+    readonly grammarWrapperSpecs: readonly LexicalGrammarWrapperSpec[];
+    readonly grammarCaptureTriviaKinds: readonly (readonly string[])[];
+    readonly boundaryTopologies: readonly LexicalBoundaryTopology[];
+    readonly controlPlans: readonly LexicalControlPlan[];
+    readonly contextSnapshots: readonly LexicalContextSnapshot[];
+    readonly ownSkipPlans: readonly LexicalOwnSkipPlan[];
+    readonly scanConsumerPolicies: readonly LexicalScanConsumerPolicy[];
+    /** False means phase B is forbidden for the entire program. */
+    readonly capabilityComplete: boolean;
+};
+export type LexicalCapabilityInventory = Pick<LexicalAlphabet, 'capabilities' | 'capabilityLanguages' | 'bindingEdges' | 'bindingProjections' | 'terminalProjections' | 'decisionFamilies' | 'decisionOutcomes' | 'decisions' | 'decisionEffectPlan' | 'transitionDiagnostics' | 'boundaryPlans' | 'materializationPlans' | 'grammarWrapperSpecs' | 'grammarCaptureTriviaKinds' | 'boundaryTopologies' | 'controlPlans' | 'contextSnapshots' | 'ownSkipPlans' | 'scanConsumerPolicies' | 'capabilityComplete'>;
+/** Numeric-only artifact projection; `TableProgram` aliases this contract. */
+export type NumericLexicalPlan = {
+    readonly recognizerOffsets: readonly number[];
+    readonly recognizerData: readonly number[];
+    readonly outcomeOffsets: readonly number[];
+    readonly outcomeData: readonly number[];
+    readonly tokenSites: readonly number[];
+    readonly sites: readonly number[];
+    readonly routes: readonly number[];
+    readonly accepted: readonly number[];
+};
+/** Lexical-family ids occupy their own published namespace. */
+export declare const FIRST_LEXICAL_FAMILY_ID = 3;
+/** Canonical, parser-free identity for one range predicate. */
+export declare function canonicalLexicalOutcomeKey(match: LexicalOutcomeMatch): string;
+/** Does a final rule-map winner bottom out at the reference being resolved? */
+export declare function winnerWrapsReference(winner: Combinator<unknown>, reference: Combinator<unknown>): boolean;
 /** Direct sub-parsers of a def, resolving a lazy through `resolve` when its own thunk is a hole. */
 export declare function tokenChildren(p: Combinator<unknown>, resolve?: (name: string) => Combinator<unknown> | undefined): Combinator<unknown>[];
 /**
@@ -58,6 +862,42 @@ export declare function tokenChildren(p: Combinator<unknown>, resolve?: (name: s
  * which is stable for a given grammar and therefore safe to switch on.
  */
 export declare function collectAlphabet(roots: ReadonlyArray<Combinator<unknown>>, resolve?: (name: string) => Combinator<unknown> | undefined): Alphabet;
+/** Re-enumerate the final graph so a dropped/filtered candidate fails closed. */
+export declare function assertLexicalCapabilityClosure(roots: ReadonlyArray<Combinator<unknown>>, alphabet: Pick<LexicalCapabilityInventory, 'capabilities' | 'capabilityLanguages' | 'bindingEdges' | 'decisionFamilies' | 'decisionOutcomes' | 'decisions'> & Partial<Pick<LexicalCapabilityInventory, 'bindingProjections' | 'terminalProjections' | 'decisionEffectPlan' | 'transitionDiagnostics' | 'boundaryPlans' | 'materializationPlans' | 'grammarWrapperSpecs' | 'grammarCaptureTriviaKinds' | 'boundaryTopologies' | 'controlPlans' | 'contextSnapshots' | 'ownSkipPlans' | 'scanConsumerPolicies'>>, resolve?: (name: string) => Combinator<unknown> | undefined): void;
+/** Phase-A only: inventory obligations without constructing unused runtime families/sites. */
+export declare function collectLexicalCapabilities(roots: ReadonlyArray<Combinator<unknown>>, resolve?: (name: string) => Combinator<unknown> | undefined): LexicalCapabilityInventory;
+/**
+ * Catalog authored lexical tokens and their dispatch views without confusing
+ * private child terminals for source tokens. This is metadata only: consumers
+ * still have to lower the canonical IR in every shipping engine before enabling
+ * an admission path.
+ */
+export declare function collectLexicalAlphabet(roots: ReadonlyArray<Combinator<unknown>>, resolve?: (name: string) => Combinator<unknown> | undefined): LexicalAlphabet;
+/**
+ * Project the compiler graph into compact numeric pools. The caller supplies
+ * only already-relocated table-site numbers and its existing const-pool intern;
+ * no combinator or identity map crosses this boundary.
+ */
+export declare function serializeLexicalPlan(alphabet: LexicalAlphabet, constant: (value: unknown) => number, tokenSites: readonly number[], dispatchSites: ReadonlyArray<{
+    readonly dsp: number;
+    readonly classifier: LexicalTokenClassifier;
+}>): NumericLexicalPlan | undefined;
+/**
+ * Metadata/reference oracle for compatible same-range views. Exact/prefix/suffix
+ * checks read char codes directly; regex matcher views slice only in this cold
+ * oracle. Production token cursors must lower matcher plans without treating
+ * this helper as their allocation contract.
+ */
+export declare function compatibleLexicalOutcomes(classifier: LexicalTokenClassifier, input: string, start: number, end: number): number[];
+/**
+ * Cold oracle for dispatch selection precedence. Compatible views are a set;
+ * selected routing is still exact cases first, then matcher source order, then
+ * otherwise, and a selected branch failure remains committed.
+ */
+export declare function selectedLexicalOutcome(classifier: LexicalTokenClassifier, input: string, start: number, end: number): {
+    route: LexicalTokenRoute;
+    outcomeId: number;
+} | undefined;
 /** The id already assigned to this terminal, if it is one. */
 export declare function terminalId(alphabet: Alphabet, p: Combinator<unknown>): number | undefined;
 /**