parseman 0.59.2 → 0.60.0

package/src/combinators/first-set.ts

@@ -1,5 +1,6 @@
 import type { CharRange, FirstSet, Combinator, ParserDef } from '../types.ts'
 import { implicitLeadsDropped, leadRanges } from '../lead-ranges.ts'
+import { triviaThroughRef } from './trivia-boundary.ts'
 
 /**
  * Resolve a NAMED cross-artifact rule reference (`g.Foo`) whose own thunk does not
@@ -27,6 +28,13 @@ export function classifyFinalChoice(
   arms: readonly Combinator<unknown>[],
   resolve?: RefResolver,
   activeTrivia?: Combinator<unknown> | null,
+  consuming = true,
+  /**
+   * Read each arm's construction-time `_meta.firstSet`, the set a disjoint
+   * interpreter `choice()` dispatches on (`choice.ts`), instead of deriving it:
+   * the table then lets in exactly the arms the interpreter lets in.
+   */
+  constructionSets = false,
 ): FinalChoiceClassification {
   const firstSets: FirstSet[] = []
   const nullable: boolean[] = []
@@ -38,7 +46,7 @@ export function classifyFinalChoice(
   for (const arm of arms) {
     const empty = matchesEmpty(arm, new Set(), resolve)
     nullable.push(empty)
-    firstSets.push(empty ? { kind: 'any' } : firstSetOf(arm, new Set(), resolve, activeTrivia))
+    firstSets.push(empty ? { kind: 'any' } : constructionSets ? arm._meta.firstSet : firstSetOf(arm, new Set(), resolve, activeTrivia, consuming))
     if (empty) exclusive = false
   }
   if (exclusive) {
@@ -73,7 +81,7 @@ export function classifyFinalChoice(
  * does not. `undefined` → no winner; fall back to the thunk. Monolithic: the winner IS
  * the thunk's target, a no-op. A winner that recurses back to `p` (recursive or
  * self-alias rule) is caught by the shared `seen` guard → `any`, a sound over-approx. */
-function winnerForRef(p: Combinator<unknown>, resolve: RefResolver | undefined): Combinator<unknown> | undefined {
+export function winnerForRef(p: Combinator<unknown>, resolve: RefResolver | undefined): Combinator<unknown> | undefined {
   if (resolve === undefined) return undefined
   const name = (p as unknown as { _ruleName?: string })._ruleName
   const w = name === undefined ? undefined : resolve(name)
@@ -316,6 +324,7 @@ function sequenceSet(
   fs: (p: Combinator<unknown>) => FirstSet,
   empties: (p: Combinator<unknown>) => boolean,
   trivia?: Combinator<unknown> | null,
+  consuming = true,
 ): FirstSet {
   const afterBoundary = (i: number): FirstSet => {
     const rest = at(i)
@@ -334,13 +343,23 @@ function sequenceSet(
     }
     const first = fs(p)
     if (!empties(p) || i + 1 === parsers.length) return first
-    return union(first, afterBoundary(i + 1))
+    // A nullable regex reports `any` as its own set (it may match empty
+    // anywhere); here the empty match is the union with the rest, so only its
+    // non-empty matches' first chars are its own contribution.
+    return union((consuming ? p._meta.consumingFirstSet : undefined) ?? first, afterBoundary(i + 1))
   }
   return at(0)
 }
 
+/**
+ * A sequence's construction-time first set, which the interpreter's `choice()`
+ * dispatches on. Trivia is unknown here, and a choice may run where leading
+ * trivia has not been skipped, so a nullable regex stands as `any` (`consuming`
+ * false): read through its non-empty matches, `choice(sequence(pad, '*'), 'x')`
+ * would not enter the pad arm at a comment before `*`.
+ */
 export function sequenceFirstSet(parsers: readonly Combinator<unknown>[]): FirstSet {
-  return sequenceSet(parsers, p => p._meta.firstSet, matchesEmpty)
+  return sequenceSet(parsers, p => p._meta.firstSet, matchesEmpty, undefined, false)
 }
 
 /**
@@ -356,6 +375,17 @@ export function sequenceFirstSet(parsers: readonly Combinator<unknown>[]): First
  * referenced rule can be replaced with a WIDER first-set, so a baked deep set
  * would wrongly skip valid input — the compose path defers dispatch to fuse time.
  *
+ * `consuming: false` reads a nullable regex as `any`, as first sets read before
+ * 0.60.0. Recovery derives its resync sentinels so (`sequence.ts`
+ * `followSentinels`, the table's follow and separator classes, with no trivia),
+ * as do a construction-time sequence set and a table choice's classes. Only a
+ * repeat item's, a gap's and a node's gate read a pad through its non-empty
+ * matches.
+ *
+ * `activeTrivia` is the live trivia, `null` for a scope known to have none (where
+ * a reference installs its rule's trivia, `triviaThroughRef`), and `undefined`
+ * where it is not known.
+ *
  * `resolve` binds NAMED cross-artifact holes (`g.Foo`) against a fused winner map —
  * see `RefResolver`. Diagnostic-only today: it is what lets the gating analysis ask
  * the question at the site where the hole actually HAS an answer.
@@ -365,6 +395,7 @@ export function firstSetOf(
   seen: Set<Combinator<unknown>> = new Set(),
   resolve?: RefResolver,
   activeTrivia?: Combinator<unknown> | null,
+  consuming = true,
 ): FirstSet {
   // `seen` is the current recursion path, not a global visited set. A completed
   // shared child must be analysed again for a sibling; only a back-edge on this
@@ -372,7 +403,7 @@ export function firstSetOf(
   if (seen.has(p)) return any()               // cycle → any (safe over-approximation)
   seen.add(p)
   try {
-    return firstSetBody(p, seen, resolve, activeTrivia)
+    return firstSetBody(p, seen, resolve, activeTrivia, consuming)
   } finally {
     seen.delete(p)
   }
@@ -383,8 +414,9 @@ function firstSetBody(
   seen: Set<Combinator<unknown>>,
   resolve: RefResolver | undefined,
   activeTrivia: Combinator<unknown> | null | undefined,
+  consuming: boolean,
 ): FirstSet {
-  const fs = (c: Combinator<unknown>): FirstSet => firstSetOf(c, seen, resolve, activeTrivia)
+  const fs = (c: Combinator<unknown>): FirstSet => firstSetOf(c, seen, resolve, activeTrivia, consuming)
   const empties = (c: Combinator<unknown>): boolean => matchesEmpty(c, new Set(), resolve)
   const d = p._def as ParserDef
   switch (d.tag) {
@@ -393,10 +425,19 @@ function firstSetBody(
     case 'keywords':  return p._meta.firstSet  // terminals: no refs, cached set is exact
     case 'dispatch':  return fs(d.selector)
     case 'lazy': {
-      const w = winnerForRef(p, resolve)
-      if (w !== undefined) return fs(w)
-      try { return fs(d.thunk()) }
-      catch { return any() }
+      let target = winnerForRef(p, resolve)
+      if (target === undefined) {
+        try { target = d.thunk() }
+        catch { return any() }
+      }
+      // Under a scope KNOWN to have no trivia (`null`), the target runs under the
+      // trivia the reference installs for it (`triviaThroughRef`). `undefined`
+      // says nothing about the scope, and the target is read as it stands. Only
+      // the table encoder (which always passes `resolve`) reads through the
+      // install; the interpreter's own callers (`sequence.ts` follow sentinels)
+      // read a reference as 0.58.0 did (#223).
+      const through = activeTrivia === null && resolve !== undefined
+      return firstSetOf(target, seen, resolve, through ? triviaThroughRef(p, target, null) : activeTrivia, consuming)
     }
     case 'choice': {
       let out: FirstSet = empty()
@@ -414,7 +455,7 @@ function firstSetBody(
       // Recurse at every term boundary because a lookahead after another
       // zero-width term runs after trivia has advanced, not at the sequence's
       // original cursor.
-      return sequenceSet(d.parsers, fs, empties, activeTrivia)
+      return sequenceSet(d.parsers, fs, empties, activeTrivia, consuming)
     }
     case 'peek': {
       // Deep-resolve the body: a `ref()` reads `any()` at CONSTRUCTION, so the
@@ -428,7 +469,6 @@ function firstSetBody(
     case 'transform':
     case 'label':
     case 'trivia':
-    case 'token':
     case 'leaf':
     case 'node':
     case 'sepBy':
@@ -437,11 +477,14 @@ function firstSetBody(
     // the inner's construction-time set, which reads `any` when the inner opens
     // with a rule reference — resolve through it like any other wrapper.
     case 'attempt':   return fs(d.parser)
+    // A token runs its body with trivia cleared (`token.ts`).
+    case 'token':     return firstSetOf(d.parser, seen, resolve, activeTrivia === undefined ? undefined : null, consuming)
     case 'grammar':   return firstSetOf(
       d.parser,
       seen,
       resolve,
       d.clearTrivia ? null : (d.triviaParser ?? activeTrivia),
+      consuming,
     )
     // A run starts with its first line's prefix; a nullable prefix starts with anything.
     case 'lines': {

package/src/combinators/gate.ts

@@ -15,7 +15,7 @@ import type { Combinator, ParseContext, ParseResult, ParserMeta } from '../types
  * Naming: this matches the `gate:` field on a gated CHOICE arm
  * (`choice({ gate, combinator }, other)`). Use the arm FIELD to SELECT a branch by
  * a cheap state predicate; use this COMBINATOR to ASSERT a state predicate
- * mid-sequence. Formerly named `guard()` — see the deprecated alias in `guard.ts`.
+ * mid-sequence. Formerly named `guard()`; that alias was removed in 0.45.0.
  *
  * Note: like all state predicates its first-set is `any`, so a `gate(...)` as the
  * LEADING term of a choice arm poisons that choice's first-char dispatch. Keep it

package/src/combinators/node.ts

@@ -187,7 +187,7 @@ export function node<N>(
   if (project !== undefined && build !== undefined) {
     throw new Error('node() options cannot combine project with a build callback')
   }
-  const baseDef = { tag: 'node' as const, parser: combinator, ...(type === undefined ? {} : { type }), ...(build === undefined ? {} : { build }) }
+  const baseDef = { tag: 'node' as const, parser: combinator, ...(type === undefined ? {} : { type }), ...(build === undefined ? {} : { build }), ...(combinator._meta.firstSet.kind !== 'any' && !matchesEmpty(combinator) ? { guarded: true as const } : {}) }
   const meta: ParserMeta = {
     firstSet: combinator._meta.firstSet,
     canMatchNewline: combinator._meta.canMatchNewline,

package/src/combinators/regex.ts

@@ -4,7 +4,7 @@ import { failAt } from './probe.ts'
 import { pushCstLeaf, cstCaptureActive } from '../cst/capture-buffer.ts'
 import { recordLineRangeFromContext } from '../line-index.ts'
 import { shorthandRanges, parseClassRanges } from '../regex/classes.ts'
-import { firstSetFromRegex } from '../regex/first-set.ts'
+import { consumingFirstSetFromRegex, firstSetFromRegex, regexCanMatchEmpty } from '../regex/first-set.ts'
 import { directTerminalFailureExpected } from './expected.ts'
 import type { ScalarParser } from './scalar.ts'
 import { PlainLeaf, PlainSpan } from '../cst/plain.js'
@@ -156,13 +156,18 @@ export function regex(pattern: string | RegExp, flags = ''): Combinator<string>
   //     pairs, so gating on the ASCII set would false-exclude them. Fall back to `any()`
   //     (always-try) — sound, and only forfeits gating for the rare Unicode-mode `/i`
   //     recognizer, never the ASCII-only `/i` at-keywords the fix targets.
-  const firstSet = !resolvedFlags.includes('i')
-    ? raw.firstSet
+  const forFlags = (fs: FirstSet): FirstSet => !resolvedFlags.includes('i')
+    ? fs
     : (resolvedFlags.includes('u') || resolvedFlags.includes('v'))
       ? any()
-      : asciiCaseFold(raw.firstSet)
+      : asciiCaseFold(fs)
+  const firstSet = forFlags(raw.firstSet)
   const canMatchNewline = raw.canMatchNewline
   const meta: ParserMeta = { firstSet, canMatchNewline, isTrivia: false }
+  if (firstSet.kind === 'any' && regexCanMatchEmpty(source)) {
+    const consuming = forFlags(consumingFirstSetFromRegex(source, resolvedFlags))
+    if (consuming.kind !== 'any') meta.consumingFirstSet = consuming
+  }
   const def = { tag: 'regex', source, flags: resolvedFlags } as const
   const expected = directTerminalFailureExpected(def)
   const parseScalar: ScalarParser = (input, pos, ctx) => {

package/src/combinators/repeat.ts

@@ -154,11 +154,11 @@ export function many<T>(combinator: Combinator<T>, opts: RepeatOptions = {}): Co
     canMatchNewline: combinator._meta.canMatchNewline,
     isTrivia: false,
   }
-  const def: { tag: 'many'; parser: Combinator<unknown>; min: 0; max?: number; valueUnused?: boolean } =
-    { tag: 'many', parser: combinator as Combinator<unknown>, min: 0, ...(max === Infinity ? {} : { max }) }
-  let expected: string[] | undefined
   // A non-nullable body can be first-set-gated per loop iteration (see repItem).
   const guardable = combinator._meta.firstSet.kind !== 'any' && !matchesEmpty(combinator)
+  const def: { tag: 'many'; parser: Combinator<unknown>; min: 0; max?: number; valueUnused?: boolean; guarded?: true } =
+    { tag: 'many', parser: combinator as Combinator<unknown>, min: 0, ...(max === Infinity ? {} : { max }), ...(guardable ? { guarded: true } : {}) }
+  let expected: string[] | undefined
   const scalarChild = scalarOf(combinator)
   const parseScalar = (input: string, pos: number, ctx: ParseContext): number =>
     repeatScalarTail(combinator, scalarChild, input, pos, ctx, guardable, def.valueUnused ? undefined : [], max)
@@ -229,11 +229,11 @@ function atLeast<T>(combinator: Combinator<T>, min: number, max: number): Combin
     canMatchNewline: combinator._meta.canMatchNewline,
     isTrivia: false,
   }
-  const def: { tag: 'oneOrMore'; parser: Combinator<unknown>; min: number; max?: number; valueUnused?: boolean } =
-    { tag: 'oneOrMore', parser: combinator as Combinator<unknown>, min, ...(max === Infinity ? {} : { max }) }
-  let expected: string[] | undefined
   // A non-nullable body can be first-set-gated per loop iteration (see repItem).
   const guardable = combinator._meta.firstSet.kind !== 'any' && !matchesEmpty(combinator)
+  const def: { tag: 'oneOrMore'; parser: Combinator<unknown>; min: number; max?: number; valueUnused?: boolean; guarded?: true } =
+    { tag: 'oneOrMore', parser: combinator as Combinator<unknown>, min, ...(max === Infinity ? {} : { max }), ...(guardable ? { guarded: true } : {}) }
+  let expected: string[] | undefined
   const scalarChild = scalarOf(combinator)
   const parseScalar = (input: string, pos: number, ctx: ParseContext): number => {
     let cur = scalarChild(input, pos, ctx)

package/src/combinators/sequence.ts

@@ -108,7 +108,7 @@ export function sequence<T extends [Combinator<unknown>, ...Combinator<unknown>[
       if (tolerant) {
         followSentinels ??= parsers.map((_, i) => firstSetSentinel(
           parsers.slice(i + 1).reduce<ReturnType<typeof firstSetOf>>(
-            (set, parser) => union(set, firstSetOf(parser)), { kind: 'empty' },
+            (set, parser) => union(set, firstSetOf(parser, new Set(), undefined, undefined, false)), { kind: 'empty' },
           ),
         ))
       }

package/src/combinators/trivia-boundary.ts

@@ -46,3 +46,21 @@ export function hasOwnTriviaBoundary(p: Combinator<unknown>, depth: number = MAX
     default: return false
   }
 }
+
+/**
+ * The trivia `target` runs under when it is reached through the reference `ref`
+ * from a scope whose live trivia is `trivia` (`null` or `undefined`: none).
+ *
+ * A reference made from a region that cleared trivia re-installs the rule's own
+ * ambient trivia, where the rule has a boundary for it to repair (`ref.ts`,
+ * `Encoder.scopedRef`). Anything derived statically through a reference — a
+ * first set, a gate, a gap — must take the same scope, or it describes the
+ * caller's scope where the body runs under its own.
+ */
+export function triviaThroughRef(
+  ref: Combinator<unknown>, target: Combinator<unknown>, trivia: Combinator<unknown> | null | undefined,
+): Combinator<unknown> | null | undefined {
+  if (trivia !== undefined && trivia !== null) return trivia
+  const amb = ref._meta.grammarTrivia ?? target._meta.grammarTrivia
+  return amb !== undefined && hasOwnTriviaBoundary(target) ? amb : trivia
+}

package/src/combinators/trivia-skip.ts

@@ -413,7 +413,7 @@ export function scanTriviaCompact(input: string, cur: number, ctx: ParseContext)
   return { end: tr.ok ? tr.span.end : cur, commit: NOOP_COMMIT }
 }
 
-/** Legacy object-shaped contract used by previously emitted assembly factories. */
+/** The object-shaped form of `scanTriviaCompact`. No engine calls it; the trivia-skip tests compare the two. */
 export function scanTrivia(input: string, cur: number, ctx: ParseContext): TriviaScan {
   const scan = scanTriviaCompact(input, cur, ctx)
   return typeof scan === 'number' ? { end: scan, commit: NOOP_COMMIT } : scan

package/src/compiler/compile-linkable-table.ts

@@ -29,9 +29,8 @@ import { PARSEMAN_VERSION } from '../version.ts'
  *                       artifact is a table, not a bag of compiled functions.
  *   `ir`              — the piece's combinator graph, serialized. This is the
  *                       composable half, and it is NOT new machinery: the plugin
- *                       already carries pieces as IR (`plugin/index.ts:1772`) and
- *                       the linker already re-lowers them (`linker.ts:640`,
- *                       `compileLinkable(evalRuleMapIR(p.ir), …)` at build time).
+ *                       already carries pieces as IR (`plugin/index.ts`) and the
+ *                       macro re-lowers them from it at build time.
  *
  * ── WHY THAT MAKES COMPOSITION THE EASY KIND ─────────────────────────────────
  * The open question was whether table-to-table composition has to merge two

package/src/compiler/degradation.ts

@@ -93,8 +93,8 @@ export function formatDegradations(list: readonly Degradation[]): string[] {
 //
 // Compilation is single-module and synchronous, so a module-level sink (opened per
 // transform, drained after) records findings without threading a field through every
-// Ctx and every compile*() return — the same shape as the regex-lowering sink in
-// codegen.ts, for the same reason.
+// Ctx and every compile*() return — the same shape as the source lowering's regex-lowering
+// sink, for the same reason.
 // ---------------------------------------------------------------------------
 //
 // A STACK, not a single slot. `transformMacro` re-enters itself when it lowers a private

package/src/compiler/ir-serialize.ts

@@ -461,7 +461,7 @@ class Serializer {
             arms.push(`when(${matcherExpr(entry)}, ${kid(entry.parser)}${entry.caseInsensitive ? ', { caseInsensitive: true }' : ''})`)
           }
         }
-        if (def.otherwise) arms.push(`otherwise(${kid(def.otherwise)})`)
+        if (def.otherwise) arms.push(`otherwise(${kid(def.otherwise)}${def.otherwiseCommit === false ? ', { commit: false }' : ''})`)
         return `dispatch(${kid(def.selector)}${arms.length === 0 ? '' : `, ${arms.join(', ')}`})`
       }
       case 'many':      return `many(${kid(def.parser)}${repeatOpts(def.min, def.max)})`

package/src/compiler/linker.ts

@@ -25,9 +25,11 @@ import { copySpan } from '../line-index.ts'
 /**
  * Compile a `rules()` map to a **linkable artifact** — the composable, shippable
  * form (RULE_ABI_PLAN §4). A package exports `linkable(rules(g => …))`; consumers
- * import that artifact and `fuse([...])` it — **no source of the base grammar is
- * ever read**. Under the macro this is precompiled to static pieces; in the
- * interpreter it compiles here at load (like `compile()`).
+ * import that artifact and `compose([...])` it — **no source of the base grammar is
+ * ever read**. Under the macro this is precompiled to static pieces, and only a
+ * build-time `compose()` can link it. In the interpreter it compiles here at load
+ * (like `compile()`) and keeps the map's `RULES_RECIPE`, which is what a runtime
+ * `compose()` links; an artifact without one is refused there (`recipesOf`).
  *
  * `ns` is a per-artifact namespace; omit it to auto-assign a process-unique one
  * (fine at runtime — the macro supplies a stable module-derived ns instead).

package/src/compiler/token-alphabet.ts

@@ -140,6 +140,8 @@ export type ExecutableLexNode =
     readonly parsers: readonly Combinator<unknown>[]
     /** Inside a no-trivia scope of the token, where the encoder's trivia is none. */
     readonly cleared: boolean
+    /** The authored choice's own `disjoint`, which decides how the encoder classifies its arms. */
+    readonly disjoint: boolean
     readonly armExpected: readonly (readonly string[])[]
     readonly expected: readonly string[]
   }
@@ -509,7 +511,7 @@ function directFixedLexNode(
       if (arms.some(arm => arm === undefined)) return undefined
       return {
         kind: 'choice', arms: arms as ExecutableLexNode[],
-        parsers: def.parsers, cleared,
+        parsers: def.parsers, cleared, disjoint: def.disjoint,
         armExpected: def.parsers.map(deriveExpected),
         expected: def.parsers.flatMap(deriveExpected),
       }

package/src/coverage.ts

@@ -309,7 +309,7 @@ function coverageEntry(entry: Combinator<unknown>, collector: GrammarCoverageCol
               arms.push(when(dispatchMatcher(entry), wrapArm(entry.parser, ids[armIndex++]), entry.caseInsensitive ? { caseInsensitive: true } : undefined))
             }
           }
-          if (def.otherwise) arms.push(otherwise(wrapArm(def.otherwise, ids[armIndex++])))
+          if (def.otherwise) arms.push(otherwise(wrapArm(def.otherwise, ids[armIndex++]), def.otherwiseCommit === false ? { commit: false } : undefined))
           const unsafeDispatch = dispatch as (selector: Combinator<string>, ...items: DispatchArm<unknown>[]) => Combinator<unknown>
           const base = unsafeDispatch(build(def.selector) as Combinator<string>, ...arms)
           return {

package/src/cst/capture-buffer.ts

@@ -270,8 +270,8 @@ export function rollbackCstCaptureAt(
   // speculative branch leaves no ghost error (see saveCstMark). Guarded on a
   // present sink + a defined mark (rollbackTrivia forwards the same field).
   //
-  // Each truncation is guarded on `length !== mark`, mirroring the compiled
-  // engine (codegen.ts `captureRestoreBody`). Assigning `array.length` runs V8's
+  // Each truncation is guarded on `length !== mark`, as the source lowering's
+  // `captureRestoreBody` did. Assigning `array.length` runs V8's
   // length setter — a backing-store trim decision — and costs the same when the
   // value is unchanged, which is the overwhelmingly common case here: a
   // speculative branch that captured nothing still pays for every store. The

package/src/cst/host-mode.ts

@@ -1,7 +1,7 @@
 /**
  * The compiled-artifact ↔ host contract, in ONE place.
  *
- * This lives on its own — rather than in `compiler/codegen.ts` where it started —
+ * This lives on its own — rather than in the source lowering where it started —
  * because the DRIVER has to enforce it too. `functional/run.ts` is the `parseman/run`
  * entry, whose whole purpose is to stay small (see `test/unit/run-entry-closure.test.ts`),
  * so it cannot import the compiler to reach an assertion. Neither can it carry its own

package/src/cst/root-trivia-scope.ts

@@ -2,7 +2,7 @@
  * The ROOT-CAPTURE POLICY of one local trivia scope, in one place.
  *
  * Three engines install a `parser({ trivia })` scope — the interpreter
- * (`combinators/grammar.ts`), source lowering (`compiler/codegen.ts`) and the
+ * (`combinators/grammar.ts`), the source lowering (removed in 0.47.0) and the
  * table (`table/assemble.ts`, `table/exec.ts`) — and each of them has to answer
  * the same two questions about selected root trivia. The refusal below was a
  * string literal in two of them and ABSENT from the third, which is how a table

package/src/index.ts

@@ -17,7 +17,7 @@ export type { KeywordsOptions, WordOptions } from './combinators/keywords.ts'
 export { sequence } from './combinators/sequence.ts'
 export { choice } from './combinators/choice.ts'
 export { dispatch, endsWith, makeWhen, matches, otherwise, routed, startsWith, when } from './combinators/dispatch.ts'
-export type { DispatchArm, DispatchOtherwise, DispatchStringMatcher, DispatchWhen, DispatchWhenFactory, DispatchWhenOptions, DispatchWhenMatcher } from './combinators/dispatch.ts'
+export type { DispatchArm, DispatchOtherwise, DispatchOtherwiseOptions, DispatchStringMatcher, DispatchWhen, DispatchWhenFactory, DispatchWhenOptions, DispatchWhenMatcher } from './combinators/dispatch.ts'
 export { attempt } from './combinators/attempt.ts'
 export type { AttemptOptions } from './combinators/attempt.ts'
 export { many, oneOrMore, optional, sepBy, oneOrMoreSep, keepSeparator } from './combinators/repeat.ts'

package/src/language-service/index.ts

@@ -6,7 +6,7 @@
  * name = node type, without ever modifying the grammar:
  *
  *   - `parse(src, { tolerant })` → CST + `ParseError[]` (recovers on the compiled
- *     fast path when `grammar` is a `compile(g, { recovery: true })` grammar).
+ *     fast path when `grammar` is a `compile()`d grammar).
  *   - `diagnostics(src)` → structural parse errors + your per-node-type lint rules.
  *   - `completionsAt(src, offset)` → the grammar's expected-token set at the cursor,
  *     mapped through your per-rule semantic completion handlers.

package/src/plugin/entry-supercompiler.ts

@@ -1,5 +1,6 @@
 /**
- * Experimental build-time Entry/SCC supercompiler for static table assemblies.
+ * Build-time Entry/SCC supercompiler for the selective assemblies a macro build
+ * emits (it runs on every macro build that selects sites).
  *
  * The table emitter remains the sole semantic backend.  This pass parses that
  * backend's ordinary JavaScript and joins uniquely-owned piece functions with

package/src/plugin/evaluator.ts

@@ -431,11 +431,16 @@ function dispatchArmValue(node: Expression, scope: XScope, code?: string, mfs?:
   }
 
   if (callee.name === 'otherwise') {
-    const [parserArg] = node.arguments
-    if (!parserArg || parserArg.type === 'SpreadElement') return null
+    const [parserArg, optsArg] = node.arguments
+    if (!parserArg || parserArg.type === 'SpreadElement' || optsArg?.type === 'SpreadElement') return null
     const parserValue = anyValue(parserArg as Expression, scope, code, mfs)
     if (!isCombinator(parserValue)) return null
-    return parseman.otherwise(parserValue)
+    if (optsArg === undefined) return parseman.otherwise(parserValue)
+    const opts = anyValue(optsArg as Expression, scope, code, mfs)
+    if (typeof opts !== 'object' || opts === null) return null
+    const commit = (opts as { commit?: unknown }).commit
+    if (commit !== undefined && typeof commit !== 'boolean') return null
+    return parseman.otherwise(parserValue, commit === undefined ? {} : { commit })
   }
 
   if (callee.name === 'when') {

package/src/plugin/index.ts

@@ -5,8 +5,9 @@
  *
  * For each such import, walks the file's AST, finds variable declarations
  * whose RHS is a pure parseman combinator call, evaluates them at build time,
- * compiles the result to an optimized inline function, and replaces the
- * declaration — removing the import entirely.
+ * compiles the result to a table program (plus generated code for its hottest
+ * sites, within `assemblyBudget`), and replaces the declaration. The macro import
+ * is removed; the module imports the shared driver from `parseman/table` instead.
  *
  * Usage:
  *   // vite.config.ts
@@ -61,7 +62,7 @@ import type {
  * The PUBLIC subpath, not a deep path into `src/`: this string is written into a
  * consumer's build output, so it has to be a specifier their resolver can see.
  * `parseman/table` is declared in package.json `exports` and re-exports
- * `tableRules` as `tableRules` (src/table/index.ts:28).
+ * `tableRules` unrenamed (`src/table/index.ts`).
  */
 const TABLE_RUNTIME_SPECIFIER = 'parseman/table'
 
@@ -119,13 +120,9 @@ export type ParsecraftPluginOptions = {
    */
   warnUnloweredRegex?: boolean
   /**
-   * Bake automatic list recovery into the macro-compiled output. **Default `false`.**
-   * When on, `many`/`oneOrMore`/`sepBy` in every compiled grammar gain a
-   * `_ctx._tolerant`-gated recovery branch (dormant on strict parses — a normal
-   * parse is unaffected and allocates nothing). The output stays macro-inlinable
-   * (sentinels build via `_ctx._rec`, not `_rp`). Turn it on for grammars you drive
-   * from an editor/language service so `run(g, src, { tolerant: true })` recovers on
-   * the fast compiled path; leave it off for build-only grammars.
+   * @deprecated Has no effect: every compiled table carries recovery, and
+   * `run(g, src, { tolerant: true })` recovers on the compiled path whatever this is
+   * set to. Removed in the next minor release.
    */
   recovery?: boolean
   /** Emit grammar-coverage hooks in macro output. Off by default, so ordinary
@@ -1456,9 +1453,8 @@ function transformMacroImpl(
 
   /**
    * Compile a `rules(factory)` call into ONE shared replacement expression
-   * for the whole call — see compileRuleMap() in codegen.ts for why this is
-   * one shared codegen pass instead of one `compile()` per entry (a `rules()`
-   * factory's entries commonly share large reachable sub-rule graphs; a
+   * for the whole call: one shared `compileRuleMap()` pass instead of one
+   * `compile()` per entry (a `rules()` factory's entries commonly share large reachable sub-rule graphs; a
    * shared pass compiles each shared sub-rule once instead of once per entry
    * that reaches it).
    */
@@ -1977,9 +1973,8 @@ function transformMacroImpl(
    * shadows the base piece which actually defines `Pair` — the encoder then finds a
    * hole where the winner should be and refuses the whole grammar with "ref() used
    * before .define()". The same filter guards `compileLinkableTable`'s `isLocal`
-   * (`compile-linkable-table.ts:96`), `itemCarried` (`linker.ts:696`) and
-   * `recoverComposedRules`; this is the fourth site that needs it, for the same
-   * reason, and skipping it is why cross-piece composition looked blocked.
+   * (`compile-linkable-table.ts`) and `recoverComposedRules`; this site needs it
+   * for the same reason, and skipping it is why cross-piece composition looked blocked.
    *
    * The reference itself is not lost: it stays inside the delta's own rule bodies and
    * `enc.winners` binds it BY NAME to whichever piece supplies the definition.

package/src/regex/first-set.ts

@@ -32,7 +32,7 @@ import { shorthandRanges, parseClassRanges, readUnicodeEscape } from './classes.
 type Node =
   | { t: 'char'; set: FirstSet; nl: boolean } // matches exactly one char from `set`
   | { t: 'any' }                              // unknown 1-char-ish construct: could start with anything
-  | { t: 'empty' }                            // zero-width: anchors, lookaround, word boundary
+  | { t: 'empty'; look?: Node }               // zero-width: anchors, lookaround, word boundary (`look`: a positive lookahead's body)
   | { t: 'seq'; parts: Node[] }
   | { t: 'alt'; arms: Node[] }
   | { t: 'rep'; node: Node; min: number }     // quantified; min 0 ⇒ nullable
@@ -142,10 +142,11 @@ function parseRegex(src: string, unicode = false, sets = false): Node {
     i++ // consume '('
     // Group prefixes: (?:…) (?=…) (?!…) (?<=…) (?<!…) (?<name>…)
     let lookaround = false
+    let lookahead = false
     if (src[i] === '?') {
       const c1 = src[i + 1]
       if (c1 === ':') i += 2
-      else if (c1 === '=' || c1 === '!') { i += 2; lookaround = true }
+      else if (c1 === '=' || c1 === '!') { lookahead = c1 === '='; i += 2; lookaround = true }
       else if (c1 === '<') {
         const c2 = src[i + 2]
         if (c2 === '=' || c2 === '!') { i += 3; lookaround = true } // lookbehind
@@ -159,7 +160,10 @@ function parseRegex(src: string, unicode = false, sets = false): Node {
     const inner = parseAlt()
     if (src[i] !== ')') throw BAIL
     i++ // consume ')'
-    return lookaround ? EMPTY : inner // lookaround is zero-width
+    // Lookaround is zero-width. A positive LOOKAHEAD keeps its body for
+    // `regexFirstTwo`; every other walker reads it as `EMPTY`.
+    if (lookaround) return lookahead ? { t: 'empty', look: inner } : EMPTY
+    return inner
   }
 
   const parseClass = (): Node => {
@@ -348,6 +352,17 @@ export function regexCanMatchEmpty(source: string): boolean {
   try { return nullable(parseRegex(source)) } catch { return true }
 }
 
+/**
+ * The first chars of a NON-EMPTY match of `source`, nullable or not: the set
+ * `firstSetFromRegex` reports for a pattern that must consume, without its
+ * widening of a nullable one to `any()`. Only a caller that accounts for the
+ * empty match itself may use it (`sequenceSet`). `any()` when unparseable.
+ * `flags` reads atoms as `firstSetFromRegex` does.
+ */
+export function consumingFirstSetFromRegex(source: string, flags = ''): FirstSet {
+  try { return firstSet(parseRegex(source, flags.includes('u') || flags.includes('v'), flags.includes('v'))) } catch { return any() }
+}
+
 /**
  * `flags` decides how atoms read: under `u`/`v` each is one code point, else one
  * code unit, so a non-`u` set holds a high surrogate where an astral character
@@ -369,3 +384,22 @@ export function firstSetFromRegex(source: string, flags = ''): { firstSet: First
   return { firstSet: firstSet(ast), canMatchNewline: canConsumeNewline(ast) }
 }
 
+/**
+ * The first TWO characters every match of `source` begins with, where the
+ * pattern says them outright: a one-character class and then another, or a
+ * positive lookahead over one (`:(?=[ \t])`). `undefined` otherwise. Every
+ * match that starts with a character of `first` has a second character, and it
+ * is in `second`. `flags` reads atoms as `firstSetFromRegex` does.
+ */
+export function regexFirstTwo(source: string, flags = ''): { first: FirstSet; second: FirstSet } | undefined {
+  let ast: Node
+  try { ast = parseRegex(source, flags.includes('u') || flags.includes('v'), flags.includes('v')) } catch { return undefined }
+  const parts = ast.t === 'seq' ? ast.parts : [ast]
+  const a = parts[0]
+  const b = parts[1]
+  if (a === undefined || a.t !== 'char' || a.set.kind !== 'ranges' || b === undefined) return undefined
+  const second = b.t === 'char'
+    ? b.set
+    : b.t === 'empty' && b.look !== undefined && !nullable(b.look) ? firstSet(b.look) : undefined
+  return second !== undefined && second.kind === 'ranges' ? { first: a.set, second } : undefined
+}