parseman 0.47.1 → 0.48.1

package/src/table/lex-program.ts

@@ -0,0 +1,431 @@
+import type { Combinator, ParseContext } from '../types.ts'
+import { failAt } from '../combinators/probe.ts'
+import type { LexBodyRecognizer } from './lex-body.ts'
+import type { LexProgramSpec, ResolvedClass } from './program.ts'
+
+/** A selected composite lexical body. Failure state is already published.
+ * `scan` is present only for the balanced shell and indexes the canonical
+ * driver-owned scan pool; it is not another recognizer implementation. */
+export type LexProgramRunner = {
+  (input: string, pos: number, ctx: ParseContext, scan?: Combinator<unknown>): number
+  readonly scan?: number
+}
+
+export const LEX_NODE_TERMINAL = 0
+export const LEX_NODE_SEQUENCE2 = 1
+export const LEX_NODE_SEQUENCE3 = 2
+export const LEX_NODE_SEQUENCE4 = 3
+export const LEX_NODE_SEQUENCE5 = 4
+export const LEX_NODE_CHOICE2 = 5
+export const LEX_NODE_CHOICE4 = 6
+export const LEX_NODE_CHOICE8 = 7
+export const LEX_NODE_NOT = 8
+export const LEX_NODE_OPTIONAL = 9
+
+export function lexProgramDigest(words: readonly number[]): number {
+  let hash = 0x811C9DC5
+  for (const word of words) {
+    hash ^= word | 0
+    hash = Math.imul(hash, 0x01000193)
+  }
+  return hash | 0
+}
+
+function classHas(cls: ResolvedClass, code: number): boolean {
+  if (code < 0) return false
+  if (code < 128) return cls.ascii[code] === 1
+  for (let i = 0; i < cls.hi.length; i += 2) {
+    if (code >= cls.hi[i]! && code <= cls.hi[i + 1]!) return true
+  }
+  return false
+}
+
+type PureLexRunner = (input: string, pos: number) => number
+type BuiltLexNode = { readonly match: PureLexRunner; readonly run: LexProgramRunner }
+const committed = (ctx: ParseContext): boolean => ctx._fc === true
+
+function trackLexLines(ctx: ParseContext, input: string, end: number): void {
+  let at = ctx._lineScannedTo ?? 0
+  const starts = ctx._lineStarts
+  if (starts === undefined || end <= at) return
+  while (at < end) {
+    if (input.charCodeAt(at) === 10) starts.push(at + 1)
+    at++
+  }
+  ctx._lineScannedTo = end
+}
+
+function terminalNode(
+  matcher: LexBodyRecognizer,
+  failure: readonly string[],
+  tracksLines: boolean,
+): BuiltLexNode {
+  const match: PureLexRunner = (input, pos) => {
+    const result = matcher(input, pos)
+    return result < 0 ? -1 : result / 2
+  }
+  return {
+    match,
+    run: (input, pos, ctx) => {
+      const end = match(input, pos)
+      if (end < 0) {
+        ctx._fe = pos
+        ctx._fx = failure as string[]
+        if (ctx._probe !== undefined) failAt(ctx, ctx._fx, pos)
+        return -1
+      }
+      if (tracksLines) trackLexLines(ctx, input, end)
+      return end
+    },
+  }
+}
+
+function sequence2(a: BuiltLexNode, b: BuiltLexNode): BuiltLexNode {
+  return {
+    match: (input, pos) => {
+      const e0 = a.match(input, pos)
+      return e0 < 0 ? -1 : b.match(input, e0)
+    },
+    run: (input, pos, ctx) => {
+      const e0 = a.run(input, pos, ctx)
+      if (e0 < 0) return -1
+      ctx._fc = false
+      return b.run(input, e0, ctx)
+    },
+  }
+}
+
+function sequence3(a: BuiltLexNode, b: BuiltLexNode, c: BuiltLexNode): BuiltLexNode {
+  return {
+    match: (input, pos) => {
+      const e0 = a.match(input, pos)
+      if (e0 < 0) return -1
+      const e1 = b.match(input, e0)
+      return e1 < 0 ? -1 : c.match(input, e1)
+    },
+    run: (input, pos, ctx) => {
+      const e0 = a.run(input, pos, ctx)
+      if (e0 < 0) return -1
+      ctx._fc = false
+      const e1 = b.run(input, e0, ctx)
+      if (e1 < 0) return -1
+      ctx._fc = false
+      return c.run(input, e1, ctx)
+    },
+  }
+}
+
+function sequence4(a: BuiltLexNode, b: BuiltLexNode, c: BuiltLexNode, d: BuiltLexNode): BuiltLexNode {
+  const abc = sequence3(a, b, c)
+  return {
+    match: (input, pos) => {
+      const end = abc.match(input, pos)
+      return end < 0 ? -1 : d.match(input, end)
+    },
+    run: (input, pos, ctx) => {
+      const end = abc.run(input, pos, ctx)
+      if (end < 0) return -1
+      ctx._fc = false
+      return d.run(input, end, ctx)
+    },
+  }
+}
+
+function sequence5(
+  a: BuiltLexNode, b: BuiltLexNode, c: BuiltLexNode, d: BuiltLexNode, e: BuiltLexNode,
+): BuiltLexNode {
+  const abcd = sequence4(a, b, c, d)
+  return {
+    match: (input, pos) => {
+      const end = abcd.match(input, pos)
+      return end < 0 ? -1 : e.match(input, end)
+    },
+    run: (input, pos, ctx) => {
+      const end = abcd.run(input, pos, ctx)
+      if (end < 0) return -1
+      ctx._fc = false
+      return e.run(input, end, ctx)
+    },
+  }
+}
+
+type ChoiceTail = (
+  input: string, pos: number, ctx: ParseContext, lead: number,
+  acc: string[] | undefined, best: number,
+) => number
+
+function appendExpected(acc: string[] | undefined, values: readonly string[] | undefined): string[] | undefined {
+  if (values === undefined || values.length === 0) return acc
+  if (acc === undefined) return values.slice()
+  acc.push(...values)
+  return acc
+}
+
+function choiceNode(
+  arms: readonly BuiltLexNode[],
+  classes: readonly (ResolvedClass | undefined)[],
+  armExpected: readonly (readonly string[])[],
+  choiceExpected: readonly string[],
+): BuiltLexNode {
+  let matchTail: PureLexRunner = () => -1
+  for (let i = arms.length - 1; i >= 0; i--) {
+    const arm = arms[i]!, next = matchTail
+    matchTail = (input, pos) => {
+      const end = arm.match(input, pos)
+      return end < 0 ? next(input, pos) : end
+    }
+  }
+  let tail: ChoiceTail = (_input, pos, ctx, _lead, acc) => {
+    ctx._fe = pos
+    ctx._fx = acc ?? choiceExpected as string[]
+    return -1
+  }
+  for (let i = arms.length - 1; i >= 0; i--) {
+    const arm = arms[i]!, cls = classes[i], staticExpected = armExpected[i]!, next = tail
+    tail = (input, pos, ctx, lead, acc, best) => {
+      if (cls !== undefined && !classHas(cls, lead)) {
+        return next(
+          input, pos, ctx, lead,
+          best === pos ? appendExpected(acc, staticExpected) : acc,
+          best,
+        )
+      }
+      ctx._fc = false
+      const end = arm.run(input, pos, ctx)
+      if (end >= 0) return end
+      const at = ctx._fe ?? pos
+      if (at > best) { best = at; acc = undefined }
+      const nextAcc = at === best ? appendExpected(acc, ctx._fx) : acc
+      if (committed(ctx)) {
+        if (nextAcc !== undefined) ctx._fx = nextAcc
+        return -1
+      }
+      return next(input, pos, ctx, lead, nextAcc, best)
+    }
+  }
+  return {
+    match: matchTail,
+    run: (input, pos, ctx) => tail(
+      input, pos, ctx, pos < input.length ? input.codePointAt(pos)! : -1, undefined, pos,
+    ),
+  }
+}
+
+function notNode(body: BuiltLexNode, failure: readonly string[]): BuiltLexNode {
+  return {
+    match: (input, pos) => body.match(input, pos) < 0 ? pos : -1,
+    run: (input, pos, ctx) => {
+      if (body.match(input, pos) < 0) return pos
+      ctx._fe = pos
+      ctx._fx = failure as string[]
+      return -1
+    },
+  }
+}
+
+function optionalNode(body: BuiltLexNode): BuiltLexNode {
+  return {
+    match: (input, pos) => {
+      const end = body.match(input, pos)
+      return end < 0 ? pos : end
+    },
+    run: (input, pos, ctx) => {
+      ctx._fc = false
+      const end = body.run(input, pos, ctx)
+      return end < 0 && !committed(ctx) ? pos : end
+    },
+  }
+}
+
+function fixedTreeRunner(
+  words: readonly number[],
+  matchers: readonly LexBodyRecognizer[],
+  classes: readonly ResolvedClass[],
+  expected: readonly (readonly string[])[],
+): LexProgramRunner {
+  const stack: BuiltLexNode[] = []
+  const pop = (): BuiltLexNode => {
+    const value = stack.pop()
+    if (value === undefined) throw new TypeError('table lexical tree has an incomplete fixed body')
+    return value
+  }
+  for (let at = 0; at < words.length;) {
+    const op = words[at++]!
+    if (op === LEX_NODE_TERMINAL) {
+      const matcher = matchers[words[at++]!], failure = expected[words[at++]!]
+      const tracks = words[at++]
+      if (matcher === undefined || failure === undefined || (tracks !== 0 && tracks !== 1)) {
+        throw new TypeError('table lexical tree terminal references an invalid pool entry')
+      }
+      stack.push(terminalNode(matcher, failure, tracks === 1))
+      continue
+    }
+    if (op >= LEX_NODE_SEQUENCE2 && op <= LEX_NODE_SEQUENCE5) {
+      const count = op - LEX_NODE_SEQUENCE2 + 2
+      const nodes = Array.from({ length: count }, pop).reverse()
+      if (count === 2) stack.push(sequence2(nodes[0]!, nodes[1]!))
+      else if (count === 3) stack.push(sequence3(nodes[0]!, nodes[1]!, nodes[2]!))
+      else if (count === 4) stack.push(sequence4(nodes[0]!, nodes[1]!, nodes[2]!, nodes[3]!))
+      else stack.push(sequence5(nodes[0]!, nodes[1]!, nodes[2]!, nodes[3]!, nodes[4]!))
+      continue
+    }
+    if (op >= LEX_NODE_CHOICE2 && op <= LEX_NODE_CHOICE8) {
+      const count = op === LEX_NODE_CHOICE2 ? 2 : op === LEX_NODE_CHOICE4 ? 4 : 8
+      const choiceFx = expected[words[at++]!]
+      if (choiceFx === undefined) throw new TypeError('table lexical tree choice has an invalid expected set')
+      const armClasses: Array<ResolvedClass | undefined> = []
+      const armFx: Array<readonly string[]> = []
+      for (let i = 0; i < count; i++) {
+        const cls = words[at++]!, fx = expected[words[at++]!]
+        if (cls < -1 || (cls >= 0 && classes[cls] === undefined) || fx === undefined) {
+          throw new TypeError('table lexical tree choice references an invalid pool entry')
+        }
+        armClasses.push(cls < 0 ? undefined : classes[cls])
+        armFx.push(fx)
+      }
+      const arms = Array.from({ length: count }, pop).reverse()
+      stack.push(choiceNode(arms, armClasses, armFx, choiceFx))
+      continue
+    }
+    if (op === LEX_NODE_NOT) {
+      const failure = expected[words[at++]!]
+      if (failure === undefined) throw new TypeError('table lexical tree assertion has an invalid expected set')
+      stack.push(notNode(pop(), failure))
+      continue
+    }
+    if (op === LEX_NODE_OPTIONAL) { stack.push(optionalNode(pop())); continue }
+    throw new TypeError('table lexical tree has an unknown fixed body')
+  }
+  if (stack.length !== 1) throw new TypeError('table lexical tree does not resolve to one root body')
+  return stack[0]!.run
+}
+
+/** Decode one fixed selected-body tuple. There is no lexical opcode loop on
+ * the parse path: each supported tuple becomes one scalar closure here. */
+export function buildLexProgramRunner(
+  spec: LexProgramSpec,
+  matchers: readonly LexBodyRecognizer[],
+  classes: readonly ResolvedClass[],
+  expected: readonly (readonly string[])[],
+): LexProgramRunner {
+  if ((spec[0] !== 0 && spec[0] !== 1 && spec[0] !== 2 && spec[0] !== 3)
+    || (spec[0] === 0 && spec.length !== 13)
+    || (spec[0] === 1 && spec.length !== 12)
+    || (spec[0] === 2 && spec.length < 6)
+    || (spec[0] === 3 && spec.length !== 3)) {
+    throw new TypeError('table lexical program has an invalid fixed body width')
+  }
+  if (spec[1] !== lexProgramDigest(spec.slice(2))) {
+    throw new TypeError('table lexical program semantic digest is inconsistent')
+  }
+  if (spec[0] === 3) {
+    if (!Number.isInteger(spec[2]) || spec[2] < 0) {
+      throw new TypeError('table balanced lexical program has an invalid scan reference')
+    }
+    const run: LexProgramRunner = (_input, _pos, ctx, scan) => {
+      if (scan === undefined) throw new TypeError('table balanced lexical program lost its scan body')
+      const result = scan.parse(_input, _pos, ctx)
+      if (!result.ok) {
+        ctx._fe = result.span.start
+        ctx._fx = (result.expected ?? []) as string[]
+        return -1
+      }
+      return result.span.end
+    }
+    return Object.assign(run, { scan: spec[2] })
+  }
+  if (spec[0] === 2) return fixedTreeRunner(spec.slice(2), matchers, classes, expected)
+  if (spec[0] === 1) {
+    const g00 = matchers[spec[2]], g01 = matchers[spec[3]]
+    const g02 = matchers[spec[4]], g03 = matchers[spec[5]]
+    const guard0Fx = expected[spec[6]]
+    const g1 = matchers[spec[7]], guard1Fx = expected[spec[8]]
+    const terminal = matchers[spec[9]], terminalFx = expected[spec[10]]
+    const tracks = spec[11]
+    if (g00 === undefined || g01 === undefined || g02 === undefined || g03 === undefined
+      || guard0Fx === undefined || g1 === undefined || guard1Fx === undefined
+      || terminal === undefined || terminalFx === undefined || (tracks !== 0 && tracks !== 1)) {
+      throw new TypeError('table lexical program references an invalid pool entry')
+    }
+    return (input, pos, ctx) => {
+      ctx._fc = false
+      if (g00(input, pos) >= 0 || g01(input, pos) >= 0
+        || g02(input, pos) >= 0 || g03(input, pos) >= 0) {
+        ctx._fe = pos
+        ctx._fx = guard0Fx as string[]
+        return -1
+      }
+      ctx._fc = false
+      if (g1(input, pos) >= 0) {
+        ctx._fe = pos
+        ctx._fx = guard1Fx as string[]
+        return -1
+      }
+      ctx._fc = false
+      const result = terminal(input, pos)
+      if (result < 0) {
+        ctx._fe = pos
+        ctx._fx = terminalFx as string[]
+        if (ctx._probe !== undefined) failAt(ctx, ctx._fx, pos)
+        return -1
+      }
+      const end = result / 2
+      if (tracks === 1) trackLexLines(ctx, input, end)
+      return end
+    }
+  }
+  const cls = classes[spec[2]]
+  const choiceFx = expected[spec[3]]
+  const m0 = matchers[spec[4]], x0 = expected[spec[5]]
+  const m1 = matchers[spec[6]], x1 = expected[spec[7]]
+  const m2 = matchers[spec[8]], x2 = expected[spec[9]]
+  const m3 = matchers[spec[10]], x3 = expected[spec[11]]
+  const tracks = spec[12]
+  if (cls === undefined || choiceFx === undefined
+    || m0 === undefined || x0 === undefined || m1 === undefined || x1 === undefined
+    || m2 === undefined || x2 === undefined || m3 === undefined || x3 === undefined
+    || (tracks !== 0 && tracks !== 1)) {
+    throw new TypeError('table lexical program references an invalid pool entry')
+  }
+
+  return (input, pos, ctx) => {
+    const lead = pos < input.length ? input.codePointAt(pos)! : -1
+    if (!classHas(cls, lead)) {
+      ctx._fe = pos
+      ctx._fx = choiceFx.slice()
+      return -1
+    }
+    let acc: string[] | undefined
+    const failed = (fx: readonly string[]): void => {
+      ctx._fe = pos
+      ctx._fx = fx as string[]
+      if (ctx._probe !== undefined) failAt(ctx, ctx._fx, pos)
+      if (acc === undefined) acc = fx.slice()
+      else acc.push(...fx)
+    }
+    ctx._fc = false
+    const success = (result: number): number => {
+      const end = result / 2
+      if (tracks === 1) trackLexLines(ctx, input, end)
+      return end
+    }
+    let result = m0(input, pos)
+    if (result >= 0) return success(result)
+    failed(x0)
+    ctx._fc = false
+    result = m1(input, pos)
+    if (result >= 0) return success(result)
+    failed(x1)
+    ctx._fc = false
+    result = m2(input, pos)
+    if (result >= 0) return success(result)
+    failed(x2)
+    ctx._fc = false
+    result = m3(input, pos)
+    if (result >= 0) return success(result)
+    failed(x3)
+    ctx._fe = pos
+    ctx._fx = acc ?? choiceFx as string[]
+    return -1
+  }
+}

package/src/table/ops.ts

@@ -30,15 +30,18 @@ export const OP_SEQV = 4
  * `CHOICE d n fx c1 … cn` — `d` indexes a dispatch table (or −1 for ordered
  * try); `fx` indexes the choice's OWN expected set.
  *
- * Both shipped engines report the UNION of the arms' expectations when a choice
- * fails. The driver reported whatever the last attempted arm left behind — or,
- * on a dispatch miss with no arm claiming the lead character, NOTHING, so a user
- * got an error naming nothing at all.
+ * Ordered failure keeps the expected sets at the deepest arm offset, merging
+ * exact ties in source order. Exclusive dispatch propagates its sole selected
+ * arm. A pure dispatch miss uses `fx`, the static union of every arm opener, so
+ * the result never names nothing merely because no arm claimed the lead.
  */
 export const OP_CHOICE = 5
 /**
- * `REP c min max sep flags` — `sep` is a child offset or −1; `max` −1 = ∞.
- * `flags` bit 0 = trailing separator allowed, bit 1 = `keepSeparators`.
+ * `REP c min max sep flags fx sepClass` — `sep` is a child offset or −1; `max`
+ * −1 = ∞. `fx` is recovery metadata. The last class is the separator sentinel
+ * for a separated list, otherwise the finite/non-nullable optional-item class
+ * (or −1). `flags` bit 0 = trailing separator allowed, bit 1 =
+ * `keepSeparators`, bit 2 = item expected reporting.
  */
 export const OP_REP = 6
 /** `REPV …` — `REP` with `valueUnused`. */
@@ -57,7 +60,8 @@ export const OP_XFORM = 9
  * none), `type` and `tags` index the const pool (`tags` −1 = none).
  *
  * `flags` is a BIT FIELD, not a boolean: bit 0 (`&1`) = grammar-owned explicit
- * `captureTrivia`, bit 2 (`&4`) = the builder reads
+ * `captureTrivia`, bit 1 (`&2`) = a direct builder is proven not to read
+ * `rawChildren`, bit 2 (`&4`) = the builder reads
  * `triviaLog`, bit 3 (`&8`) = it reads `ctx.state`, bit 4 (`&16`) = the node has
  * read fields, bit 5 (`&32`) = `collapse`, bit 6 (`&64`) = `unwrap`, bit 7
  * (`&128`) = `trailingTrivia`. Bit 0 distinguishes capture a host may not
@@ -110,8 +114,12 @@ export const OP_NODE_TRACK = 19
  * on `ctx.trivia` for the duration and restores the outer one after, which is
  * how the runtime's own `advanceTrivia` fast scanner gets reached — the same
  * shared machinery the interpreter uses, not a second copy of it.
- * Synthetic rule-entry/restoration scopes use `OP_SCOPE_PLAIN` so a zero
- * policy stays a three-word row without changing this opcode's ABI.
+ *
+ * `policy` bit 0 suppresses selected root capture for an opaque scope; bit 1
+ * refuses an unclassified local scope while selected root capture is active.
+ * Synthetic rule-entry and cross-rule restoration scopes have no policy and use
+ * the three-word `OP_SCOPE_PLAIN` row below. Keeping the opcodes distinct means
+ * a zero policy costs no extra program word without changing this opcode's ABI.
  */
 export const OP_SCOPE = 20
 /**
@@ -122,6 +130,11 @@ export const OP_EXPECT = 21
 /**
  * `SEQX f n c1 … cn` — a `transform()` whose child is a `sequence()`.
  *
+ * `f >= 0` indexes the reducer in `prog.fns`. `f < 0` is the descriptor
+ * `~childIndex`: the transform is the exact direct projection
+ * `([…, value, …]) => value`, so the row returns that already-parsed child and
+ * carries no reducer.  Both forms have the same row width.
+ *
  * That pair is the dominant shape in every grammar here (json is nine of them),
  * and running it as two rows costs two switch dispatches and two JS call frames
  * per rule invocation where the emitted code pays neither. Fusing them into one
@@ -428,10 +441,33 @@ export const OP_LABEL = 39
  */
 export const OP_COV = 40
 /**
- * `SCOPE_PLAIN k c` — a synthetic ambient-trivia scope with policy fixed to zero.
- * A distinct opcode prevents the next row's opcode from being read as policy.
+ * `SCOPE_PLAIN k c` — a synthetic zero-policy ambient-trivia scope.
+ *
+ * This is the same scanner/context swap as `OP_SCOPE`, with policy fixed to
+ * zero at assembly. `encodeRule()` uses it for a rule entry and `scopedRef()`
+ * for the lexical-trivia restoration around a reference reached from
+ * `noTrivia()`. Neither row represents an authored `parser()` scope, so neither
+ * can be opaque or require the unclassified-scope refusal.
+ *
+ * A separate opcode is load-bearing: treating a three-word synthetic row as the
+ * four-word `OP_SCOPE` makes the following row's opcode become the policy bits.
  */
 export const OP_SCOPE_PLAIN = 41
+/**
+ * `LEX_BODY b fx suffixFx` — one compiler-selected childless lexical replacement
+ * body. `b` indexes `prog.lex`; `fx` is the exact base-recognizer failure set;
+ * `suffixFx` is optional()'s swallowed literal failure publication.
+ *
+ * There is deliberately no child operand and no mode bit. The complete
+ * CHARACTER and TOKEN candidates were compared before serialization; this row
+ * is only the winning TOKEN body. A program without a selected TOKEN body has
+ * no row or lexical pool.
+ */
+/** `LEX_BODY body fx suffixFx lineFlags` — selected childless lexical body.
+ * lineFlags bit 0 publishes the regex range; bit 1 publishes a matched suffix. */
+export const OP_LEX_BODY = 42
+/** `LEX_PROGRAM p` — selected fixed composite lexical body, childless. */
+export const OP_LEX_PROGRAM = 43
 /**
  * `DISPATCH sel d other otherRouted n a1 … an` — `dispatch()`.
  *
@@ -464,4 +500,6 @@ export const OP_NAMES: Record<number, string> = {
   [OP_GREEDY]: 'GREEDY', [OP_REJECT]: 'REJECT', [OP_ARMGATE]: 'ARMGATE',
   [OP_LIVE]: 'LIVE', [OP_ATTEMPT]: 'ATTEMPT', [OP_LABEL]: 'LABEL', [OP_COV]: 'COV',
   [OP_SCOPE_PLAIN]: 'SCOPE_PLAIN',
+  [OP_LEX_BODY]: 'LEX_BODY',
+  [OP_LEX_PROGRAM]: 'LEX_PROGRAM',
 }

package/src/table/program.ts

@@ -5,6 +5,8 @@ import { regex } from '../combinators/regex.ts'
 import { choice } from '../combinators/choice.ts'
 import { many } from '../combinators/repeat.ts'
 import type { EmittedFactory, PoolPlan } from './emit-assembly.ts'
+import { buildLexBodyRecognizer, type LexBodyRecognizer } from './lex-body.ts'
+import { buildLexProgramRunner, type LexProgramRunner } from './lex-program.ts'
 
 /**
  * One assembly the BUILD already compiled, so the run does not have to.
@@ -57,6 +59,11 @@ export type TableProgram = {
    * at `resolveTable`, exactly like the char-class ASCII lookups.
    */
   readonly dsp: readonly DispatchSpec[]
+  /** Selected childless lexical bodies. Losing compiler candidates never enter
+   * this pool. The first row is `[regexConst, suffixCodeUnit]`. */
+  readonly lex?: readonly LexBodySpec[]
+  /** Selected composite lexical programs. Losing candidates never enter it. */
+  readonly lexPrograms?: readonly LexProgramSpec[]
   /** Rule name → entry offset in `code`. */
   readonly rules: Readonly<Record<string, number>>
   /**
@@ -291,6 +298,8 @@ export type CompactProgram = {
   readonly f: readonly unknown[]
   readonly l?: 0 | 1
   readonly p?: readonly DispatchSpec[]
+  readonly lx?: readonly LexBodySpec[]
+  readonly lp?: readonly LexProgramSpec[]
   readonly lb?: readonly string[]
   readonly rc?: 0 | 1
   readonly h?: 'ast' | 'cst'
@@ -309,6 +318,8 @@ export function expandCompact(p: TableProgram | CompactProgram): TableProgram {
   return ownTableProgram({
     code: p.c, k: p.k, cc: p.x, fx: p.e, disp: p.d, rules: p.r, fns: p.f,
     lines: p.l ?? 0, dsp: p.p ?? [],
+    ...(p.lx === undefined ? {} : { lex: p.lx }),
+    ...(p.lp === undefined ? {} : { lexPrograms: p.lp }),
     ...(p.lb === undefined ? {} : { labels: p.lb }),
     ...(p.rc === undefined ? {} : { classified: p.rc }),
     ...(p.h === undefined ? {} : { hostMode: p.h }),
@@ -322,6 +333,28 @@ export function expandCompact(p: TableProgram | CompactProgram): TableProgram {
   })
 }
 
+/** Selected lexical body. `suffixCodeUnit === -1` is a direct regex terminal;
+ * 0..0xFFFF is the legacy regex + optional(one code unit) spelling. */
+export type LexBodySpec = readonly [regexConst: number, suffixCodeUnit: number]
+
+/** `kind=0`: fixed ordered four-terminal selected body. */
+export type LexProgramSpec = readonly [
+  kind: 0, digest: number, commonClass: number, choiceFx: number,
+  arm0Matcher: number, arm0Fx: number,
+  arm1Matcher: number, arm1Fx: number,
+  arm2Matcher: number, arm2Fx: number,
+  arm3Matcher: number, arm3Fx: number,
+  trackLines: 0 | 1,
+] | readonly [
+  kind: 1, digest: number,
+  guard0Arm0: number, guard0Arm1: number, guard0Arm2: number, guard0Arm3: number,
+  guard0Fx: number,
+  guard1Matcher: number, guard1Fx: number,
+  terminalMatcher: number, terminalFx: number,
+  trackLines: 0 | 1,
+] | readonly [kind: 2, digest: number, ...fixedTreeWords: number[]]
+  | readonly [kind: 3, digest: number, scan: number]
+
 /** One `dispatch()`'s routing tables, in serialisable form. */
 export type DispatchSpec = {
   /** Exact keys, parallel to `keyArm`. */
@@ -353,7 +386,12 @@ export type ResolvedDispatchSpec = {
   readonly expected: readonly string[]
 }
 
-/** Refuse malformed dispatch arm references before closure/emitted linking. */
+/**
+ * Refuse malformed fixed arm references before closure or emitted linking.
+ *
+ * This is deliberately an assembly-time trust boundary. `execRules()` stays an
+ * independent semantic oracle and never pays artifact validation per parse.
+ */
 export function validateDispatchSpec(
   spec: ResolvedDispatchSpec | undefined,
   arity: number,
@@ -428,6 +466,8 @@ export type ResolvedTable = {
   readonly fx: readonly (readonly string[])[]
   readonly disp: readonly ResolvedDispatch[]
   readonly dsp: readonly ResolvedDispatchSpec[]
+  readonly lex: readonly LexBodyRecognizer[]
+  readonly lexPrograms: readonly LexProgramRunner[]
   /** Trivia combinators rebuilt from `triviaSpecs`, once per table. */
   readonly trivia: readonly Combinator<unknown>[]
   /**
@@ -605,6 +645,7 @@ export function resolveTable(prog: TableProgram): ResolvedTable {
   if (hit !== undefined) return hit
   const cc = prog.cc.map(resolveClass)
   const triviaBuilt = (prog.triviaSpecs ?? []).map(buildTrivia)
+  const lex = (prog.lex ?? []).map(spec => buildLexBodyRecognizer(spec, prog.k))
   const built: ResolvedTable = {
     prog,
     code: Int32Array.from(prog.code),
@@ -623,6 +664,9 @@ export function resolveTable(prog: TableProgram): ResolvedTable {
       routed: d.routed,
       expected: d.expected,
     })),
+    lex,
+    lexPrograms: (prog.lexPrograms ?? []).map(spec =>
+      buildLexProgramRunner(spec, lex, cc, prog.fx)),
     rules: prog.rules,
   }
   if (owner !== undefined) owner.resolved = built
@@ -687,7 +731,7 @@ export type CompactFolded = {
  * Named rather than derived so the refusal can say which one broke.
  */
 const SHARED_FIELDS = [
-  'k', 'cc', 'fx', 'disp', 'dsp', 'rules', 'labels', 'classified',
+  'k', 'cc', 'fx', 'disp', 'dsp', 'lex', 'lexPrograms', 'rules', 'labels', 'classified',
   'scanSkip', 'scanSkipOf', 'scans', 'triviaSpecs', 'runtimeOnly',
   // `rec` is a property of the GRAMMAR BUILD, not of the trackLines/hostMode
   // axis a fold varies — every variant of one export is encoded with the same