parseman 0.59.3 → 0.60.1

package/src/table/exec.ts

@@ -21,8 +21,8 @@ import {
   OP_LIT_TRACK, OP_RX_TRACK, OP_NODE_TRACK, OP_SCOPE, OP_SCOPE_CAP, OP_SCOPE_PLAIN, OP_EXPECT, OP_SEQX, OP_SCAN,
   OP_LIVE,
   OP_FIELD, OP_DISPATCH, OP_ROUTED, OP_LIT_CI, OP_LIT_CI_TRACK, OP_TOKEN, OP_WITHCTX, OP_GUARD, OP_ATTEMPT, OP_LABEL,
-  OP_COV,
-  OP_ADJ, OP_GREEDY, OP_REJECT, OP_ARMGATE, OP_RGATE,
+  OP_COV, OP_GAPGATE, OP_STRICT,
+  OP_ADJ, OP_GREEDY, OP_REJECT, OP_ARMGATE, OP_RGATE, OP_RGATE2,
   OP_LEX_BODY, OP_LEX_PROGRAM,
 } from './ops.ts'
 import { adjacencyHolds, adjacencyMisuse } from '../combinators/adjacency.ts'
@@ -75,60 +75,6 @@ import { FAIL, newEndCell, type EndCell } from './cell.ts'
 import { fieldSpec, leadingTriviaFieldAt } from './field-spec.ts'
 import { FIELD_LEADING_TRIVIA, FIELD_TRAILING_TRIVIA, LEAD, triviaFieldCapture } from '../combinators/field-trivia.ts'
 
-/**
- * DIAGNOSTIC ROW COUNTER — off unless `PM_TABLE_COUNT=1` at process start.
- *
- * Read ONCE, at module load, into a module const, so this is not the per-parse
- * option branch G5 forbids: with the variable false the whole thing folds away
- * after tier-up. It exists because "too many rows" and "each row too slow" are
- * different defects with the same symptom, and only a count separates them.
- * No timing run in this repo may set it — see `bench/jess/table-rows.ts`.
- */
-export const tableCounters: {
-  rows: number
-  byOp: Int32Array
-  /** Distinct reducer/host functions actually reaching each shared call site. */
-  sites: Map<string, Set<unknown>>
-  /**
-   * SPECULATION AT UNGATED CHOICES. `encode.ts:368-376` gives a choice its
-   * first-char dispatch only when NO arm is nullable, ALL arms are pairwise
-   * disjoint and ALL map to a char class; any one failure ungates every arm of
-   * the site, which then runs the linear loop below. Codegen has no such
-   * all-or-nothing rule — `emitFirstMatch` guards each arm on its OWN first set
-   * — so these counts are work the table does and the compiled engine does not.
-   *
-   * `ungatedEntries` is arms entered there; `ungatedFails` is those that then
-   * failed, which is pure waste by construction: a failing arm consumed nothing
-   * the winning arm will not scan again.
-   */
-  ungatedEntries: number
-  ungatedFails: number
-  /** Rows executed INSIDE failed ungated arms — the subtree each wasted entry drags. */
-  ungatedFailRows: number
-  gatedEntries: number
-  /** Arm entries DECLINED by the per-arm class gate — the work this change removes. */
-  armGateSkips: number
-} = { rows: 0, byOp: new Int32Array(64), sites: new Map(), ungatedEntries: 0, ungatedFails: 0, ungatedFailRows: 0, gatedEntries: 0, armGateSkips: 0 }
-
-const COUNT = process.env.PM_TABLE_COUNT === '1'
-
-function siteFn(site: string, fn: unknown): void {
-  let s = tableCounters.sites.get(site)
-  if (s === undefined) { s = new Set(); tableCounters.sites.set(site, s) }
-  s.add(fn)
-}
-
-export function resetTableCounters(): void {
-  tableCounters.rows = 0
-  tableCounters.byOp = new Int32Array(64)
-  tableCounters.sites = new Map()
-  tableCounters.ungatedEntries = 0
-  tableCounters.ungatedFails = 0
-  tableCounters.ungatedFailRows = 0
-  tableCounters.gatedEntries = 0
-  tableCounters.armGateSkips = 0
-}
-
 const EMPTY_TL: readonly number[] = Object.freeze([])
 const EMPTY_FX: string[] = []
 const ROUTED_FX: string[] = ['routed()']
@@ -375,7 +321,11 @@ function makeDriver(
   }
 
   function exec(ip: number, input: string, pos: number, ctx: ParseContext): unknown {
-    if (COUNT) { tableCounters.rows++; tableCounters.byOp[code[ip]!]!++ }
+    // A row that only hands on to one child at the same position (a gate that
+    // passes, a rule trampoline, `OP_STRICT`) continues here rather than
+    // recursing, so the reference driver's stack grows with the grammar's
+    // nesting, not with its gate rows.
+    rows: for (;;) {
     switch (code[ip]) {
       case OP_LIT: {
         const s = k[code[ip + 1]!] as string
@@ -492,7 +442,7 @@ function makeDriver(
           ctx._fe = pos; ctx._fx = fx[code[ip + 3]!] as string[]
           return FAIL
         }
-        return exec(code[ip + 2]!, input, pos, ctx)
+        ip = code[ip + 2]!; continue rows
       }
 
       case OP_LABEL: {
@@ -502,7 +452,7 @@ function makeDriver(
         return v
       }
       case OP_RULE:
-        return exec(code[ip + 1]!, input, pos, ctx)
+        ip = code[ip + 1]!; continue rows
 
       /**
        * A COUNTER ROW IS TRANSPARENT HERE, and that is the whole of its handling.
@@ -518,8 +468,14 @@ function makeDriver(
        * a second implementation of it here would be a second place for the phase
        * semantics (entry for a rule, success for an arm) to drift.
        */
+      // The reference runs a gap-gated item as written: the gate is the repeat's,
+      // in the shipped engines, and this driver is the leg it is compared with.
       case OP_COV:
-        return exec(code[ip + 1]!, input, pos, ctx)
+      case OP_GAPGATE:
+        ip = code[ip + 1]!; continue rows
+
+      case OP_STRICT:
+        ip = code[ctx._tolerant || ctx._probe !== undefined ? ip + 2 : ip + 1]!; continue rows
 
       case OP_LIVE: {
         // A HAND-WRITTEN COMBINATOR, run through its own `.parse` — mirrors
@@ -601,6 +557,7 @@ function makeDriver(
 
         let target: number
         let usesRouted: boolean
+        let commits = true
         if (arm === undefined) {
           const other = code[ip + 3]!
           if (other < 0) {
@@ -610,7 +567,8 @@ function makeDriver(
             return FAIL
           }
           target = other
-          usesRouted = code[ip + 4]! === 1
+          usesRouted = (code[ip + 4]! & 1) === 1
+          commits = (code[ip + 4]! & 2) === 0
         } else {
           target = code[armBase + arm]!
           usesRouted = spec.routed[arm] === 1
@@ -637,8 +595,8 @@ function makeDriver(
           rollbackTrivia(ctx, mark)
           // The interpreter marks a failed dispatch branch COMMITTED: the
           // selector already matched, so an enclosing choice must not treat this
-          // as "try the next arm".
-          ctx._fc = true
+          // as "try the next arm". An uncommitted fallback keeps its own flag.
+          if (commits) ctx._fc = true
           return FAIL
         }
         // The end cell already holds the branch's end — dispatch's span runs from `pos`
@@ -878,7 +836,6 @@ function makeDriver(
           // Running it through `exec` at the POST-scan position would find the gap
           // already consumed and answer "adjacent" unconditionally.
           if (code[child] === OP_ADJ) {
-            if (COUNT) { tableCounters.rows++; tableCounters.byOp[OP_ADJ]!++ }
             const ki = code[child + 2]!
             if (!adjacencyHolds(input, cur, ctx, code[child + 1] === 1, ki < 0 ? undefined : k[ki] as readonly string[])) {
               ctx._fe = cur; ctx._fx = fx[code[child + 3]!] as string[]
@@ -914,10 +871,6 @@ function makeDriver(
           // LIT/RX in place removes both. The duplication is in the DRIVER,
           // which ships once for every grammar — the cost this design trades on.
           const cop = code[child]
-          // The two inline terminals below execute a ROW without going through
-          // `exec`, so the counter has to see them or the row count understates
-          // exactly where the driver already won.
-          if (COUNT && (cop === OP_LIT || cop === OP_RX)) { tableCounters.rows++; tableCounters.byOp[cop]!++ }
           if (cop === OP_LIT) {
             const lit = k[code[child + 1]!] as string
             if (!input.startsWith(lit, cur)) {
@@ -958,7 +911,6 @@ function makeDriver(
           const reducer = code[ip + 1]!
           if (reducer < 0) return values![~reducer]
           const fn = fns[reducer] as (value: unknown, span: { start: number; end: number }) => unknown
-          if (COUNT) siteFn('SEQX fn()', fn)
           return fn(values, { start: pos, end: cur })
         }
         return values
@@ -990,7 +942,6 @@ function makeDriver(
           }
           if (arm >= 0) {
             ctx._fc = false
-            if (COUNT) tableCounters.gatedEntries++
             const v = exec(code[base + arm]!, input, pos, ctx)
             if (v !== FAIL) return v
             // THE CUT. `dispatch()` is the library's one true cut: once its
@@ -1029,6 +980,12 @@ function makeDriver(
           // its `g.X` arms were unresolved refs (choice.ts:35 — the same staleness
           // `encode.ts` recomputes around), so they firstMatch all seven arms where
           // this driver dispatches to one. The arm sets differ, not the rule.
+          //
+          // Under a completions probe the interpreter runs every arm on a miss
+          // (`choice.ts`), so each opener reaches the probe; so does this.
+          if (arm < 0 && ctx._probe !== undefined) {
+            for (let i = 0; i < code[ip + 2]!; i++) exec(code[base + i]!, input, pos, ctx)
+          }
           return failChoice()
         }
         // THE PER-ARM GATE. Arms in source order, each skipped when its own
@@ -1063,8 +1020,9 @@ function makeDriver(
         const armFx = base + n
         for (let i = 0; i < n; i++) {
           const cls = armCls[i]
-          if (cls !== undefined && cls !== null && !classHas(cls, c)) {
-            if (COUNT) tableCounters.armGateSkips++
+          // Under a completion probe every arm is entered, as the interpreter's
+          // `firstMatch` enters it: its opener then reaches the probe (`failAt`).
+          if (ctx._probe === undefined && cls !== undefined && cls !== null && !classHas(cls, c)) {
             // THE GATED ARM STILL REPORTS. The interpreter has no char gate: it
             // enters the arm, the arm's opener fails, and its static set is what
             // lands in the accumulation. Declining to enter must not also decline
@@ -1073,11 +1031,8 @@ function makeDriver(
             continue
           }
           ctx._fc = false
-          if (COUNT) tableCounters.ungatedEntries++
-          const rows0 = COUNT ? tableCounters.rows : 0
           const v = exec(code[base + i]!, input, pos, ctx)
           if (v !== FAIL) return v
-          if (COUNT) { tableCounters.ungatedFails++; tableCounters.ungatedFailRows += tableCounters.rows - rows0 }
           const at = ctx._fe ?? pos
           if (at > best) { best = at; acc = undefined }
           if (at === best) push(ctx._fx)
@@ -1119,14 +1074,17 @@ function makeDriver(
         return sup
       }
 
-      case OP_RGATE: {
+      // The reference tests the first character only: the second is the shipped
+      // engines' (`OP_RGATE2`), and this driver is the leg they are compared with.
+      case OP_RGATE:
+      case OP_RGATE2: {
         const item = ctx._routed
         if (item !== undefined && item.span.start === pos && ctx._probe === undefined && !ctx._tolerant
           && !classHas(cc[code[ip + 1]!]!, lead(input, item.span.end))) {
           ctx._fe = item.span.end; ctx._fx = fx[code[ip + 3]!] as string[]
           return FAIL
         }
-        return exec(code[ip + 2]!, input, pos, ctx)
+        ip = code[ip + 2]!; continue rows
       }
 
       case OP_ARMGATE: {
@@ -1141,7 +1099,7 @@ function makeDriver(
           ctx._fx = fx[code[ip + 3]!] as string[]
           return FAIL
         }
-        return exec(code[ip + 2]!, input, pos, ctx)
+        ip = code[ip + 2]!; continue rows
       }
 
       case OP_REJECT: {
@@ -1225,7 +1183,7 @@ function makeDriver(
         // own sync. The item's expected set is at ip + 6 and the separator's
         // sentinel class at ip + 7, both laid down only for a recovery table.
         const mySync = REC ? ctx._sync : undefined
-        const recFx = REC ? fx[code[ip + 6]!] as string[] : undefined
+        const recFx = !REC ? undefined : (code[ip + 5]! & 16) !== 0 ? [] : fx[code[ip + 6]!] as string[]
         const sepSent = REC ? sentinelFor(code[ip + 7]!) : undefined
         const itemClassIndex = sep < 0 ? code[ip + 7]! : -1
         const itemClass = REC && itemClassIndex >= 0 ? cc[itemClassIndex]! : undefined
@@ -1397,7 +1355,6 @@ function makeDriver(
         const v = exec(code[ip + 2]!, input, pos, ctx)
         if (v === FAIL) return FAIL
         const fn = fns[code[ip + 1]!] as (value: unknown, span: { start: number; end: number }) => unknown
-        if (COUNT) siteFn('XFORM fn()', fn)
         return fn(v, { start: pos, end: EC.e })
       }
 
@@ -1439,7 +1396,6 @@ function makeDriver(
         if (v === FAIL) return FAIL
         const end = EC.e
         const fn = fns[code[ip + 1]!] as (value: unknown, span: { start: number; end: number }) => unknown
-        if (COUNT) siteFn('LEAF fn()', fn)
         const out = fn(v, { start: pos, end })
         if (wasCapturing) pushCstLeaf(ctx, {
           _tag: 'leaf',
@@ -1542,7 +1498,6 @@ function makeDriver(
               children: readonly unknown[], fields: FieldMap | undefined, span: { start: number; end: number },
               rawChildren: readonly unknown[], triviaLog: readonly number[], state: unknown,
             ) => unknown
-            if (COUNT) siteFn('NODE build()', build)
             nd = build(kids, fieldMap, span, cap.rawChildren, (flags & 4) !== 0 || hostCst ? cap.triviaLog : EMPTY_TL, st)
           }
         } else if (host !== undefined) {
@@ -1644,6 +1599,7 @@ function makeDriver(
       default:
         throw new Error(`table driver: unknown opcode ${String(code[ip])} at ${ip}`)
     }
+    }
   }
 
   /**

package/src/table/factor-choice.ts

@@ -35,8 +35,16 @@ import { sequence } from '../combinators/sequence.ts'
 import { leaf, sourceLeaf } from '../combinators/token.ts'
 import { noTrivia, parser } from '../combinators/grammar.ts'
 import { attempt } from '../combinators/attempt.ts'
+import { optional } from '../combinators/repeat.ts'
 import { mayCommitFailure } from '../analysis/commitment.ts'
 
+/**
+ * The choices this module builds. The interpreter never runs them: it runs the
+ * authored choice, so a factored one's `disjoint` says nothing about which arms
+ * the interpreter lets in (`Encoder` classifies them by their derived sets).
+ */
+export const factoredChoices = new WeakSet<Combinator<unknown>>()
+
 type ChoiceDef = Extract<ParserDef, { tag: 'choice' }>
 
 /** Own keys every combinator may carry beside its definition; all derived from it. */
@@ -151,6 +159,7 @@ export function factorSharedEnds(d: ChoiceDef): Combinator<unknown> | undefined
   // it walks everything the arms reach.
   if (arms.some(a => mayCommitFailure(a))) return undefined
   const middle = choice(...(shapes.map(s => s.terms[at]!) as [Combinator<unknown>, ...Combinator<unknown>[]]))
+  factoredChoices.add(middle)
   // The inner choice must keep arm order: a strategy that reorders (longest
   // literal first) or classifies would pick a different arm. Nor may it add an
   // automatic lookahead the arms as written do not have: `'start'` before
@@ -171,3 +180,56 @@ export function factorSharedEnds(d: ChoiceDef): Combinator<unknown> | undefined
   // `attempt`. The sequence alone would report where its inner choice failed.
   return out === undefined ? undefined : attempt(out)
 }
+
+/**
+ * AN ARM WHOSE NEXT SIBLING IS ITS OWN PREFIX — `choice(sequence(X, a), X)`.
+ *
+ * On input where `X` matches and `a` does not, the choice parses `X`, fails `a`,
+ * rolls back, and parses `X` again for the next arm. Where the choice's value is
+ * unused (its sequence arms are `valueUnused`, so only their captures reach the
+ * enclosing node), the first arm may be `sequence(X, optional(a))`:
+ *
+ * - `X` then `a` matches: the arm as it was.
+ * - `X` matches, `a` fails uncommitted: the optional matches empty, rolls `a`'s
+ *   captures back and leaves the gap before it unrecorded, so the arm ends where
+ *   `X` did with `X`'s captures, which is what the retried next arm produced.
+ * - `a` fails committed: the arm fails committed, as it did.
+ * - `X` fails: the arm fails as before, and the next arm, kept as it is, fails the
+ *   same way, so the choice merges the same sets in the same order.
+ *
+ * `X` is the next arm's sequence of terms, or that arm itself as a single term,
+ * and "the same" is `sameComb`. Gated arms, `autoNot` and strategies that reorder
+ * are left alone.
+ */
+export function factorPrefixArms(d: ChoiceDef): Combinator<unknown> | undefined {
+  const arms = d.parsers
+  if (arms.length < 2) return undefined
+  if (d.strategy.tag !== 'firstMatch' && d.strategy.tag !== 'sharedPrefix') return undefined
+  if (d.gates.some(g => g !== null) || d.autoNot.some(a => a !== null)) return undefined
+  const out = [...arms]
+  let changed = false
+  for (let i = 0; i + 1 < arms.length; i++) {
+    const arm = arms[i]!
+    if (arm._def.tag !== 'sequence' || arm._def.valueUnused !== true) continue
+    const terms = arm._def.parsers
+    const next = arms[i + 1]!
+    const prefix = next._def.tag === 'sequence' && next._def.valueUnused === true ? next._def.parsers : [next]
+    if (prefix.length >= terms.length) continue
+    if (!prefix.every((t, j) => sameComb(t, terms[j]!, 0))) continue
+    const rest = terms.slice(prefix.length)
+    // A rest that cannot fail (already `optional`, or `many`) has nothing to factor.
+    if (rest.every(never)) continue
+    const tail = rest.length === 1 ? rest[0]! : sequence(...(rest as [Combinator<unknown>, ...Combinator<unknown>[]]))
+    if (tail !== rest[0]) (tail._def as { valueUnused?: boolean }).valueUnused = true
+    const parts: Combinator<unknown>[] = [...terms.slice(0, prefix.length), optional(tail)]
+    const seq = sequence(...(parts as [Combinator<unknown>, ...Combinator<unknown>[]]))
+    ;(seq._def as { valueUnused?: boolean }).valueUnused = true
+    out[i] = seq
+    changed = true
+  }
+  if (!changed) return undefined
+  const rebuilt = choice(...(out as [Combinator<unknown>, ...Combinator<unknown>[]]))
+  factoredChoices.add(rebuilt)
+  const st = rebuilt._def.tag === 'choice' ? rebuilt._def.strategy : undefined
+  return st !== undefined && (st.tag === 'firstMatch' || st.tag === 'sharedPrefix') ? rebuilt : undefined
+}

package/src/table/fail-clean.ts

@@ -24,7 +24,7 @@ import {
   OP_ATTEMPT, OP_CHOICE, OP_COV, OP_DISPATCH, OP_EMPTY, OP_EXPECT, OP_FIELD, OP_GATE, OP_GUARD, OP_LABEL,
   OP_LEAF, OP_LIT, OP_LIT_CI, OP_LIT_CI_TRACK, OP_LIT_TRACK, OP_NODE, OP_NODE_TRACK, OP_NOT, OP_OPT, OP_PEEK,
   OP_REP, OP_REPV, OP_ROUTED, OP_RULE, OP_RX, OP_RX_TRACK, OP_SCOPE, OP_SCOPE_CAP, OP_SCOPE_PLAIN, OP_SEQ,
-  OP_SEQV, OP_SEQX, OP_SOURCE_LEAF, OP_TOKEN, OP_WITHCTX, OP_XFORM,
+  OP_SEQV, OP_SEQX, OP_SOURCE_LEAF, OP_TOKEN, OP_WITHCTX, OP_XFORM, OP_GAPGATE,
 } from './ops.ts'
 
 export type FailClean = (ip: number) => boolean
@@ -35,7 +35,7 @@ export type FailClean = (ip: number) => boolean
  */
 function passChild(code: ArrayLike<number>, ip: number): number {
   switch (code[ip]) {
-    case OP_RULE: case OP_LABEL: case OP_COV: case OP_TOKEN:
+    case OP_RULE: case OP_LABEL: case OP_COV: case OP_TOKEN: case OP_GAPGATE:
       return code[ip + 1]!
     case OP_GATE: case OP_NODE: case OP_NODE_TRACK: case OP_XFORM: case OP_FIELD: case OP_SCOPE:
     case OP_SCOPE_CAP: case OP_SCOPE_PLAIN: case OP_WITHCTX: case OP_LEAF: case OP_SOURCE_LEAF:

package/src/table/gap-gate.ts

@@ -0,0 +1,55 @@
+/**
+ * AN OPTIONAL REPEAT ITEM THAT OPENS WITH A GAP (`OP_GAPGATE`).
+ *
+ * SCSS writes its operators as `sequence(pad, '*', pad)` with `pad` a regex that
+ * may match empty, and a value chain has a repeat of them at every level. After
+ * a value followed by a space, each level entered its operator, scanned the
+ * space and failed on the character after it: the same gap read once per level,
+ * and an operator node's frame opened each time.
+ *
+ * The repeat now reads the gap itself, ONCE, with the item's own regex, and
+ * looks at the character after it: an item that opens with the gap must
+ * continue with a character of `cls` there (and an arm without the gap must
+ * start with one of `other` where the gap starts). Where neither holds the item
+ * cannot match, so the repeat ends, exactly as it ends on a strict item whose
+ * class excludes the lead. Otherwise it enters the item and leaves the gap's end
+ * in the GAP SLOT, from which the regex row takes it instead of scanning again.
+ *
+ * The slot is a memo of one pure function: the end of regex row `row` matched
+ * at `at` in `input`. A row reads it only when all three are its own, so a
+ * different row, position or input cannot take it, and a slot left set can
+ * only ever give a row the end it would have computed. Only rows the table
+ * names as a gap (`gapRows`) test it at all. It is set for the item and cleared
+ * when the item returns.
+ *
+ * Not under tolerant recovery or a completions probe, which observe an item's
+ * failure: there the repeat enters the item as written. The reference driver
+ * (`exec.ts`) always does, which is what makes it the ungated leg.
+ */
+import { childSlots } from './child-slots.ts'
+import { OP_GAPGATE } from './ops.ts'
+
+/** The gap slot one assembly shares between its closure and emitted pieces. */
+export type GapSlot = { row: number; at: number; end: number; input: string | undefined }
+
+export function newGapSlot(): GapSlot {
+  return { row: -1, at: -1, end: -1, input: undefined }
+}
+
+/** The regex rows some reachable `OP_GAPGATE` names as its gap. */
+export function gapRows(code: ArrayLike<number>, roots: Iterable<number>): Set<number> {
+  const rows = new Set<number>()
+  const seen = new Set<number>()
+  const stack = [...roots]
+  const kids: number[] = []
+  while (stack.length > 0) {
+    const ip = stack.pop()!
+    if (seen.has(ip)) continue
+    seen.add(ip)
+    if (code[ip] === OP_GAPGATE) rows.add(code[ip + 2]!)
+    kids.length = 0
+    childSlots(code, ip, kids)
+    for (const c of kids) stack.push(c)
+  }
+  return rows
+}

package/src/table/hold-gap.ts

@@ -17,14 +17,14 @@
 import {
   OP_ADJ, OP_ATTEMPT, OP_CHOICE, OP_COV, OP_EXPECT, OP_FIELD, OP_GATE, OP_LABEL, OP_LEAF, OP_LIT, OP_LIT_CI,
   OP_NODE, OP_NOT, OP_PEEK, OP_REP, OP_REPV, OP_RULE, OP_RX, OP_SCOPE, OP_SCOPE_CAP, OP_SCOPE_PLAIN, OP_SEQ,
-  OP_SEQV, OP_SEQX, OP_SOURCE_LEAF, OP_TOKEN, OP_WITHCTX, OP_XFORM,
+  OP_SEQV, OP_SEQX, OP_SOURCE_LEAF, OP_TOKEN, OP_WITHCTX, OP_XFORM, OP_GAPGATE,
 } from './ops.ts'
 import { childSlots } from './child-slots.ts'
 import { leadingTriviaFieldAt } from './field-spec.ts'
 import { regexCanMatchEmpty } from '../regex/first-set.ts'
 
 const WRAPPERS = new Set([OP_NODE, OP_XFORM, OP_LEAF, OP_SOURCE_LEAF, OP_TOKEN, OP_SCOPE, OP_SCOPE_CAP,
-  OP_SCOPE_PLAIN, OP_FIELD, OP_LABEL, OP_EXPECT, OP_WITHCTX, OP_RULE, OP_ATTEMPT, OP_COV])
+  OP_SCOPE_PLAIN, OP_FIELD, OP_LABEL, OP_EXPECT, OP_WITHCTX, OP_RULE, OP_ATTEMPT, OP_COV, OP_GAPGATE])
 
 /**
  * Can the row at `ip` match empty? `true` wherever it cannot tell. A first-set

package/src/table/lex-program.ts

@@ -187,7 +187,7 @@ function sequence5(
 
 type ChoiceTail = (
   input: string, pos: number, ctx: ParseContext, lead: number,
-  acc: string[] | undefined, best: number,
+  acc: string[] | undefined, best: number, entered: boolean,
 ) => number
 
 function appendExpected(acc: string[] | undefined, values: readonly string[] | undefined): string[] | undefined {
@@ -211,19 +211,22 @@ function choiceNode(
       return end < 0 ? next(input, pos) : end
     }
   }
-  let tail: ChoiceTail = (_input, pos, ctx, _lead, acc) => {
+  let tail: ChoiceTail = (input, pos, ctx, _lead, acc, _best, entered) => {
+    // No arm was entered. A completion probe still sees each one fail on its
+    // opener, as the interpreter's entered arms leave it.
+    if (!entered && ctx._probe !== undefined) for (const arm of arms) arm.run(input, pos, ctx)
     ctx._fe = pos
     ctx._fx = acc ?? choiceExpected as string[]
     return -1
   }
   for (let i = arms.length - 1; i >= 0; i--) {
     const armRun = arms[i]!.run, cls = classes[i], staticExpected = armExpected[i]!, next = tail
-    tail = (input, pos, ctx, lead, acc, best) => {
+    tail = (input, pos, ctx, lead, acc, best, entered) => {
       if (cls !== undefined && !classHas(cls, lead)) {
         return next(
           input, pos, ctx, lead,
           best === pos ? appendExpected(acc, staticExpected) : acc,
-          best,
+          best, entered,
         )
       }
       ctx._fc = false
@@ -236,13 +239,13 @@ function choiceNode(
         if (nextAcc !== undefined) ctx._fx = nextAcc
         return -1
       }
-      return next(input, pos, ctx, lead, nextAcc, best)
+      return next(input, pos, ctx, lead, nextAcc, best, true)
     }
   }
   return {
     match: matchTail,
     run: (input, pos, ctx) => tail(
-      input, pos, ctx, pos < input.length ? input.codePointAt(pos)! : -1, undefined, pos,
+      input, pos, ctx, pos < input.length ? input.codePointAt(pos)! : -1, undefined, pos, false,
     ),
   }
 }
@@ -290,6 +293,9 @@ function exclusiveChoiceNode(
       const failed = ctx._fx
       if (failed !== undefined && failed.length > 0) { ctx._fe = pos; return -1 }
     }
+    // A miss under a completions probe: each arm fails on its opener, as the
+    // interpreter's arms do when it runs them all (`choice.ts`).
+    if (arm < 0 && ctx._probe !== undefined) for (const a of arms) a.run(input, pos, ctx)
     ctx._fe = pos
     ctx._fx = choiceFx as string[]
     return -1
@@ -547,6 +553,9 @@ export function buildLexProgramRunner(
     if (!classHas(cls, lead)) {
       ctx._fe = pos
       ctx._fx = choiceFx.slice()
+      // Each arm fails here; under a probe their openers are recorded as the
+      // interpreter's entered arms record them.
+      if (ctx._probe !== undefined) failAt(ctx, [...x0, ...x1, ...x2, ...x3], pos)
       return -1
     }
     let acc: string[] | undefined

package/src/table/ops.ts

@@ -38,11 +38,15 @@ export const OP_SEQV = 4
  */
 export const OP_CHOICE = 5
 /**
- * `REP c min max sep flags fx sepClass` — `sep` is a child offset or −1; `max`
- * −1 = ∞. `fx` is recovery metadata. The last class is the separator sentinel
- * for a separated list, otherwise the finite/non-nullable optional-item class
- * (or −1). `flags` bit 0 = trailing separator allowed, bit 1 =
- * `keepSeparators`, bit 2 = item expected reporting.
+ * `REP c min max sep flags fx sepClass [sepGate]` — EIGHT words, or NINE with
+ * flag bit 3. `sep` is a child offset or −1; `max` −1 = ∞. `fx` is the item's
+ * expected set. `sepClass` is the separator sentinel for a separated list,
+ * otherwise the finite/non-nullable optional-item class (or −1). `flags`: bit 0
+ * (`&1`) trailing separator allowed; bit 1 (`&2`) `keepSeparators`; bit 2 (`&4`)
+ * item expected reporting; bit 3 (`&8`) a ninth word at `ip + 8`, the class of
+ * characters the separator can start, so a strict list past its minimum ends
+ * where it cannot (`sepGate`); bit 4 (`&16`) the item's derived set is empty, so
+ * `fx` holds the item's tag for the minimum check and a resync error carries [].
  */
 export const OP_REP = 6
 /** `REPV …` — `REP` with `valueUnused`. */
@@ -70,13 +74,7 @@ export const OP_XFORM = 9
  * from the reducer's declared arity by the same analysis codegen runs, and
  * forced on under `hostMode: 'cst'`. The driver reads the bits and re-derives
  * nothing.
- *
- * TWO ENCODERS SHARE THIS OPCODE and they do NOT agree on its length. The layout
- * above is `encode.ts` / `exec.ts`. `encode-baseline.ts` / `exec-baseline.ts` emit
- * the FOUR-word form `NODE b c flags` with bits 2 and 3 only, and refuse
- * `collapse` / `unwrap` / `project` outright. Neither driver walks the stream by
- * instruction length, so the divergence is contained — but read the pair you are
- * in, not this comment alone. */
+ */
 export const OP_NODE = 10
 /** `RULE r` — `r` indexes `prog.rules`. */
 export const OP_RULE = 11
@@ -89,9 +87,6 @@ export const OP_NOT = 13
  * `['peek(<child tag>)']`, because `peek.ts:60` DISCARDS the body's expectation:
  * a lookahead's failure is "the guard did not hold", not a request for whatever
  * token the body happened to stop on.
- *
- * `encode-baseline.ts` / `exec-baseline.ts` emit and read the two-word form and
- * are internally consistent; read the pair you are in.
  */
 export const OP_PEEK = 14
 /** `LEAF f c` — `leaf()`: reduce, then record as a spanned leaf. */
@@ -486,11 +481,35 @@ export const OP_SOURCE_LEAF = 44
  * at this position, a completions probe, or tolerant recovery enters the arm.
  */
 export const OP_RGATE = 45
+/**
+ * `GAPGATE c p cls other` — an optional repeat ITEM that opens with a gap: the
+ * pure regex row `p`, which may match empty, and then a term whose first set is
+ * the class `cls` (`other`: the class of any choice arm that does not open with
+ * `p`, or −1). `c` is the item. The row is TRANSPARENT wherever it runs: only
+ * the repeat that holds it reads its operands (`gap-gate.ts`).
+ */
+export const OP_GAPGATE = 46
+/**
+ * `RGATE2 cls child fx first second` — `OP_RGATE` with a SECOND character: where
+ * the character after the routed token is in `first`, the one after it must be
+ * in `second` too (the next term reads both, `regexFirstTwo`), or the arm fails
+ * there with `fx`, as entering it would.
+ */
+export const OP_RGATE2 = 47
+/**
+ * `STRICT strict tolerant` — one construct encoded two ways: `strict` for a
+ * strict parse, `tolerant` for tolerant recovery and a completions probe, which
+ * must see the construct as written (`factor-choice.ts` `factorPrefixArms`). An
+ * assembly resolves it to one child; it has no body of its own.
+ */
+export const OP_STRICT = 48
 /**
  * `DISPATCH sel d other otherRouted n a1 … an` — `dispatch()`.
  *
  * `sel` is the selector's offset, `d` indexes a dispatch table in `prog.dsp`,
- * `other` is the `otherwise()` offset (or −1), `otherRouted` is 1 when the
+ * `other` is the `otherwise()` offset (or −1), `otherRouted` bit 1 (`&2`) is set
+ * when the fallback's failure is NOT committed (`otherwise(p, { commit: false })`)
+ * and bit 0 when the
  * fallback consumes the routed token, and `a1…an` are the arm offsets. Arm
  * `usesRouted` bits live in the `dsp` entry beside the key maps.
  *
@@ -520,4 +539,7 @@ export const OP_NAMES: Record<number, string> = {
   [OP_SCOPE_PLAIN]: 'SCOPE_PLAIN',
   [OP_LEX_BODY]: 'LEX_BODY',
   [OP_LEX_PROGRAM]: 'LEX_PROGRAM',
+  [OP_GAPGATE]: 'GAPGATE',
+  [OP_RGATE2]: 'RGATE2',
+  [OP_STRICT]: 'STRICT',
 }

package/src/table/program.ts

@@ -17,10 +17,15 @@ import { buildLexProgramRunner, type LexProgramRunner } from './lex-program.ts'
  * `EMITTED_PARAMS` in order. `plan` rebuilds the three data pools the factory
  * expects (see `PoolPlan`), and `reached` is the emitter's site set, which
  * `Assembly.reached` publishes and `test/unit/table-assemble.test.ts`
- * asserts on.
+ * asserts on. `arity` is the number of helpers the factory was printed against
+ * (`EMITTED_PARAMS.length` of the printing parseman); the runtime compares it with
+ * its own list before calling the factory. It is printed data, not
+ * `factory.length`, which a minifier that drops unused trailing parameters lowers.
+ * An assembly printed before 0.60.0 has none.
  */
 export type PrecompiledAssembly = {
   readonly key: number
+  readonly arity?: number
   readonly factory: EmittedFactory
   readonly plan: PoolPlan
   readonly reached: readonly number[]
@@ -433,7 +438,7 @@ export function validateDispatchSpec(
     return false
   }
   if (spec === undefined || !Number.isInteger(arity) || arity < 0
-    || (fallbackRouted !== 0 && fallbackRouted !== 1)
+    || !Number.isInteger(fallbackRouted) || fallbackRouted < 0 || fallbackRouted > 3
     || spec.routed.length !== arity
     || spec.routed.some(value => value !== 0 && value !== 1)
     || invalid(spec.byKey.values()) || invalid(spec.byFold.values())

package/src/table/select-hot.ts

@@ -15,7 +15,7 @@
  * A build given a corpus (`assemblyProfile`) ranks by measured calls instead.
  */
 import { childSlots } from './child-slots.ts'
-import { OP_CHOICE, OP_DISPATCH, OP_REP, OP_REPV } from './ops.ts'
+import { OP_CHOICE, OP_DISPATCH, OP_GAPGATE, OP_REP, OP_REPV } from './ops.ts'
 
 const LOOP = 8
 const ROUNDS = 60
@@ -62,7 +62,12 @@ function staticWeights(code: readonly number[], entries: readonly number[]): Map
       const scale = op === OP_REP || op === OP_REPV ? LOOP
         : op === OP_CHOICE || op === OP_DISPATCH ? 1 / Math.max(1, kids.length)
         : 1
-      for (const kid of kids) next.set(kid, Math.min(CAP, (next.get(kid) ?? 0) + w * scale))
+      for (const k of kids) {
+        // A gap gate has no body: weigh its item as the repeat's own child, so
+        // adding the row moves no other site's estimate (`gap-gate.ts`).
+        const kid = code[k] === OP_GAPGATE ? code[k + 1]! : k
+        next.set(kid, Math.min(CAP, (next.get(kid) ?? 0) + w * scale))
+      }
     }
     weight = next
   }