parseman 0.59.3 → 0.60.0

package/src/table/encode.ts

@@ -1,7 +1,8 @@
 import { scanRefs } from './program.ts'
-import { factorSharedEnds } from './factor-choice.ts'
+import { regexFirstTwo } from '../regex/first-set.ts'
+import { factorPrefixArms, factorSharedEnds, factoredChoices } from './factor-choice.ts'
 import type { AutoNotCheck, Combinator, FirstSet, ParserDef } from '../types.ts'
-import { classifyFinalChoice, firstSetOf, intersects, matchesEmpty, union, type RefResolver } from '../combinators/first-set.ts'
+import { classifyFinalChoice, empty, firstSetOf, intersects, matchesEmpty, union, winnerForRef, type RefResolver } from '../combinators/first-set.ts'
 import { childrenOf } from '../analysis/gating.ts'
 import { capturesLeaf, mayCommitFailure, mayLeavePartialCapture, reachesMarked } from '../analysis/commitment.ts'
 import { getCoreLiteralValue } from '../combinators/choice.ts'
@@ -17,13 +18,13 @@ import {
   OP_LIT_TRACK, OP_RX_TRACK, OP_NODE_TRACK, OP_SCOPE, OP_SCOPE_CAP, OP_SCOPE_PLAIN, OP_EXPECT, OP_SEQX, OP_SCAN,
   OP_LIVE, OP_ATTEMPT, OP_LABEL,
   OP_FIELD, OP_DISPATCH, OP_ROUTED, OP_LIT_CI, OP_LIT_CI_TRACK, OP_TOKEN, OP_WITHCTX, OP_GUARD,
-  OP_ADJ, OP_GREEDY, OP_REJECT, OP_ARMGATE, OP_RGATE, OP_COV,
+  OP_ADJ, OP_GREEDY, OP_REJECT, OP_ARMGATE, OP_RGATE, OP_COV, OP_GAPGATE, OP_RGATE2, OP_STRICT,
   OP_LEX_BODY, OP_LEX_PROGRAM,
 } from './ops.ts'
 import { adjacencyExpected } from '../combinators/adjacency.ts'
 import { resolveAdjacencyKindMask } from '../cst/trivia-kinds.ts'
 import { missingInferredType } from '../combinators/node.ts'
-import { hasOwnTriviaBoundary } from '../combinators/trivia-boundary.ts'
+import { triviaThroughRef } from '../combinators/trivia-boundary.ts'
 import type { BalancedSpec } from '../combinators/scanTo.ts'
 import type { DispatchSpec, LexBodySpec, LexProgramSpec, ScanSpec, SubtreeRef, TableProgram, TriviaSpec } from './program.ts'
 import {
@@ -116,6 +117,48 @@ export type TableSettings = {
 type Emitted = { readonly ip: number }
 
 
+/**
+ * The first two characters a term reads, where it says them outright: a
+ * literal of two or more characters, or a case-sensitive regex
+ * (`regexFirstTwo`), through a label or a trivia scope. The first must be BMP
+ * and not a surrogate, so the second starts at the next code unit. The gate
+ * reads that second character as a code point (`lead`); a class holding a high
+ * surrogate admits the astral points it begins (`leadRanges`), so a code-unit
+ * second set reads them too.
+ */
+function firstTwo(p: Combinator<unknown>): { first: FirstSet; second: FirstSet } | undefined {
+  const d = p._def as ParserDef
+  if (d.tag === 'label' || d.tag === 'grammar') return firstTwo(d.parser)
+  let two: { first: FirstSet; second: FirstSet } | undefined
+  if (d.tag === 'literal' && d.value.length >= 2 && d.caseInsensitive !== true) {
+    const second = d.value.codePointAt(1)!
+    two = { first: { kind: 'ranges', ranges: [{ lo: d.value.charCodeAt(0), hi: d.value.charCodeAt(0) }] }, second: { kind: 'ranges', ranges: [{ lo: second, hi: second }] } }
+  } else if (d.tag === 'regex' && !d.flags.includes('i')) {
+    const units = regexFirstTwo(d.source, d.flags)
+    if (units !== undefined) two = units
+  }
+  if (two === undefined || two.first.kind !== 'ranges') return undefined
+  return two.first.ranges.every(r => r.hi < 0xd800 || (r.lo > 0xdfff && r.hi <= 0xffff)) ? two : undefined
+}
+
+/** ASCII whitespace and the `/` a comment opens with: what a gap starts with. */
+const GAP_CHARS = new Set([9, 10, 12, 13, 32, 47])
+
+/**
+ * A regex that reads a GAP: it may match empty, or every non-empty match starts
+ * with whitespace or `/`. Only such a regex leads an item the repeat gap-gates;
+ * a word or a number before the deciding character is the item's own class
+ * test's business, not a gap's.
+ */
+function isGap(p: Combinator<unknown>): boolean {
+  if (matchesEmpty(p)) return true
+  const fs = p._meta.firstSet
+  return fs.kind === 'ranges' && fs.ranges.every(r => {
+    for (let c = r.lo; c <= r.hi; c++) if (!GAP_CHARS.has(c)) return false
+    return true
+  })
+}
+
 class Encoder {
   code: number[] = []
   k: unknown[] = []
@@ -163,8 +206,33 @@ class Encoder {
    */
   private commitFailureMemo = new Map<Combinator<unknown>, boolean>()
   labels: readonly string[] | undefined = undefined
+  /** The choice being encoded as written, for `OP_STRICT`'s tolerant child. */
+  private unfactoredChoice: Combinator<unknown> | undefined = undefined
   /** Trivia table in scope at the row being encoded — codegen's `ctx.activeTrivia`. */
   activeTrivia: Combinator<unknown> | undefined = undefined
+  /**
+   * Is the trivia live at this row known? It is not in the body of an entry
+   * that installs none of its own: that body runs under whatever trivia its
+   * caller has (a compiled rule inside another grammar, `run({ trivia })`).
+   * Every scope (`parser({ trivia })`, `noTrivia`, `token()`, a rule's install)
+   * makes it known again.
+   */
+  triviaKnown = true
+  /**
+   * The trivia a gate's first set is derived under: the live table, `null` for a
+   * scope known to have none, or `undefined` where it is not known.
+   */
+  private gateTrivia(): Combinator<unknown> | null | undefined {
+    return this.activeTrivia ?? (this.triviaKnown ? null : undefined)
+  }
+  /**
+   * Whether a gate may read a nullable pad through its non-empty matches
+   * (`firstSetOf`'s `consuming`): only where the trivia that may follow the pad
+   * is known. Elsewhere a gate reads first sets as 0.58.0 did.
+   */
+  private gateConsuming(): boolean {
+    return this.activeTrivia !== undefined || this.triviaKnown
+  }
   classified = false
   scans: ScanSpec[] = []
   /** Ambient `scanSkip` sets, pooled by the ARRAY's identity (stable per grammar). */
@@ -303,10 +371,11 @@ class Encoder {
             // encoder would hold at that row, so both lowerings rank the same
             // failures.
             const final = classifyFinalChoice(
-              node.parsers, this.refResolver(), node.cleared ? undefined : this.activeTrivia,
+              node.parsers, this.refResolver(), node.cleared ? null : this.gateTrivia(), false, node.disjoint,
             )
             const classes = final.firstSets.map((first, i) => final.nullable[i] ? -1 : this.charClass(first))
-            const exclusive = final.exclusive && classes.every(cls => cls >= 0)
+            const exclusive = final.exclusive && classes.every(cls => cls >= 0) &&
+              this.wideArms(node.parsers, final.firstSets, node.disjoint, node.cleared ? null : this.gateTrivia(), node.cleared ? undefined : this.activeTrivia).length === 0
             words.push(LEX_NODE_CHOICE, node.arms.length, exclusive ? 1 : 0, this.expected(node.expected))
             for (let i = 0; i < node.arms.length; i++) {
               words.push(classes[i]!, this.expected(node.armExpected[i]!))
@@ -499,16 +568,104 @@ class Encoder {
   private kIndex = new Map<unknown, number>()
   private ccIndex = new Map<string, number>()
   private fxIndex = new Map<string, number>()
-  /** Memoized by combinator identity: a shared sub-combinator is ONE row. */
-  memo = new Map<Combinator<unknown>, number>()
   /**
-   * Memo for `lazy` REFERENCES, keyed by the trivia scope the reference is made
-   * from. A cross-rule reference's row depends on that scope (`scopedRef`), so the
-   * single `memo` slot every other construct uses would freeze the FIRST
-   * reference's scope onto every later one and undo the fix.
+   * Memoized by combinator identity AND the trivia scope it is encoded under
+   * (`scopeKey`): a shared sub-combinator is ONE row per scope it runs in.
+   *
+   * A construct's rows depend on that scope. A first set derived through a
+   * nullable term takes in the trivia that may follow it, a gap gate holds only
+   * where no trivia runs, a routed gate reads what may follow its token, and a
+   * cross-rule reference installs its rule's trivia only from a cleared region
+   * (`scopedRef`). One memo slot per combinator froze the FIRST scope it was
+   * reached under onto every later one: a rule body encoded as an entry with no
+   * trivia, then run under a caller's comment trivia, gated its repeats as if no
+   * comment could come between their terms.
+   */
+  memo = new Map<Combinator<unknown>, Map<string, number>>()
+  pending = new Map<Combinator<unknown>, Map<string, number[]>>()
+  private triviaSignatures = new Map<Combinator<unknown>, string>()
+  /**
+   * What of the live trivia an encoding can depend on: none, or its first set.
+   * Every derivation reads trivia only through `firstSetOf(trivia)` or whether it
+   * is live, so two scopes whose trivia opens on the same characters encode the
+   * same rows and share them. A construct that reads no scope (`readsScope`)
+   * has one encoding.
+   */
+  private scopeKey(p: Combinator<unknown>): string {
+    const t = this.activeTrivia
+    if (!this.readsScope(p)) return ''
+    if (t === undefined) return this.triviaKnown ? '' : '?'
+    let sig = this.triviaSignatures.get(t)
+    if (sig === undefined) {
+      sig = `t${JSON.stringify(firstSetOf(t, new Set(), this.refResolver()))}`
+      this.triviaSignatures.set(t, sig)
+    }
+    return sig
+  }
+
+  private readsScopeMemo = new Map<Combinator<unknown>, boolean>()
+  /**
+   * Can `p`'s rows depend on the trivia scope it is encoded under? No for a
+   * terminal, a scope of its own (`parser({ trivia })`, `noTrivia`, `token()`),
+   * and a wrapper over a construct that reads none; yes wherever a first set, a gate or
+   * a reference's install is derived with the scope: a sequence with a nullable
+   * term before another (the trivia between them), a routed arm (the trivia after
+   * its token), a reference that installs its rule's trivia from a cleared
+   * region, and anything not modelled here — a cycle included.
    */
-  refMemo = new Map<Combinator<unknown>, Map<Combinator<unknown> | undefined, number>>()
-  pending = new Map<Combinator<unknown>, number[]>()
+  private readsScope(p: Combinator<unknown>): boolean {
+    const hit = this.readsScopeMemo.get(p)
+    if (hit !== undefined) return hit
+    this.readsScopeMemo.set(p, true)
+    const d = p._def as ParserDef
+    const rr = this.refResolver()
+    let out: boolean
+    switch (d.tag) {
+      case 'literal': case 'regex': case 'keywords': case 'token':
+        out = false
+        break
+      // `routed()` encodes its fallback under the scope that reaches it.
+      case 'routed':
+        out = d.fallback !== undefined && this.readsScope(d.fallback)
+        break
+      case 'grammar':
+        out = d.clearTrivia !== true && d.triviaParser === undefined && this.readsScope(d.parser)
+        break
+      case 'node': case 'label': case 'leaf': case 'transform': case 'attempt': case 'expect':
+      case 'not': case 'peek': case 'optional': case 'many': case 'oneOrMore': case 'field':
+        out = this.readsScope(d.parser)
+        break
+      case 'sepBy':
+        out = this.readsScope(d.parser) || this.readsScope(d.separator)
+        break
+      case 'choice':
+        out = d.parsers.some(a => this.readsScope(a))
+        break
+      case 'dispatch':
+        out = this.readsScope(d.selector) || d.cases.some(c => this.readsScope(c.parser))
+          || (d.matchers?.some(m => this.readsScope(m.parser)) ?? false)
+          || (d.otherwise !== undefined && this.readsScope(d.otherwise))
+        break
+      case 'sequence': {
+        const ps = d.parsers
+        // A gap-led sequence: the gap walk holds only where no trivia runs.
+        out = ps.some(t => this.readsScope(t))
+          || (ps[0] !== undefined && ((ps[0]._def as ParserDef).tag === 'routed' || ((ps[0]._def as ParserDef).tag === 'regex' && isGap(ps[0]))))
+          || ps.some((t, i) => i + 1 < ps.length && matchesEmpty(t, new Set(), rr))
+        break
+      }
+      case 'lazy': {
+        let target: Combinator<unknown> | undefined = winnerForRef(p, rr)
+        if (target === undefined) { try { target = d.thunk() } catch { target = undefined } }
+        out = target === undefined || triviaThroughRef(p, target, null) !== null || this.readsScope(target)
+        break
+      }
+      default:
+        out = true
+    }
+    this.readsScopeMemo.set(p, out)
+    return out
+  }
 
   readonly settings: TableSettings
   /** Resolved once, HERE, at table-build time — never consulted at run time. */
@@ -540,7 +697,9 @@ class Encoder {
     let fs: FirstSet = { kind: 'empty' }
     const rr = this.refResolver()
     for (let j = i + 1; j < parsers.length; j++) {
-      fs = union(fs, firstSetOf(parsers[j]!, new Set(), rr, this.activeTrivia))
+      // As the interpreter's `followSentinels` derive them (`sequence.ts`): with no
+      // trivia, whatever scope the sequence runs in.
+      fs = union(fs, firstSetOf(parsers[j]!, new Set(), rr, undefined, false))
     }
     return this.charClass(fs)
   }
@@ -724,7 +883,7 @@ class Encoder {
       || op === OP_RX || op === OP_RX_TRACK) return child
     const rr = this.refResolver()
     if (matchesEmpty(p, new Set(), rr)) return child
-    const cls = this.charClass(firstSetOf(p, new Set(), rr, this.activeTrivia))
+    const cls = this.charClass(firstSetOf(p, new Set(), rr, this.activeTrivia, false))
     if (cls < 0) return child
     const e = deriveExpected(p)
     return this.emit(OP_GATE, cls, child, this.expected(e.length > 0 ? e : [p._tag]))
@@ -756,11 +915,39 @@ class Encoder {
    * `{`, `Call` at `}`). The resolved set is the one a choice arm's gate uses,
    * over the final merged map.
    */
-  private gateFirstSet(p: Combinator<unknown>): FirstSet {
+  /**
+   * The arms whose first set under `gateTrivia()` is wider than under the
+   * active trivia alone — a reference's installed trivia read through from a
+   * region with none. Empty for construction-set (disjoint) choices, whose
+   * sets do not depend on trivia.
+   */
+  private wideArms(
+    arms: readonly Combinator<unknown>[], wide: readonly FirstSet[], constructionSets: boolean,
+    wideTrivia: Combinator<unknown> | null | undefined = this.gateTrivia(),
+    base: Combinator<unknown> | undefined = this.activeTrivia,
+  ): number[] {
+    if (constructionSets || wideTrivia === base) return []
+    const narrow = classifyFinalChoice(arms, this.refResolver(), base, false, false).firstSets
+    const out: number[] = []
+    for (let i = 0; i < arms.length; i++) {
+      if (this.charClass(wide[i]!) !== this.charClass(narrow[i]!)) out.push(i)
+    }
+    return out
+  }
+
+  private gateFirstSet(p: Combinator<unknown>, guarded: boolean, narrowed: boolean): FirstSet {
     const cached = p._meta.firstSet
-    return p._def.tag === 'lazy' || cached.kind === 'any'
-      ? firstSetOf(p, new Set(), this.refResolver(), this.activeTrivia)
-      : cached
+    if (p._def.tag !== 'lazy' && cached.kind !== 'any') return cached
+    const rr = this.refResolver()
+    // GUARDED: the interpreter tests this child's set before entering it (its set
+    // was finite when the wrapper was built), so the table tests the same set: a
+    // reference's resolved target, as its construction-time set reads it, without
+    // the trivia it installs. Otherwise the interpreter enters the child, and the
+    // gate is the set the child can start with where it runs. Only a node's and a
+    // repeat item's gate read a nullable pad through its non-empty matches (the
+    // operator of `A (pad op pad A)*`); an attempt's reads first sets as 0.58.0 did.
+    if (guarded) return firstSetOf(p, new Set(), rr, undefined, false)
+    return firstSetOf(p, new Set(), rr, this.gateTrivia(), narrowed && this.gateConsuming())
   }
 
   /**
@@ -792,6 +979,133 @@ class Encoder {
     return ip
   }
 
+  /**
+   * An optional repeat item tests the item's class before it runs it (`OP_REP`
+   * ip+7), wherever the item's own first-set gate would also run. So a sequence
+   * item whose first term is a gate on that same class gets its own copy of the
+   * sequence row with the gate's child as that term: the test runs once. The
+   * shared row is left as it is for its other references.
+   */
+  private withoutLeadGate(seq: number, cls: number): number {
+    const op = this.code[seq]
+    if (op !== OP_SEQ && op !== OP_SEQV) return seq
+    const first = this.code[seq + 2]!
+    if (this.code[first] !== OP_GATE || this.code[first + 1] !== cls) return seq
+    const n = this.code[seq + 1]!
+    const width = 1 + 2 * n
+    const head = this.emitHead(op, width)
+    for (let i = 1; i <= width; i++) this.code[head + i] = this.code[seq + i]!
+    this.code[head + 2] = this.code[first + 2]!
+    return head
+  }
+
+  /**
+   * An optional repeat item that opens with a GAP — a regex that may match
+   * empty, or one that starts only with whitespace or a comment's `/` — and
+   * then a term with a finite first set gets an `OP_GAPGATE` row
+   * (`gap-gate.ts`): the repeat reads the gap once, ends where the character
+   * after it opens neither that term nor an arm without the gap, and hands the
+   * gap's end to the regex row. Anything the walk cannot prove keeps the item as
+   * written.
+   */
+  private gapGated(item: Combinator<unknown>, ip: number): number {
+    // The gate holds only where no trivia runs between the gap and its term, so
+    // not where the live trivia is unknown.
+    if (this.activeTrivia === undefined && !this.triviaKnown) return ip
+    const lead = this.gapLead(item, this.activeTrivia, 0)
+    if (lead === undefined) return ip
+    const cls = this.charClass(lead.follow)
+    if (cls < 0) return ip
+    const other = lead.other.kind === 'empty' ? -1 : this.charClass(lead.other)
+    if (lead.other.kind !== 'empty' && other < 0) return ip
+    const p = this.node(lead.p).ip
+    if (this.code[p] !== OP_RX && this.code[p] !== OP_RX_TRACK) return ip
+    return this.emit(OP_GAPGATE, ip, p, cls, other)
+  }
+
+  /**
+   * The gap an item opens with: the regex `p`, the first set `follow` of what
+   * must come straight after it, and the first set `other` of choice arms that
+   * do not open with `p`. Only through wrappers that consume nothing before
+   * their body, a sequence's first term, and a choice's arms; the gap must sit
+   * in a sequence with no trivia between its terms.
+   */
+  private gapLead(
+    c: Combinator<unknown>, trivia: Combinator<unknown> | null | undefined, depth: number,
+  ): { p: Combinator<unknown>; follow: FirstSet; other: FirstSet } | undefined {
+    if (depth > 24) return undefined
+    const rr = this.refResolver()
+    const d = c._def as ParserDef
+    switch (d.tag) {
+      case 'lazy': {
+        let target: Combinator<unknown> | undefined = winnerForRef(c, rr)
+        if (target === undefined) { try { target = d.thunk() } catch { return undefined } }
+        return this.gapLead(target, triviaThroughRef(c, target, trivia), depth + 1)
+      }
+      case 'node': case 'leaf': case 'label': case 'attempt':
+        return this.gapLead(d.parser, trivia, depth + 1)
+      case 'field':
+        return d.leadingTrivia === true ? undefined : this.gapLead(d.parser, trivia, depth + 1)
+      case 'grammar':
+        return this.gapLead(d.parser, d.clearTrivia === true ? null : (d.triviaParser ?? trivia), depth + 1)
+      case 'sequence': {
+        const [first, ...rest] = d.parsers
+        if (first === undefined) return undefined
+        if ((first._def as ParserDef).tag !== 'regex' || !isGap(first)) {
+          return this.gapLead(first, trivia, depth + 1)
+        }
+        // No trivia between the gap and the term after it (`undefined`: none).
+        if ((trivia !== undefined && trivia !== null) || rest.length === 0) return undefined
+        let follow: FirstSet = empty()
+        let closed = false
+        for (const t of rest) {
+          const fs = firstSetOf(t, new Set(), rr, null)
+          if (fs.kind === 'any') return undefined
+          follow = union(follow, fs)
+          if (!matchesEmpty(t, new Set(), rr)) { closed = true; break }
+        }
+        return closed ? { p: first, follow, other: empty() } : undefined
+      }
+      case 'choice': {
+        let p: Combinator<unknown> | undefined
+        let follow: FirstSet = empty()
+        let other: FirstSet = empty()
+        for (const arm of d.parsers) {
+          const lead = this.gapLead(arm, trivia, depth + 1)
+          if (lead !== undefined && (p === undefined || lead.p === p)) {
+            p = lead.p
+            follow = union(follow, lead.follow)
+            other = union(other, lead.other)
+            continue
+          }
+          if (matchesEmpty(arm, new Set(), rr)) return undefined
+          // Tested where the gate runs, which is only where no trivia is live.
+          const fs = firstSetOf(arm, new Set(), rr, trivia ?? null)
+          if (fs.kind === 'any') return undefined
+          other = union(other, fs)
+        }
+        return p === undefined ? undefined : { p, follow, other }
+      }
+      default:
+        return undefined
+    }
+  }
+
+  /**
+   * The class a strict list may end on — a character its separator `p` (row
+   * `ip`) cannot start — without entering the separator, or −1.
+   *
+   * The test runs before every separator, to save the one that fails at the
+   * list's end, so it is laid down only where entering the separator costs more
+   * than the test: not on a terminal row, which is itself one character test.
+   */
+  private sepGate(p: Combinator<unknown>, ip: number): number {
+    const op = this.code[ip]
+    if (op === OP_LIT || op === OP_LIT_CI || op === OP_RX || op === OP_LIT_TRACK || op === OP_RX_TRACK || op === OP_LIT_CI_TRACK) return -1
+    if (matchesEmpty(p, new Set(), this.refResolver())) return -1
+    return this.charClass(firstSetOf(p, new Set(), this.refResolver(), this.gateTrivia(), this.gateConsuming()))
+  }
+
   /** Reserve `n` operand slots to patch once children are laid out. */
   private emitHead(op: number, n: number): number {
     const ip = this.code.length
@@ -845,8 +1159,10 @@ class Encoder {
     // scope for a `notAdjacent({ kinds })` anywhere inside it — see the same
     // save/restore in `case 'grammar'`.
     this.activeTrivia = p._meta.grammarTrivia
+    this.triviaKnown = p._meta.grammarTrivia !== undefined
     const body = this.node(p).ip
     this.activeTrivia = undefined
+    this.triviaKnown = true
     // GRAMMAR-LEVEL AMBIENT TRIVIA (`rules({ trivia }, …)`).
     //
     // `run()` installs this from `entry._meta.grammarTrivia`, but ONLY for a
@@ -930,13 +1246,19 @@ class Encoder {
    * silently produced the ONE keyword `bc`.
    */
   private scopedRef(p: Combinator<unknown>, target: Combinator<unknown>): number {
-    const ip = this.node(target).ip
     // `rules()` stamps `grammarTrivia` on the map entry, which for a proxied rule
     // is the REF itself; a composed `winners` entry carries it on the target.
-    const amb = p._meta.grammarTrivia ?? target._meta.grammarTrivia
-    if (amb === undefined || this.activeTrivia !== undefined) return ip
-    if (!hasOwnTriviaBoundary(target)) return ip
-    return this.emit(OP_SCOPE_PLAIN, this.triviaSlot(amb), ip)
+    const runsUnder = triviaThroughRef(p, target, this.activeTrivia)
+    if (this.activeTrivia !== undefined || runsUnder === undefined || runsUnder === null) return this.node(target).ip
+    // The body is encoded under the trivia this row installs for it, so its own
+    // gates are derived under the scope they run in.
+    const savedKnown = this.triviaKnown
+    this.activeTrivia = runsUnder
+    this.triviaKnown = true
+    const ip = this.node(target).ip
+    this.activeTrivia = undefined
+    this.triviaKnown = savedKnown
+    return this.emit(OP_SCOPE_PLAIN, this.triviaSlot(runsUnder), ip)
   }
 
   /**
@@ -948,7 +1270,7 @@ class Encoder {
    * unable to match empty or to commit, so a miss is exactly that term failing
    * at the token's end with its own set.
    */
-  private routedArmGate(arm: Combinator<unknown>): { cls: number; fx: number } | undefined {
+  private routedArmGate(arm: Combinator<unknown>): { cls: number; fx: number; two?: readonly [number, number] } | undefined {
     type Follow = 'end' | { next: Combinator<unknown>; boundary: Combinator<unknown> | undefined } | undefined
     const walk = (p: Combinator<unknown>, trivia: Combinator<unknown> | undefined, depth: number): Follow => {
       if (depth > 16) return undefined
@@ -983,45 +1305,51 @@ class Encoder {
     // reports that opener once statically, once per arm when it runs.
     const nextDef = next._def as ParserDef
     if (nextDef.tag !== 'literal' && nextDef.tag !== 'regex' && nextDef.tag !== 'keywords' && nextDef.tag !== 'label') return undefined
-    const first = firstSetOf(next, new Set(), rr, boundary)
+    // Where the trivia after the token is not known, the gate cannot say what may
+    // follow it.
+    if (boundary === undefined && !this.triviaKnown) return undefined
+    const first = firstSetOf(next, new Set(), rr, boundary ?? null)
     const cls = this.charClass(boundary === undefined ? first : union(first, firstSetOf(boundary, new Set(), rr)))
     if (cls < 0) return undefined
-    const miss = nextDef.tag === 'label' ? deriveExpected(next) : directTerminalFailureExpected(nextDef)
-    return { cls, fx: this.expected(miss) }
+    const fx = this.expected(nextDef.tag === 'label' ? deriveExpected(next) : directTerminalFailureExpected(nextDef))
+    // THE SECOND CHARACTER. Where the next term starts with a character of
+    // `first` — which no gap before it can start with, so the term starts at the
+    // token's end — it also reads the one after: test both, still with no scan.
+    const two = firstTwo(next)
+    if (two === undefined) return { cls, fx }
+    if (boundary !== undefined && intersects(two.first, firstSetOf(boundary, new Set(), rr))) return { cls, fx }
+    const a = this.charClass(two.first)
+    const b = this.charClass(two.second)
+    return a < 0 || b < 0 ? { cls, fx } : { cls, fx, two: [a, b] }
   }
 
   node(p: Combinator<unknown>): Emitted {
-    // See `refMemo`: a reference is memoised per REFERENCING trivia scope, every
-    // other construct once for the program.
-    const lazyRef = (p._def as ParserDef).tag === 'lazy'
-    const scopeKey = lazyRef ? this.activeTrivia : undefined
-    const refMemo = this.refMemo
-    const hit = lazyRef ? refMemo.get(p)?.get(scopeKey) : this.memo.get(p)
+    // See `memo`: one encoding per combinator per trivia scope.
+    const key = this.scopeKey(p)
+    let byScope = this.memo.get(p)
+    const hit = byScope?.get(key)
     if (hit !== undefined) return { ip: hit }
     // Recursion: reserve a trampoline row now, patch its target when the real
     // body lands. One extra indirection per recursive rule, zero per call site.
-    const inFlight = this.pending.get(p)
+    let inFlightByScope = this.pending.get(p)
+    const inFlight = inFlightByScope?.get(key)
     if (inFlight !== undefined) {
       const ip = this.emitHead(OP_RULE, 1)
       inFlight.push(ip + 1)
       return { ip }
     }
     const patches: number[] = []
-    this.pending.set(p, patches)
+    if (inFlightByScope === undefined) { inFlightByScope = new Map(); this.pending.set(p, inFlightByScope) }
+    inFlightByScope.set(key, patches)
     // A RULE IS CREDITED ON ENTRY, not on success — the source lowering emitted the
     // hook as the first statement of the named function body, so a rule that is
     // reached and then fails still counts as exercised. The counter goes on the
     // MEMOISED offset, which is the one every reference to this combinator lands
     // on, and the recursion trampolines below are patched with it too.
     const ip = this.covWrap(this.encodeDef(p), this.plan?.rules.get(p), 0)
-    this.pending.delete(p)
-    if (lazyRef) {
-      let byScope = refMemo.get(p)
-      if (byScope === undefined) { byScope = new Map(); refMemo.set(p, byScope) }
-      byScope.set(scopeKey, ip)
-    } else {
-      this.memo.set(p, ip)
-    }
+    inFlightByScope.delete(key)
+    if (byScope === undefined) { byScope = new Map(); this.memo.set(p, byScope) }
+    byScope.set(key, ip)
     for (const slot of patches) {
       // A TRAMPOLINE PATCHED TO ITSELF IS NOT A TABLE. `OP_RULE ip → ip` makes no
       // progress in either driver: `exec.ts`'s `case OP_RULE` recurses on the same
@@ -1150,8 +1478,24 @@ class Encoder {
         // Arms that differ in one term are that term's choice (`factor-choice.ts`).
         // Not under a coverage plan, which credits the arms as written.
         if (this.plan === undefined) {
-          const factored = factorSharedEnds(d)
-          if (factored !== undefined) return this.node(factored).ip
+          const shared = factorSharedEnds(d)
+          if (shared !== undefined) return this.node(shared).ip
+          // The prefix arm reads `X` once only in a strict parse. Tolerant
+          // recovery and a completions probe run the choice as written: inside
+          // `sequence(X, optional(a))`, `X`'s own recovery would see `a`'s first
+          // set as all that may follow it, where the next arm's `X` sees the
+          // choice's.
+          if (this.unfactoredChoice !== p) {
+            const prefix = factorPrefixArms(d)
+            if (prefix !== undefined) {
+              const strict = this.node(prefix).ip
+              const saved = this.unfactoredChoice
+              this.unfactoredChoice = p
+              const tolerant = this.encodeDef(p)
+              this.unfactoredChoice = saved
+              return this.emit(OP_STRICT, strict, tolerant)
+            }
+          }
         }
         // `greedyClassify` is NOT a choice at all — one arm runs and another arm
         // is credited — so it gets its own row rather than an arm ordering.
@@ -1226,7 +1570,9 @@ class Encoder {
             && (gate === null || gate === undefined)
           const routedGate = plain ? this.routedArmGate(c) : undefined
           const body = this.node(c).ip
-          const inner = routedGate === undefined ? body : this.emit(OP_RGATE, routedGate.cls, body, routedGate.fx)
+          const inner = routedGate === undefined ? body
+            : routedGate.two === undefined ? this.emit(OP_RGATE, routedGate.cls, body, routedGate.fx)
+            : this.emit(OP_RGATE2, routedGate.cls, body, routedGate.fx, routedGate.two[0], routedGate.two[1])
           const ip = checks === null || checks === undefined || checks.length === 0
             ? inner
             : this.reject(inner, checks)
@@ -1269,7 +1615,13 @@ class Encoder {
         // the O(1) table (`exclusive`) or falls to the ordered per-arm path.
         // Arm ORDER is preserved on both, which is what makes this a PEG-safe
         // change rather than a reordering.
-        const finalChoice = classifyFinalChoice(arms, rr, this.activeTrivia)
+        // The arms' classes let in what the interpreter's `choice()` lets in. A
+        // disjoint one dispatches on its arms' construction-time sets (no trivia,
+        // a nullable pad as `any`), and the table reads those same sets. Any other
+        // tries each arm under the trivia that runs there; see `wideArms` below.
+        const constructionSets = d.disjoint && !factoredChoices.has(p)
+        const finalChoice = classifyFinalChoice(arms, rr, this.gateTrivia(), false, constructionSets)
+        const wideArms = this.wideArms(arms, finalChoice.firstSets, constructionSets)
         // attempt() preserves its child's success language, but the broad
         // classifier deliberately treats the transaction as nullable/unknown so
         // it can never by itself authorize exclusive one-arm selection. Recover
@@ -1281,7 +1633,7 @@ class Encoder {
           if (!finalChoice.nullable[i]) return undefined
           const def = arm._def as ParserDef
           if (def.tag !== 'attempt' || matchesEmpty(def.parser, new Set(), rr)) return undefined
-          return firstSetOf(def.parser, new Set(), rr, this.activeTrivia)
+          return firstSetOf(def.parser, new Set(), rr, this.activeTrivia, false)
         })
         const gateFirst = finalChoice.firstSets.map((first, i) => transactionalFirst[i] ?? first)
         const classes = gateFirst.map((first, i) => finalChoice.nullable[i] && transactionalFirst[i] === undefined
@@ -1304,6 +1656,13 @@ class Encoder {
         // ordered site leaves the decision to `OP_ARMGATE`. An exclusive site keeps
         // its classes: there the dispatched arm's gate failing IS the choice
         // failing, with that arm's set, as in the interpreter's disjoint path.
+        // A class widened by a reference's installed trivia can make arms look
+        // disjoint where the interpreter still tries each one and merges their
+        // failures at the choice position. Such a site stays ordered: the widened
+        // arm is always entered, so the arms beside it keep their opener sets.
+        if (wideArms.length > 0 && dispatchIsExclusive(classes, this.cc)) {
+          for (const i of wideArms) classes[i] = -1
+        }
         if (d.gates.some(g => g !== null && g !== undefined) && !dispatchIsExclusive(classes, this.cc)) {
           for (let k = 0; k < classes.length; k++) {
             const gate = d.gates[order[k]!]
@@ -1349,13 +1708,18 @@ class Encoder {
       }
       case 'many':
       case 'oneOrMore': {
-        const child = this.node(d.parser).ip
         const rr = this.refResolver()
+        // The item's gate tests what the interpreter's repeat guard tests where it
+        // has one (`repeat.ts`), and otherwise what the item can start with.
         const itemClass = matchesEmpty(d.parser, new Set(), rr)
           ? -1
-          : this.charClass(firstSetOf(d.parser, new Set(), rr, this.activeTrivia))
+          : this.charClass(this.gateFirstSet(d.parser, d.guarded === true, true))
         const op = d.valueUnused ? OP_REPV : OP_REP
         const min = d.tag === 'many' ? 0 : d.min
+        const item = min === 0 && itemClass >= 0
+          ? this.withoutLeadGate(this.node(d.parser).ip, itemClass)
+          : this.node(d.parser).ip
+        const child = min === 0 && itemClass >= 0 ? this.gapGated(d.parser, item) : item
         // ip + 6 is the ITEM's expected set and ip + 7 the item's sentinel class:
         // the resync error's payload is the ITEM's set (`deriveExpected` in
         // `combinators/repeat.ts`).
@@ -1371,20 +1735,33 @@ class Encoder {
         // under `min` reports what would have let it CONTINUE — the ITEM, which
         // is what `failAt` (repeat.ts) and `deriveExpectedArr([def.parser])`
         // (codegen emitSepBy) both use; the driver otherwise reports whatever
-        // sub-parse failed last, which is the SEPARATOR. Only `min >= 2` can
-        // reach that end under `min` with items already taken — at `min === 1`
-        // it means the FIRST item failed, and that failure already set the
-        // item's own set. So no committed grammar grows by a word.
-        const flags = (d.trailing === 'allow' ? 1 : 0) | (d.keepSeparators === true ? 2 : 0) | (d.min >= 2 ? 4 : 0)
-        // Every list row carries ip + 6 (the ITEM's expected set, which
-        // the resync error reports) and ip + 7 (the SEPARATOR's sentinel class, so
+        // sub-parse failed last. At `min === 1` that end is the FIRST item
+        // failing, and `failAt` reports the item's derived set at the list's
+        // start there too, not whatever the item's own failure left (a deeper
+        // set the interpreter discards, `repeat.ts` `failAt(pos)`).
+        const gate = this.sepGate(d.separator, sep)
+        // The ITEM's set, reported where the list ends under `min`, falls back to
+        // the item's tag where it derives empty (`repeat.ts` `failAt`); a resync
+        // error carries the set as derived. Bit 16 says that set is empty.
+        const itemSet = deriveExpected(d.parser)
+        const itemFx = this.expected(itemSet.length > 0 ? itemSet : [d.parser._tag])
+        const flags = (d.trailing === 'allow' ? 1 : 0) | (d.keepSeparators === true ? 2 : 0) | (d.min >= 1 ? 4 : 0) | (gate >= 0 ? 8 : 0) | (itemSet.length === 0 ? 16 : 0)
+        // Every list row carries ip + 6 (the ITEM's expected set, which the
+        // resync error reports) and ip + 7 (the SEPARATOR's sentinel class, so
         // a tolerant list resyncs to its own separator OR the enclosing
         // delimiter — `orSentinel(separator, term)` in repeat.ts:454 and
-        // `_rec.or(sepSent, mySync)` in codegen's emitSepBy).
-        return this.emit(
-          OP_REP, child, d.min, d.max ?? -1, sep, flags,
-          this.expected(deriveExpected(d.parser)), this.charClass(firstSetOf(d.separator, new Set(), this.refResolver(), this.activeTrivia)),
-        )
+        // `_rec.or(sepSent, mySync)` in codegen's emitSepBy). With flag bit 8,
+        // ip + 8 is the separator's first set for a strict list to end on
+        // (`sepGate`).
+        const ops = [
+          child, d.min, d.max ?? -1, sep, flags,
+          itemFx,
+          // As 0.58.0: the interpreter resyncs on the separator itself
+          // (`orSentinel`), which runs under the live trivia.
+          this.charClass(firstSetOf(d.separator, new Set(), this.refResolver(), this.activeTrivia, false)),
+        ]
+        if (gate >= 0) ops.push(gate)
+        return this.emit(OP_REP, ...ops)
       }
       case 'optional': {
         const head = this.emit(OP_OPT, this.lookaheadGate(d.parser, this.node(d.parser).ip))
@@ -1400,7 +1777,7 @@ class Encoder {
         // rule invocation. The inner sequence must not be shared with anything
         // else, or fusing it here would steal it from the other reference.
         const inner = d.parser._def as ParserDef
-        if (inner.tag === 'sequence' && !this.memo.has(d.parser) && !this.pending.has(d.parser)) {
+        if (inner.tag === 'sequence' && !this.memo.get(d.parser)?.has(this.scopeKey(d.parser)) && !this.pending.get(d.parser)?.has(this.scopeKey(d.parser))) {
           const kids = inner.parsers.map(c => this.node(c).ip)
           const head = this.emitHead(OP_SEQX, 2 + 2 * kids.length)
           const projection = directArrayProjection(d.fn, d.fnSrc, kids.length)
@@ -1513,7 +1890,11 @@ class Encoder {
         // accepts (the reducer reroutes via `winners`, but this recognizer gate did
         // not). Resolve the ref's gate through `winners`, exactly as the `lazy` case
         // resolves the call. Non-ref bodies keep the cached set (byte-identical).
-        const fs = this.gateFirstSet(d.parser)
+        // Not narrowed through a nullable pad: a node's gate miss reports a static
+        // set, and only where the body's first set is finite without the pad does
+        // that set match what entering the body reports (a `keywords()` term, a
+        // list's item set, an optional lead are only known by running it).
+        const fs = this.gateFirstSet(d.parser, d.guarded === true, false)
         if (fs.kind === 'any') return body
         const cls = this.charClass(fs)
         if (cls < 0) return body
@@ -1521,7 +1902,16 @@ class Encoder {
         return this.emit(OP_GATE, cls, body, this.expected(e.length > 0 ? e : [d.parser._tag]))
       }
       case 'token': {
-        return this.emit(OP_TOKEN, this.node(d.parser).ip)
+        // A token runs its body with trivia cleared (`token.ts`), so its body is
+        // encoded under no trivia.
+        const saved = this.activeTrivia
+        const savedKnown = this.triviaKnown
+        this.activeTrivia = undefined
+        this.triviaKnown = true
+        const body = this.node(d.parser).ip
+        this.activeTrivia = saved
+        this.triviaKnown = savedKnown
+        return this.emit(OP_TOKEN, body)
       }
       case 'routed':
         return this.emit(OP_ROUTED, d.fallback === undefined ? -1 : this.node(d.fallback).ip)
@@ -1610,7 +2000,9 @@ class Encoder {
         const otherRouted = d.otherwise === undefined
           ? false
           : branchUsesRouted({ parser: d.otherwise as Combinator<unknown>, usesRouted: d.otherwiseUsesRouted === true })
-        this.code[head + 4] = otherRouted ? 1 : 0
+        // Bit 0: the fallback reads the routed token. Bit 1: its failure is not
+        // committed (`otherwise(p, { commit: false })`).
+        this.code[head + 4] = (otherRouted ? 1 : 0) | (d.otherwiseCommit === false ? 2 : 0)
         this.code[head + 5] = arms.length
         for (let i = 0; i < arms.length; i++) this.code[head + 6 + i] = arms[i]!
         return head
@@ -1714,7 +2106,7 @@ class Encoder {
         // not a slower equivalent.
         // Same override-aware resolution as the node gate: a bare-ref inner resolves
         // its gate through `winners` so a widened winner is not gated out.
-        const fs = this.gateFirstSet(d.parser)
+        const fs = this.gateFirstSet(d.parser, d.guarded === true, false)
         if (fs.kind === 'any' || matchesEmpty(d.parser, new Set(), this.refResolver())) return inner
         const cls = this.charClass(fs)
         if (cls < 0) return inner
@@ -1793,10 +2185,12 @@ class Encoder {
         // `ctx.activeTrivia` does. Saved and restored rather than assigned,
         // because a scope is nestable and an inner one does not leak outward.
         const savedTrivia = this.activeTrivia
-        if (d.clearTrivia === true) this.activeTrivia = undefined
-        else if (d.triviaParser !== undefined) this.activeTrivia = d.triviaParser
+        const savedKnown = this.triviaKnown
+        if (d.clearTrivia === true) { this.activeTrivia = undefined; this.triviaKnown = true }
+        else if (d.triviaParser !== undefined) { this.activeTrivia = d.triviaParser; this.triviaKnown = true }
         const bodyIp = this.node(d.parser).ip
         this.activeTrivia = savedTrivia
+        this.triviaKnown = savedKnown
         if (d.clearTrivia === true) return this.emit(op, -1, bodyIp, flags)
         if (d.triviaParser === undefined) {
           return cap || flags !== 0 ? this.emit(op, -1, bodyIp, flags) : bodyIp
@@ -1839,8 +2233,8 @@ class Encoder {
     }
     const slots = (ip: number): number[] => {
       switch (this.code[ip]) {
-        case OP_GATE: case OP_RGATE: return [ip + 2]
-        case OP_RULE: case OP_OPT: case OP_NOT: case OP_PEEK: case OP_EXPECT: case OP_ATTEMPT: case OP_LABEL: case OP_COV: return [ip + 1]
+        case OP_GATE: case OP_RGATE: case OP_RGATE2: return [ip + 2]
+        case OP_RULE: case OP_OPT: case OP_NOT: case OP_PEEK: case OP_EXPECT: case OP_ATTEMPT: case OP_LABEL: case OP_COV: case OP_GAPGATE: return [ip + 1]
         case OP_SCOPE: case OP_SCOPE_CAP: case OP_SCOPE_PLAIN: case OP_WITHCTX: case OP_XFORM: case OP_LEAF: case OP_SOURCE_LEAF: case OP_NODE: case OP_NODE_TRACK: return [ip + 2]
         case OP_SEQ: case OP_SEQV: return Array.from({ length: this.code[ip + 1]! }, (_, i) => ip + 2 + i)
         case OP_SEQX: return Array.from({ length: this.code[ip + 2]! }, (_, i) => ip + 3 + i)
@@ -1859,6 +2253,7 @@ class Encoder {
         // REJECT's check operands are const/class indices, never offsets.
         case OP_REJECT: return [ip + 1]
         case OP_REP: case OP_REPV: return this.code[ip + 4]! >= 0 ? [ip + 1, ip + 4] : [ip + 1]
+        case OP_STRICT: return [ip + 1, ip + 2]
         default: return []
       }
     }