parseman 0.61.6 → 0.61.8

package/dist/version.d.ts

@@ -13,5 +13,5 @@
  * Kept in sync with `package.json` by `test/unit/version-sync.test.ts`, which fails
  * the build if this constant drifts from the published version.
  */
-export declare const PARSEMAN_VERSION = "0.61.6";
+export declare const PARSEMAN_VERSION = "0.61.8";
 //# sourceMappingURL=version.d.ts.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "parseman",
-  "version": "0.61.6",
+  "version": "0.61.8",
   "packageManager": "pnpm@10.15.1",
   "engines": {
     "node": "^20.19.0 || >=22.13.0"

package/src/combinators/expect.ts

@@ -1,7 +1,7 @@
 import type { Combinator, ParseContext, ParseResult, ParserMeta, ParseError } from '../types.ts'
 import { captureError } from '../recovery/scan.ts'
 import { annotateSpanFromLineContext } from '../line-index.ts'
-import { matchesEmpty, isZeroWidthAssertion } from './first-set.ts'
+import { any, matchesEmpty, isZeroWidthAssertion } from './first-set.ts'
 
 export type { ParseError }
 export { assertionFailureExpected, directTerminalFailureExpected } from './expected.ts'
@@ -133,7 +133,8 @@ function derive(c: Combinator<unknown>, seen: Set<Combinator<unknown>>): string[
 export function expect<T>(combinator: Combinator<T>, label?: string): Combinator<T | ParseError> {
   const expected = label !== undefined ? [label] : deriveExpected(combinator)
   const meta: ParserMeta = {
-    firstSet: combinator._meta.firstSet,
+    // It runs on any character: on one its child can't start, it reports the error.
+    firstSet: any(),
     canMatchNewline: combinator._meta.canMatchNewline,
     isTrivia: false,
   }

package/src/combinators/first-set.ts

@@ -202,7 +202,8 @@ function matchesEmptyBody(
     case 'many':
     case 'optional':
     case 'not':
-    case 'peek':     return true          // zero repetitions / absent / lookahead
+    case 'peek':
+    case 'expect':   return true          // zero repetitions / absent / lookahead / recovered
     // Default `sepBy` is `(item (sep item)*)?` — it MATCHES THE EMPTY STRING.
     // Any `min >= 1` requires that many ITEMS, so it is nullable only when the
     // item is. (Keying this off `min === 1` reported every `{ min: 2 }` list as
@@ -220,7 +221,6 @@ function matchesEmptyBody(
     case 'trivia':
     case 'token':
     case 'leaf':
-    case 'expect':
     case 'withCtx':
     case 'node':
     case 'grammar':
@@ -382,6 +382,8 @@ export function sequenceFirstSet(parsers: readonly Combinator<unknown>[]): First
  * repeat item's, a gap's and a node's gate read a pad through its non-empty
  * matches.
  *
+ * `follow: true` asks where a resync sentinel stops, which reads through `expect`.
+ *
  * `activeTrivia` is the live trivia, `null` for a scope known to have none (where
  * a reference installs its rule's trivia, `triviaThroughRef`), and `undefined`
  * where it is not known.
@@ -396,6 +398,7 @@ export function firstSetOf(
   resolve?: RefResolver,
   activeTrivia?: Combinator<unknown> | null,
   consuming = true,
+  follow = false,
 ): FirstSet {
   // `seen` is the current recursion path, not a global visited set. A completed
   // shared child must be analysed again for a sibling; only a back-edge on this
@@ -403,7 +406,7 @@ export function firstSetOf(
   if (seen.has(p)) return any()               // cycle → any (safe over-approximation)
   seen.add(p)
   try {
-    return firstSetBody(p, seen, resolve, activeTrivia, consuming)
+    return firstSetBody(p, seen, resolve, activeTrivia, consuming, follow)
   } finally {
     seen.delete(p)
   }
@@ -415,8 +418,9 @@ function firstSetBody(
   resolve: RefResolver | undefined,
   activeTrivia: Combinator<unknown> | null | undefined,
   consuming: boolean,
+  follow: boolean,
 ): FirstSet {
-  const fs = (c: Combinator<unknown>): FirstSet => firstSetOf(c, seen, resolve, activeTrivia, consuming)
+  const fs = (c: Combinator<unknown>): FirstSet => firstSetOf(c, seen, resolve, activeTrivia, consuming, follow)
   const empties = (c: Combinator<unknown>): boolean => matchesEmpty(c, new Set(), resolve)
   const d = p._def as ParserDef
   switch (d.tag) {
@@ -437,7 +441,7 @@ function firstSetBody(
       // install; the interpreter's own callers (`sequence.ts` follow sentinels)
       // read a reference as 0.58.0 did (#223).
       const through = activeTrivia === null && resolve !== undefined
-      return firstSetOf(target, seen, resolve, through ? triviaThroughRef(p, target, null) : activeTrivia, consuming)
+      return firstSetOf(target, seen, resolve, through ? triviaThroughRef(p, target, null) : activeTrivia, consuming, follow)
     }
     case 'choice': {
       let out: FirstSet = empty()
@@ -457,6 +461,9 @@ function firstSetBody(
       // original cursor.
       return sequenceSet(d.parsers, fs, empties, activeTrivia, consuming)
     }
+    // Runs on any character: on one its child can't start, it reports the error. A
+    // resync sentinel still stops where the child could start.
+    case 'expect':    return follow ? fs(d.parser) : any()
     case 'peek': {
       // Deep-resolve the body: a `ref()` reads `any()` at CONSTRUCTION, so the
       // shallow `_meta.firstSet` baked into the assertion would lose the gate.
@@ -472,19 +479,19 @@ function firstSetBody(
     case 'leaf':
     case 'node':
     case 'sepBy':
-    case 'expect':
     // A transaction accepts exactly what its inner accepts. Its own `_meta` is
     // the inner's construction-time set, which reads `any` when the inner opens
     // with a rule reference — resolve through it like any other wrapper.
     case 'attempt':   return fs(d.parser)
     // A token runs its body with trivia cleared (`token.ts`).
-    case 'token':     return firstSetOf(d.parser, seen, resolve, activeTrivia === undefined ? undefined : null, consuming)
+    case 'token':     return firstSetOf(d.parser, seen, resolve, activeTrivia === undefined ? undefined : null, consuming, follow)
     case 'grammar':   return firstSetOf(
       d.parser,
       seen,
       resolve,
       d.clearTrivia ? null : (d.triviaParser ?? activeTrivia),
       consuming,
+      follow,
     )
     // A run starts with its first line's prefix; a nullable prefix starts with anything.
     case 'lines': {

package/src/combinators/sequence.ts

@@ -108,7 +108,7 @@ export function sequence<T extends [Combinator<unknown>, ...Combinator<unknown>[
       if (tolerant) {
         followSentinels ??= parsers.map((_, i) => firstSetSentinel(
           parsers.slice(i + 1).reduce<ReturnType<typeof firstSetOf>>(
-            (set, parser) => union(set, firstSetOf(parser, new Set(), undefined, undefined, false)), { kind: 'empty' },
+            (set, parser) => union(set, firstSetOf(parser, new Set(), undefined, undefined, false, true)), { kind: 'empty' },
           ),
         ))
       }

package/src/table/assemble.ts

@@ -2844,8 +2844,46 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
         }
         if (hasAdj) {
           const first = linkFirst()
+          // What a held gap needs undone if the next term recovers at zero width.
+          // Written after the lookahead returns and read before the next term runs,
+          // so nothing can re-enter this sequence in between.
+          let heldFrom = 0, heldTl = 0, heldScanTl = 0, heldLog = 0, heldScanLog = 0, heldRoot = 0, heldScanRoot = 0
           const runnerAt = (i: number): TermRunner => {
             const kidIp = code[base + i]!
+            if (i > 1 && holdsGap(code, k, base, n, i - 1, NULLABLE) && rowNullable(code, k, kidIp, NULLABLE)) {
+              const kid = link(kidIp)
+              return (input, cur, ctx) => {
+                const from = heldFrom, tl = heldTl, scanTl = heldScanTl, log = heldLog
+                const scanLog = heldScanLog, root = heldRoot, scanRoot = heldScanRoot
+                const next = step(kid, input, cur, ctx)
+                if (next !== cur || from === cur) return next
+                rollbackScannedTriviaAt(ctx, tl, scanTl, log, scanLog, root, scanRoot)
+                return from
+              }
+            }
+            if (holdsGap(code, k, base, n, i, NULLABLE) && rowNullable(code, k, code[base + i + 1]!, NULLABLE)) {
+              const kid = link(kidIp)
+              return (input, cur, ctx) => {
+                if (ctx.trivia === undefined) {
+                  const v = kid(input, cur, ctx)
+                  if (v === FAIL) return -1
+                  TERMV = v
+                  heldFrom = cur
+                  return EC.e
+                }
+                const tl = cstTlLen(ctx), log = ctx._triviaLog?.length ?? 0, root = ctx._rootTriviaLog?.length ?? 0
+                const scanEnd = skipTrivia(input, cur, ctx)
+                const scanTl = cstTlLen(ctx), scanLog = ctx._triviaLog?.length ?? 0, scanRoot = ctx._rootTriviaLog?.length ?? 0
+                const v = kid(input, scanEnd, ctx)
+                if (v === FAIL) return -1
+                TERMV = v
+                heldFrom = cur
+                heldTl = tl; heldScanTl = scanTl
+                heldLog = log; heldScanLog = scanLog
+                heldRoot = root; heldScanRoot = scanRoot
+                return scanEnd
+              }
+            }
             if (holdsGap(code, k, base, n, i, NULLABLE)) {
               const kid = link(kidIp)
               return (input, cur, ctx) => {

package/src/table/emit-assembly.ts

@@ -467,16 +467,40 @@ ${scanned}
 }`
 }
 
+const TRIVIA_LOG_LENGTH = 'ctx._cstBuf!==undefined?(ctx._cstBuf.tl!==undefined?ctx._cstBuf.tl.length:0):(ctx._cstTriviaLog!==undefined?ctx._cstTriviaLog.length:0)'
+
 /**
  * A lookahead term that HOLDS its gap (`hold-gap.ts`): the gap is scanned and
  * kept, the lookahead runs at its end, and the cursor stays there for the next
- * term, which cannot match empty.
+ * term, which cannot match empty unless it `recovers`. Then the marks stay in
+ * scope for `emitHeldGapUndo`, which follows the next term.
  */
-function emitHoldTerm(callee: string, dst: string, t: string, l: SiteLabel, skip: string): string {
+function emitHoldTerm(callee: string, dst: string, t: string, l: SiteLabel, skip: string, recovers: boolean): string {
   const fast = `const ${t}v=${callee}(input,cur,ctx)
 if(${t}v===FAIL)return FAIL
 ${dst}=${t}v
 cur=EC.e`
+  if (recovers) {
+    const marked = `let ${t}c=cur,${t}s=cur,${t}tl=0,${t}stl=0,${t}lg=0,${t}slg=0,${t}rt=0,${t}srt=0`
+    if (l.tri === TRI_NONE) return `${marked}\n${fast}`
+    const held = `${t}tl=${TRIVIA_LOG_LENGTH}
+${t}lg=ctx._triviaLog!==undefined?ctx._triviaLog.length:0
+${t}rt=ctx._rootTriviaLog!==undefined?ctx._rootTriviaLog.length:0
+${t}s=${skip}(input,cur,ctx)
+${t}stl=${TRIVIA_LOG_LENGTH}
+${t}slg=ctx._triviaLog!==undefined?ctx._triviaLog.length:0
+${t}srt=ctx._rootTriviaLog!==undefined?ctx._rootTriviaLog.length:0
+const ${t}v=${callee}(input,${t}s,ctx)
+if(${t}v===FAIL)return FAIL
+${dst}=${t}v
+cur=${t}s`
+    return l.tri !== TRI_UNKNOWN ? `${marked}\n${held}` : `${marked}
+if(ctx.trivia===undefined){
+${fast}
+}else{
+${held}
+}`
+  }
   if (l.tri === TRI_NONE) return fast
   const held = `const ${t}s=${skip}(input,cur,ctx)
 const ${t}v=${callee}(input,${t}s,ctx)
@@ -491,6 +515,11 @@ ${held}
 }`
 }
 
+/** The next term recovered at zero width, so the gap held for it was never consumed. */
+function emitHeldGapUndo(t: string): string {
+  return `if(cur===${t}s&&${t}s!==${t}c){rollbackScannedTriviaAt(ctx,${t}tl,${t}stl,${t}lg,${t}slg,${t}rt,${t}srt);cur=${t}c}`
+}
+
 /**
  * The repetition's LEAD-TRIVIA skip, as a BLOCK.
  *
@@ -2019,6 +2048,7 @@ return v
         }
         parts.push('let cur=EC.e')
         const names: string[] = ['v0']
+        let heldUndo: string | undefined
         for (let i = 1; i < n; i++) {
           const vn = `v${i}`
           parts.push(`let ${vn}`)
@@ -2038,9 +2068,16 @@ return v
             // The callee expression hands it over AFTER the scan statement, right
             // before the call, so a field inside the trivia grammar cannot take it.
             const callee = leadingTriviaFieldAt(code, k, childIp) ? `(LEAD.at=cur,${kids[i]!})` : kids[i]!
-            parts.push(holdsGap(code, k, base, n, i, nullable)
-              ? emitHoldTerm(callee, vn, tmp(), L, skipFor(L))
-              : emitTerm(callee, vn, tmp(), L, skipFor(L), L.tri >= 0 ? triviaMiss(L.tri) : undefined))
+            if (holdsGap(code, k, base, n, i, nullable)) {
+              const t = tmp()
+              const recovers = rowNullable(code, k, code[base + i + 1]!, nullable)
+              parts.push(emitHoldTerm(callee, vn, t, L, skipFor(L), recovers))
+              heldUndo = recovers ? t : undefined
+            } else {
+              parts.push(emitTerm(callee, vn, tmp(), L, skipFor(L), L.tri >= 0 ? triviaMiss(L.tri) : undefined))
+              if (heldUndo !== undefined) parts.push(emitHeldGapUndo(heldUndo))
+              heldUndo = undefined
+            }
           }
           names.push(vn)
         }

package/src/table/encode.ts

@@ -751,7 +751,7 @@ class Encoder {
     for (let j = i + 1; j < parsers.length; j++) {
       // As the interpreter's `followSentinels` derive them (`sequence.ts`): with no
       // trivia, whatever scope the sequence runs in.
-      fs = union(fs, firstSetOf(parsers[j]!, new Set(), rr, undefined, false))
+      fs = union(fs, firstSetOf(parsers[j]!, new Set(), rr, undefined, false, true))
     }
     return this.charClass(fs)
   }

package/src/table/fail-clean.ts

@@ -120,11 +120,15 @@ export function failCleanAnalysis(code: ArrayLike<number>): FailClean {
       case OP_CONT:
         out = clean(code[ip + 1]!) && neverFails(code[ip + 3]!)
         break
-      // An arm's failure and the fallback's are committed; only the selector's is
-      // not, and without a fallback an unknown key fails after the selector wrote.
-      case OP_DISPATCH:
-        out = code[ip + 3]! >= 0 && clean(code[ip + 1]!)
+      // An arm's failure and a committing fallback's are committed; only the
+      // selector's is not. An uncommitted fallback's failure leaves the selector's
+      // capture, unless the fallback reads it through `routed()`, which rolls that
+      // capture back first. With no fallback, the dispatch fails after the selector wrote.
+      case OP_DISPATCH: {
+        const flags = code[ip + 4]!
+        out = code[ip + 3]! >= 0 && ((flags & 2) === 0 || (flags & 1) === 1) && clean(code[ip + 1]!)
         break
+      }
       default: {
         const c = passChild(code, ip)
         out = c >= 0 && clean(c)

package/src/table/hold-gap.ts

@@ -13,6 +13,10 @@
  *
  * Not before an adjacency assertion or a leading-trivia field, which read the
  * cursor from before the gap.
+ *
+ * A next term that can match empty only by recovering (`expect`) holds the gap
+ * too, so valid text still scans it once. If it does recover, the gap is
+ * unrecorded and the cursor goes back, where only invalid text pays for it.
  */
 import {
   OP_ADJ, OP_ATTEMPT, OP_CHOICE, OP_COV, OP_EXPECT, OP_FIELD, OP_GATE, OP_LABEL, OP_LEAF, OP_LIT, OP_LIT_CI,
@@ -24,16 +28,21 @@ import { leadingTriviaFieldAt } from './field-spec.ts'
 import { regexCanMatchEmpty } from '../regex/first-set.ts'
 
 const WRAPPERS = new Set([OP_NODE, OP_XFORM, OP_LEAF, OP_SOURCE_LEAF, OP_TOKEN, OP_SCOPE, OP_SCOPE_CAP,
-  OP_SCOPE_PLAIN, OP_FIELD, OP_LABEL, OP_EXPECT, OP_WITHCTX, OP_RULE, OP_ATTEMPT, OP_COV, OP_GAPGATE])
+  OP_SCOPE_PLAIN, OP_FIELD, OP_LABEL, OP_WITHCTX, OP_RULE, OP_ATTEMPT, OP_COV, OP_GAPGATE])
 
 /**
  * Can the row at `ip` match empty? `true` wherever it cannot tell. A first-set
  * gate row wraps only a child that cannot (`encode.ts` gates nothing nullable).
+ * An `expect` recovers at zero width whatever its child needs; with `recovers`
+ * false, this asks what the row can match empty without recovering.
  */
-export function rowNullable(code: ArrayLike<number>, k: readonly unknown[], ip: number, memo: Map<number, boolean>): boolean {
-  const hit = memo.get(ip)
+export function rowNullable(
+  code: ArrayLike<number>, k: readonly unknown[], ip: number, memo: Map<number, boolean>, recovers = true,
+): boolean {
+  const key = recovers ? ip : ~ip
+  const hit = memo.get(key)
   if (hit !== undefined) return hit
-  memo.set(ip, true) // a cycle is answered conservatively
+  memo.set(key, true) // a cycle is answered conservatively
   const op = code[ip]!
   const kids: number[] = []
   let out = true
@@ -42,20 +51,20 @@ export function rowNullable(code: ArrayLike<number>, k: readonly unknown[], ip:
   else if (op === OP_RX) { const re = k[code[ip + 1]!]; out = !(re instanceof RegExp) || regexCanMatchEmpty(re.source) }
   else if (op === OP_SEQ || op === OP_SEQV || op === OP_SEQX) {
     childSlots(code, ip, kids)
-    out = kids.every(c => rowNullable(code, k, c, memo))
+    out = kids.every(c => rowNullable(code, k, c, memo, recovers))
   } else if (op === OP_CHOICE) {
     childSlots(code, ip, kids)
-    out = kids.length === 0 || kids.some(c => rowNullable(code, k, c, memo))
+    out = kids.length === 0 || kids.some(c => rowNullable(code, k, c, memo, recovers))
   } else if (op === OP_REP || op === OP_REPV) {
-    out = code[ip + 2] === 0 || rowNullable(code, k, code[ip + 1]!, memo)
+    out = code[ip + 2] === 0 || rowNullable(code, k, code[ip + 1]!, memo, recovers)
   } else if (op === OP_SEED) {
-    out = rowNullable(code, k, code[ip + 1]!, memo)
+    out = rowNullable(code, k, code[ip + 1]!, memo, recovers)
   } else if (op === OP_CONT) {
-    out = rowNullable(code, k, code[ip + 1]!, memo) && rowNullable(code, k, code[ip + 3]!, memo)
-  } else if (WRAPPERS.has(op)) {
-    if (childSlots(code, ip, kids) && kids.length === 1) out = rowNullable(code, k, kids[0]!, memo)
+    out = rowNullable(code, k, code[ip + 1]!, memo, recovers) && rowNullable(code, k, code[ip + 3]!, memo, recovers)
+  } else if (WRAPPERS.has(op) || (op === OP_EXPECT && !recovers)) {
+    if (childSlots(code, ip, kids) && kids.length === 1) out = rowNullable(code, k, kids[0]!, memo, recovers)
   }
-  memo.set(ip, out)
+  memo.set(key, out)
   return out
 }
 
@@ -68,5 +77,5 @@ export function holdsGap(
   if (code[term] !== OP_NOT && code[term] !== OP_PEEK) return false
   const next = code[base + i + 1]!
   if (code[next] === OP_ADJ || leadingTriviaFieldAt(code, k, next)) return false
-  return !rowNullable(code, k, next, memo)
+  return !rowNullable(code, k, next, memo, false)
 }

package/src/version.ts

@@ -13,4 +13,4 @@
  * Kept in sync with `package.json` by `test/unit/version-sync.test.ts`, which fails
  * the build if this constant drifts from the published version.
  */
-export const PARSEMAN_VERSION = '0.61.6'
+export const PARSEMAN_VERSION = '0.61.8'