parseman 0.60.2 → 0.60.4

package/src/table/emit-assembly.ts

@@ -54,7 +54,7 @@ import {
   OP_LEX_BODY, OP_LEX_PROGRAM, OP_GAPGATE, OP_STRICT,
 } from './ops.ts'
 import {
-  choiceCannotCommit, choiceRollbackMask, failureRollbackClean, validateDispatchSpec,
+  choiceCannotCommit, choiceLogRollbackMask, choiceRollbackMask, failureLogRollback, failureRollbackClean, validateDispatchSpec,
   type ResolvedClass, type ResolvedTable, type TableProgram,
 } from './program.ts'
 import { emitShapeMatch, scanShapeFromRegex } from './scan-shapes.ts'
@@ -406,6 +406,19 @@ function emitRollback(t: string, buf: boolean, rawMode: number, s: Sinks = NO_SI
   return `if(${t}n){rollbackTriviaAt(ctx,${t}raw,${t}tl,${t}lv,${s.fd ? `${t}fd` : 'undefined'},${s.er ? `${t}er` : 'undefined'},${t}lg,${t}rt)}`
 }
 
+/**
+ * A mark of the two trivia logs alone, for a child the census counts as clean but
+ * whose failure can leave trivia rows a failed node recorded
+ * (`program.ts` `choiceLogRollbackMask`, `failureLogRollback`).
+ */
+function emitLogMark(t: string): string {
+  return `const ${t}lg=ctx._triviaLog!==undefined?ctx._triviaLog.length:0,${t}rt=ctx._rootTriviaLog!==undefined?ctx._rootTriviaLog.length:0`
+}
+function emitLogRollback(t: string): string {
+  return `{const g=ctx._triviaLog;if(g!==undefined)while(g.length>${t}lg)g.pop()}
+{const r=ctx._rootTriviaLog;if(r!==undefined)while(r.length>${t}rt)r.pop()}`
+}
+
 /**
  * ONE SEQUENCE TERM — `assemble.ts`'s `nextTerm`, inlined at the site.
  *
@@ -597,6 +610,7 @@ export function emitAssemblySource(
     tolerant: boolean
     coverage: boolean
     probe: boolean
+    inheritedTrivia?: boolean
   },
   extraIps: readonly number[] = [],
   staticBuild = false,
@@ -618,6 +632,10 @@ export function emitAssemblySource(
   const { code, k, fx, disp, dsp, triviaLabelled } = t
   const swapLegal = !cfg.trackLines
   const hostCst = cfg.hostCst
+  // Trivia rows are recorded only by a program with trivia of its own, or under
+  // trivia a caller supplied (`RunCfg.inheritedTrivia`). Otherwise no site
+  // restores the trivia logs alone: nothing can have written them.
+  const LOGS = (prog.triviaSpecs?.length ?? 0) > 0 || cfg.inheritedTrivia === true
 
   /**
    * RECOVERY IS EMITTED, and the refusal this replaces was wrong in SCOPE.
@@ -1769,12 +1787,13 @@ return v
       case OP_ATTEMPT: {
         const child = link(code[ip + 1]!)
         const clean = !REC && failureRollbackClean(prog, ip)
+        const logs = clean && LOGS && failureLogRollback(prog, ip)
         const p = tmp()
         return `${head}
-${clean ? '' : emitMark(p, L.buf, L.raw, sinks)}
+${clean ? logs ? emitLogMark(p) : '' : emitMark(p, L.buf, L.raw, sinks)}
 const v=${child}(input,pos,ctx)
 if(v!==FAIL)return v
-${clean ? '' : emitRollback(p, L.buf, L.raw, sinks)}
+${clean ? logs ? emitLogRollback(p) : '' : emitRollback(p, L.buf, L.raw, sinks)}
 ${code[ip + 2] === 1 ? 'ctx._fc=false' : 'if(ctx._fc===true)return FAIL'}
 ctx._fe=pos
 return FAIL
@@ -1801,6 +1820,7 @@ return FAIL
 ${lookaheadMiss(code[ip + 1]!, 'EC.e=pos;return null')}${emitMark(p, L.buf, L.raw, sinks)}
 ${cfg.probe ? 'const pb=ctx._probe?.best\n' : ''}const v=${child}(input,pos,ctx)
 ${emitRollback(p, L.buf, L.raw, sinks)}
+ctx._fc=false
 ${cfg.probe ? 'if(ctx._probe!==undefined)ctx._probe.best=pb??null\n' : ''}if(v===FAIL){EC.e=pos;return null}
 ctx._fe=pos
 ctx._fx=${xf}
@@ -1826,6 +1846,7 @@ return null
 ${lookaheadMiss(code[ip + 1]!, `ctx._fe=pos;ctx._fx=${xf};return FAIL`)}${emitMark(p, L.buf, L.raw, sinks)}
 ${cfg.probe ? 'const pb=ctx._probe?.best\n' : ''}const v=${child}(input,pos,ctx)
 ${emitRollback(p, L.buf, L.raw, sinks)}
+ctx._fc=false
 ${cfg.probe ? 'if(ctx._probe!==undefined)ctx._probe.best=pb??null\n' : ''}if(v===FAIL){ctx._fe=pos;ctx._fx=${xf};return FAIL}
 EC.e=pos
 return null
@@ -1835,14 +1856,15 @@ return null
       case OP_OPT: {
         const child = link(code[ip + 1]!)
         const clean = !REC && failureRollbackClean(prog, ip)
+        const logs = clean && LOGS && failureLogRollback(prog, ip)
         const p = tmp()
         return `${head}
-${lookaheadMiss(code[ip + 1]!, 'EC.e=pos;return null')}${clean ? '' : emitMark(p, L.buf, L.raw, sinks)}
+${lookaheadMiss(code[ip + 1]!, 'EC.e=pos;return null')}${clean ? logs ? emitLogMark(p) : '' : emitMark(p, L.buf, L.raw, sinks)}
 ctx._fc=false
 const v=${child}(input,pos,ctx)
 if(v===FAIL){
 if(ctx._fc===true)return FAIL
-${clean ? '' : emitRollback(p, L.buf, L.raw, sinks)}
+${clean ? logs ? emitLogRollback(p) : '' : emitRollback(p, L.buf, L.raw, sinks)}
 EC.e=pos
 return null
 }
@@ -2052,6 +2074,13 @@ return FAIL
           const encodedRollbackMask = choiceRollbackMask(prog, ip) ?? -1
           const rollbackMask = REC ? -1 : encodedRollbackMask
           const hasRollback = rollbackMask !== 0
+          // Arms that leave only trivia-log rows: restored from the full mark when
+          // one is taken, else from a mark of the two logs taken as that arm is
+          // entered. An earlier arm that failed was restored, so the lengths there
+          // are the choice's own, and an arm the char gate skips takes no mark.
+          const logMask = rollbackMask === -1 || !LOGS ? 0 : choiceLogRollbackMask(prog, ip)
+          const choiceMark = hasRollback ? emitMark(p, L.buf, L.raw, sinks) : ''
+          const armMark = (i: number): string => !hasRollback && (logMask & (1 << i)) !== 0 ? `${emitLogMark(p)}\n` : ''
           // An always-consuming leading scalar makes a failure at `pos`
           // statically exact: it is the arm's derived opener set, and the
           // choice's own `choiceFx` already concatenates every such set in
@@ -2082,7 +2111,8 @@ return FAIL
           const rollbackFor = (i: number): string =>
             rollbackMask === -1 || (rollbackMask & (1 << i)) !== 0
               ? emitRollback(p, L.buf, L.raw, sinks)
-              : ''
+              : (logMask & (1 << i)) === 0 ? ''
+                : hasRollback ? emitRollback(p, L.buf, L.raw, sinks) : emitLogRollback(p)
           if (choiceCannotCommit(prog, ip) && !startFailureExact) {
             // A successful ordered choice does not expose any losing arm's
             // diagnostic. Snapshot each entered arm's failure in scalar locals;
@@ -2121,7 +2151,7 @@ ctx._fc=false
 ${remember(i, 'at', pretest.sequence.expected.map((value, at) => `${decision}===${~at}?${value}:`).join('') + pretest.sequence.expected.at(-1))}}
 }`
               return `${decision === '' ? '' : `let ${decision}=${pretest?.sequence === undefined ? -1 : 0}\n`}if(${gate}){
-ctx._fc=false
+${armMark(i)}ctx._fc=false
 {const v=${arm}(input,pos,ctx)
 if(v!==FAIL)return v}
 {const at=ctx._fe??pos
@@ -2140,7 +2170,7 @@ ctx._fe=pos;ctx._fx=acc??${choiceFx}
 return FAIL`
             return `${head}
 const c=${directAsciiMask ? 'input.charCodeAt(pos)' : 'lead(input,pos)'}
-${hasRollback ? emitMark(p, L.buf, L.raw, sinks) : ''}
+${choiceMark}
 ${deepAt.map((at, i) => `let ${at},${deepFx[i]}`).join('\n')}
 ${maskable ? `if(c<128){
 const bits=${maskName}[${directAsciiMask ? 'c' : 'c<0?128:c'}]
@@ -2191,7 +2221,7 @@ if(at>best){best=at;acc=undefined}
 if(at===best${startFailureExact ? '&&at>pos' : ''})acc=_accSet(${pretest.sequence.expected.map((value, at) => `${decision}===${~at}?${value}:`).join('')}${pretest.sequence.expected.at(-1)},acc)}
 }`
             return `${decision === '' ? '' : `let ${decision}=${pretest?.sequence === undefined ? -1 : 0}\n`}if((bits&${1 << i})!==0${condition}){
-ctx._fc=false
+${armMark(i)}ctx._fc=false
 {const v=${arm}(input,pos,ctx)
 if(v!==FAIL)return v}
 ${startFailureExact ? '' : `if(best===pos)acc=${catchName}(${i},prev,acc)
@@ -2231,7 +2261,7 @@ if(at>best){best=at;acc=undefined}
 if(at===best${startFailureExact ? '&&at>pos' : ''})acc=_accSet(${pretest.token.expected},acc)
 }${startFailureExact ? '' : `else if(best===pos)acc=_accSet(${expected[i]},acc)`}`
             return `${decision === '' ? '' : `let ${decision}=${pretest?.sequence === undefined ? -1 : 0}\n`}if((${gateRefs[i]}===null||classHas(${gateRefs[i]},c))${condition}){
-ctx._fc=false
+${armMark(i)}ctx._fc=false
 {const v=${arm}(input,pos,ctx)
 if(v!==FAIL)return v}
 {const at=ctx._fe??pos
@@ -2250,7 +2280,7 @@ ${rollbackFor(i)}
           // `choiceFx`, the arms' flatMap, set by set.
           return `${head}
 const c=${directAsciiMask ? 'input.charCodeAt(pos)' : 'lead(input,pos)'}
-${hasRollback ? emitMark(p, L.buf, L.raw, sinks) : ''}
+${choiceMark}
 let acc
 let best=pos
 ${maskable ? `if(c<128){

package/src/table/encode.ts

@@ -190,6 +190,33 @@ class Encoder {
   private choiceRollbackMasks = new Map<number, number>()
   private nonCommittingChoiceSites = new Set<number>()
   private failureRollbackCleanSites = new Set<number>()
+  /**
+   * THE TRIVIA LOGS A FAILED NODE LEAVES. A node drops its own capture frame when
+   * it fails, but the trivia its body recorded into `_triviaLog`/`_rootTriviaLog`
+   * is not part of that frame. So a child the census above counts as clean can
+   * still leave trivia rows when it reaches a node whose body can fail after a
+   * trivia scan (`mayLeavePartialCapture`'s `throughNodes`). These are the arms
+   * and sites that need only the two logs restored. A program with no trivia of
+   * its own records rows only under trivia its caller supplies, so its emitted
+   * assembly takes these marks only for such a parse (`RunCfg.inheritedTrivia`).
+   */
+  private choiceLogRollbackMasks = new Map<number, number>()
+  private failureLogRollbackSites = new Set<number>()
+  /**
+   * HOW EACH CHOICE WAS LOWERED, for the census above. A choice on the ordered
+   * path whose arms cannot commit a failure restores the logs after every arm
+   * it enters (its full mark, or its own two-log mark), so it leaves no rows.
+   * An exclusive dispatch runs its one arm with no mark, and an ordered choice
+   * returns a committed arm's failure before restoring, so either leaves what an
+   * arm leaves. A choice in neither set, or in both, is taken to leave rows. Both
+   * sets are complete only once every rule is encoded, so the trivia-log census
+   * runs then (`logCensus`).
+   */
+  private restoringChoices = new Set<Combinator<unknown>>()
+  private leakingChoices = new Set<Combinator<unknown>>()
+  private logCensus: (() => void)[] = []
+  private readonly restoresArms = (choice: Combinator<unknown>): boolean =>
+    this.restoringChoices.has(choice) && !this.leakingChoices.has(choice)
   /**
    * `reachesSideWriter`'s answers, shared by every `failureNeedsRollback` call
    * for the lifetime of this `Encoder`. Every entry is exact — it depends only on
@@ -816,15 +843,25 @@ class Encoder {
   private failureNeedsRollback(
     p: Combinator<unknown>,
     autoNot: readonly AutoNotCheck[] | null | undefined,
+    throughNodes: false | ((choice: Combinator<unknown>) => boolean),
   ): boolean {
     const refName = (p as unknown as { _ruleName?: string })._ruleName
     const winner = refName === undefined ? undefined : this.winners?.[refName]
     const root = winner !== undefined && !winnerWrapsReference(winner, p) ? winner : p
-    return mayLeavePartialCapture(root, new Set(), true)
+    return mayLeavePartialCapture(root, new Set(), true, throughNodes)
       || this.reachesSideWriter(root)
       || ((autoNot?.length ?? 0) > 0 && capturesLeaf(root))
   }
 
+  /** The rollback census of an `optional()` or `attempt()` row at `ip` over `child`. */
+  private censusTransaction(child: Combinator<unknown>, ip: number): void {
+    if (this.failureNeedsRollback(child, undefined, false)) return
+    this.failureRollbackCleanSites.add(ip)
+    this.logCensus.push(() => {
+      if (this.failureNeedsRollback(child, undefined, this.restoresArms)) this.failureLogRollbackSites.add(ip)
+    })
+  }
+
   /**
    * Whether a field, recovered error, interpreted scanner or unresolvable shape
    * is reachable from `root` — the writer census of `failureNeedsRollback`,
@@ -1671,6 +1708,7 @@ class Encoder {
         }
         const dispIdx = this.disp.length
         this.disp.push(classes)
+        const exclusive = dispatchIsExclusive(classes, this.cc)
         // PER-ARM EXPECTED SETS RIDE ALONG, after the arm offsets. While the best
         // failure remains at the choice position, an arm declined by the driver's
         // char gate contributes the static opener set the interpreter would get by
@@ -1683,10 +1721,7 @@ class Encoder {
         let rollbackMask = kids.length > 31 ? -1 : 0
         if (rollbackMask === 0) {
           for (let i = 0; i < arms.length; i++) {
-            const src = order[i]!
-            if (this.failureNeedsRollback(arms[i]!, d.autoNot[src])) {
-              rollbackMask |= 1 << i
-            }
+            if (this.failureNeedsRollback(arms[i]!, d.autoNot[order[i]!], false)) rollbackMask |= 1 << i
           }
         }
         const head = this.emitHead(OP_CHOICE, 3 + 2 * kids.length)
@@ -1701,9 +1736,19 @@ class Encoder {
           this.code[head + 4 + kids.length + i] = this.expected(deriveExpected(arms[i]!))
         }
         this.choiceRollbackMasks.set(head, rollbackMask)
-        if (arms.every(arm => !mayCommitFailure(arm, rr, this.commitFailureMemo))) {
-          this.nonCommittingChoiceSites.add(head)
+        if (rollbackMask !== -1) {
+          const full = rollbackMask
+          this.logCensus.push(() => {
+            let logMask = 0
+            for (let i = 0; i < arms.length; i++) {
+              if ((full & (1 << i)) === 0 && this.failureNeedsRollback(arms[i]!, d.autoNot[order[i]!], this.restoresArms)) logMask |= 1 << i
+            }
+            if (logMask !== 0) this.choiceLogRollbackMasks.set(head, logMask)
+          })
         }
+        const nonCommitting = arms.every(arm => !mayCommitFailure(arm, rr, this.commitFailureMemo))
+        if (nonCommitting) this.nonCommittingChoiceSites.add(head)
+        ;(!exclusive && nonCommitting ? this.restoringChoices : this.leakingChoices).add(p)
         return head
       }
       case 'many':
@@ -1765,7 +1810,7 @@ class Encoder {
       }
       case 'optional': {
         const head = this.emit(OP_OPT, this.lookaheadGate(d.parser, this.node(d.parser).ip))
-        if (!this.failureNeedsRollback(d.parser, undefined)) this.failureRollbackCleanSites.add(head)
+        this.censusTransaction(d.parser, head)
         return head
       }
       case 'transform': {
@@ -2095,7 +2140,7 @@ class Encoder {
       // lowering was correct only for a choice arm.
       case 'attempt': {
         const inner = this.emit(OP_ATTEMPT, this.node(d.parser).ip, d.contain === true ? 1 : 0)
-        if (!this.failureNeedsRollback(d.parser, undefined)) this.failureRollbackCleanSites.add(inner)
+        this.censusTransaction(d.parser, inner)
         // THE FIRST-SET FAIL-FAST GUARD, lowered as the `OP_GATE` row the `node()`
         // case already uses — it is the same guard, written twice in the
         // interpreter (`attempt.ts:22`, `node.ts:239`). It is NOT merely an
@@ -2295,6 +2340,7 @@ class Encoder {
   finish(): TableProgram {
     if (this.code.length === 0) this.emit(OP_EMPTY)
     this.collapseIndirection()
+    for (const census of this.logCensus) census()
     return ownTableProgram({
       code: this.code, k: this.k, fns: this.fns, cc: this.cc,
       fx: this.fx, disp: this.disp, dsp: this.dsp, rules: this.rules,
@@ -2315,6 +2361,8 @@ class Encoder {
     }, undefined, {
       choiceRollbackMasks: this.choiceRollbackMasks,
       failureRollbackCleanSites: this.failureRollbackCleanSites,
+      choiceLogRollbackMasks: this.choiceLogRollbackMasks,
+      failureLogRollbackSites: this.failureLogRollbackSites,
       nonCommittingChoiceSites: this.nonCommittingChoiceSites,
     })
   }

package/src/table/exec.ts

@@ -1558,8 +1558,10 @@ function makeDriver(
         const mRoot = need ? ctx._rootTriviaLog?.length ?? 0 : 0
         const v = exec(code[ip + 1]!, input, pos, ctx)
         if (need) rollbackTriviaAt(ctx, mRaw, mTl, mLv, mFl, mEr, mLog, mRoot)
+        // A lookahead contains its child's commitment (`not.ts` returns a fresh
+        // result): a generic child's committed failure is cleared here.
+        ctx._fc = savedFc
         if (scalarChild) {
-          ctx._fc = savedFc
           ctx._fe = savedFe
           ctx._fx = savedFx
         }
@@ -1588,7 +1590,8 @@ function makeDriver(
         const mRoot = need ? ctx._rootTriviaLog?.length ?? 0 : 0
         const v = exec(code[ip + 1]!, input, pos, ctx)
         if (need) rollbackTriviaAt(ctx, mRaw, mTl, mLv, mFl, mEr, mLog, mRoot)
-        if (scalarChild) ctx._fc = savedFc
+        // A lookahead contains its child's commitment, as under `not()`.
+        ctx._fc = savedFc
         if (ctx._probe !== undefined) ctx._probe.best = savedProbe ?? null
         // `peek.ts:60` — the ASSERTION's own set, at the assertion's position.
         if (v === FAIL) { ctx._fe = pos; ctx._fx = fx[code[ip + 2]!] as string[]; return FAIL }

package/src/table/program.ts

@@ -531,6 +531,8 @@ type RuntimeTableProgram = TableProgram & {
     readonly choiceRollbackMasks?: ReadonlyMap<number, number>
     readonly nonCommittingChoiceSites?: ReadonlySet<number>
     readonly failureRollbackCleanSites?: ReadonlySet<number>
+    readonly choiceLogRollbackMasks?: ReadonlyMap<number, number>
+    readonly failureLogRollbackSites?: ReadonlySet<number>
   }
 }
 
@@ -542,6 +544,8 @@ export function ownTableProgram(
     readonly choiceRollbackMasks?: ReadonlyMap<number, number>
     readonly failureRollbackCleanSites?: ReadonlySet<number>
     readonly nonCommittingChoiceSites?: ReadonlySet<number>
+    readonly choiceLogRollbackMasks?: ReadonlyMap<number, number>
+    readonly failureLogRollbackSites?: ReadonlySet<number>
   } = {},
 ): TableProgram {
   const runtime = (prog as RuntimeTableProgram)[TABLE_RUNTIME]
@@ -552,6 +556,8 @@ export function ownTableProgram(
       choiceRollbackMasks: caches.choiceRollbackMasks ?? runtime?.choiceRollbackMasks,
       nonCommittingChoiceSites: caches.nonCommittingChoiceSites ?? runtime?.nonCommittingChoiceSites,
       failureRollbackCleanSites: caches.failureRollbackCleanSites ?? runtime?.failureRollbackCleanSites,
+      choiceLogRollbackMasks: caches.choiceLogRollbackMasks ?? runtime?.choiceLogRollbackMasks,
+      failureLogRollbackSites: caches.failureLogRollbackSites ?? runtime?.failureLogRollbackSites,
     },
   } as RuntimeTableProgram
 }
@@ -575,6 +581,19 @@ export function failureRollbackClean(prog: TableProgram, ip: number): boolean {
   return (prog as RuntimeTableProgram)[TABLE_RUNTIME]?.failureRollbackCleanSites?.has(ip) === true
 }
 
+/** The arms of the choice at `ip` the census counts as clean whose failure can
+ * still leave trivia-log rows: a node they reach fails after recording trivia
+ * (`encode.ts` `choiceLogRollbackMasks`). Zero without that authority. */
+export function choiceLogRollbackMask(prog: TableProgram, ip: number): number {
+  return (prog as RuntimeTableProgram)[TABLE_RUNTIME]?.choiceLogRollbackMasks?.get(ip) ?? 0
+}
+
+/** An `optional()` or `attempt()` row `failureRollbackClean` admits whose failed
+ * child can still leave trivia-log rows, for the same reason. */
+export function failureLogRollback(prog: TableProgram, ip: number): boolean {
+  return (prog as RuntimeTableProgram)[TABLE_RUNTIME]?.failureLogRollbackSites?.has(ip) === true
+}
+
 /** A compiled entry, shaped exactly like a codegen rule function. */
 export type TableRule = (input: string, pos: number, ctx: ParseContext) => ParseResult<unknown>
 

package/src/version.ts

@@ -13,4 +13,4 @@
  * Kept in sync with `package.json` by `test/unit/version-sync.test.ts`, which fails
  * the build if this constant drifts from the published version.
  */
-export const PARSEMAN_VERSION = '0.60.2'
+export const PARSEMAN_VERSION = '0.60.4'