parseman 0.61.4 → 0.61.5

package/src/table/encode.ts

@@ -18,7 +18,7 @@ import {
   OP_LIT_TRACK, OP_RX_TRACK, OP_NODE_TRACK, OP_SCOPE, OP_SCOPE_CAP, OP_SCOPE_PLAIN, SCOPE_INHERIT, OP_EXPECT, OP_SEQX, OP_SCAN,
   OP_LIVE, OP_ATTEMPT, OP_LABEL,
   OP_FIELD, OP_DISPATCH, OP_ROUTED, OP_LIT_CI, OP_LIT_CI_TRACK, OP_TOKEN, OP_WITHCTX, OP_GUARD,
-  OP_ADJ, OP_GREEDY, OP_REJECT, OP_ARMGATE, OP_RGATE, OP_COV, OP_GAPGATE, OP_RGATE2, OP_STRICT,
+  OP_ADJ, OP_GREEDY, OP_REJECT, OP_ARMGATE, OP_RGATE, OP_COV, OP_GAPGATE, OP_RGATE2, OP_STRICT, OP_CONT, OP_SEED,
   OP_LEX_BODY, OP_LEX_PROGRAM,
 } from './ops.ts'
 import { adjacencyExpected } from '../combinators/adjacency.ts'
@@ -41,6 +41,7 @@ import {
   collectLexicalCapabilities, winnerWrapsReference,
 } from '../compiler/token-capability.ts'
 import { directExecutableTokenBody, type ExecutableLexNode } from '../compiler/token-alphabet.ts'
+import { continuationGate } from './late-frame.ts'
 
 /**
  * Can `emitConst` print this? Mirrors the guard in `emit.ts` — scalars, arrays
@@ -159,6 +160,30 @@ function isGap(p: Combinator<unknown>): boolean {
   })
 }
 
+/**
+ * Does this combinator publish exactly one node child and return that same value, on
+ * every path? Only then can a late frame (`OP_CONT`) take the corner's publication back
+ * and be the child it would have captured. A `transform` returns what its function
+ * makes of the child; a `dispatch` returns `[key, arm]` and its selector can publish too;
+ * an `expect` publishes nothing when its child fails. A `ref()` not defined yet proves nothing.
+ */
+function publishesOneNode(p: Combinator<unknown>, rr: RefResolver | undefined, seen = new Set<Combinator<unknown>>()): boolean {
+  if (seen.has(p)) return true
+  seen.add(p)
+  const d = p._def
+  switch (d.tag) {
+    case 'node': return true
+    case 'lazy': {
+      let target = winnerForRef(p, rr)
+      if (target === undefined) { try { target = d.thunk() } catch { return false } }
+      return publishesOneNode(target, rr, seen)
+    }
+    case 'choice': return d.parsers.every(a => publishesOneNode(a, rr, seen))
+    case 'grammar': case 'withCtx': case 'label': return publishesOneNode(d.parser, rr, seen)
+    default: return false
+  }
+}
+
 class Encoder {
   code: number[] = []
   k: unknown[] = []
@@ -1859,7 +1884,8 @@ class Encoder {
         // without, on match and on failure.
         // An AUTHORING error, not a table gap — raise what the other two engines do.
         if (d.type === undefined) missingInferredType()
-        const child = this.node(d.parser).ip
+        const late = this.opensLate(d)
+        const child = late ? this.lateBody(d.parser._def as Extract<ParserDef, { tag: 'sequence' }>) : this.node(d.parser).ip
         // Capture flags, resolved HERE from the reducer's declared arity using the
         // same analysis codegen runs (`src/compiler/build-arity.ts`). `hostMode:
         // 'cst'` forces them on, exactly as the emitted `cstOut` path does. The
@@ -1928,7 +1954,15 @@ class Encoder {
         // `deriveExpected(body)`, which for a body with nullable leading terms
         // names tokens the run would have skipped — `['"@"', '"(", '"x"']` where
         // running the body reports `['"x"']` alone.
-        if (matchesEmpty(d.parser, new Set(), this.refResolver())) return body
+        // `child` is `SEQ 2 seed tail …`, and `seed` is `SEED corner`. A tail with no first
+        // test has nothing to decide on: the node runs as written, over its corner.
+        let run = body
+        if (late) {
+          const corner = this.code[this.code[child + 2]! + 1]!
+          if (continuationGate(this.code, this.code[child + 3]!) === undefined) this.code[child + 2] = corner
+          else run = this.emit(OP_CONT, corner, body, this.code[child + 3]!)
+        }
+        if (matchesEmpty(d.parser, new Set(), this.refResolver())) return run
         // A node whose BODY is a bare cross-rule ref (`node('X', g.Y, …)`) caches its
         // first set from the ref's base binding. Under a compose OVERRIDE the winner
         // for `Y` may be WIDER, so the cached set gates out input the merged rule
@@ -1940,11 +1974,11 @@ class Encoder {
         // that set match what entering the body reports (a `keywords()` term, a
         // list's item set, an optional lead are only known by running it).
         const fs = this.gateFirstSet(d.parser, d.guarded === true, false)
-        if (fs.kind === 'any') return body
+        if (fs.kind === 'any') return run
         const cls = this.charClass(fs)
-        if (cls < 0) return body
+        if (cls < 0) return run
         const e = deriveExpected(d.parser)
-        return this.emit(OP_GATE, cls, body, this.expected(e.length > 0 ? e : [d.parser._tag]))
+        return this.emit(OP_GATE, cls, run, this.expected(e.length > 0 ? e : [d.parser._tag]))
       }
       case 'token': {
         // A token runs its body with trivia cleared (`token.ts`), so its body is
@@ -2271,6 +2305,33 @@ class Encoder {
     }
   }
 
+  /**
+   * `node(sequence(corner, tail), build, { collapse: true })` with a tail that can take
+   * nothing and a corner that is one node: the frame can open after the corner, only if
+   * the tail can start. Any other node keeps its frame from the start, and so does one
+   * with no reducer, whose host can scope the corner's trivia to its type.
+   */
+  private opensLate(d: Extract<ParserDef, { tag: 'node' }>): boolean {
+    if (this.plan !== undefined || d.collapse !== true || d.build === undefined || d.trailingTrivia === true) return false
+    const seq = d.parser._def
+    return seq.tag === 'sequence' && seq.parsers.length === 2
+      && matchesEmpty(seq.parsers[1]!, new Set(), this.refResolver())
+      && publishesOneNode(seq.parsers[0]!, this.refResolver())
+  }
+
+  /** The body of a late frame: the sequence as written, its corner behind `OP_SEED`. */
+  private lateBody(seq: Extract<ParserDef, { tag: 'sequence' }>): number {
+    const seed = this.emit(OP_SEED, this.node(seq.parsers[0]!).ip)
+    const tailIp = this.node(seq.parsers[1]!).ip
+    const head = this.emitHead(seq.valueUnused ? OP_SEQV : OP_SEQ, 5)
+    this.code[head + 1] = 2
+    this.code[head + 2] = seed
+    this.code[head + 3] = tailIp
+    this.code[head + 4] = this.followClass(seq.parsers, 0)
+    this.code[head + 5] = this.followClass(seq.parsers, 1)
+    return head
+  }
+
   /**
    * Peephole: an `OP_RULE` row is pure indirection — a switch dispatch and a
    * call frame to do nothing but jump. Rewrite every child slot that points at
@@ -2306,6 +2367,8 @@ class Encoder {
         case OP_REJECT: return [ip + 1]
         case OP_REP: case OP_REPV: return this.code[ip + 4]! >= 0 ? [ip + 1, ip + 4] : [ip + 1]
         case OP_STRICT: return [ip + 1, ip + 2]
+        case OP_CONT: return [ip + 1, ip + 2, ip + 3]
+        case OP_SEED: return [ip + 1]
         default: return []
       }
     }

package/src/table/exec.ts

@@ -21,7 +21,7 @@ import {
   OP_LIT_TRACK, OP_RX_TRACK, OP_NODE_TRACK, OP_SCOPE, OP_SCOPE_CAP, OP_SCOPE_PLAIN, SCOPE_INHERIT, OP_EXPECT, OP_SEQX, OP_SCAN,
   OP_LIVE,
   OP_FIELD, OP_DISPATCH, OP_ROUTED, OP_LIT_CI, OP_LIT_CI_TRACK, OP_TOKEN, OP_WITHCTX, OP_GUARD, OP_ATTEMPT, OP_LABEL,
-  OP_COV, OP_GAPGATE, OP_STRICT,
+  OP_COV, OP_GAPGATE, OP_STRICT, OP_CONT, OP_SEED,
   OP_ADJ, OP_GREEDY, OP_REJECT, OP_ARMGATE, OP_RGATE, OP_RGATE2,
   OP_LEX_BODY, OP_LEX_PROGRAM,
 } from './ops.ts'
@@ -477,6 +477,12 @@ function makeDriver(
       case OP_STRICT:
         ip = code[ctx._tolerant || ctx._probe !== undefined ? ip + 2 : ip + 1]!; continue rows
 
+      // The reference runs a late frame's node as written, and is the leg it is compared with.
+      case OP_CONT:
+        ip = code[ip + 2]!; continue rows
+      case OP_SEED:
+        ip = code[ip + 1]!; continue rows
+
       case OP_LIVE: {
         // A HAND-WRITTEN COMBINATOR, run through its own `.parse` — mirrors
         // codegen's `emitRuntimeFallback`. Its result IS the interpreter's, so

package/src/table/fail-clean.ts

@@ -24,7 +24,7 @@ import {
   OP_ATTEMPT, OP_CHOICE, OP_COV, OP_DISPATCH, OP_EMPTY, OP_EXPECT, OP_FIELD, OP_GATE, OP_GUARD, OP_LABEL,
   OP_LEAF, OP_LIT, OP_LIT_CI, OP_LIT_CI_TRACK, OP_LIT_TRACK, OP_NODE, OP_NODE_TRACK, OP_NOT, OP_OPT, OP_PEEK,
   OP_REP, OP_REPV, OP_ROUTED, OP_RULE, OP_RX, OP_RX_TRACK, OP_SCOPE, OP_SCOPE_CAP, OP_SCOPE_PLAIN, OP_SEQ,
-  OP_SEQV, OP_SEQX, OP_SOURCE_LEAF, OP_TOKEN, OP_WITHCTX, OP_XFORM, OP_GAPGATE,
+  OP_SEQV, OP_SEQX, OP_SOURCE_LEAF, OP_TOKEN, OP_WITHCTX, OP_XFORM, OP_GAPGATE, OP_CONT, OP_SEED,
 } from './ops.ts'
 
 export type FailClean = (ip: number) => boolean
@@ -113,6 +113,13 @@ export function failCleanAnalysis(code: ArrayLike<number>): FailClean {
       case OP_ROUTED:
         out = code[ip + 1]! < 0 || clean(code[ip + 1]!)
         break
+      case OP_SEED:
+        out = clean(code[ip + 1]!)
+        break
+      // The corner may fail; the wrapper fails only through a tail that can.
+      case OP_CONT:
+        out = clean(code[ip + 1]!) && neverFails(code[ip + 3]!)
+        break
       // An arm's failure and the fallback's are committed; only the selector's is
       // not, and without a fallback an unknown key fails after the selector wrote.
       case OP_DISPATCH:

package/src/table/hold-gap.ts

@@ -17,7 +17,7 @@
 import {
   OP_ADJ, OP_ATTEMPT, OP_CHOICE, OP_COV, OP_EXPECT, OP_FIELD, OP_GATE, OP_LABEL, OP_LEAF, OP_LIT, OP_LIT_CI,
   OP_NODE, OP_NOT, OP_PEEK, OP_REP, OP_REPV, OP_RULE, OP_RX, OP_SCOPE, OP_SCOPE_CAP, OP_SCOPE_PLAIN, OP_SEQ,
-  OP_SEQV, OP_SEQX, OP_SOURCE_LEAF, OP_TOKEN, OP_WITHCTX, OP_XFORM, OP_GAPGATE,
+  OP_SEQV, OP_SEQX, OP_SOURCE_LEAF, OP_TOKEN, OP_WITHCTX, OP_XFORM, OP_GAPGATE, OP_CONT, OP_SEED,
 } from './ops.ts'
 import { childSlots } from './child-slots.ts'
 import { leadingTriviaFieldAt } from './field-spec.ts'
@@ -48,6 +48,10 @@ export function rowNullable(code: ArrayLike<number>, k: readonly unknown[], ip:
     out = kids.length === 0 || kids.some(c => rowNullable(code, k, c, memo))
   } else if (op === OP_REP || op === OP_REPV) {
     out = code[ip + 2] === 0 || rowNullable(code, k, code[ip + 1]!, memo)
+  } else if (op === OP_SEED) {
+    out = rowNullable(code, k, code[ip + 1]!, memo)
+  } else if (op === OP_CONT) {
+    out = rowNullable(code, k, code[ip + 1]!, memo) && rowNullable(code, k, code[ip + 3]!, memo)
   } else if (WRAPPERS.has(op)) {
     if (childSlots(code, ip, kids) && kids.length === 1) out = rowNullable(code, k, kids[0]!, memo)
   }

package/src/table/late-frame.ts

@@ -0,0 +1,54 @@
+/**
+ * WHAT THE TAIL OF A LATE FRAME CAN START WITH, read off the row that runs it.
+ *
+ * `OP_CONT` asks, after the corner, whether the tail could do anything at the next
+ * token. The answer is the repeat's or the optional's own first test (`OP_REP` ip+7
+ * and its `OP_GAPGATE`, an `OP_OPT` over `OP_GATE`), so it holds exactly where the row
+ * would have ended without entering its item. Any other tail has no answer, and with
+ * no answer the frame always opens: that site runs as written.
+ */
+import { OP_GAPGATE, OP_GATE, OP_OPT, OP_REP, OP_REPV, OP_RULE, OP_WITHCTX } from './ops.ts'
+import { TRI_NONE } from './site-labels.ts'
+
+export type ContinuationGate = {
+  /** Class of the first character, past any trivia. */
+  readonly cls: number
+  /** The regex row of a leading gap, or −1. */
+  readonly gapRow: number
+  /** Class of the character after the gap. */
+  readonly follow: number
+  /** Class of an arm that opens without the gap, or −1. */
+  readonly other: number
+}
+
+export function continuationGate(code: ArrayLike<number>, ip: number): ContinuationGate | undefined {
+  let at = ip
+  for (let n = 0; n < 8; n++) {
+    const op = code[at]
+    if (op === OP_RULE) at = code[at + 1]!
+    else if (op === OP_WITHCTX) at = code[at + 2]!
+    else break
+  }
+  const op = code[at]
+  if (op === OP_OPT) {
+    const c = code[at + 1]!
+    return code[c] === OP_GATE ? { cls: code[c + 1]!, gapRow: -1, follow: -1, other: -1 } : undefined
+  }
+  if (op !== OP_REP && op !== OP_REPV) return undefined
+  // A separated list keeps ip+7 for its recovery sentinel; a mandatory item reports.
+  if (code[at + 2] !== 0 || code[at + 4]! >= 0 || code[at + 7]! < 0) return undefined
+  const item = code[at + 1]!
+  return code[item] === OP_GAPGATE
+    ? { cls: code[at + 7]!, gapRow: code[item + 2]!, follow: code[item + 3]!, other: code[item + 4]! }
+    : { cls: code[at + 7]!, gapRow: -1, follow: -1, other: -1 }
+}
+
+/**
+ * Can this assembly decide at this `OP_CONT`? It needs the tail's test, and the
+ * trivia the label names: a scope it cannot name, or one with no scanner, would cost a
+ * hit more calls than the frame it saves on a miss.
+ */
+export function canDecide(code: ArrayLike<number>, ip: number, tri: number, triviaScan: readonly unknown[]): boolean {
+  if (tri !== TRI_NONE && (tri < 0 || triviaScan[tri] == null)) return false
+  return continuationGate(code, code[ip + 3]!) !== undefined
+}

package/src/table/ops.ts

@@ -509,6 +509,23 @@ export const OP_RGATE2 = 47
  * assembly resolves it to one child; it has no body of its own.
  */
 export const OP_STRICT = 48
+/**
+ * `CONT corner wrapper tail` — a `collapse` node that is `sequence(corner, tail)`, with
+ * a tail that can take nothing and a corner that is one node. A strict assembly runs
+ * `corner` in the PARENT's collector and looks at the next token. Where `tail` cannot
+ * start there, the corner is the result: no frame, no children, no builder. Where it
+ * can, the corner's publication is taken back and `wrapper`'s frame opens around it and
+ * `tail`, in this one function. `tail` is read for the decision (`late-frame.ts`). Where
+ * no decision can be made, and under recovery, a probe or the reference driver, this
+ * is `wrapper`, the node as written.
+ */
+export const OP_CONT = 49
+/**
+ * `SEED corner` — marks the first term of `wrapper`'s body. Run, it is `corner`. It
+ * exists so the labels can leave out the edge from a late frame to its corner, which
+ * never runs there.
+ */
+export const OP_SEED = 50
 /**
  * `DISPATCH sel d other otherRouted n a1 … an` — `dispatch()`.
  *
@@ -548,4 +565,6 @@ export const OP_NAMES: Record<number, string> = {
   [OP_GAPGATE]: 'GAPGATE',
   [OP_RGATE2]: 'RGATE2',
   [OP_STRICT]: 'STRICT',
+  [OP_CONT]: 'CONT',
+  [OP_SEED]: 'SEED',
 }

package/src/table/select-hot.ts

@@ -15,7 +15,8 @@
  * A build given a corpus (`assemblyProfile`) ranks by measured calls instead.
  */
 import { childSlots } from './child-slots.ts'
-import { OP_CHOICE, OP_DISPATCH, OP_GAPGATE, OP_REP, OP_REPV } from './ops.ts'
+import { continuationGate } from './late-frame.ts'
+import { OP_CHOICE, OP_CONT, OP_DISPATCH, OP_GAPGATE, OP_REP, OP_REPV } from './ops.ts'
 
 const LOOP = 8
 const ROUNDS = 60
@@ -90,6 +91,11 @@ function staticWeights(code: readonly number[], entries: readonly number[]): Map
       kids.length = 0
       childSlots(code, ip, kids)
       const op = code[ip]
+      // A late frame is lowered with its wrapper in one function, calling these two.
+      if (op === OP_CONT && continuationGate(code, code[ip + 3]!) !== undefined) {
+        kids.length = 0
+        kids.push(code[ip + 1]!, code[ip + 3]!)
+      }
       const scale = op === OP_REP || op === OP_REPV ? LOOP
         : op === OP_CHOICE || op === OP_DISPATCH ? 1 / Math.max(1, kids.length)
         : 1

package/src/table/site-labels.ts

@@ -60,7 +60,7 @@
  * label licenses at a leaf is dropping the TEST — never the capture.
  */
 import {
-  OP_ATTEMPT, OP_CHOICE, OP_DISPATCH, OP_EXPECT, OP_FIELD, OP_GATE, OP_LABEL, OP_LEAF, OP_LIT, OP_SOURCE_LEAF,
+  OP_CONT, OP_SEED, OP_ATTEMPT, OP_CHOICE, OP_DISPATCH, OP_EXPECT, OP_FIELD, OP_GATE, OP_LABEL, OP_LEAF, OP_LIT, OP_SOURCE_LEAF,
   OP_NODE, OP_NODE_TRACK, OP_NOT, OP_OPT, OP_PEEK, OP_REP, OP_REPV, OP_ROUTED, OP_RULE,
   OP_SCAN, OP_SCOPE, OP_SCOPE_CAP, OP_SCOPE_PLAIN, SCOPE_INHERIT, OP_SEQ, OP_SEQV, OP_SEQX, OP_TOKEN, OP_XFORM,
 } from './ops.ts'
@@ -161,7 +161,7 @@ export function passCollapse(code: Int32Array, ip: number, hostCst: boolean): bo
     else if (op === OP_RULE) at = code[at + 1]!
     else break
   }
-  return code[at] === OP_NODE
+  return code[at] === OP_NODE || code[at] === OP_CONT
 }
 
 /**
@@ -266,7 +266,11 @@ function transfer(code: Int32Array, ip: number, at: SiteLabel, hostCst: boolean)
 }
 
 /** A site's label, or `TOP` for anything the walk did not reach. */
-export type SiteLabels = { at: (ip: number) => SiteLabel }
+export type SiteLabels = {
+  at: (ip: number) => SiteLabel
+  /** The `OP_CONT` sites whose corner runs in the parent's collector in every engine. */
+  late: ReadonlySet<number>
+}
 
 /**
  * Compute a label for every site reachable from `roots`.
@@ -284,10 +288,11 @@ export function computeSiteLabels(
   code: Int32Array,
   roots: Iterable<number>,
   hostCst: boolean,
+  /** Does this assembly run this `OP_CONT` late, in EVERY engine it holds? Absent: none. */
+  late?: (ip: number, tri: number) => boolean,
 ): SiteLabels {
   const labels = new Map<number, SiteLabel>()
   const work: number[] = []
-
   const push = (ip: number, l: SiteLabel): void => {
     const cur = labels.get(ip)
     if (cur === undefined) {
@@ -300,19 +305,38 @@ export function computeSiteLabels(
     labels.set(ip, next)
     work.push(ip)
   }
-
-  for (const r of roots) push(r, TOP)
-
+  // A corner a late frame never runs takes no label from that frame. So every frame is
+  // late until a site proves undecidable, when its seed's edge joins the same fixpoint: a
+  // frame does not change the trivia in force, so no other site's answer moves.
+  const opened = new Set<number>()
   const kids: number[] = []
-  while (work.length > 0) {
-    const ip = work.pop()!
-    const down = transfer(code, ip, labels.get(ip)!, hostCst)
-    kids.length = 0
-    childSlots(code, ip, kids)
-    for (const c of kids) push(c, down)
+  const lateIps = new Set<number>()
+  for (const r of roots) push(r, TOP)
+  for (let pass = 0; pass < 2; pass++) {
+    while (work.length > 0) {
+      const ip = work.pop()!
+      const down = transfer(code, ip, labels.get(ip)!, hostCst)
+      kids.length = 0
+      if (late === undefined || code[ip] !== OP_SEED || opened.has(ip)) childSlots(code, ip, kids)
+      for (const c of kids) push(c, down)
+    }
+    if (late === undefined || pass === 1) break
+    for (const [ip, l] of [...labels]) {
+      if (code[ip] !== OP_CONT) continue
+      if (late(ip, l.tri)) lateIps.add(ip)
+      else {
+        const seed = seedOf(code, ip)
+        opened.add(seed)
+        work.push(seed)
+      }
+    }
   }
+  return { at: (ip: number): SiteLabel => labels.get(ip) ?? TOP, late: lateIps }
+}
 
-  return { at: (ip: number): SiteLabel => labels.get(ip) ?? TOP }
+/** `CONT corner wrapper tail`, `NODE … body`, `SEQ n seed tail`. */
+function seedOf(code: ArrayLike<number>, cont: number): number {
+  return code[code[code[cont + 2]! + 2]! + 2]!
 }
 
 /**

package/src/version.ts

@@ -13,4 +13,4 @@
  * Kept in sync with `package.json` by `test/unit/version-sync.test.ts`, which fails
  * the build if this constant drifts from the published version.
  */
-export const PARSEMAN_VERSION = '0.61.4'
+export const PARSEMAN_VERSION = '0.61.5'