parseman 0.55.4 → 0.57.0

package/src/table/encode.ts

@@ -1,3 +1,4 @@
+import { scanRefs } from './program.ts'
 import type { AutoNotCheck, Combinator, FirstSet, ParserDef } from '../types.ts'
 import { classifyFinalChoice, firstSetOf, intersects, matchesEmpty, union, type RefResolver } from '../combinators/first-set.ts'
 import { childrenOf } from '../analysis/gating.ts'
@@ -31,7 +32,7 @@ import {
   LEX_NODE_SEQUENCE2, LEX_NODE_SEQUENCE3, LEX_NODE_SEQUENCE4, LEX_NODE_SEQUENCE5,
   LEX_NODE_TERMINAL,
 } from './lex-program.ts'
-import { covKindCode, encodeClassSpec, ownTableProgram } from './program.ts'
+import { covKindCode, dispatchIsExclusive, encodeClassSpec, ownTableProgram } from './program.ts'
 import type { GrammarCoveragePlan } from '../compiler/grammar-coverage-ids.ts'
 import { directArrayProjection } from '../compiler/direct-projection.ts'
 import {
@@ -683,6 +684,7 @@ class Encoder {
       case 'expect':
       case 'recover':
       case 'scanTo':
+      case 'lines':
       case 'unknown':
         return true
       case 'lazy': {
@@ -697,6 +699,38 @@ class Encoder {
     }
   }
 
+  /**
+   * THE FIRST-CHARACTER GATE OF AN `optional`, `not` OR `peek` CHILD.
+   *
+   * A non-nullable child cannot match at a character outside its first set, so
+   * the wrapper's outcome there is already decided: `optional` yields `null`,
+   * `not` succeeds, `peek` fails with its own set. The interpreter's `optional`
+   * makes that test itself (`repeat.ts` `startsFirstSet`); the table entered the
+   * child — a rollback mark, the call, the child's own failure — to learn it.
+   *
+   * The gate is an `OP_GATE` row around the child, so every engine runs it
+   * unchanged, a probe or tolerant parse forwards through it as for a node's
+   * gate, and the wrapper's row keeps its layout. The engines test the class
+   * before taking the wrapper's mark. The first set is resolved through
+   * `winners`, the analysis a choice arm's gate uses, so a composed rule's set
+   * is the merged one. A scalar `not` child stays bare: `scalarTerminalNotChild`
+   * already reduces it to one recognizer call.
+   */
+  private lookaheadGate(p: Combinator<unknown>, child: number): number {
+    // By opcode FAMILY: a line-tracked variant spells the same terminal
+    // `OP_LIT_TRACK`/`OP_RX_TRACK`, and every variant of one grammar must lay
+    // down the same words (`foldPrograms`).
+    const op = this.code[child]
+    if (op === OP_GATE || op === OP_LIT || op === OP_LIT_CI || op === OP_LIT_TRACK || op === OP_LIT_CI_TRACK
+      || op === OP_RX || op === OP_RX_TRACK) return child
+    const rr = this.refResolver()
+    if (matchesEmpty(p, new Set(), rr)) return child
+    const cls = this.charClass(firstSetOf(p, new Set(), rr, this.activeTrivia))
+    if (cls < 0) return child
+    const e = deriveExpected(p)
+    return this.emit(OP_GATE, cls, child, this.expected(e.length > 0 ? e : [p._tag]))
+  }
+
   /** A first set becomes a char class string, or −1 for `any`. */
   private charClass(fs: FirstSet): number {
     if (fs.kind !== 'ranges' || fs.ranges.length === 0) return -1
@@ -713,14 +747,21 @@ class Encoder {
     return i
   }
 
-  /** Bind a guard only when its class is already part of the program. Repeat
-   * gating must not grow the compact artifact merely to avoid a failed call. */
-  private existingCharClass(fs: FirstSet): number {
-    if (fs.kind !== 'ranges' || fs.ranges.length === 0) return -1
-    const hit = this.ccIndex.get(encodeClassSpec(fs.ranges))
-    // This reuses the recovery row's existing `-1` slot. Keep its decimal width
-    // at two bytes or less so a speed hint cannot grow a compact artifact.
-    return hit !== undefined && hit < 100 ? hit : -1
+  /**
+   * The first set a node's or an attempt's fail-fast gate tests. A bare
+   * cross-rule ref resolves through `winners` (a compose OVERRIDE may widen
+   * it). Any other body keeps its construction-time `_meta.firstSet`, except
+   * where that is `any`: under `compose()` a body that reaches a rule of
+   * another piece is built before that rule exists, so its cached set is
+   * `any` and the node got NO gate (jess's `CompoundSelector` was entered at
+   * `{`, `Call` at `}`). The resolved set is the one a choice arm's gate uses,
+   * over the final merged map.
+   */
+  private gateFirstSet(p: Combinator<unknown>): FirstSet {
+    const cached = p._meta.firstSet
+    return p._def.tag === 'lazy' || cached.kind === 'any'
+      ? firstSetOf(p, new Set(), this.refResolver(), this.activeTrivia)
+      : cached
   }
 
   /**
@@ -987,6 +1028,17 @@ class Encoder {
       open: bal.open,
       close: bal.close,
     }))
+    // `lines()` and `columns()` lower as their constructor arguments too: the
+    // table rebuilds each and runs it, the body and prefixes being table subtrees.
+    if (d.tag === 'lines') {
+      return this.emit(OP_SCAN, this.scanSlot({
+        kind: 2, flags: 0, skip: [],
+        prefix: this.subtree(d.prefix), body: this.subtree(d.body),
+        ...(d.first === undefined ? {} : { first: this.subtree(d.first) }),
+        ...(d.lazy === undefined ? {} : { lazy: this.subtree(d.lazy) }),
+      }))
+    }
+    if (d.tag === 'columns') return this.emit(OP_SCAN, this.scanSlot({ kind: 3, flags: 0, skip: [], n: d.n }))
     if (d.tag === 'scanTo') {
       const sentDef = d.sentinel._def
       return this.emit(OP_SCAN, this.scanSlot({
@@ -1189,6 +1241,19 @@ class Encoder {
             if (transactionalFirst[i] !== undefined) classes[i] = -1
           }
         }
+        // A STATE-GATED ARM ON THE ORDERED PATH IS ALWAYS ENTERED. The interpreter
+        // skips a gated-off arm and contributes nothing for it (choice.ts
+        // firstMatch), and gates it on only by running it. A char-gated skip would
+        // instead add the arm's static opener set whatever the gate says, so an
+        // ordered site leaves the decision to `OP_ARMGATE`. An exclusive site keeps
+        // its classes: there the dispatched arm's gate failing IS the choice
+        // failing, with that arm's set, as in the interpreter's disjoint path.
+        if (d.gates.some(g => g !== null && g !== undefined) && !dispatchIsExclusive(classes, this.cc)) {
+          for (let k = 0; k < classes.length; k++) {
+            const gate = d.gates[order[k]!]
+            if (gate !== null && gate !== undefined) classes[k] = -1
+          }
+        }
         const dispIdx = this.disp.length
         this.disp.push(classes)
         // PER-ARM EXPECTED SETS RIDE ALONG, after the arm offsets. While the best
@@ -1232,7 +1297,7 @@ class Encoder {
         const rr = this.refResolver()
         const itemClass = matchesEmpty(d.parser, new Set(), rr)
           ? -1
-          : this.existingCharClass(firstSetOf(d.parser, new Set(), rr, this.activeTrivia))
+          : this.charClass(firstSetOf(d.parser, new Set(), rr, this.activeTrivia))
         const op = d.valueUnused ? OP_REPV : OP_REP
         const min = d.tag === 'many' ? 0 : d.min
         // ip + 6 is the ITEM's expected set and ip + 7 the separator's sentinel
@@ -1275,7 +1340,7 @@ class Encoder {
           : this.emit(OP_REP, child, d.min, d.max ?? -1, sep, flags)
       }
       case 'optional': {
-        const head = this.emit(OP_OPT, this.node(d.parser).ip)
+        const head = this.emit(OP_OPT, this.lookaheadGate(d.parser, this.node(d.parser).ip))
         if (!this.failureNeedsRollback(d.parser, undefined)) this.failureRollbackCleanSites.add(head)
         return head
       }
@@ -1396,16 +1461,14 @@ class Encoder {
         // `deriveExpected(body)`, which for a body with nullable leading terms
         // names tokens the run would have skipped — `['"@"', '"(", '"x"']` where
         // running the body reports `['"x"']` alone.
-        if (matchesEmpty(d.parser)) return body
+        if (matchesEmpty(d.parser, new Set(), this.refResolver())) return body
         // A node whose BODY is a bare cross-rule ref (`node('X', g.Y, …)`) caches its
         // first set from the ref's base binding. Under a compose OVERRIDE the winner
         // for `Y` may be WIDER, so the cached set gates out input the merged rule
         // accepts (the reducer reroutes via `winners`, but this recognizer gate did
         // not). Resolve the ref's gate through `winners`, exactly as the `lazy` case
         // resolves the call. Non-ref bodies keep the cached set (byte-identical).
-        const fs = d.parser._def.tag === 'lazy'
-          ? firstSetOf(d.parser, new Set(), this.refResolver(), this.activeTrivia)
-          : d.parser._meta.firstSet
+        const fs = this.gateFirstSet(d.parser)
         if (fs.kind === 'any') return body
         const cls = this.charClass(fs)
         if (cls < 0) return body
@@ -1545,7 +1608,7 @@ class Encoder {
       // other engine reported `['not(literal)']`. It hid because `expected` is a
       // TOP-LEVEL field on `RunResult` and is NOT part of the identity digest.
       case 'not':
-        return this.emit(OP_NOT, this.node(d.parser).ip, this.expected(assertionFailureExpected(false, d.parser._tag)))
+        return this.emit(OP_NOT, this.lookaheadGate(d.parser, this.node(d.parser).ip), this.expected(assertionFailureExpected(false, d.parser._tag)))
       case 'guard':
         // `'gate'`, the public name — see gate.ts. The def tag stays `'guard'`.
         return this.emit(OP_GUARD, this.fn(d.predicate, d.predSrc ?? null), this.expected(['gate']))
@@ -1590,7 +1653,7 @@ class Encoder {
         // lookahead's failure is "the guard did not hold", and naming the body's
         // internals offers a token the parse never asked the author for. The
         // table propagated the body's set instead.
-        return this.emit(OP_PEEK, this.node(d.parser).ip, this.expected(assertionFailureExpected(true, d.parser._tag)))
+        return this.emit(OP_PEEK, this.lookaheadGate(d.parser, this.node(d.parser).ip), this.expected(assertionFailureExpected(true, d.parser._tag)))
       // A TRANSACTION IS A ROW. See `OP_ATTEMPT` for why the transparent
       // lowering was correct only for a choice arm.
       case 'attempt': {
@@ -1606,10 +1669,8 @@ class Encoder {
         // not a slower equivalent.
         // Same override-aware resolution as the node gate: a bare-ref inner resolves
         // its gate through `winners` so a widened winner is not gated out.
-        const fs = d.parser._def.tag === 'lazy'
-          ? firstSetOf(d.parser, new Set(), this.refResolver(), this.activeTrivia)
-          : d.parser._meta.firstSet
-        if (fs.kind === 'any' || matchesEmpty(d.parser)) return inner
+        const fs = this.gateFirstSet(d.parser)
+        if (fs.kind === 'any' || matchesEmpty(d.parser, new Set(), this.refResolver())) return inner
         const cls = this.charClass(fs)
         if (cls < 0) return inner
         const e = deriveExpected(d.parser)
@@ -1762,12 +1823,7 @@ class Encoder {
     // interiors uncollapsed and — worse — left the spec's own offset pointing at
     // a trampoline whose target had been rewritten around it.
     const refs: SubtreeRef[] = []
-    for (const s of this.scans) {
-      if (s.sentinel !== undefined) refs.push(s.sentinel)
-      if (s.recoverAt !== undefined) refs.push(s.recoverAt)
-      if (s.stopAt !== undefined) refs.push(s.stopAt)
-      refs.push(...s.skip)
-    }
+    for (const s of this.scans) refs.push(...scanRefs(s))
     for (const set of this.scanSkipSets) refs.push(...set)
     const stack = [...Object.values(this.rules), ...refs.map(r => r[0])]
     while (stack.length > 0) {
@@ -1788,6 +1844,10 @@ class Encoder {
       ...(s.sentinel === undefined ? {} : { sentinel: res(s.sentinel) }),
       ...(s.recoverAt === undefined ? {} : { recoverAt: res(s.recoverAt) }),
       ...(s.stopAt === undefined ? {} : { stopAt: res(s.stopAt) }),
+      ...(s.prefix === undefined ? {} : { prefix: res(s.prefix) }),
+      ...(s.body === undefined ? {} : { body: res(s.body) }),
+      ...(s.first === undefined ? {} : { first: res(s.first) }),
+      ...(s.lazy === undefined ? {} : { lazy: res(s.lazy) }),
     }))
     this.scanSkipSets = this.scanSkipSets.map(set => set.map(res))
   }

package/src/table/exec-baseline.ts

@@ -487,9 +487,7 @@ function makeDriver(
           children: readonly unknown[], fields: undefined, span: { start: number; end: number },
           rawChildren: readonly unknown[], triviaLog: readonly number[], state: unknown,
         ) => unknown
-        const st = (flags & 8) !== 0 && ctx.state !== undefined
-          ? Object.assign({}, ctx.state as Record<string, unknown>)
-          : undefined
+        const st = (flags & 8) !== 0 ? ctx.state : undefined
         const nd = build(
           cap.children, undefined,
           code[ip] === OP_NODE_TRACK ? spanLines(ctx, pos, end) : { start: pos, end },

package/src/table/exec.ts

@@ -1,3 +1,4 @@
+import { rebuildLineScan } from './line-scan.ts'
 import type { Combinator, FieldMap, FirstSet, ParseContext, ParseResult, ParserDef } from '../types.ts'
 import { balanced, scanTo } from '../combinators/scanTo.ts'
 import { buildFieldMap } from '../compiler/fields.ts'
@@ -1122,7 +1123,10 @@ function makeDriver(
           // choice.ts:150 is a `continue`, not a failure — the arm is treated as
           // never entered, so a cut it might have raised must not cut the choice.
           ctx._fc = false
-          ctx._fe = pos
+          // No failure POSITION: an ordered choice ranks arms by `_fe`, and a skipped
+          // arm must not rank. A dispatched choice reads `_fx` alone, so this arm's
+          // set is still the answer when it is the one selected.
+          ctx._fe = -1
           ctx._fx = fx[code[ip + 3]!] as string[]
           return FAIL
         }
@@ -1478,9 +1482,7 @@ function makeDriver(
         if (v === FAIL) return FAIL
         const end = EC.e
         const span = code[ip] === OP_NODE_TRACK ? spanLines(ctx, pos, end) : { start: pos, end }
-        const st = (flags & 8) !== 0 && ctx.state !== undefined
-          ? Object.assign({}, ctx.state as Record<string, unknown>)
-          : undefined
+        const st = (flags & 8) !== 0 ? ctx.state : undefined
         const tagIdx = code[ip + 6]!
         const tags = tagIdx < 0 ? undefined : k[tagIdx] as readonly string[]
 
@@ -1517,13 +1519,13 @@ function makeDriver(
           nd = kids[0]
         } else if (proj >= 0) {
           nd = hostCst && host !== undefined
-            ? host(type, kids, fieldMap, span, cap.rawChildren, cap.triviaLog, (flags & 8) !== 0 ? st : ctx.state !== undefined ? Object.assign({}, ctx.state as Record<string, unknown>) : undefined, tags)
+            ? host(type, kids, fieldMap, span, cap.rawChildren, cap.triviaLog, ctx.state, tags)
             : projectChild(kids, proj, type)
         } else if (buildIdx >= 0) {
           if (hostCst && host !== undefined) {
             // A direct builder is bypassed under a CST host: the host must never
             // receive an arbitrary AST object as a child of a CST node.
-            nd = host(type, kids, fieldMap, span, cap.rawChildren, cap.triviaLog, (flags & 8) !== 0 ? st : ctx.state !== undefined ? Object.assign({}, ctx.state as Record<string, unknown>) : undefined, tags)
+            nd = host(type, kids, fieldMap, span, cap.rawChildren, cap.triviaLog, ctx.state, tags)
           } else {
             const build = fns[buildIdx] as (
               children: readonly unknown[], fields: FieldMap | undefined, span: { start: number; end: number },
@@ -1682,6 +1684,8 @@ function makeDriver(
    * drift.
    */
   const scans: readonly Combinator<unknown>[] = (prog.scans ?? []).map(s => {
+    const rebuilt = rebuildLineScan(s, subtreeComb)
+    if (rebuilt !== undefined) return rebuilt
     const skip = s.skip.map(r => subtreeComb(r))
     const raw = (s.flags & 1) !== 0
     if (s.kind === 1) {

package/src/table/line-scan.ts

@@ -0,0 +1,19 @@
+import type { Combinator } from '../types.ts'
+import type { ScanSpec, SubtreeRef } from './program.ts'
+import { columns, lines } from '../combinators/lines.ts'
+
+/**
+ * Rebuild a `lines()` (kind 2) or `columns()` (kind 3) scan spec as the
+ * combinator it was constructed as, its subtrees bound to the table's pieces.
+ * Both drivers run the result through its own `parse`, as they do `scanTo`.
+ */
+export function rebuildLineScan(s: ScanSpec, subtree: (r: SubtreeRef) => Combinator<unknown>): Combinator<unknown> | undefined {
+  if (s.kind === 2) {
+    return lines(subtree(s.prefix!), subtree(s.body!), {
+      ...(s.first === undefined ? {} : { first: subtree(s.first) }),
+      ...(s.lazy === undefined ? {} : { lazy: subtree(s.lazy) }),
+    }) as Combinator<unknown>
+  }
+  if (s.kind === 3) return columns(s.n!) as Combinator<unknown>
+  return undefined
+}

package/src/table/ops.ts

@@ -339,13 +339,16 @@ export const OP_REJECT = 35
  * arm INSTEAD of sitting inside it, so the encoder still reads the arm's own
  * first set for `disp` and the site keeps its dispatch slot.
  *
- * A blocked arm is SKIPPED, not failed — `choice.ts:150` is `continue`. The row
- * therefore clears `_fc` on the gate-false path exactly as `OP_REJECT` does:
+ * A blocked arm is SKIPPED, not failed — `choice.ts` firstMatch `continue`s. The
+ * row therefore clears `_fc` on the gate-false path exactly as `OP_REJECT` does:
  * "pretend this arm was never entered", so no cut it might have raised survives
- * to cut the choice. It still reports `e`, because the OTHER path a blocked arm
- * can be reached by is first-char dispatch, where skip-and-retry and
- * fail-the-choice are the same thing (choice.ts:23-34, :100-105) and the
- * interpreter answers with `deriveExpected(arm)` — this set.
+ * to cut the choice. It sets `_fe` to −1, so an ORDERED choice, which ranks arms
+ * by failure position, takes nothing from it, as the interpreter does. It still
+ * leaves `e` in `_fx`, because the other path a blocked arm can be reached by is
+ * exclusive first-char dispatch, where skip-and-retry and fail-the-choice are the
+ * same thing and the interpreter answers with `deriveExpected(arm)` — this set.
+ * On an ordered site the encoder gives a gated arm no char class, so it is always
+ * entered and this row, not a char gate, decides what it contributes.
  *
  * The predicate is a live function, like `OP_GUARD`'s, so a grammar using one is
  * runtime-only for `emitTableModule` unless `fnSources` are supplied.

package/src/table/program.ts

@@ -241,13 +241,13 @@ export type SubtreeRef = readonly [ip: number, cls: number]
 /**
  * A scanning construct as its CONSTRUCTOR ARGUMENTS.
  *
- * `kind` 0 is `scanTo`, 1 is `balanced`. `flags` bit 0 = `raw`, bit 1 = `orEOF`
+ * `kind` 0 is `scanTo`, 1 is `balanced`, 2 is `lines`, 3 is `columns`. `flags` bit 0 = `raw`, bit 1 = `orEOF`
  * (scanTo), bit 2 = `strict` (balanced). `sent` is the sentinel's literal text
  * when it is a `literal()`, so the rebuilt scan reports the same expected set as
  * the interpreter, and `null` otherwise (which reports `"sentinel"`).
  */
 export type ScanSpec = {
-  readonly kind: 0 | 1
+  readonly kind: 0 | 1 | 2 | 3
   readonly flags: number
   readonly skip: readonly SubtreeRef[]
   readonly sentinel?: SubtreeRef
@@ -256,6 +256,34 @@ export type ScanSpec = {
   readonly sent?: string | null
   readonly open?: string
   readonly close?: string
+  /** `lines()` (kind 2): the continuation prefix, the body, and the optional first-line prefix and lazy check. */
+  readonly prefix?: SubtreeRef
+  readonly body?: SubtreeRef
+  readonly first?: SubtreeRef
+  readonly lazy?: SubtreeRef
+  /** `columns()` (kind 3): the column count. */
+  readonly n?: number
+}
+
+/** Every subtree a scan spec references, which the table links as roots. */
+export function scanRefs(s: ScanSpec): SubtreeRef[] {
+  const refs: SubtreeRef[] = []
+  for (const r of [s.sentinel, s.recoverAt, s.stopAt, s.prefix, s.body, s.first, s.lazy]) if (r !== undefined) refs.push(r)
+  refs.push(...s.skip)
+  return refs
+}
+
+/**
+ * The sites an assembly links from OUTSIDE its rule entries — the scan pool's
+ * subtrees, then the scan-skip sets — in the one order every assembler names
+ * them. The emitted source is keyed by this order, so a second copy of the list
+ * that disagrees prints different source for the same program.
+ */
+export function assemblyExtraIps(prog: TableProgram): number[] {
+  const ips: number[] = []
+  for (const s of prog.scans ?? []) for (const r of scanRefs(s)) ips.push(r[0])
+  for (const set of prog.scanSkip ?? []) for (const r of set) ips.push(r[0])
+  return ips
 }
 
 /** A trivia combinator as data. `arms` empty means a plain `trivia(regex)`. */
@@ -629,6 +657,17 @@ export function classHas(cls: ResolvedClass, code: number): boolean {
   return false
 }
 
+/**
+ * Whether a choice with these arm classes (indices into the class-spec pool `cc`)
+ * takes the O(1) exclusive dispatch — the same decision `resolveDispatch` makes,
+ * for an encoder that has only the specs.
+ */
+export function dispatchIsExclusive(arms: readonly number[], cc: readonly string[]): boolean {
+  const resolved: ResolvedClass[] = []
+  for (const ci of arms) if (ci >= 0) resolved[ci] = resolveClass(cc[ci]!)
+  return resolveDispatch(arms, resolved).exclusive
+}
+
 function resolveDispatch(arms: readonly number[], cc: readonly ResolvedClass[]): ResolvedDispatch {
   const ascii = new Uint8Array(128)
   const hi: number[] = []

package/src/table/select-hot.ts

@@ -0,0 +1,70 @@
+/**
+ * WHICH SITES A SELECTIVE BUILD-TIME ASSEMBLY EMITS.
+ *
+ * The emitted engine beats the closure engine per site, but its bytes are paid by
+ * every process that loads the module. Emitting every site made jess's Less AST
+ * artifact 20.8x its grammar source. So a build emits only the sites with the
+ * highest ESTIMATED hotness per byte, up to a byte budget, and the closure engine
+ * runs the rest (`assemble.ts`, the `HYBRID` path).
+ *
+ * Hotness is estimated statically, since a macro build has no workload to
+ * profile: every rule entry starts at 1, a repetition multiplies its body by
+ * `LOOP`, and a choice or dispatch splits its weight evenly across its arms. The
+ * weights are propagated `ROUNDS` times, so a recursive cycle saturates at `CAP`
+ * instead of diverging. Sites are then taken greedily by weight per emitted byte.
+ * A build given a corpus (`assemblyProfile`) ranks by measured calls instead.
+ */
+import { childSlots } from './child-slots.ts'
+import { OP_CHOICE, OP_DISPATCH, OP_REP, OP_REPV } from './ops.ts'
+
+const LOOP = 8
+const ROUNDS = 60
+const CAP = 1e12
+
+/** The sites to emit: highest weight per byte first, while they fit in `budget` bytes. */
+export function selectHotSites(
+  code: readonly number[],
+  entries: readonly number[],
+  siteBytes: ReadonlyMap<number, number>,
+  budget: number,
+  /** Measured calls per site (`countSiteVisits` over a corpus). Replaces the
+   * static estimate when given. */
+  profile?: ArrayLike<number>,
+): Set<number> {
+  const selected = new Set<number>()
+  if (budget <= 0) return selected
+  const weight = profile === undefined ? staticWeights(code, entries) : undefined
+  const w = (ip: number): number => (weight !== undefined ? weight.get(ip) : profile![ip]) ?? 0
+  // Ties (saturated cycles) break by site offset, so a build is deterministic.
+  const ranked = [...siteBytes.keys()].sort((a, b) =>
+    w(b) / siteBytes.get(b)! - w(a) / siteBytes.get(a)! || a - b)
+  let spent = 0
+  for (const ip of ranked) {
+    if (w(ip) === 0) break
+    const bytes = siteBytes.get(ip)!
+    if (spent + bytes > budget) continue
+    spent += bytes
+    selected.add(ip)
+  }
+  return selected
+}
+
+function staticWeights(code: readonly number[], entries: readonly number[]): Map<number, number> {
+  let weight = new Map<number, number>()
+  const kids: number[] = []
+  for (let round = 0; round < ROUNDS; round++) {
+    const next = new Map<number, number>()
+    for (const ip of entries) next.set(ip, 1)
+    for (const [ip, w] of weight) {
+      kids.length = 0
+      childSlots(code, ip, kids)
+      const op = code[ip]
+      const scale = op === OP_REP || op === OP_REPV ? LOOP
+        : op === OP_CHOICE || op === OP_DISPATCH ? 1 / Math.max(1, kids.length)
+        : 1
+      for (const kid of kids) next.set(kid, Math.min(CAP, (next.get(kid) ?? 0) + w * scale))
+    }
+    weight = next
+  }
+  return weight
+}

package/src/table/trivia-gap.ts

@@ -0,0 +1,109 @@
+import type { CharRange, Combinator, ParseContext } from '../types.ts'
+import type { LabeledTriviaSpec } from '../cst/trivia-kinds.ts'
+import { commitTriviaScan, fastTriviaScanner, scanTriviaCompact } from '../combinators/trivia-skip.ts'
+
+/**
+ * THE TABLE ENGINES' DEFERRED-COMMIT TRIVIA SKIP: `commitTriviaScan(scanTriviaCompact(…))`,
+ * the path every labelled trivia scan takes once a log is live, answered without
+ * the scan where the answer is already known. Both engines call it: the closure
+ * engine's `skipTrivia` and the emitted assembly's skips.
+ *
+ * ONLY THE ROOT LOG IS LIVE, so only a root-selected chunk can be recorded,
+ * and a run in which no character can start a selected arm holds none: its end
+ * is the whole answer, and the unlabelled scanner gives it without the
+ * chunk-by-chunk visit. Every chunk starts inside the run, so a run free of
+ * selected leaders is free of selected chunks. jess parses with
+ * `rootTrivia: { select: [comment labels] }`, which is this case for every run
+ * of plain whitespace.
+ *
+ * A SEQUENCE TERM THAT MATCHES NOTHING hands back the cursor from BEFORE its
+ * trivia, so the next term asks for the same gap again: on `benchmark.less`,
+ * 21,741 of the closure engine's 53,607 trivia scans per parse started where
+ * the previous one did. A scan that recorded nothing records nothing again from
+ * the same position, in the same parse, under the same trivia, root selection,
+ * root-capture policy, trivia log and state, and with no CST capture, and it
+ * ends at the same place, so the last such scan answers the repeat (`ctx._gap`).
+ * The state is part of the key because a trivia arm can read it (`gate`), and
+ * the log because a `leaf` sets it aside while its child runs. The memo lives
+ * on the context so it is released with the parse.
+ *
+ * Neither applies with `trackLines` (a scan records line ranges) or under CST
+ * capture (a scan records into the open buffer).
+ */
+export function skipDeferredTrivia(input: string, cur: number, ctx: ParseContext): number {
+  const triviaP = ctx.trivia
+  const plain = ctx.trackLines !== true && ctx.captureTrivia !== true
+  const gap = ctx._gap
+  if (plain && gap !== undefined && cur === gap.at && input === gap.input && triviaP === gap.of
+    && ctx._rootTriviaCapture === gap.root && ctx.state === gap.state
+    && ctx._rootTriviaKindIndex === gap.rootKinds && ctx._triviaLog === gap.log) return gap.end
+  const end = clearRunEnd(input, cur, ctx, triviaP, plain)
+  if (end >= 0) {
+    remember(cur, end, ctx, input, triviaP)
+    return end
+  }
+  const scan = scanTriviaCompact(input, cur, ctx)
+  if (typeof scan !== 'number') return commitTriviaScan(scan)
+  if (plain) remember(cur, scan, ctx, input, triviaP)
+  return scan
+}
+
+/** The end of the run at `cur` when nothing root-selected lies in it, else −1. */
+function clearRunEnd(input: string, cur: number, ctx: ParseContext, triviaP: Combinator<unknown> | undefined, plain: boolean): number {
+  const rootKinds = ctx._rootTriviaKindIndex
+  const spec = triviaP?._meta.labeledTriviaSpec
+  if (!plain || triviaP === undefined || spec == null || rootKinds === undefined || ctx._triviaLog !== undefined) return -1
+  if (spec.rootKinds !== rootKinds) prime(spec, triviaP, rootKinds)
+  const fast = spec.rootScan
+  if (fast === null) return -1
+  const end = fast(input, cur)
+  return ctx._rootTriviaCapture === false || clearOf(spec.rootLeaders, input, cur, end) ? end : -1
+}
+
+function remember(cur: number, end: number, ctx: ParseContext, input: string, triviaP: Combinator<unknown> | undefined): void {
+  const gap = ctx._gap
+  if (gap === undefined) {
+    ctx._gap = { at: cur, end, input, of: triviaP, root: ctx._rootTriviaCapture, rootKinds: ctx._rootTriviaKindIndex, log: ctx._triviaLog, state: ctx.state }
+    return
+  }
+  gap.at = cur
+  gap.end = end
+  gap.input = input
+  gap.of = triviaP
+  gap.root = ctx._rootTriviaCapture
+  gap.rootKinds = ctx._rootTriviaKindIndex
+  gap.log = ctx._triviaLog
+  gap.state = ctx.state
+}
+
+/**
+ * Prime `spec` for the selection `rootKinds`: the first-character ranges of the
+ * arms it selects, and the scanner, which is `null` where one of those arms is
+ * unbounded or the trivia has no unlabelled scanner. `run()` builds one
+ * selection per parse, so this runs once per parse per trivia.
+ */
+function prime(spec: LabeledTriviaSpec, triviaP: Combinator<unknown>, rootKinds: Readonly<Record<string, number>>): void {
+  const ranges: CharRange[] = []
+  let bounded = true
+  for (const arm of spec.arms) {
+    if (rootKinds[arm.label] === undefined) continue
+    const fs = arm.parser._meta.firstSet
+    if (fs.kind === 'any') bounded = false
+    else if (fs.kind === 'ranges') ranges.push(...fs.ranges)
+  }
+  const classified = triviaP._meta.triviaScanner
+  spec.rootKinds = rootKinds
+  spec.rootLeaders = ranges
+  spec.rootScan = !bounded ? null : classified === undefined ? fastTriviaScanner(triviaP) : classified
+}
+
+/** No character in `[from, to)` falls in `ranges`. A surrogate is never clear:
+ * the ranges are code points, and a code unit cannot be checked against them. */
+function clearOf(ranges: readonly CharRange[], input: string, from: number, to: number): boolean {
+  for (let i = from; i < to; i++) {
+    const c = input.charCodeAt(i)
+    if (c >= 0xD800 && c <= 0xDFFF) return false
+    for (const r of ranges) if (c >= r.lo && c <= r.hi) return false
+  }
+  return true
+}

package/src/types.ts

@@ -1,3 +1,4 @@
+import type { LineView } from './combinators/lines.ts'
 import type { LabeledTriviaSpec } from './cst/trivia-kinds.ts'
 
 export type Span = {
@@ -113,6 +114,11 @@ export type ParserDef =
   // pre-ambient raw byte-walk.
   | { tag: 'scanTo';   sentinel: Combinator<unknown>; skip: Combinator<unknown>[]; raw: boolean; orEOF: boolean; recoverAt?: Combinator<unknown>; stopAt?: Combinator<unknown> }
   | { tag: 'keywords'; words: readonly string[]; caseInsensitive: boolean; boundary: string | undefined }
+  // `lines()`: `body` parses a run of lines with `prefix` (`first` on the first
+  // line) removed from each; `lazy` admits a line the prefix does not match.
+  | { tag: 'lines';    prefix: Combinator<unknown>; body: Combinator<unknown>; first?: Combinator<unknown>; lazy?: Combinator<unknown> }
+  // `columns(n)`: n columns of indentation, a tab reaching the next stop of four.
+  | { tag: 'columns';  n: number }
   | { tag: 'unknown' }
 
 export type Combinator<T> = {
@@ -280,6 +286,14 @@ export type ParseContext = {
   trackLines: boolean
   /** Grammar-author-provided state, scoped with withCtx() and read in guard(). */
   state?: unknown
+  /**
+   * Inside a `lines()` body, the run's lines: where each line's content sits in
+   * the body's input and in the run's own input, and the column it starts at.
+   * Undefined outside any run.
+   */
+  lineView?: LineView | undefined
+  /** Framework-internal: where `columns()` last stopped inside a tab, for `lines()`. */
+  _columnsSplit?: { at: number; split: number } | undefined
   /**
    * Mode host (RULE_ABI_PLAN §7): when set, a linkable/fused grammar's `node()`
    * rules build via `build(type, children, fields, span, rawChildren, triviaLog, state)`
@@ -384,6 +398,12 @@ export type ParseContext = {
    * selected markers into the document-root capture.
    */
   _rootTriviaCapture?: boolean | undefined
+  /**
+   * Framework-internal: the last deferred-commit trivia scan in this parse that
+   * recorded nothing (`table/trivia-gap.ts`). Held on the context so it is freed
+   * with the parse.
+   */
+  _gap?: TriviaGapMemo | undefined
   /**
    * Framework-internal: flat per-node trivia log for CST capture mode.
    * When set alongside _cstRawChildren, each trivia entry is recorded as three
@@ -592,3 +612,15 @@ export type CompiledParser<T> = {
   /** Present only when compiled with `{ coverage: true }`. */
   coverageDefinitions?: readonly import('./compiler/grammar-coverage-ids.ts').GrammarCoverageDefinition[]
 }
+
+/** A trivia gap scanned from `at` to `end` that recorded nothing, and the key it holds for. */
+export interface TriviaGapMemo {
+  at: number
+  end: number
+  input: string
+  of: Combinator<unknown> | undefined
+  root: boolean | undefined
+  rootKinds: Readonly<Record<string, number>> | undefined
+  log: number[] | undefined
+  state: unknown
+}

package/src/version.ts

@@ -13,4 +13,4 @@
  * Kept in sync with `package.json` by `test/unit/version-sync.test.ts`, which fails
  * the build if this constant drifts from the published version.
  */
-export const PARSEMAN_VERSION = '0.55.4'
+export const PARSEMAN_VERSION = '0.57.0'