parseman 0.56.0 → 0.58.0

package/src/table/run-support.ts

@@ -13,6 +13,7 @@
  * reads only its arguments.
  */
 import type { ParseContext } from '../types.ts'
+import { popCstChild, pushCstChild } from '../cst/capture-buffer.ts'
 
 export type Span = { start: number; end: number }
 export type LineSpan = Span & {
@@ -64,3 +65,37 @@ export function lead(input: string, pos: number): number {
   if (c < 0xd800 || c > 0xdbff) return c
   return input.codePointAt(pos) ?? c
 }
+
+const EMPTY_KIDS: readonly unknown[] = Object.freeze([])
+const EMPTY_TRIVIA: readonly number[] = Object.freeze([])
+
+type CollapseBuild = (
+  children: readonly unknown[], fields: undefined, span: { start: number; end: number },
+  rawChildren: readonly unknown[], triviaLog: readonly number[], state: undefined,
+) => unknown
+
+/**
+ * A PASS-THROUGH COLLAPSE whose one child built `undefined` (`site-labels.ts`
+ * `passCollapse`).
+ *
+ * A collapse over a single node is that node: the node publishes its value into
+ * the collector the collapse's parent opened, exactly where the collapse would
+ * have published it, over the same span. So both engines run the body with no
+ * frame of the collapse's own and hand its value up. The one value a frame would
+ * have treated differently is `undefined`, which a frame does not count as a
+ * child: the collapse then builds from no children. This is that case — take the
+ * child's publication back and publish the build instead, as the frame would
+ * have. The caller has already set `EC.e`.
+ */
+export function collapseEmpty(
+  ctx: ParseContext, build: CollapseBuild, input: string, pos: number, end: number, readsRaw: boolean,
+): unknown {
+  const collected = ctx._cstBuf !== undefined || ctx._cstChildren !== undefined
+  if (collected) popCstChild(ctx)
+  // A frame keeps the raw entry of the child it does not count, for a reducer
+  // that reads `rawChildren`.
+  const raw = readsRaw ? [rawEntry(undefined, input, pos, end)] : EMPTY_KIDS
+  const nd = build(EMPTY_KIDS, undefined, { start: pos, end }, raw, EMPTY_TRIVIA, undefined)
+  if (collected) pushCstChild(ctx, nd, ctx._cstBuf?.noRaw === true ? undefined : rawEntry(nd, input, pos, end))
+  return nd
+}

package/src/table/scalar-terminal.ts

@@ -54,6 +54,14 @@ export function scalarTerminalNotChild(code: ArrayLike<number>, ip: number): num
   return op === OP_RX || op === OP_LIT ? child : -1
 }
 
+/** The same terminal under `peek()`: its match decides the lookahead, and nothing it captured survives it. */
+export function scalarTerminalPeekChild(code: ArrayLike<number>, ip: number): number {
+  if (code[ip] !== OP_PEEK) return -1
+  const child = code[ip + 1]!
+  const op = code[child]
+  return op === OP_RX || op === OP_LIT ? child : -1
+}
+
 /**
  * A direct scalar terminal that every successful execution of `ip` must match
  * first, through wrappers that cannot consume or select before their child.

package/src/table/select-hot.ts

@@ -0,0 +1,70 @@
+/**
+ * WHICH SITES A SELECTIVE BUILD-TIME ASSEMBLY EMITS.
+ *
+ * The emitted engine beats the closure engine per site, but its bytes are paid by
+ * every process that loads the module. Emitting every site made jess's Less AST
+ * artifact 20.8x its grammar source. So a build emits only the sites with the
+ * highest ESTIMATED hotness per byte, up to a byte budget, and the closure engine
+ * runs the rest (`assemble.ts`, the `HYBRID` path).
+ *
+ * Hotness is estimated statically, since a macro build has no workload to
+ * profile: every rule entry starts at 1, a repetition multiplies its body by
+ * `LOOP`, and a choice or dispatch splits its weight evenly across its arms. The
+ * weights are propagated `ROUNDS` times, so a recursive cycle saturates at `CAP`
+ * instead of diverging. Sites are then taken greedily by weight per emitted byte.
+ * A build given a corpus (`assemblyProfile`) ranks by measured calls instead.
+ */
+import { childSlots } from './child-slots.ts'
+import { OP_CHOICE, OP_DISPATCH, OP_REP, OP_REPV } from './ops.ts'
+
+const LOOP = 8
+const ROUNDS = 60
+const CAP = 1e12
+
+/** The sites to emit: highest weight per byte first, while they fit in `budget` bytes. */
+export function selectHotSites(
+  code: readonly number[],
+  entries: readonly number[],
+  siteBytes: ReadonlyMap<number, number>,
+  budget: number,
+  /** Measured calls per site (`countSiteVisits` over a corpus). Replaces the
+   * static estimate when given. */
+  profile?: ArrayLike<number>,
+): Set<number> {
+  const selected = new Set<number>()
+  if (budget <= 0) return selected
+  const weight = profile === undefined ? staticWeights(code, entries) : undefined
+  const w = (ip: number): number => (weight !== undefined ? weight.get(ip) : profile![ip]) ?? 0
+  // Ties (saturated cycles) break by site offset, so a build is deterministic.
+  const ranked = [...siteBytes.keys()].sort((a, b) =>
+    w(b) / siteBytes.get(b)! - w(a) / siteBytes.get(a)! || a - b)
+  let spent = 0
+  for (const ip of ranked) {
+    if (w(ip) === 0) break
+    const bytes = siteBytes.get(ip)!
+    if (spent + bytes > budget) continue
+    spent += bytes
+    selected.add(ip)
+  }
+  return selected
+}
+
+function staticWeights(code: readonly number[], entries: readonly number[]): Map<number, number> {
+  let weight = new Map<number, number>()
+  const kids: number[] = []
+  for (let round = 0; round < ROUNDS; round++) {
+    const next = new Map<number, number>()
+    for (const ip of entries) next.set(ip, 1)
+    for (const [ip, w] of weight) {
+      kids.length = 0
+      childSlots(code, ip, kids)
+      const op = code[ip]
+      const scale = op === OP_REP || op === OP_REPV ? LOOP
+        : op === OP_CHOICE || op === OP_DISPATCH ? 1 / Math.max(1, kids.length)
+        : 1
+      for (const kid of kids) next.set(kid, Math.min(CAP, (next.get(kid) ?? 0) + w * scale))
+    }
+    weight = next
+  }
+  return weight
+}

package/src/table/site-labels.ts

@@ -60,9 +60,9 @@
  * label licenses at a leaf is dropping the TEST — never the capture.
  */
 import {
-  OP_ATTEMPT, OP_CHOICE, OP_DISPATCH, OP_EXPECT, OP_FIELD, OP_GATE, OP_LABEL, OP_LEAF, OP_SOURCE_LEAF,
+  OP_ATTEMPT, OP_CHOICE, OP_DISPATCH, OP_EXPECT, OP_FIELD, OP_GATE, OP_LABEL, OP_LEAF, OP_LIT, OP_SOURCE_LEAF,
   OP_NODE, OP_NODE_TRACK, OP_NOT, OP_OPT, OP_PEEK, OP_REP, OP_REPV, OP_ROUTED, OP_RULE,
-  OP_SCOPE, OP_SCOPE_CAP, OP_SCOPE_PLAIN, OP_SEQ, OP_SEQV, OP_SEQX, OP_TOKEN, OP_XFORM,
+  OP_SCAN, OP_SCOPE, OP_SCOPE_CAP, OP_SCOPE_PLAIN, OP_SEQ, OP_SEQV, OP_SEQX, OP_TOKEN, OP_XFORM,
 } from './ops.ts'
 import { childSlots } from './child-slots.ts'
 
@@ -142,6 +142,61 @@ export function reachableSites(code: Int32Array, roots: Iterable<number>): Set<n
   return seen
 }
 
+/**
+ * A node both engines lower as a PASS-THROUGH COLLAPSE (`run-support.ts`
+ * `collapseEmpty`): `collapse: true` over a direct reducer, no projection, no CST
+ * host, no line tracking, whose body is one node behind wrappers that consume and
+ * capture nothing (a first-set gate, a trivia scope, a rule reference). One
+ * predicate, so the two engines and the labels cannot disagree on a site.
+ */
+export function passCollapse(code: Int32Array, ip: number, hostCst: boolean): boolean {
+  const flags = code[ip + 3]!
+  if (hostCst || code[ip] !== OP_NODE || code[ip + 1]! < 0 || code[ip + 4]! >= 0) return false
+  if (flags !== 32 && flags !== 34) return false
+  let at = code[ip + 2]!
+  for (let n = 0; n < 8; n++) {
+    const op = code[at]
+    if (op === OP_GATE || op === OP_SCOPE_PLAIN || (op === OP_SCOPE && code[at + 3] === 0)) at = code[at + 2]!
+    else if (op === OP_RULE) at = code[at + 1]!
+    else break
+  }
+  return code[at] === OP_NODE
+}
+
+/**
+ * A collapse that a FIRST CHARACTER can reduce to a pass-through: its body is a
+ * sequence of `optional`/`not` terms, each behind a first-character gate, then
+ * one node. Where the character at the collapse's start opens none of those
+ * terms (nor its sequence's trivia — the engines test that against the site's
+ * label), every one of them matches empty there and the body is that node alone.
+ * `leads` are the gate rows, `lits` the constant indices of direct literal terms,
+ * `last` the node term. Not a `passCollapse`.
+ */
+export function gatedPassCollapse(code: Int32Array, ip: number, hostCst: boolean): { leads: number[]; lits: number[]; last: number } | undefined {
+  const flags = code[ip + 3]!
+  if (hostCst || code[ip] !== OP_NODE || code[ip + 1]! < 0 || code[ip + 4]! >= 0) return undefined
+  if (flags !== 32 && flags !== 34) return undefined
+  const seq = code[ip + 2]!
+  if (code[seq] !== OP_SEQ && code[seq] !== OP_SEQV) return undefined
+  const n = code[seq + 1]!
+  if (n < 2) return undefined
+  const leads: number[] = []
+  const lits: number[] = []
+  for (let i = 0; i < n - 1; i++) {
+    const t = code[seq + 2 + i]!
+    if (code[t] !== OP_OPT && code[t] !== OP_NOT) return undefined
+    const g = code[t + 1]!
+    // A gate row carries the term's first characters; a direct literal is its own.
+    if (code[g] === OP_GATE) leads.push(g)
+    else if (code[g] === OP_LIT) lits.push(code[g + 1]!)
+    else return undefined
+  }
+  const last = code[seq + 2 + n - 1]!
+  let at = last
+  for (let k = 0; k < 8 && (code[at] === OP_GATE || code[at] === OP_RULE); k++) at = code[at] === OP_RULE ? code[at + 1]! : code[at + 2]!
+  return code[at] === OP_NODE ? { leads, lits, last } : undefined
+}
+
 /**
  * What a site hands DOWN to its children. Every op but the two that write the
  * context passes its own label through unchanged.
@@ -159,6 +214,10 @@ function transfer(code: Int32Array, ip: number, at: SiteLabel, hostCst: boolean)
   }
   if (op === OP_NODE || op === OP_NODE_TRACK) {
     const flags = code[ip + 3]!
+    // A pass-through collapse opens nothing: its body runs in its parent's
+    // collector and capture setting.
+    if (passCollapse(code, ip, hostCst)) return at
+    const gated = gatedPassCollapse(code, ip, hostCst) !== undefined
     // The same selection `emit-assembly.ts` makes: the trailing-trivia bit (128)
     // keeps the children-only shape.
     const directChildren = !hostCst && op === OP_NODE && code[ip + 1]! >= 0 && code[ip + 4]! < 0
@@ -168,7 +227,9 @@ function transfer(code: Int32Array, ip: number, at: SiteLabel, hostCst: boolean)
       // arrays instead of a CstCaptureBuf. Descendants still capture, but they
       // must take the generic collector branch rather than dereference `_cstBuf`.
       // False means UNKNOWN in this lattice, which is exactly the safe answer.
-      return { tri: at.tri, buf: false, raw: RAW_UNKNOWN, cap: CAP_OFF }
+      const framed = { tri: at.tri, buf: false, raw: RAW_UNKNOWN, cap: CAP_OFF }
+      // A gated pass collapse runs its node term in its parent's collector too.
+      return gated ? meet(at, framed) : framed
     }
     // `ctx._cstBuf = buf` and `ctx.captureTrivia = <literal>`, both unconditional
     // — `emit-assembly.ts`'s `OP_NODE` body opens the buffer before it descends
@@ -178,7 +239,8 @@ function transfer(code: Int32Array, ip: number, at: SiteLabel, hostCst: boolean)
       ? RAW_OMIT
       : RAW_CAPTURE
     if (at.buf && raw === at.raw && cap === at.cap) return at
-    return { tri: at.tri, buf: true, raw, cap }
+    const framed = { tri: at.tri, buf: true, raw, cap }
+    return gated ? meet(at, framed) : framed
   }
   // THE TWO BOUNDARIES clear every capture sink for their child and contribute
   // ONE leaf for the whole match; `token()` clears `ctx.trivia` as well. So a
@@ -251,3 +313,72 @@ export function computeSiteLabels(
 
   return { at: (ip: number): SiteLabel => labels.get(ip) ?? TOP }
 }
+
+/**
+ * Does a piece entered at `ip` need the caller's INSTALLED trivia scanner handed
+ * across an engine boundary?
+ *
+ * Each table engine keeps its own installed-scanner slot, and the two hand theirs
+ * across at every call between them. A site whose label names its trivia (`tri`)
+ * reads no slot: both engines lower its skips onto that trivia's own scanner, and
+ * a call out of it can hand that scanner across instead of the slot. Only a site
+ * whose trivia the labels cannot name (`TRI_UNKNOWN`) reads the slot. So the
+ * handoff is needed exactly when such a site is reachable from `ip` before a
+ * scope installs a scanner of its own (an aliased scope, which installs what is
+ * already installed, does not). `false` is also the answer for a body with no
+ * trivia at all.
+ */
+/**
+ * A trivia scope that writes only what its label proves is already in force, so
+ * it runs as its child. A slot scope under that same slot is one; so is a scope
+ * with no trivia where the label proves none (inside `token()` or another such
+ * scope), since every way to `TRI_NONE` leaves no trivia, no kind labels and no
+ * scanner — what this scope would write. `OP_SCOPE_PLAIN` has no root policy; a
+ * policy-bearing `OP_SCOPE` is one only with policy zero, and `OP_SCOPE_CAP`
+ * never is, since it raises `captureTrivia`. `TRI_UNKNOWN` matches nothing.
+ */
+export function scopeIsIdentity(code: Int32Array, ip: number, at: SiteLabel): boolean {
+  const op = code[ip]
+  if (op !== OP_SCOPE_PLAIN && !(op === OP_SCOPE && code[ip + 3]! === 0)) return false
+  const ki = code[ip + 1]!
+  return at.tri === (ki < 0 ? TRI_NONE : ki)
+}
+
+export function scanHandoffNeeded(code: Int32Array, ip: number, labels: SiteLabels): boolean {
+  const seen = new Set<number>()
+  const stack = [ip]
+  const kids: number[] = []
+  while (stack.length > 0) {
+    const at = stack.pop()!
+    if (seen.has(at)) continue
+    seen.add(at)
+    const l = labels.at(at)
+    if (l.tri === TRI_UNKNOWN) return true
+    const op = code[at]
+    // A scan runs interpreted combinators that reach pieces through `prog.scans`.
+    if (op === OP_SCAN) return true
+    if (op === OP_TOKEN || op === OP_SCOPE_CAP) continue
+    if ((op === OP_SCOPE_PLAIN || op === OP_SCOPE) && !(code[at + 1]! >= 0 && l.tri === code[at + 1]!
+      && (op === OP_SCOPE_PLAIN || code[at + 3]! === 0))) continue
+    kids.length = 0
+    if (!childSlots(code, at, kids)) return true
+    for (const c of kids) stack.push(c)
+  }
+  return false
+}
+
+/**
+ * The first characters that can start trivia at a site labelled `tri`, as code
+ * point ranges: none for `TRI_NONE`, `undefined` where the labels cannot name the
+ * trivia or its first set is unbounded.
+ */
+export function triviaLeadRanges(
+  tri: number, trivia: readonly ({ _meta: { firstSet: { kind: string; ranges?: readonly { lo: number; hi: number }[] } } } | undefined)[],
+): readonly { lo: number; hi: number }[] | undefined {
+  if (tri === TRI_NONE) return []
+  if (tri < 0) return undefined
+  const fs = trivia[tri]?._meta.firstSet
+  if (fs === undefined) return undefined
+  if (fs.kind === 'empty') return []
+  return fs.kind === 'ranges' && fs.ranges !== undefined ? fs.ranges : undefined
+}

package/src/table/trivia-gap.ts

@@ -0,0 +1,109 @@
+import type { CharRange, Combinator, ParseContext } from '../types.ts'
+import type { LabeledTriviaSpec } from '../cst/trivia-kinds.ts'
+import { commitTriviaScan, fastTriviaScanner, scanTriviaCompact } from '../combinators/trivia-skip.ts'
+
+/**
+ * THE TABLE ENGINES' DEFERRED-COMMIT TRIVIA SKIP: `commitTriviaScan(scanTriviaCompact(…))`,
+ * the path every labelled trivia scan takes once a log is live, answered without
+ * the scan where the answer is already known. Both engines call it: the closure
+ * engine's `skipTrivia` and the emitted assembly's skips.
+ *
+ * ONLY THE ROOT LOG IS LIVE, so only a root-selected chunk can be recorded,
+ * and a run in which no character can start a selected arm holds none: its end
+ * is the whole answer, and the unlabelled scanner gives it without the
+ * chunk-by-chunk visit. Every chunk starts inside the run, so a run free of
+ * selected leaders is free of selected chunks. jess parses with
+ * `rootTrivia: { select: [comment labels] }`, which is this case for every run
+ * of plain whitespace.
+ *
+ * A SEQUENCE TERM THAT MATCHES NOTHING hands back the cursor from BEFORE its
+ * trivia, so the next term asks for the same gap again: on `benchmark.less`,
+ * 21,741 of the closure engine's 53,607 trivia scans per parse started where
+ * the previous one did. A scan that recorded nothing records nothing again from
+ * the same position, in the same parse, under the same trivia, root selection,
+ * root-capture policy, trivia log and state, and with no CST capture, and it
+ * ends at the same place, so the last such scan answers the repeat (`ctx._gap`).
+ * The state is part of the key because a trivia arm can read it (`gate`), and
+ * the log because a `leaf` sets it aside while its child runs. The memo lives
+ * on the context so it is released with the parse.
+ *
+ * Neither applies with `trackLines` (a scan records line ranges) or under CST
+ * capture (a scan records into the open buffer).
+ */
+export function skipDeferredTrivia(input: string, cur: number, ctx: ParseContext): number {
+  const triviaP = ctx.trivia
+  const plain = ctx.trackLines !== true && ctx.captureTrivia !== true
+  const gap = ctx._gap
+  if (plain && gap !== undefined && cur === gap.at && input === gap.input && triviaP === gap.of
+    && ctx._rootTriviaCapture === gap.root && ctx.state === gap.state
+    && ctx._rootTriviaKindIndex === gap.rootKinds && ctx._triviaLog === gap.log) return gap.end
+  const end = clearRunEnd(input, cur, ctx, triviaP, plain)
+  if (end >= 0) {
+    remember(cur, end, ctx, input, triviaP)
+    return end
+  }
+  const scan = scanTriviaCompact(input, cur, ctx)
+  if (typeof scan !== 'number') return commitTriviaScan(scan)
+  if (plain) remember(cur, scan, ctx, input, triviaP)
+  return scan
+}
+
+/** The end of the run at `cur` when nothing root-selected lies in it, else −1. */
+function clearRunEnd(input: string, cur: number, ctx: ParseContext, triviaP: Combinator<unknown> | undefined, plain: boolean): number {
+  const rootKinds = ctx._rootTriviaKindIndex
+  const spec = triviaP?._meta.labeledTriviaSpec
+  if (!plain || triviaP === undefined || spec == null || rootKinds === undefined || ctx._triviaLog !== undefined) return -1
+  if (spec.rootKinds !== rootKinds) prime(spec, triviaP, rootKinds)
+  const fast = spec.rootScan
+  if (fast === null) return -1
+  const end = fast(input, cur)
+  return ctx._rootTriviaCapture === false || clearOf(spec.rootLeaders, input, cur, end) ? end : -1
+}
+
+function remember(cur: number, end: number, ctx: ParseContext, input: string, triviaP: Combinator<unknown> | undefined): void {
+  const gap = ctx._gap
+  if (gap === undefined) {
+    ctx._gap = { at: cur, end, input, of: triviaP, root: ctx._rootTriviaCapture, rootKinds: ctx._rootTriviaKindIndex, log: ctx._triviaLog, state: ctx.state }
+    return
+  }
+  gap.at = cur
+  gap.end = end
+  gap.input = input
+  gap.of = triviaP
+  gap.root = ctx._rootTriviaCapture
+  gap.rootKinds = ctx._rootTriviaKindIndex
+  gap.log = ctx._triviaLog
+  gap.state = ctx.state
+}
+
+/**
+ * Prime `spec` for the selection `rootKinds`: the first-character ranges of the
+ * arms it selects, and the scanner, which is `null` where one of those arms is
+ * unbounded or the trivia has no unlabelled scanner. `run()` builds one
+ * selection per parse, so this runs once per parse per trivia.
+ */
+function prime(spec: LabeledTriviaSpec, triviaP: Combinator<unknown>, rootKinds: Readonly<Record<string, number>>): void {
+  const ranges: CharRange[] = []
+  let bounded = true
+  for (const arm of spec.arms) {
+    if (rootKinds[arm.label] === undefined) continue
+    const fs = arm.parser._meta.firstSet
+    if (fs.kind === 'any') bounded = false
+    else if (fs.kind === 'ranges') ranges.push(...fs.ranges)
+  }
+  const classified = triviaP._meta.triviaScanner
+  spec.rootKinds = rootKinds
+  spec.rootLeaders = ranges
+  spec.rootScan = !bounded ? null : classified === undefined ? fastTriviaScanner(triviaP) : classified
+}
+
+/** No character in `[from, to)` falls in `ranges`. A surrogate is never clear:
+ * the ranges are code points, and a code unit cannot be checked against them. */
+function clearOf(ranges: readonly CharRange[], input: string, from: number, to: number): boolean {
+  for (let i = from; i < to; i++) {
+    const c = input.charCodeAt(i)
+    if (c >= 0xD800 && c <= 0xDFFF) return false
+    for (const r of ranges) if (c >= r.lo && c <= r.hi) return false
+  }
+  return true
+}

package/src/types.ts

@@ -398,6 +398,12 @@ export type ParseContext = {
    * selected markers into the document-root capture.
    */
   _rootTriviaCapture?: boolean | undefined
+  /**
+   * Framework-internal: the last deferred-commit trivia scan in this parse that
+   * recorded nothing (`table/trivia-gap.ts`). Held on the context so it is freed
+   * with the parse.
+   */
+  _gap?: TriviaGapMemo | undefined
   /**
    * Framework-internal: flat per-node trivia log for CST capture mode.
    * When set alongside _cstRawChildren, each trivia entry is recorded as three
@@ -606,3 +612,15 @@ export type CompiledParser<T> = {
   /** Present only when compiled with `{ coverage: true }`. */
   coverageDefinitions?: readonly import('./compiler/grammar-coverage-ids.ts').GrammarCoverageDefinition[]
 }
+
+/** A trivia gap scanned from `at` to `end` that recorded nothing, and the key it holds for. */
+export interface TriviaGapMemo {
+  at: number
+  end: number
+  input: string
+  of: Combinator<unknown> | undefined
+  root: boolean | undefined
+  rootKinds: Readonly<Record<string, number>> | undefined
+  log: number[] | undefined
+  state: unknown
+}

package/src/version.ts

@@ -13,4 +13,4 @@
  * Kept in sync with `package.json` by `test/unit/version-sync.test.ts`, which fails
  * the build if this constant drifts from the published version.
  */
-export const PARSEMAN_VERSION = '0.56.0'
+export const PARSEMAN_VERSION = '0.58.0'