parseman 0.45.0 → 0.47.0

package/src/table/site-labels.ts

@@ -0,0 +1,228 @@
+/**
+ * SITE LABELS — the encoder's downward pass over an encoded program.
+ *
+ * ## The criterion
+ *
+ * *Any consulting of options at parse time, per rule or per combinator, is a
+ * FAIL.* After U4 four consultations remained, and three of them are here:
+ * `ctx.trivia` per sequence term, `cstCaptureActive` per leaf, and
+ * `hasTrivia`/`needMark` per repetition. (`forCtx`, once per parse at the
+ * boundary, is not one of these — it selects the assembly rather than running
+ * inside one.)
+ *
+ * ## Why a label and not a `RunCfg` bit
+ *
+ * `assemble.ts:213-277` states the rule `RunCfg` is held to: a bit belongs there
+ * only when it is FIXED FOR THE LIFETIME OF A PARSE. Neither of these facts is.
+ *
+ *   - `ctx.trivia` is per-SCOPE. `OP_SCOPE` swaps it mid-parse, so a `RunCfg`
+ *     bit would be answering a question that has different answers at different
+ *     sites in the same parse.
+ *   - `ctx._cstBuf` is per-NODE — `assemble.ts:247-249` records that a previous
+ *     lane proposed keying on it and that doing so would have been *incorrect,
+ *     not merely redundant*.
+ *
+ * A site label is strictly stronger than either, and it is what makes both facts
+ * legal to resolve: it is computed from the PROGRAM STRUCTURE at encode time,
+ * before any option set exists, and it says what is true *at that site* rather
+ * than what is true for the parse. `OP_NODE` sets `ctx._cstBuf` unconditionally
+ * on entry and restores it on exit, so every site dynamically inside one has
+ * `_cstBuf !== undefined` — not because of an option, but because of where it
+ * sits in the program.
+ *
+ * ## The two axes, kept apart
+ *
+ * The capture label does not compress into one bit, and an earlier attempt to
+ * make it one was wrong in both directions:
+ *
+ *   - `cap` is THREE-VALUED. `OP_NODE` writes `ctx.captureTrivia` a LITERAL —
+ *     `readsTrivia || hostCst`, resolved at emit — so a site under a node has it
+ *     definitely true OR definitely false. `OP_SCOPE_CAP` only ever sets it true.
+ *     Collapsing "definitely off" into "unknown" loses the case that lets
+ *     `_skipTrivia` drop its capture arm entirely.
+ *   - `buf` is a separate FLAG, because `_cstBuf` and `captureTrivia` are set by
+ *     different ops and neither implies the other: `OP_SCOPE_CAP` turns capture
+ *     on without opening a buffer, and a builder-less `OP_NODE` opens a buffer
+ *     with capture off.
+ *
+ * ## Asymmetry, deliberately
+ *
+ * `buf` is TRUE only for "guaranteed present". FALSE means UNKNOWN, never
+ * "guaranteed absent" — no root can prove absence, because every entry point
+ * (`prog.rules`, and the `extraIps` a scan pool reaches from outside the emitted
+ * scope) is called with a context this pass cannot see. Nothing is elided on
+ * `buf === false`; it only declines to elide. The same holds for `TRI_UNKNOWN`
+ * and `CAP_UNKNOWN`, which are the lattice's top element, not a third fact.
+ *
+ * That asymmetry is what rules out the unsound version of this pass: eliding a
+ * leaf's capture when `hostCst === false`. `OP_NODE` opens `_cstBuf` whatever the
+ * host mode is, and those leaves feed `kids` → `build(...)`, so the ONLY thing a
+ * label licenses at a leaf is dropping the TEST — never the capture.
+ */
+import {
+  OP_ATTEMPT, OP_CHOICE, OP_DISPATCH, OP_EXPECT, OP_FIELD, OP_GATE, OP_LABEL, OP_LEAF,
+  OP_NODE, OP_NODE_TRACK, OP_NOT, OP_OPT, OP_PEEK, OP_REP, OP_REPV, OP_ROUTED, OP_RULE,
+  OP_SCOPE, OP_SCOPE_CAP, OP_SEQ, OP_SEQV, OP_SEQX, OP_TOKEN, OP_XFORM,
+} from './ops.ts'
+import { childSlots } from './child-slots.ts'
+
+/** No scope on this path installs a known trivia — the lattice's top element. */
+export const TRI_UNKNOWN = -2
+/** `ctx.trivia === undefined` is guaranteed here. */
+export const TRI_NONE = -1
+/** Any value `>= 0` is a trivia slot index, guaranteed installed here. */
+
+/** `ctx.captureTrivia` is not known at this site. */
+export const CAP_UNKNOWN = 0
+/** `ctx.captureTrivia` is guaranteed NOT `true` here. */
+export const CAP_OFF = 1
+/** `ctx.captureTrivia === true` is guaranteed here. */
+export const CAP_ON = 2
+
+export type SiteLabel = {
+  /** `TRI_UNKNOWN`, `TRI_NONE`, or the trivia slot the enclosing scope installed. */
+  readonly tri: number
+  /**
+   * `ctx._cstBuf !== undefined` is GUARANTEED. False means unknown — see the
+   * asymmetry note in this file's header. Never read as "guaranteed absent".
+   */
+  readonly buf: boolean
+  /** `CAP_UNKNOWN` / `CAP_OFF` / `CAP_ON`. */
+  readonly cap: number
+}
+
+/** The lattice's top: every entry point starts here. */
+export const TOP: SiteLabel = { tri: TRI_UNKNOWN, buf: false, cap: CAP_UNKNOWN }
+
+function meet(a: SiteLabel, b: SiteLabel): SiteLabel {
+  const tri = a.tri === b.tri ? a.tri : TRI_UNKNOWN
+  const buf = a.buf && b.buf
+  const cap = a.cap === b.cap ? a.cap : CAP_UNKNOWN
+  if (tri === a.tri && buf === a.buf && cap === a.cap) return a
+  return { tri, buf, cap }
+}
+
+/**
+ * The sites reachable from `roots` THROUGH THE SHARED EDGE TABLE.
+ *
+ * Not `inspect.ts`'s `reachableIps`: that one walks a whole program from its rule
+ * entries, and the emitter's question is about the set IT will lower — the rule
+ * entries plus the scan pool's `extraIps`. Different ROOTS, same EDGES, and the
+ * edges now live in `child-slots.ts` so the two answers cannot drift when an
+ * opcode is added to one and not the other. That drift is the failure this file's
+ * header used to warn about while guarding only against it happening within this
+ * file; the copy it warned about was in `inspect.ts` the whole time.
+ *
+ * `childSlots`'s false return (an opcode it does not know) is ignored here: the
+ * site resolves to `TOP` through `labelAt`, and the enclosing assembly is
+ * unemittable regardless.
+ */
+export function reachableSites(code: Int32Array, roots: Iterable<number>): Set<number> {
+  const seen = new Set<number>()
+  const stack = [...roots]
+  const kids: number[] = []
+  while (stack.length > 0) {
+    const ip = stack.pop()!
+    if (seen.has(ip)) continue
+    seen.add(ip)
+    kids.length = 0
+    childSlots(code, ip, kids)
+    for (const c of kids) stack.push(c)
+  }
+  return seen
+}
+
+/**
+ * What a site hands DOWN to its children. Every op but the two that write the
+ * context passes its own label through unchanged.
+ */
+function transfer(code: Int32Array, ip: number, at: SiteLabel, hostCst: boolean): SiteLabel {
+  const op = code[ip]
+  if (op === OP_SCOPE || op === OP_SCOPE_CAP) {
+    const ki = code[ip + 1]!
+    const tri = ki < 0 ? TRI_NONE : ki
+    // `OP_SCOPE_CAP` is an OR with the inherited context, never a switch-off
+    // (`encode.ts:1153-1158`), so it can only ever raise `cap` to `CAP_ON`.
+    const cap = op === OP_SCOPE_CAP ? CAP_ON : at.cap
+    if (tri === at.tri && cap === at.cap) return at
+    return { tri, buf: at.buf, cap }
+  }
+  if (op === OP_NODE || op === OP_NODE_TRACK) {
+    // `ctx._cstBuf = buf` and `ctx.captureTrivia = <literal>`, both unconditional
+    // — `emit-assembly.ts`'s `OP_NODE` body opens the buffer before it descends
+    // and closes it after, whatever the host mode is.
+    const cap = ((code[ip + 3]! & 4) !== 0 || hostCst) ? CAP_ON : CAP_OFF
+    if (at.buf && cap === at.cap) return at
+    return { tri: at.tri, buf: true, cap }
+  }
+  // THE TWO BOUNDARIES clear every capture sink for their child and contribute
+  // ONE leaf for the whole match; `token()` clears `ctx.trivia` as well. So a
+  // site inside either has NO open `_cstBuf`.
+  //
+  // The label can only say `buf: false`, which this lattice reads as UNKNOWN
+  // rather than as "guaranteed absent" (see the asymmetry note above). That is
+  // the SOUND direction and it is the one that matters: `_rbBuf` and
+  // `_pushLeafBuf` dereference `ctx._cstBuf` with no null test, so a site that
+  // inherited `buf: true` through a `token()` would throw on the first rollback.
+  // Losing the elision inside a token is the price; being wrong there is not
+  // an option.
+  if (op === OP_TOKEN) {
+    if (at.tri === TRI_NONE && !at.buf) return at
+    return { tri: TRI_NONE, buf: false, cap: at.cap }
+  }
+  if (op === OP_LEAF) {
+    if (!at.buf) return at
+    return { tri: at.tri, buf: false, cap: at.cap }
+  }
+  return at
+}
+
+/** A site's label, or `TOP` for anything the walk did not reach. */
+export type SiteLabels = { at: (ip: number) => SiteLabel }
+
+/**
+ * Compute a label for every site reachable from `roots`.
+ *
+ * A worklist to a fixpoint, because the program has cycles (a recursive rule is
+ * a back-edge into a site already in flight) and because a site shared by two
+ * parents must carry the MEET of what both hand it — a downward pass that simply
+ * overwrote would give a shared site whichever parent it happened to visit last,
+ * and the emitted body for that site is ONE body serving both.
+ *
+ * The meet only ever moves a label toward `TOP`, and `TOP` is absorbing, so the
+ * lattice has height three and the walk terminates.
+ */
+export function computeSiteLabels(
+  code: Int32Array,
+  roots: Iterable<number>,
+  hostCst: boolean,
+): SiteLabels {
+  const labels = new Map<number, SiteLabel>()
+  const work: number[] = []
+
+  const push = (ip: number, l: SiteLabel): void => {
+    const cur = labels.get(ip)
+    if (cur === undefined) {
+      labels.set(ip, l)
+      work.push(ip)
+      return
+    }
+    const next = meet(cur, l)
+    if (next === cur) return
+    labels.set(ip, next)
+    work.push(ip)
+  }
+
+  for (const r of roots) push(r, TOP)
+
+  const kids: number[] = []
+  while (work.length > 0) {
+    const ip = work.pop()!
+    const down = transfer(code, ip, labels.get(ip)!, hostCst)
+    kids.length = 0
+    childSlots(code, ip, kids)
+    for (const c of kids) push(c, down)
+  }
+
+  return { at: (ip: number): SiteLabel => labels.get(ip) ?? TOP }
+}

package/src/table/stamp.ts

@@ -0,0 +1,229 @@
+/**
+ * The rule-map envelope both table drivers share.
+ *
+ * Everything here is about what a table ENTRY has to look like to `run()` and to
+ * the linker's public wrappers — trivia metadata, host-mode stamps, the
+ * `_lineStarts` seeding, the ambient `scanSkip` install and the failure-to-
+ * `ParseResult` conversion. None of it is recognition, and having two copies of
+ * it (one in `exec.ts`, one in `assemble.ts`) would be two places for a
+ * divergence the three-way identity sweep would then have to catch.
+ *
+ * The drivers supply only what differs: how a rule RUNS.
+ */
+import type { Combinator, ParseContext, ParseResult } from '../types.ts'
+import { FUSED_HOST_ELIDED, FUSED_HOST_MODE } from '../cst/host-mode.ts'
+import { GRAMMAR_COVERAGE_DEFINITIONS } from '../grammar-metadata.ts'
+import { reachableIps } from './inspect.ts'
+import { OP_NODE, OP_NODE_TRACK, OP_SCOPE } from './ops.ts'
+import { covDefinitions, resolveTable, type ResolvedTable, type TableProgram, type TableRule } from './program.ts'
+
+const EMPTY_FX: string[] = []
+
+/**
+ * Did something inside CUT?
+ *
+ * Read through a call so TypeScript does not narrow `_fc` to `false` from the
+ * assignment that precedes each speculative attempt — the driver mutates it and
+ * the checker cannot see that. Lives here rather than in a driver because the
+ * ENVELOPE reads it too, and a third copy would be a third place to drift.
+ */
+export function committed(c: ParseContext): boolean {
+  return c._fc === true
+}
+
+/**
+ * The ENTRY result's span, line-annotated when the table tracks lines.
+ *
+ * Codegen annotates the root result the moment `ctx.lineTracking` is on — both
+ * the success span and the failure span (`_spanLines(_ctx, …)`), and so does the
+ * interpreter's `parser({ trackLines: true })` scope. The table's envelope built
+ * a bare `{ start, end }` regardless, so a tracked table parse handed back a
+ * result with no `startLine`/`endColumn` at all while paying for the tracking.
+ *
+ * The binary search over `_lineStarts` now lives in `run-support.ts`. The
+ * private copy that sat here was justified by "not worth giving the hot node
+ * piece a cross-module call"; the EMITTED engine needs the same answer, the
+ * node piece no longer calls this copy at all, and a THIRD copy is how a
+ * reported column drifts between two engines that are gated against each other
+ * precisely so that it cannot.
+ */
+import { spanLines } from './run-support.ts'
+export { spanLines }
+
+/**
+ * What a driver must provide per parse, in the order the entry needs it.
+ *
+ * `run` returns the driver's own FAIL sentinel for a failed parse. The sentinel
+ * is module-private on each side, so it is never compared here — `runRule`
+ * answers with `undefined` for failure instead, and the caller's sentinel never
+ * leaves its module.
+ */
+export type RuleRunner = {
+  /**
+   * Run rule `ri` and return its end position, or −1 on failure.
+   *
+   * Returning the position rather than the value keeps the driver's FAIL
+   * sentinel private; the value is fetched separately only on success.
+   */
+  readonly runRule: (ri: number, input: string, pos: number, ctx: ParseContext) => number
+  /** The value the last successful `runRule` produced. */
+  readonly lastValue: () => unknown
+  /** The ambient `scanSkip` set for rule `ri`, if the program declares one. */
+  readonly scanSkipFor: (ri: number, ctx: ParseContext) => readonly Combinator<unknown>[] | undefined
+}
+
+/**
+ * Wrap a driver's per-rule run into the map an artifact exports.
+ *
+ * The entries have the SAME signature as codegen rule functions, so `run()`, the
+ * linker's public wrappers and every consumer are unchanged.
+ */
+export function stampRuleMap(
+  prog: TableProgram,
+  d: RuleRunner,
+  artifactMetadata: Readonly<Record<symbol, unknown>> = {},
+  resolved: ResolvedTable = resolveTable(prog),
+): Record<string, TableRule> {
+  // `run()` reads trivia metadata off the ENTRY and takes its
+  // `typeof r === 'function'` branch for compiled entries, which codegen stamps
+  // with `_meta`. A table entry is a function too, so it must be stamped or
+  // `run({ rootTrivia })` rejects a grammar that plainly has labelled trivia.
+  //
+  // `grammarTrivia` joins them for the same reason and with a sharper edge: `run()`
+  // now consumes the DOCUMENT ROOT's trailing trivia off the entry's own ambient
+  // trivia (see `ambientTriviaFromRunnable`), which a combinator entry carries on
+  // `_meta.grammarTrivia`. A table entry that did not carry it would leave the tail
+  // unconsumed and report a short `span` / a non-null `unconsumedFrom` where the
+  // interpreter reports the whole file — a three-way identity divergence that shows
+  // up as `ok: true` with bytes missing, never as an error.
+  //
+  // Read from the ENTRY ROW, not from one program-wide slot: `encodeRule` wraps each
+  // rule's entry in `OP_SCOPE <triviaSlot> <body>` iff THAT rule has ambient trivia,
+  // and a `composeLeaf` grammar's pieces do not agree (the same disagreement
+  // `scanSkipOf` exists for). The owning driver passes its resolved table, so
+  // this is the exact trivia array it already built.
+  const triviaOfRule = (ip: number): Combinator<unknown> | undefined => {
+    if (prog.code[ip] !== OP_SCOPE) return undefined
+    const slot = prog.code[ip + 1]!
+    return slot < 0 ? undefined : resolved.trivia[slot]
+  }
+  const baseMeta = prog.labels === undefined && prog.classified !== 1
+    ? undefined
+    : {
+        ...(prog.labels === undefined ? {} : { triviaKindLabels: prog.labels }),
+        ...(prog.classified === 1 ? { rootTriviaClassified: true as const } : {}),
+      }
+  // Chosen ONCE, from table data, at rule-map construction. Not a per-parse
+  // branch on an option: a plain table never has this wrapper at all.
+  const lines = prog.lines === 1
+  const mode = prog.hostMode ?? 'ast'
+  const elided = mode === 'ast' && hasDirectBuilder(prog)
+  const coverageDefinitions = prog.cov === undefined
+    ? undefined
+    : Object.freeze(covDefinitions(prog).map(Object.freeze))
+  // Metadata lives on the map's prototype FROM BIRTH. Symbol reads keep their
+  // existing contract, while Object.keys, object spread and Object.assign see
+  // only the map's own rule entries. This is the no-descriptor equivalent of the
+  // old non-enumerable stamps: no post-construction shape transition, WeakMap or
+  // generated wrapper is involved.
+  const metadataPrototype = {
+    ...artifactMetadata,
+    [FUSED_HOST_MODE]: mode,
+    [FUSED_HOST_ELIDED]: elided,
+    ...(coverageDefinitions === undefined
+      ? {}
+      : { [GRAMMAR_COVERAGE_DEFINITIONS]: coverageDefinitions }),
+  }
+  const out = Object.create(metadataPrototype) as Record<string, TableRule>
+  const names = Object.keys(prog.rules)
+  for (let ri = 0; ri < names.length; ri++) {
+    const name = names[ri]!
+    const index = ri
+    const entryFn = (input: string, pos: number, ctx: ParseContext): ParseResult<unknown> => {
+      // The table lowering ships coverage counters in 0.47 but not codegen's
+      // fine-grained trace phases. Failing at the artifact boundary is deliberate:
+      // accepting the public sink and writing no events produces a plausible empty
+      // trace, which is indistinguishable from a genuinely unvisited grammar.
+      if (ctx._grammarTrace !== undefined) {
+        throw new TypeError(
+          'parseman: grammar tracing is not supported by table-backed parsers in 0.47; '
+            + 'remove the _grammarTrace sink or run the combinator through the interpreter',
+        )
+      }
+      // Ambient scanSkip, which `run()` cannot install for a function entry.
+      // Chosen PER RULE, because that is what `run()` does — it reads the ENTRY
+      // rule's own `_meta.grammarScanSkip`. One program-wide set instead gave a
+      // `composeLeaf` piece's rules a skip list the interpreter never gives them.
+      const ownSkip = d.scanSkipFor(index, ctx)
+      if (ownSkip !== undefined && ctx.scanSkip === undefined) {
+        ctx.scanSkip = ownSkip as ParseContext['scanSkip']
+      }
+      ctx._fe = -1
+      ctx._fx = EMPTY_FX
+      // `_fc` is a SPECULATION-LOCAL bit cleared at the boundary so a stale
+      // `true` from a PREVIOUS parse on a reused `ctx` cannot read as "the cut
+      // fired".
+      ctx._fc = false
+      if (lines && ctx._lineStarts === undefined) { ctx._lineStarts = [0]; ctx._lineScannedTo = 0 }
+      const end = d.runRule(index, input, pos, ctx)
+      if (end < 0) {
+        const fe = ctx._fe
+        const at = fe === undefined || fe < 0 ? pos : fe
+        const fspan = lines ? spanLines(ctx, at, at) : { start: at, end: at }
+        // `committed` IS PART OF THE FAILURE ENVELOPE, not a diagnostic extra.
+        // The interpreter carries it on the result (`{ ok: false, …, committed: true }`)
+        // and codegen re-derives it from the same bit this driver uses
+        // (`codegen.ts`: `...(_ctx._fc ? { committed: true } : {})`). The table
+        // dropped it, so a table entry embedded as a CHILD of another parser — the
+        // one place the field is read rather than merely reported — lost the cut.
+        return committed(ctx)
+          ? { ok: false, expected: (ctx._fx ?? EMPTY_FX) as string[], span: fspan, committed: true }
+          : { ok: false, expected: (ctx._fx ?? EMPTY_FX) as string[], span: fspan }
+      }
+      return { ok: true, value: d.lastValue(), span: lines ? spanLines(ctx, pos, end) : { start: pos, end } }
+    }
+    const ownTrivia = triviaOfRule(prog.rules[name]!)
+    const meta = ownTrivia === undefined
+      ? baseMeta
+      : { ...baseMeta, grammarTrivia: ownTrivia }
+    // A standalone entry has no registry from which `run()` could recover host
+    // mode, so keep the two ordinary symbol fields on the function itself. Every
+    // entry receives them in the same construction-time order.
+    out[name] = Object.assign(entryFn, {
+      ...(meta === undefined ? {} : { _meta: meta }),
+      [FUSED_HOST_MODE]: mode,
+      [FUSED_HOST_ELIDED]: elided,
+    })
+  }
+  // STAMP THE HOST MODE. `run()` reads it off the entry through `FUSED_HOST_MODE`
+  // and `assertHostModeCompatible` throws when a 'cst' artifact runs without a
+  // CST host. Encoding with `hostMode: 'cst'` set the capture flags but nothing
+  // stamped the mode, so such a table returned the grammar's own AST objects with
+  // `ok: true` while paying full CST capture.
+  // WHAT `FUSED_HOST_ELIDED` MEANS is "a DIRECT BUILDER's positioned-CST branch was
+  // dropped" — it is what makes `'ast' artifact + CST host` an error rather than a
+  // preference. `mode === 'ast'` alone over-reports it: an all-STRUCTURAL grammar has
+  // no direct builder, so no branch was dropped and it stays usable with either host
+  // (the `node(parser)` contract). A node row carries `-1` in its build slot when it
+  // has no direct builder, so the program answers this itself — and it MUST agree with
+  // the macro's emitted stamp, or re-stamping the same map throws
+  // "Cannot redefine property".
+  return out
+}
+
+/**
+ * Does any REACHABLE node row own a direct builder?
+ *
+ * Reachability, not a scan for the opcode's numeric value: operands are ordinary
+ * integers and collide with opcodes, so counting words reports confident nonsense
+ * (`inspect.ts` states the same thing about `opHistogram`). One linear pass at load,
+ * beside the assembly pass that already visits every row.
+ */
+function hasDirectBuilder(prog: TableProgram): boolean {
+  const code = prog.code
+  for (const ip of reachableIps(prog)) {
+    const op = code[ip]
+    if ((op === OP_NODE || op === OP_NODE_TRACK) && code[ip + 1] !== -1) return true
+  }
+  return false
+}