parseman 0.45.0 → 0.47.0

package/src/cst/host-mode.ts

@@ -0,0 +1,69 @@
+/**
+ * The compiled-artifact ↔ host contract, in ONE place.
+ *
+ * This lives on its own — rather than in `compiler/codegen.ts` where it started —
+ * because the DRIVER has to enforce it too. `functional/run.ts` is the `parseman/run`
+ * entry, whose whole purpose is to stay small (see `test/unit/run-entry-closure.test.ts`),
+ * so it cannot import the compiler to reach an assertion. Neither can it carry its own
+ * copy: two spellings of one contract is how the two engines drift, and this contract
+ * exists precisely because the failure it prevents is silent.
+ *
+ * No imports, by design — that is what keeps it free for both sides to depend on.
+ */
+
+/**
+ * Which consumer a compiled artifact was lowered FOR.
+ *
+ * - `'ast'` — the grammar's own `build` callbacks produce the result (the eval consumer).
+ * - `'cst'` — a positioned-CST `ctx.build` host produces it (linter / IDE / language
+ *   service), and every node's capture is sized for that host.
+ *
+ * ONE grammar source serves both; they are two COMPILATIONS of it, not two grammars.
+ * Deciding at compile time is what keeps the eval-AST artifact free of per-node host
+ * probing — see the `hostMode` note on the codegen Ctx.
+ */
+export type HostMode = 'ast' | 'cst'
+
+/** Host mode a fused rule map (and each of its rule functions) was lowered for;
+ * absent on hand-built maps → 'ast'. */
+export const FUSED_HOST_MODE = Symbol.for('parseman.fusedHostMode')
+
+/** Whether any fused piece omitted a direct builder's positioned-CST branch. */
+export const FUSED_HOST_ELIDED = Symbol.for('parseman.fusedHostElided')
+
+/**
+ * A compiled artifact and the host it is driven with must agree — and when they do not,
+ * that has to be an ERROR, never a quietly-degraded tree.
+ *
+ * An `'ast'` artifact does not emit the positioned-CST branch at all, so attaching a
+ * `_parsemanCstOutput` host to one would silently get the grammar's own AST objects
+ * where the caller asked for a CST. A `'cst'` artifact builds every node through the
+ * host, so running one without a host would call `undefined`. Both are compile/run
+ * mismatches with an obvious fix, so both say so.
+ *
+ * Called once per parse from the TypeScript entry points, never from generated code.
+ */
+export function assertHostModeCompatible(mode: HostMode, build: unknown, hostBranchElided = true): void {
+  const isCstHost = (build as { _parsemanCstOutput?: true } | undefined)?._parsemanCstOutput === true
+  // A purely STRUCTURAL artifact never had a direct-builder host branch to drop, so it
+  // serves a CST host in either mode — that is the long-standing `node(parser)` contract
+  // and this change does not touch it. The INTERPRETER passes `false` for the same
+  // reason from the other direction: it re-decides per parse and never elides anything.
+  if (mode === 'ast' && isCstHost && hostBranchElided) {
+    throw new Error(
+      'parseman: this parser was compiled for host mode "ast" (the default), so its nodes '
+        + 'build through their own `build` callbacks and no positioned-CST branch was emitted. '
+        + 'It cannot be driven with a positioned-CST host (cstBuildHost / a language-service '
+        + 'host). Compile a second artifact from the SAME grammar with '
+        + "`compile(grammar, { hostMode: 'cst' })` — or, under the macro, a second "
+        + "`rules({ hostMode: 'cst' }, factory)` call site — and drive that one instead.",
+    )
+  }
+  if (mode === 'cst' && !isCstHost) {
+    throw new Error(
+      'parseman: this parser was compiled for host mode "cst", so every node builds through '
+        + 'a positioned-CST `ctx.build` host. Pass one (e.g. `{ build: cstBuildHost }`), or use '
+        + 'an artifact compiled with the default `hostMode: "ast"` to get the grammar\'s own AST.',
+    )
+  }
+}

package/src/cst/reflection-attach.ts

@@ -0,0 +1,11 @@
+import { GRAMMAR_REFLECTION, type GrammarReflection } from './reflection.ts'
+
+/**
+ * Legacy interpreted-rule-map attachment. It is deliberately outside the shared
+ * table runtime graph: macro/table artifacts construct reflection in their
+ * metadata prototype instead of mutating an already-built map.
+ */
+export function attachGrammarReflection<T extends object>(grammar: T, reflection: GrammarReflection): T {
+  Object.defineProperty(grammar, GRAMMAR_REFLECTION, { value: reflection, enumerable: false, configurable: true })
+  return grammar
+}

package/src/cst/reflection-symbols.ts

@@ -0,0 +1,9 @@
+/**
+ * Type-level CST identity lives on the hot table-runtime import graph.  Keep the
+ * symbols separate from reflection collection/attachment: the latter is a
+ * grammar-construction concern and must not make every emitted parser load its
+ * descriptor-based legacy helper.
+ */
+export const GRAMMAR_REFLECTION: unique symbol = Symbol.for('parseman.grammarReflection') as never
+export const NODE_TYPE: unique symbol = Symbol.for('parseman.type.nodeType') as never
+export const NODE_TAG: unique symbol = Symbol.for('parseman.type.nodeTag') as never

package/src/cst/reflection.ts

@@ -0,0 +1,149 @@
+import type { Combinator, ParserDef } from '../types.ts'
+import { GRAMMAR_REFLECTION, NODE_TAG, NODE_TYPE } from './reflection-symbols.ts'
+
+export { GRAMMAR_REFLECTION, NODE_TYPE, NODE_TAG } from './reflection-symbols.ts'
+
+export type GrammarNodeReflection = {
+  type: string
+  tags: readonly string[]
+}
+
+export type GrammarReflection = {
+  nodes: readonly GrammarNodeReflection[]
+}
+
+export type GrammarWithReflection<
+  NodeType extends string = string,
+  NodeTag extends string = string,
+> = {
+  readonly [GRAMMAR_REFLECTION]?: GrammarReflection
+  readonly [NODE_TYPE]?: NodeType
+  readonly [NODE_TAG]?: NodeTag
+}
+
+type NodeDef = Extract<ParserDef, { tag: 'node' }>
+
+function childrenOf(def: ParserDef): readonly Combinator<unknown>[] {
+  switch (def.tag) {
+    case 'sequence':
+    case 'choice':
+      return def.parsers
+    case 'dispatch':
+      return [
+        def.selector,
+        ...def.cases.map(c => c.parser),
+        ...(def.matchers ? def.matchers.map(c => c.parser) : []),
+        ...(def.otherwise ? [def.otherwise] : []),
+      ]
+    case 'many':
+    case 'oneOrMore':
+    case 'optional':
+    case 'attempt':
+    case 'transform':
+    case 'trivia':
+    case 'token':
+    case 'leaf':
+    case 'label':
+    case 'field':
+    case 'not':
+    case 'peek':
+    case 'node':
+    case 'withCtx':
+    case 'expect':
+      return [def.parser]
+    case 'recover':
+      return [def.parser, def.sentinel]
+    case 'grammar':
+      return def.triviaParser ? [def.parser, def.triviaParser] : [def.parser]
+    case 'sepBy':
+      return [def.parser, def.separator]
+    case 'scanTo':
+      return [def.sentinel, ...def.skip]
+    case 'routed':
+      return def.fallback ? [def.fallback] : []
+    case 'literal':
+    case 'regex':
+    case 'keywords':
+    case 'guard':
+    case 'adjacency':
+    case 'lazy':
+    case 'unknown':
+      return []
+  }
+}
+
+function addNode(out: Map<string, string[]>, def: NodeDef): void {
+  if (def.type === undefined) return
+  const existing = out.get(def.type)
+  if (existing === undefined) {
+    out.set(def.type, def.tags === undefined ? [] : [...def.tags])
+    return
+  }
+  if (def.tags === undefined) return
+  for (const tag of def.tags) {
+    if (!existing.includes(tag)) existing.push(tag)
+  }
+}
+
+function collectFromCombinator(
+  parser: Combinator<unknown>,
+  out: Map<string, string[]>,
+  seen: Set<Combinator<unknown>>,
+  followLazy = true,
+): void {
+  if (seen.has(parser)) return
+  seen.add(parser)
+  const def = parser._def
+  if (def.tag === 'lazy') {
+    if (followLazy) {
+      try { collectFromCombinator(def.thunk(), out, seen, followLazy) } catch {}
+    }
+    return
+  }
+  if (def.tag === 'node') addNode(out, def)
+  for (const child of childrenOf(def)) collectFromCombinator(child, out, seen, followLazy)
+}
+
+export function collectGrammarReflection(
+  ruleMap: ReadonlyArray<readonly [string, Combinator<unknown>]>,
+  opts?: { followLazy?: boolean },
+): GrammarReflection {
+  const byType = new Map<string, string[]>()
+  const followLazy = opts?.followLazy !== false
+  for (const [, rule] of ruleMap) {
+    if (!followLazy && rule._def.tag === 'lazy') {
+      try {
+        const resolved = rule._def.thunk()
+        if (resolved._def.tag !== 'lazy') collectFromCombinator(resolved, byType, new Set(), false)
+      } catch {}
+      continue
+    }
+    collectFromCombinator(rule, byType, new Set(), followLazy)
+  }
+  return { nodes: [...byType].map(([type, tags]) => ({ type, tags })) }
+}
+
+export function mergeGrammarReflections(reflections: readonly GrammarReflection[]): GrammarReflection {
+  const byType = new Map<string, string[]>()
+  for (const reflection of reflections) {
+    for (const node of reflection.nodes) {
+      const tags = byType.get(node.type)
+      if (tags === undefined) {
+        byType.set(node.type, [...node.tags])
+        continue
+      }
+      for (const tag of node.tags) {
+        if (!tags.includes(tag)) tags.push(tag)
+      }
+    }
+  }
+  return { nodes: [...byType].map(([type, tags]) => ({ type, tags })) }
+}
+
+export function grammarReflectionOf(grammar: object): GrammarReflection | undefined {
+  return (grammar as GrammarWithReflection)[GRAMMAR_REFLECTION]
+}
+
+export function grammarReflectionSource(reflection: GrammarReflection): string {
+  return `{ nodes: ${JSON.stringify(reflection.nodes)} }`
+}

package/src/cst/relative-spans.ts

@@ -0,0 +1,207 @@
+/**
+ * Relative span model for incremental edits.
+ *
+ * Absolute source offsets are simple but fragile: an insertion at offset `at`
+ * shifts EVERY downstream offset, so a whole-tree renumber is O(nodes-after-edit).
+ * Storing each node's span RELATIVE to its parent's start makes edits local: an
+ * insertion inside child k of a node only changes
+ *   (a) child k's relative end,
+ *   (b) the relative start of siblings after k,
+ * and leaves the *children* of those shifted siblings untouched — they move with
+ * their parent for free. So the mutation count is O(depth + trailing siblings),
+ * not O(all following nodes), and every unaffected subtree is shared by identity.
+ *
+ * An absolute view is recovered on query by accumulating the parent base as you
+ * descend (the `absolutize` / `absoluteSpanAt` cursor) — one source of truth,
+ * two manifestations.
+ */
+
+/** A node with an ABSOLUTE source span and ordered children. */
+export type AbsNode = {
+  readonly start: number
+  readonly end: number
+  readonly children: readonly AbsNode[]
+}
+
+/** A node whose `start`/`end` are RELATIVE to its parent's start (root base = 0). */
+export type RelNode = {
+  readonly start: number
+  readonly end: number
+  readonly children: readonly RelNode[]
+}
+
+/** Convert an absolute tree to parent-relative spans. */
+export function relativize(node: AbsNode, parentBase = 0): RelNode {
+  return {
+    start: node.start - parentBase,
+    end: node.end - parentBase,
+    children: node.children.map((c) => relativize(c, node.start)),
+  }
+}
+
+/** Reconstruct absolute spans from a relative tree. Inverse of `relativize`. */
+export function absolutize(node: RelNode, parentBase = 0): AbsNode {
+  const start = parentBase + node.start
+  return {
+    start,
+    end: parentBase + node.end,
+    children: node.children.map((c) => absolutize(c, start)),
+  }
+}
+
+/**
+ * Absolute span of the node at `path` (child indices from the root), accumulating
+ * the base as it descends — O(path length), no full absolutize. The relative-form
+ * cursor: query any node's absolute position without materializing the whole tree.
+ */
+export function absoluteSpanAt(root: RelNode, path: readonly number[]): { start: number; end: number } {
+  let base = 0
+  let node = root
+  for (const idx of path) {
+    base += node.start
+    const child = node.children[idx]
+    if (!child) throw new RangeError(`absoluteSpanAt: no child ${idx} on path ${path.join('.')}`)
+    node = child
+  }
+  return { start: base + node.start, end: base + node.end }
+}
+
+// ---------------------------------------------------------------------------
+// CST-shaped projection (parseDoc incremental trees)
+//
+// The bare Abs/RelNode helpers above are the reference model. The functions
+// below apply the same relative-span algebra to real CST children — nodes,
+// leaves, and error nodes — where the span is a nested `{ start, end }` and
+// children are heterogeneous (leaves carry a span but no `children`). Trivia
+// never appears in structural `children`, so every child has a span.
+// ---------------------------------------------------------------------------
+
+/**
+ * A CST child as far as span projection cares: a nested span + optional children.
+ * `children` is typed loosely (`unknown[]`) so the real heterogeneous CST unions
+ * (nodes, leaves, error nodes — a leaf has no `children`) satisfy it directly.
+ */
+type Spanned = {
+  readonly span: { readonly start: number; readonly end: number }
+  readonly children?: readonly unknown[]
+}
+
+/**
+ * Rewrite a CST subtree's spans to be RELATIVE to `parentBase` (the absolute
+ * start of the node's parent; root base = 0). Each child's own children are
+ * relativized against *this* node's absolute start. Preserves every other field
+ * (`_tag`, `type`, `state`, leaf `value`, …) by shallow spread.
+ */
+export function relativizeCST<T extends Spanned>(node: T, parentBase = 0): T {
+  const span = { start: node.span.start - parentBase, end: node.span.end - parentBase }
+  if (node.children) {
+    const base = node.span.start
+    return { ...node, span, children: node.children.map((c) => relativizeCST(c as Spanned, base)) } as T
+  }
+  return { ...node, span } as T
+}
+
+/** Reconstruct absolute CST spans from a relative subtree. Inverse of `relativizeCST`. */
+export function absolutizeCST<T extends Spanned>(node: T, parentBase = 0): T {
+  const start = parentBase + node.span.start
+  const span = { start, end: parentBase + node.span.end }
+  if (node.children) {
+    return { ...node, span, children: node.children.map((c) => absolutizeCST(c as Spanned, start)) } as T
+  }
+  return { ...node, span } as T
+}
+
+/**
+ * Absolute span of the CST node at `path` (child indices from the root),
+ * accumulating the base as it descends — O(path length), no full absolutize.
+ * The relative-form cursor for a positioned CST: query one node's absolute
+ * position without materializing the whole tree.
+ */
+export function absoluteSpanCST(root: Spanned, path: readonly number[]): { start: number; end: number } {
+  let base = 0
+  let node: Spanned = root
+  for (const idx of path) {
+    base += node.span.start
+    const child = node.children?.[idx] as Spanned | undefined
+    if (!child) throw new RangeError(`absoluteSpanCST: no child ${idx} on path ${path.join('.')}`)
+    node = child
+  }
+  return { start: base + node.span.start, end: base + node.span.end }
+}
+
+/** Naive absolute reshift (baseline): shift every offset >= `at` by `delta`. */
+export function shiftAbsolute(node: AbsNode, at: number, delta: number): AbsNode {
+  const s = node.start >= at ? node.start + delta : node.start
+  const e = node.end >= at ? node.end + delta : node.end
+  return {
+    start: s,
+    end: e,
+    children: node.children.map((c) => shiftAbsolute(c, at, delta)),
+  }
+}
+
+/**
+ * Apply an edit of `delta` characters at absolute offset `at` to a RELATIVE tree.
+ * Returns a new relative tree; every subtree that ends at/before `at` (in absolute
+ * terms) is returned by IDENTITY (unchanged), so the result structurally shares
+ * all unaffected subtrees with the input.
+ *
+ * `stats.allocated` counts nodes that had to be re-created — the incremental cost.
+ */
+export function applyEdit(
+  root: RelNode,
+  at: number,
+  delta: number,
+  stats?: { allocated: number },
+): RelNode {
+  // The root has no parent; its base never shifts (base 0).
+  return editNode(root, at, delta, 0, 0, stats)
+}
+
+function editNode(
+  node: RelNode,
+  at: number,
+  delta: number,
+  parentBase: number,
+  parentShift: number, // how much this node's parent base moved (0 or delta)
+  stats?: { allocated: number },
+): RelNode {
+  const absStart = parentBase + node.start
+  const absEnd = parentBase + node.end
+
+  // Strictly before the edit → unchanged (its parent base is also unshifted).
+  // Uses `< at` (not `<=`) to match the shift convention: offsets `>= at` move,
+  // so a node ending exactly at `at` grows and must be rewritten.
+  if (absEnd < at) return node
+
+  // Fully after the edit AND the parent moved by `delta` too → this node moves as
+  // one unit with its parent, so its parent-relative offsets are identical. Share
+  // the ENTIRE subtree by identity — this is the O(depth) locality win.
+  const startShifts = absStart >= at
+  if (startShifts && parentShift === delta) return node
+
+  if (stats) stats.allocated++
+
+  const newAbsStart = startShifts ? absStart + delta : absStart
+  const newAbsEnd = absEnd + delta // absEnd > at here (fully-before returned above)
+  const newParentBase = parentBase + parentShift
+  const newStart = newAbsStart - newParentBase
+  const newEnd = newAbsEnd - newParentBase
+
+  // Children are relative to this node's absolute start. Our own shift (0 or delta)
+  // becomes their parentShift.
+  const ownShift = newAbsStart - absStart
+  let changed = false
+  const newChildren: RelNode[] = []
+  for (const c of node.children) {
+    const nc = editNode(c, at, delta, absStart, ownShift, stats)
+    if (nc !== c) changed = true
+    newChildren.push(nc)
+  }
+
+  return {
+    start: newStart,
+    end: newEnd,
+    children: changed ? newChildren : node.children,
+  }
+}

package/src/cst/root-trivia-scope.ts

@@ -0,0 +1,30 @@
+/**
+ * The ROOT-CAPTURE POLICY of one local trivia scope, in one place.
+ *
+ * Three engines install a `parser({ trivia })` scope — the interpreter
+ * (`combinators/grammar.ts`), source lowering (`compiler/codegen.ts`) and the
+ * table (`table/assemble.ts`, `table/exec.ts`) — and each of them has to answer
+ * the same two questions about selected root trivia. The refusal below was a
+ * string literal in two of them and ABSENT from the third, which is how a table
+ * parse came to accept a grammar the other two reject.
+ *
+ * Import-free, like the rest of `cst/`, so the drivers can reach it without
+ * pulling the combinator set in.
+ */
+
+/** The message `parser()` raises for an unclassified local scope. Exported so the
+ *  three engines cannot drift a word apart. */
+export const ROOT_TRIVIA_SCOPE_REFUSAL
+  = 'parser(): selected root trivia requires classifiedTrivia() for every local trivia scope, or rootCapture: \'opaque\'.'
+
+/**
+ * Refuse a local trivia scope that cannot be reported faithfully.
+ *
+ * With selected root capture active, a scope whose trivia is not classified and
+ * is not declared opaque would silently drop the markers inside it. Throwing is
+ * the contract (`combinators/grammar.ts:98`); returning a parse that quietly
+ * omits them is the failure this exists to prevent.
+ */
+export function refuseUnclassifiedRootScope(strict: boolean | undefined): void {
+  if (strict === true) throw new TypeError(ROOT_TRIVIA_SCOPE_REFUSAL)
+}