parseman 0.52.0 → 0.54.0

package/src/compiler/borrowed-rules.ts

@@ -0,0 +1,88 @@
+/**
+ * Where the interpreter and a compiled table would bind one reference differently.
+ *
+ * A rule reference (`g.X`) is a slot object. The interpreter follows the slot it
+ * holds. The table encoder binds a reference to the map's rule of the same NAME
+ * (`encode.ts`, `winners`). Within one linked map the two are the same rule. They
+ * differ only for a rule object BORROWED from another grammar (`Entry: base.Entry`):
+ * its graph holds `base`'s own slots, so the interpreter keeps `base`'s `Atom` while
+ * the encoder binds this map's `Atom`. Both parse, and they accept different input.
+ *
+ * ponytail: refuses the pattern instead of rebinding borrowed graphs, which is the
+ * planned follow-up that lets both engines agree.
+ */
+import type { Combinator } from '../types.ts'
+import { childrenOf } from '../analysis/gating.ts'
+import { winnerWrapsReference } from './token-alphabet.ts'
+
+type Comb = Combinator<unknown>
+
+const nameOf = (c: Comb): string | undefined => (c as unknown as { _ruleName?: string })._ruleName
+
+/** Follow a slot to the combinator it resolves to, or `undefined` for a hole. */
+function deref(c: Comb): Comb | undefined {
+  const seen = new Set<Comb>()
+  let current: Comb = c
+  while (current._def.tag === 'lazy' && !seen.has(current)) {
+    seen.add(current)
+    try { current = current._def.thunk() } catch { return undefined }
+  }
+  return current
+}
+
+/**
+ * The first rule of `map` whose graph reaches another grammar's slot for a rule
+ * `map` also defines — the rule, and the name it references — or `undefined`.
+ */
+export function borrowedRuleReference(
+  map: Readonly<Record<string, Comb>>,
+): { rule: string; reference: string } | undefined {
+  const seen = new Set<Comb>()
+  for (const [rule, self] of Object.entries(map)) {
+    let found: string | undefined
+    const visit = (c: Comb): void => {
+      if (found !== undefined) return
+      if (c._def.tag === 'lazy') {
+        // The root's own slot leads to this rule's body; any other named slot is a
+        // reference, judged before it is ever marked seen (another root's own slot
+        // is that root's body, which its own walk covers).
+        if (!winnerWrapsReference(self, c)) {
+          const name = nameOf(c)
+          // OWN rules only: an inherited name (`toString`) is not a rule of the map.
+          const winner = name !== undefined && Object.hasOwn(map, name) ? map[name] : undefined
+          if (winner !== undefined) {
+            // This map's own reference, or another slot to the very same rule.
+            if (winnerWrapsReference(winner, c) || deref(c) === deref(winner)) return
+            found = name
+            return
+          }
+        }
+        // The root's body, or a rule this map does not define: both engines follow
+        // the slot, so its body matters only for what IT references. ONE step: the
+        // slot may resolve to another slot (an alias, `Entry: g.Atom`), and that one is
+        // a reference to judge too, not a hop to skip.
+        if (seen.has(c)) return
+        seen.add(c)
+        let body: Comb
+        try { body = c._def.thunk() } catch { return }
+        visit(body)
+        return
+      }
+      if (seen.has(c)) return
+      seen.add(c)
+      for (const child of childrenOf(c._def)) visit(child)
+    }
+    visit(self)
+    if (found !== undefined) return { rule, reference: found }
+  }
+  return undefined
+}
+
+/** The refusal, worded for the API that refused. */
+export function borrowedRuleMessage(api: string, hit: { rule: string; reference: string }): string {
+  return `${api}: rule "${hit.rule}" is a rule object borrowed from another grammar, and its graph references `
+    + `"${hit.reference}", which this grammar also defines. The interpreter would keep the other grammar's `
+    + `"${hit.reference}" while a compiled table binds this grammar's, so the two would accept different input. `
+    + `Compose the grammar that defines "${hit.rule}" instead of borrowing its rule object, so its `
+    + `references resolve to this grammar's rules.`
+}

package/src/compiler/compile-linkable-table.ts

@@ -31,7 +31,7 @@ import { PARSEMAN_VERSION } from '../version.ts'
  *                       composable half, and it is NOT new machinery: the plugin
  *                       already carries pieces as IR (`plugin/index.ts:1772`) and
  *                       the linker already re-lowers them (`linker.ts:640`,
- *                       `compileLinkable(evalRuleMapIR(p.ir), …)`).
+ *                       `compileLinkable(evalRuleMapIR(p.ir), …)` at build time).
  *
  * ── WHY THAT MAKES COMPOSITION THE EASY KIND ─────────────────────────────────
  * The open question was whether table-to-table composition has to merge two

package/src/compiler/fields.ts

@@ -183,11 +183,13 @@ export function parserEnablesTriviaCapture(p: Combinator<unknown>, seen: Set<Com
   }
 }
 
-export function buildFieldMap(captures: ReadonlyArray<{ name: string; value: unknown; span: { start: number; end: number } }> | undefined): FieldMap | undefined {
+export function buildFieldMap(captures: ReadonlyArray<{ name: string; value: unknown; span: { start: number; end: number }; leading?: string; trailing?: string }> | undefined): FieldMap | undefined {
   if (!captures?.length) return undefined
   const out: FieldMap = {}
   for (const cap of captures) {
-    const entry = { value: cap.value, span: cap.span }
+    const entry = cap.leading === undefined
+      ? { value: cap.value, span: cap.span }
+      : { value: cap.value, span: cap.span, leading: cap.leading, trailing: cap.trailing! }
     const current = out[cap.name]
     if (current === undefined) out[cap.name] = entry
     else if (Array.isArray(current)) current.push(entry)
@@ -195,3 +197,4 @@ export function buildFieldMap(captures: ReadonlyArray<{ name: string; value: unk
   }
   return out
 }
+

package/src/compiler/grammar-coverage-ids.ts

@@ -60,7 +60,10 @@ function children(def: ParserDef, winners?: Record<string, Combinator<unknown>>)
         return []
       }
       const name = (resolved as Combinator<unknown> & { _ruleName?: string })._ruleName
-      return name && winners?.[name] ? [winners[name]!] : [resolved]
+      const winner = name ? winners?.[name] : undefined
+      // A slot that IS its rule's winner (a runtime composition's) descends into
+      // its own body; any other reference reroutes to the winner.
+      return winner !== undefined && winner._def !== def ? [winner] : [resolved]
     }
     default: return []
   }
@@ -92,18 +95,41 @@ export function buildGrammarPlan(entry: Combinator<unknown> | readonly Combinato
       }
     }
   }
+  /** The body behind a slot that is itself its rule's winner — how a runtime
+   * composition holds every rule. Transparent: its body owns the rule. */
+  const selfWinnerBody = (parser: Combinator<unknown>): Combinator<unknown> | undefined => {
+    if (parser._def.tag !== 'lazy' || winners === undefined) return undefined
+    const name = winnerNames.get(parser)
+    if (name === undefined || winners[name] !== parser) return undefined
+    try { return parser._def.thunk() } catch { return undefined }
+  }
+  // A body that SEVERAL winners' slots resolve to (`X: shared, Y: shared`) can carry
+  // only one name, so there each slot owns its own rule and the body owns none.
+  const slotsPerBody = new Map<Combinator<unknown>, number>()
+  if (winners) {
+    for (const winner of Object.values(winners)) {
+      const body = selfWinnerBody(winner)
+      if (body !== undefined) slotsPerBody.set(body, (slotsPerBody.get(body) ?? 0) + 1)
+    }
+  }
+  const sharedBody = (parser: Combinator<unknown>): boolean => (slotsPerBody.get(parser) ?? 0) > 1
+  const ownRule = (parser: Combinator<unknown>, rule: string): void => {
+    const id = `rule:${rule}`
+    definitions.set(id, { id, kind: 'rule' })
+    rules.set(parser, id)
+  }
   const seen = new Set<Combinator<unknown>>()
   const visit = (parser: Combinator<unknown>, path: string): void => {
     if (seen.has(parser)) return
     seen.add(parser)
     const rule = winnerNames.get(parser) ?? (parser as Combinator<unknown> & { _ruleName?: string })._ruleName
+    const body = selfWinnerBody(parser)
     // `rules()` references are tagged too for linker naming, but only the final
     // rule definition owns execution coverage; otherwise ref + target double-hit.
-    if (rule && (winnerNames.has(parser) || parser._def.tag !== 'lazy')) {
-      const id = `rule:${rule}`
-      definitions.set(id, { id, kind: 'rule' })
-      rules.set(parser, id)
+    if (rule && body === undefined && !sharedBody(parser) && (winnerNames.has(parser) || parser._def.tag !== 'lazy')) {
+      ownRule(parser, rule)
     }
+    if (rule && body !== undefined && sharedBody(body)) ownRule(parser, rule)
     if (parser._def.tag === 'choice') {
       const ids = parser._def.parsers.map((_, index) => `choice:${path}/arm:${index}`)
       choices.set(parser, ids)
@@ -128,7 +154,10 @@ export function buildGrammarPlan(entry: Combinator<unknown> | readonly Combinato
     // caller's winner map is then the only durable public identity for each
     // root; using the generic `entry` path would merge separate roots' choice
     // IDs and omit their rule definitions.
-    visit(root, winnerNames.get(root) ?? (root as Combinator<unknown> & { _ruleName?: string })._ruleName ?? 'entry')
+    const name = winnerNames.get(root) ?? (root as Combinator<unknown> & { _ruleName?: string })._ruleName ?? 'entry'
+    const body = selfWinnerBody(root)
+    if (body !== undefined && sharedBody(body) && !seen.has(root)) { seen.add(root); ownRule(root, name) }
+    visit(body ?? root, name)
   }
   return { definitions: [...definitions.values()].sort((a, b) => a.id.localeCompare(b.id)), choices, dispatches, attempts, labels, rules }
 }

package/src/compiler/ir-serialize.ts

@@ -33,7 +33,7 @@ import { peek } from '../combinators/peek.ts'
 import { node } from '../combinators/node.ts'
 import { parser } from '../combinators/grammar.ts'
 import { scanTo, balanced, type BalancedAmbient } from '../combinators/scanTo.ts'
-import { token, leaf, sourceLeaf } from '../combinators/token.ts'
+import { token, leaf } from '../combinators/token.ts'
 import { classifiedTrivia, transform, trivia, label, field } from '../combinators/map.ts'
 import { analyzeLabeledTrivia } from '../cst/trivia-kinds.ts'
 import { expect as expectC } from '../combinators/expect.ts'
@@ -157,123 +157,17 @@ export function serializeRuleMap(
   }
 }
 
-/** Reconstruct a rule map from serialized IR (the inverse of `serializeRuleMap`) —
- * evaluate the combinator-construction expression with every constructor in scope.
- * Used at fuse time (runtime linker + build-time plugin) to re-lower carried IR. */
-export function evalRuleMapIR(ir: string): Array<[string, Comb]> {
-  // `_tf`/`_nd` reconstruct a transform/node AND restore its captured callback
-  // source (`_def.fnSrc`/`buildSrc`) so re-lowering inlines it statically. The live
-  // fn is only needed for interpreted mode; a self-contained transform source is
-  // eval'd, a node build (which may reference imported AST classes) is left to its
-  // source only. Macro-only validation metadata is carried as plain data: this
-  // runtime module never parses callback source or imports a compiler frontend.
-  const _tf = (child: Comb, src: string, recognitionOnly = false): Comb => {
-    let fn: (...a: unknown[]) => unknown
-    // eslint-disable-next-line no-eval
-    try { fn = (0, eval)(`(${src})`) } catch { fn = () => { throw new Error('IR transform fn not materialized') } }
-    const t = transform(child as never, fn as never)
-    ;(t._def as { fnSrc?: string }).fnSrc = src
-    if (recognitionOnly) (t._def as { recognitionOnly?: boolean }).recognitionOnly = true
-    return t as Comb
-  }
-  const _lf = (child: Comb, src: string, source = false): Comb => {
-    let fn: (...a: unknown[]) => unknown
-    // eslint-disable-next-line no-eval
-    try { fn = (0, eval)(`(${src})`) } catch { fn = () => { throw new Error('IR leaf fn not materialized') } }
-    const l = source ? sourceLeaf(child as never, fn as never) : leaf(child as never, fn as never)
-    ;(l._def as { fnSrc?: string }).fnSrc = src
-    return l as Comb
-  }
-  const _nd = (type: string, child: Comb, src: string, opts?: unknown, staticError?: readonly string[], sigSrc?: string, buildImports?: ReadonlyArray<{ local: string; source: string; imported: string }>, rawUnused?: true): Comb => {
-    if (staticError !== undefined && staticError.length > 0) {
-      // Fail closed: a builder with an un-rescuable binding is NOT fused. The
-      // plugin catches this and leaves the runtime compose() in place rather than
-      // shipping a fused table that would ReferenceError on import. Import-rescued
-      // free names never reach here — they ride in `buildImports`, not `staticError`.
-      throw new Error(`IR direct node builder for ${type} must be macro-static and self-contained; unsupported binding(s): ${staticError.join(', ')}`)
-    }
-    // A serialized direct builder needs an inert sentinel as well as buildSrc.
-    // `node(..., undefined)` is structural, so re-lowering a composed artifact
-    // silently routes it through ctx.build/default CST even though the IR still
-    // carries the callback source. The compiler is the only consumer that may
-    // materialize `buildSrc`; raw IR interpretation deliberately rejects direct
-    // builders rather than evaluating arbitrary captured source at runtime.
-    const n = node(type, child as never, (() => { throw new Error('IR node build requires static re-lowering') }) as never, opts as never)
-    ;(n._def as { buildSrc?: string; buildStaticError?: readonly string[] }).buildSrc = src
-    if (staticError !== undefined) (n._def as { buildStaticError?: readonly string[] }).buildStaticError = staticError
-    // Analysis-only resolved reducer signature (see `buildAnalysisSrc`). Without it a
-    // re-lowered composed artifact silently re-acquires the fail-open capture cost the
-    // authoring module had already resolved away.
-    if (sigSrc !== undefined) (n._def as { buildSigSrc?: string }).buildSigSrc = sigSrc
-    if (rawUnused === true) (n._def as { buildRawUnused?: true }).buildRawUnused = true
-    // Re-attach the direct-builder import provenance so the plugin's re-lower pass
-    // can re-emit the imports into the consuming module. Plain data — this runtime
-    // module never resolves or emits imports itself.
-    if (buildImports !== undefined) (n._def as { buildImports?: ReadonlyArray<{ local: string; source: string; imported: string }> }).buildImports = buildImports
-    return n as Comb
-  }
-  // `_gch` rebuilds a GATED choice AND restores its `_def.gateSrcs` (parallel to the
-  // arms) — the gate mirror of `_tf`/`_nd`. Each item is either a plain arm or a
-  // `[gateSource, arm]` tuple: the source is eval'd to the live predicate (interpreted
-  // mode) and recorded as the arm's gateSrcs entry so re-lowering INLINES the gate
-  // statically (keeping the artifact fusible via `emitFusedSource`). A missing/failed
-  // source falls back to a throwing predicate — the same contract as `_tf`.
-  const _gch = (items: Array<Comb | [string, Comb]>): Comb => {
-    const gateSrcs: (string | null)[] = []
-    const arms = items.map(it => {
-      if (Array.isArray(it)) {
-        const [src, comb] = it
-        gateSrcs.push(src)
-        let gate: (s: unknown) => boolean
-        // eslint-disable-next-line no-eval
-        try { gate = (0, eval)(`(${src})`) } catch { gate = () => { throw new Error('IR gate fn not materialized') } }
-        return { gate, combinator: comb }
-      }
-      gateSrcs.push(null)
-      return it
-    })
-    const c = (choice as (...a: unknown[]) => Comb)(...arms)
-    // `choice` is the authority on gate alignment; `gateSrcs` must line up 1:1 with
-    // the constructed `_def.gates`. They always do on the normal path (both from the
-    // same `items.map`), but assert it so a future change in `choice` that coalesces
-    // or drops arms can't silently ship a mis-aligned gate-source array.
-    const gates = (c._def as { gates?: unknown[] }).gates
-    if (gates && gates.length !== gateSrcs.length) {
-      throw new Error(`_gch: gateSrcs length ${gateSrcs.length} != choice gates length ${gates.length}`)
-    }
-    ;(c._def as { gateSrcs?: (string | null)[] }).gateSrcs = gateSrcs
-    return c
-  }
-  // `_wc` rebuilds a `withCtx` AND restores its `_def.extraSrc` (the source of the
-  // `extra`/state value) — the withCtx mirror of `_tf`/`_nd`/`_gch`. The source is
-  // eval'd to the live `extra` value (interpreted mode) and recorded on the def so
-  // re-lowering INLINES the state getter statically (`() => (extraSrc)`), keeping the
-  // artifact fusible via `emitFusedSource`. A plain `withCtx(value, inner)` would
-  // leave `extraSrc` unset → codegen emits a source-less runtime closure (a non-static
-  // callback) → `emitFusedSource` fails and a downstream `compose()` silently falls
-  // back to a runtime fuse. A missing/failed source falls back to `undefined` state —
-  // the same best-effort contract as `_tf`.
-  const _wc = (src: string, inner: Comb): Comb => {
-    let extra: unknown
-    // eslint-disable-next-line no-eval
-    try { extra = (0, eval)(`(${src})`) } catch { extra = undefined }
-    const w = (withCtx as (e: unknown, c: Comb) => Comb)(extra, inner)
-    ;(w._def as { extraSrc?: string }).extraSrc = src
-    return w
-  }
-  // eslint-disable-next-line no-new-func
-  const fn = new Function(
-    'rules', 'ref', 'regex', 'literal', 'keywords', 'sequence', 'choice', 'dispatch', 'when', 'startsWith', 'endsWith', 'matches', 'otherwise', 'routed', 'attempt',
-    'many', 'oneOrMore', 'optional', 'sepBy', 'keepSeparator', 'not', 'peek', 'node', 'parser',
-    'scanTo', 'balanced', 'token', 'leaf', 'transform', 'trivia', 'classifiedTrivia', 'label', 'field', 'expect', 'adjacent', 'notAdjacent', '_tf', '_lf', '_nd', '_gch', '_wc',
-    `return (${ir})`,
-  )
-  const map = fn(
-    rules, ref, regex, literal, keywords, sequence, choice, dispatch, when, startsWith, endsWith, matches, otherwise, routed, attempt,
-    many, oneOrMore, optional, sepBy, keepSeparator, not, peek, node, parser,
-    scanTo, balanced, token, leaf, transform, trivia, classifiedTrivia, label, field, expectC, adjacent, notAdjacent, _tf, _lf, _nd, _gch, _wc,
-  ) as Record<string, Comb>
-  return Object.entries(map)
+/**
+ * ONE plain combinator as IR — what a compiled module carries on an exported terminal
+ * so another module can import it into a grammar (the macro evaluates grammars, and a
+ * compiled parser function is not a combinator it can evaluate).
+ *
+ * No new format: it is `serializeRuleMap`'s `scanSkip` channel over an empty map, which
+ * already serializes rule-free terminals OUTSIDE any factory — exactly this shape. Its
+ * inverse, `evalCombinatorIR`, evaluates source and so lives in `src/plugin/ir-eval.ts`.
+ */
+export function serializeCombinator(c: Comb): string | null {
+  return serializeRuleMap([], [c])
 }
 
 /** A recursive combinator currently being emitted, and the local `ref()` var that
@@ -590,7 +484,7 @@ class Serializer {
         return `_lf(${kid(def.parser)}, ${JSON.stringify(def.fnSrc)}${def.cstValue === 'source' ? ', true' : ''})`
       }
       case 'label':     return `label(${JSON.stringify(def.label)}, ${kid(def.parser)})`
-      case 'field':     return `field(${JSON.stringify(def.name)}, ${kid(def.parser)})`
+      case 'field':     return `field(${JSON.stringify(def.name)}, ${kid(def.parser)}${fieldOptionsSrc(def)})`
       case 'expect':    return `expect(${kid(def.parser)}${def.label !== undefined ? `, ${JSON.stringify(def.label)}` : ''})`
       case 'scanTo':
         return `scanTo(${kid(def.sentinel)}, { skip: [${def.skip.map(kid).join(', ')}]${def.raw ? ', raw: true' : ''}, orEOF: ${def.orEOF}${def.recoverAt === undefined ? '' : `, recoverAt: ${kid(def.recoverAt)}`}${def.stopAt === undefined ? '' : `, stopAt: ${kid(def.stopAt)}`} })`
@@ -675,3 +569,9 @@ class Serializer {
     }
   }
 }
+
+/** The options argument of a trivia-attributing `field()`, or nothing for a plain one. */
+function fieldOptionsSrc(def: Extract<ParserDef, { tag: 'field' }>): string {
+  if (def.leadingTrivia !== true && def.trailingTrivia !== true) return ''
+  return `, { ${[def.leadingTrivia === true ? 'leadingTrivia: true' : '', def.trailingTrivia === true ? 'trailingTrivia: true' : ''].filter(Boolean).join(', ')} }`
+}