parseman 0.61.0 → 0.61.1

package/dist/version.d.ts

@@ -13,5 +13,5 @@
  * Kept in sync with `package.json` by `test/unit/version-sync.test.ts`, which fails
  * the build if this constant drifts from the published version.
  */
-export declare const PARSEMAN_VERSION = "0.61.0";
+export declare const PARSEMAN_VERSION = "0.61.1";
 //# sourceMappingURL=version.d.ts.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "parseman",
-  "version": "0.61.0",
+  "version": "0.61.1",
   "packageManager": "pnpm@10.15.1",
   "engines": {
     "node": "^20.19.0 || >=22.13.0"
@@ -119,6 +119,10 @@
     "size:guard:jess": "node --import tsx/esm bench/jess-grammar-size-guard.ts",
     "vendor:jess-grammars": "node scripts/vendor-jess-grammars.mjs",
     "size:probe": "node --import tsx/esm bench/size/probe.ts",
+    "callcount:guard": "node --import tsx/esm bench/call-count-guard.ts",
+    "callcount:baseline": "node --import tsx/esm bench/call-count-guard.ts --update",
+    "dupfn:guard": "node --import tsx/esm bench/dup-function-guard.ts",
+    "dupfn:baseline": "node --import tsx/esm bench/dup-function-guard.ts --update",
     "choicecost:guard": "node --import tsx/esm bench/choice-cost-guard.ts",
     "choicecost:baseline": "node --import tsx/esm bench/choice-cost-guard.ts --update",
     "choicecost:report": "node --import tsx/esm bench/choice-cost-guard.ts --report",

package/src/analysis/backtrack.ts

@@ -0,0 +1,249 @@
+/**
+ * MEASURE BACKTRACKING among the terminals a parse matched.
+ *
+ * `backtrackProbe(grammar, engine)` parses with an instrumented copy of the grammar
+ * and reports, per parse, how often a matched terminal started behind the one before
+ * it and how many bytes matched twice, plus the lookahead it did (`BacktrackCounts`,
+ * `table/backtrack-probe.ts`). Each `measure` also runs the plain grammar, returns
+ * that result, and refuses to report if the instrumented parse returned anything else.
+ *
+ * Counts of zero do not show a parse wasted nothing. Failed reads, trivia re-scans,
+ * the choice predecide pretest and table `token()` bodies go uncounted; the list and
+ * an example are in `table/backtrack-probe.ts`.
+ *
+ * Three engines, one rule. `closure` and `emitted` run a rewritten copy of the
+ * table (`probeProgram`); `interpreter` runs a rebuilt copy of the combinator graph
+ * whose terminals report to the same recorder. Nothing here is reachable from a
+ * parse that did not ask for it, which is why it costs an ordinary parse nothing.
+ */
+import type { Combinator, ParseContext, ParseResult } from '../types.ts'
+import { run, type RunOptions, type RunResult } from '../functional/run.ts'
+import { digestValue } from '../oracle/digest.ts'
+import { compileRuleMapRunnable } from '../table/compile-rule-map.ts'
+import { expandCompact, type CompactProgram, type TableProgram } from '../table/program.ts'
+import { AssemblyCache, tableRules } from '../table/assemble.ts'
+import {
+  BacktrackRecorder, hasLookaround, probeCfg, probeProgram, type BacktrackCounts,
+} from '../table/backtrack-probe.ts'
+import { rebuildCombinator } from './rebuild.ts'
+
+export type { BacktrackCounts } from '../table/backtrack-probe.ts'
+
+export type BacktrackEngine = 'interpreter' | 'closure'
+
+export type BacktrackMeasurement = {
+  readonly result: RunResult
+  readonly counts: BacktrackCounts
+}
+
+export type BacktrackProbe = {
+  /** The engine probed parses run on. */
+  readonly engine: { readonly ran: BacktrackEngine | 'emitted' }
+  /**
+   * Parse `input` from rule `entry`, plainly and probed, count the probed parse and
+   * return the plain parse's result. Throws if the two parses differ in outcome, value
+   * or where they stopped.
+   *
+   * Both parses get the same `options.state`, the plain one first. A grammar that
+   * writes state (a `gate()` that mutates it, say) sees the first parse's writes in the
+   * second, so the measure either throws or counts a different parse, possibly with
+   * fewer rewinds. Measure such a grammar only if resetting its state makes no difference.
+   */
+  measure(entry: string, input: string, options?: RunOptions): BacktrackMeasurement
+}
+
+type Runnable = Parameters<typeof run>[0]
+
+function isCombinator(x: unknown): x is Combinator<unknown> {
+  return typeof x === 'object' && x !== null && '_def' in x
+    && typeof (x as { parse?: unknown }).parse === 'function'
+}
+
+function isProgram(x: object): x is TableProgram | CompactProgram {
+  return Array.isArray((x as { code?: unknown }).code) || Array.isArray((x as { c?: unknown }).c)
+}
+
+/** The rules a `compile()` of this map would encode: combinators, minus undefined refs. */
+function ruleEntries(grammar: Record<string, unknown>): Array<[string, Combinator<unknown>]> {
+  const out: Array<[string, Combinator<unknown>]> = []
+  for (const [name, value] of Object.entries(grammar)) {
+    if (!isCombinator(value)) continue
+    if (value._def.tag === 'lazy') {
+      try { value._def.thunk() } catch { continue }
+    }
+    out.push([name, value])
+  }
+  return out
+}
+
+function outcome(r: RunResult): string {
+  return digestValue({ ok: r.ok, value: r.value, span: r.span, unconsumedFrom: r.unconsumedFrom })
+}
+
+type Grammar = Readonly<Record<string, unknown>> | TableProgram | CompactProgram
+
+function tableOf(grammar: Grammar): TableProgram {
+  if (isProgram(grammar)) return expandCompact(grammar)
+  const refusals: string[] = []
+  const compiled = compileRuleMapRunnable(ruleEntries(grammar), { refusals })
+  if (compiled === null) throw new Error(`backtrackProbe: this grammar could not be encoded to a table: ${refusals.join('; ')}`)
+  return compiled.prog
+}
+
+/**
+ * Probe a grammar on one engine.
+ *
+ * `grammar` is a rule map of combinators (a `rules()` or `compose()` result), or a
+ * table program for the closure engine. A rule map runs on the table through
+ * `compileRuleMapRunnable`, the lowering `compile()` uses.
+ *
+ * The emitted engine is not offered here: it needs code built from source, which
+ * at runtime only `compile()` may do (`docs/design/runtime-and-size-contract.md`,
+ * rule 1). Tests and benches reach it through `backtrackProbeAssembled` with a
+ * build-time module (`bench/emitted-assembly.ts`). It counts what the closure
+ * engine counts, because both run the same rewritten program.
+ */
+export function backtrackProbe(grammar: Grammar, engine: BacktrackEngine = 'closure'): BacktrackProbe {
+  const rec = new BacktrackRecorder()
+  if (engine === 'interpreter') {
+    if (isProgram(grammar)) throw new TypeError('backtrackProbe: the interpreter runs a rule map of combinators, not a table program')
+    const rules = Object.fromEntries(ruleEntries(grammar))
+    const built = new Map<string, Combinator<unknown>>()
+    const instrument = instrumentFor(rec)
+    return probe(rec, 'interpreter', entry => ruleOf(rules, entry), (entry) => {
+      let hit = built.get(entry)
+      if (hit === undefined) {
+        hit = rebuildCombinator(ruleOf(rules, entry), new Map(), { thaw: true, after: instrument }).root
+        built.set(entry, hit)
+      }
+      return hit
+    })
+  }
+  if (engine !== 'closure') throw new TypeError(`backtrackProbe: no engine ${JSON.stringify(engine)}; it runs 'closure' or 'interpreter'`)
+  const prog = tableOf(grammar)
+  const base = tableRules({ ...prog, asm: [] })
+  const rules = tableRules(probeProgram(prog, rec))
+  return probe(rec, 'closure', entry => ruleOf(base, entry), entry => ruleOf(rules, entry))
+}
+
+/**
+ * For tests and benches, not `parseman/diagnostics`: the probe on the emitted
+ * engine. `assemble` hands the rewritten program back with a pre-compiled
+ * assembly for `probeCfg(program)`, built as a module the way a macro build ships
+ * one. Nothing here turns a string into code, and a program with no assembly is
+ * refused, because the assembler would.
+ */
+export async function backtrackProbeAssembled(
+  grammar: Grammar,
+  assemble: (probed: TableProgram) => Promise<TableProgram>,
+): Promise<BacktrackProbe> {
+  const rec = new BacktrackRecorder()
+  const prog = tableOf(grammar)
+  const base = tableRules({ ...prog, asm: [] })
+  const assembled = await assemble(probeProgram(prog, rec))
+  // A program with no inventory would make the assembler build the engine from source.
+  if (assembled.asm === undefined) throw new TypeError('backtrackProbeAssembled: `assemble` returned a program with no pre-compiled assembly')
+  // Ask the assembler which engine it built, rather than assume the factory was taken.
+  const refusal = new AssemblyCache(assembled).for(probeCfg(assembled)).emitRefusal
+  if (refusal !== undefined) throw new Error(`backtrackProbeAssembled: the assembly was not used: ${refusal}`)
+  const rules = tableRules(assembled)
+  const emitted = probe(rec, 'emitted', entry => ruleOf(base, entry), entry => ruleOf(rules, entry))
+  return {
+    engine: emitted.engine,
+    measure(entry, input, options = {}) {
+      // The assembly serves `probeCfg`'s option set alone. An option that selects
+      // another one (tolerant, completions, a build host…) would run closures under
+      // an `emitted` label, so only the parse state may be passed.
+      const other = Object.keys(options).filter(k => k !== 'state')
+      if (other.length > 0) throw new TypeError(`backtrackProbeAssembled: the emitted probe runs a plain parse; it takes no ${other.join(', ')}`)
+      return emitted.measure(entry, input, options)
+    },
+  }
+}
+
+function probe(
+  rec: BacktrackRecorder,
+  ran: BacktrackProbe['engine']['ran'],
+  plain: (entry: string) => Runnable,
+  probed: (entry: string) => Runnable,
+): BacktrackProbe {
+  return {
+    engine: { ran },
+    measure(entry, input, options = {}) {
+      const result = run(plain(entry), input, options)
+      rec.begin(input)
+      const probedResult = run(probed(entry), input, options)
+      const counts = rec.end()
+      if (outcome(probedResult) !== outcome(result)) {
+        throw new Error(`backtrackProbe: the probed ${ran} parse of rule ${JSON.stringify(entry)} differs from the plain one; its counts would describe another parse`)
+      }
+      return { result, counts }
+    },
+  }
+}
+
+function ruleOf<T>(rules: Readonly<Record<string, T>>, entry: string): T {
+  if (!Object.hasOwn(rules, entry)) throw new Error(`backtrackProbe: the grammar has no rule ${JSON.stringify(entry)}`)
+  return rules[entry]!
+}
+
+type Parse = (input: string, pos: number, ctx: ParseContext) => ParseResult<unknown>
+
+function wrapped(out: Combinator<unknown>, parse: Parse): Combinator<unknown> {
+  const w: Combinator<unknown> = { _tag: out._tag, _meta: out._meta, _def: out._def, parse }
+  const name = (out as { _ruleName?: string })._ruleName
+  if (name !== undefined) (w as { _ruleName?: string })._ruleName = name
+  return w
+}
+
+/**
+ * The interpreter's half of the rewrite `probeProgram` makes to a table: terminals
+ * report their span, a lookahead marks its extent, and a scan's own reads stay
+ * inside it. The rebuilt node is wrapped, never patched, so the grammar the caller
+ * holds is untouched.
+ */
+function instrumentFor(rec: BacktrackRecorder): (original: Combinator<unknown>, rebuilt: Combinator<unknown>) => Combinator<unknown> {
+  return (_original, out) => {
+    const d = out._def
+    switch (d.tag) {
+      case 'literal': case 'keywords':
+        return wrapped(out, (input, pos, ctx) => {
+          const r = out.parse(input, pos, ctx)
+          if (r.ok) rec.consume(r.span.start, r.span.end)
+          return r
+        })
+      case 'regex': {
+        const re = new RegExp(d.source, d.flags)
+        if (!hasLookaround(re)) {
+          return wrapped(out, (input, pos, ctx) => {
+            const r = out.parse(input, pos, ctx)
+            if (r.ok) rec.consume(r.span.start, r.span.end)
+            return r
+          })
+        }
+        return wrapped(out, (input, pos, ctx) => {
+          rec.regexAttempt()
+          const r = out.parse(input, pos, ctx)
+          if (r.ok) rec.regexMatch(re, r.span.start, r.span.end)
+          return r
+        })
+      }
+      case 'scanTo': case 'lines': case 'columns': case 'recover': case 'unknown':
+        return wrapped(out, (input, pos, ctx) => {
+          rec.enterOpaque()
+          let r: ParseResult<unknown>
+          try { r = out.parse(input, pos, ctx) } finally { rec.exitOpaque() }
+          if (r.ok) rec.consume(r.span.start, r.span.end)
+          return r
+        })
+      case 'not': case 'peek':
+        return wrapped(out, (input, pos, ctx) => {
+          rec.enterLookahead()
+          try { return out.parse(input, pos, ctx) } finally { rec.exitLookahead() }
+        })
+      default:
+        return out
+    }
+  }
+}
+

package/src/analysis/diagnostics.ts

@@ -30,4 +30,8 @@ export { measureChoiceCost, armFirstSets } from './corpus.ts'
 export type { CorpusSample, CorpusSite, ChoiceCorpusCost, ArmCorpusCost } from './corpus.ts'
 
 export { rebuildCombinator } from './rebuild.ts'
+export type { RebuildOptions } from './rebuild.ts'
+
+export { backtrackProbe } from './backtrack.ts'
+export type { BacktrackProbe, BacktrackEngine, BacktrackMeasurement, BacktrackCounts } from './backtrack.ts'
 export type { RebuildResult, FrozenSubtree } from './rebuild.ts'

package/src/analysis/rebuild.ts

@@ -53,12 +53,17 @@ import { token, leaf, sourceLeaf } from '../combinators/token.ts'
 import { expect } from '../combinators/expect.ts'
 import { adjacent, notAdjacent } from '../combinators/adjacency.ts'
 import { parser } from '../combinators/grammar.ts'
+import { dispatch, dispatchMatcher, otherwise, routed, when, type DispatchArm } from '../combinators/dispatch.ts'
+import { withCtx } from '../combinators/withCtx.ts'
 
 /** A node kind reused verbatim: no public factory reconstructs it without loss. */
 const FROZEN_TAGS: ReadonlySet<string> = new Set([
   'dispatch', 'recover', 'scanTo', 'guard', 'withCtx', 'routed', 'trivia', 'unknown', 'lines', 'columns',
 ])
 
+/** The frozen kinds `RebuildOptions.thaw` rebuilds. */
+const THAWED_TAGS: ReadonlySet<string> = new Set(['dispatch', 'withCtx', 'routed', 'guard'])
+
 /** A subtree that was reused verbatim instead of rebuilt. */
 export type FrozenSubtree = {
   tag: string
@@ -117,6 +122,23 @@ function subtreeContains(p: Combinator<unknown>, targets: ReadonlySet<Combinator
   return hit
 }
 
+export type RebuildOptions = {
+  /**
+   * Also rebuild `dispatch`, a gated `choice`, `withCtx` and `routed` instead of
+   * freezing them. Each is reconstructed from its own def fields — the arms, gates,
+   * matchers and `extra` the grammar built it with. Off for `proposeFixes()`, whose
+   * reports name the frozen kinds; on for an instrumented rebuild, which must reach
+   * every terminal (`backtrack.ts`).
+   */
+  readonly thaw?: boolean
+  /**
+   * Called with each node and what it was rebuilt to (itself, for a frozen or
+   * childless one), and its result is used in its place. Not called for a
+   * replacement or a `lazy` slot.
+   */
+  readonly after?: (original: Combinator<unknown>, rebuilt: Combinator<unknown>) => Combinator<unknown>
+}
+
 /**
  * Rebuild `root`, replacing every combinator that is a key of `replacements` with its
  * value. The replacement is spliced WITHOUT descending into the original target.
@@ -124,7 +146,10 @@ function subtreeContains(p: Combinator<unknown>, targets: ReadonlySet<Combinator
 export function rebuildCombinator(
   root: Combinator<unknown>,
   replacements: ReadonlyMap<Combinator<unknown>, Combinator<unknown>>,
+  options: RebuildOptions = {},
 ): RebuildResult {
+  const thaw = options.thaw === true
+  const after = options.after ?? ((_p: Combinator<unknown>, out: Combinator<unknown>) => out)
   const memo = new Map<Combinator<unknown>, Combinator<unknown>>()
   const frozenKeys = new Set<string>()
   const frozen: FrozenSubtree[] = []
@@ -141,12 +166,13 @@ export function rebuildCombinator(
     if (d === undefined || typeof d !== 'object') { memo.set(p, p); return p }
     const here = (p as { _ruleName?: string })._ruleName ?? rule
 
-    if (FROZEN_TAGS.has(d.tag)) {
+    if (FROZEN_TAGS.has(d.tag) && !(thaw && THAWED_TAGS.has(d.tag))) {
       const key = JSON.stringify([here, d.tag])
       if (!frozenKeys.has(key)) { frozenKeys.add(key); frozen.push({ tag: d.tag, rule: here }) }
       for (const t of subtreeContains(p, targets)) if (!unapplied.includes(t)) unapplied.push(t)
-      memo.set(p, p)
-      return p
+      const kept = after(p, p)
+      memo.set(p, kept)
+      return kept
     }
 
     // A ref/lazy is the only place a cycle can close. Publish an EMPTY slot into the
@@ -188,12 +214,21 @@ export function rebuildCombinator(
         // arity (it takes GatedArm objects the grammar built), and the whole point of
         // rebuilding a choice is that its strategy is recomputed — which the gated path
         // suppresses anyway. Freeze it rather than rebuild it wrong.
+        if (d.gates.some(g => g !== null) && thaw) {
+          const unsafeChoice = choice as unknown as (...items: unknown[]) => Combinator<unknown>
+          out = unsafeChoice(...d.parsers.map((arm, i) => {
+            const gate = d.gates[i]
+            return gate ? { combinator: one(arm), gate } : one(arm)
+          }))
+          break
+        }
         if (d.gates.some(g => g !== null)) {
           const key = JSON.stringify([here, "choice(gated)"])
           if (!frozenKeys.has(key)) { frozenKeys.add(key); frozen.push({ tag: 'choice(gated)', rule: here }) }
           for (const t of subtreeContains(p, targets)) if (!unapplied.includes(t)) unapplied.push(t)
-          memo.set(p, p)
-          return p
+          const kept = after(p, p)
+          memo.set(p, kept)
+          return kept
         }
         const kids = d.parsers.map(one) as [Combinator<unknown>, ...Combinator<unknown>[]]
         out = choice(...kids); break
@@ -250,8 +285,9 @@ export function rebuildCombinator(
           const key = `${here} node(untyped+build+opts)`
           if (!frozenKeys.has(key)) { frozenKeys.add(key); frozen.push({ tag: 'node(untyped+build+opts)', rule: here }) }
           for (const t of subtreeContains(p, targets)) if (!unapplied.includes(t)) unapplied.push(t)
-          memo.set(p, p)
-          return p
+          const kept = after(p, p)
+          memo.set(p, kept)
+          return kept
         }
         out = d.type === undefined
           ? (d.build === undefined ? node(one(d.parser), opts) : node(one(d.parser), d.build))
@@ -259,6 +295,20 @@ export function rebuildCombinator(
         if (d.buildRawUnused === true && out._def.tag === 'node') out._def.buildRawUnused = true
         break
       }
+      case 'dispatch': {
+        const arms: DispatchArm<unknown>[] = d.cases.map(e =>
+          when(e.keys, one(e.parser), e.caseInsensitive ? { caseInsensitive: true } : undefined))
+        for (const e of d.matchers ?? []) {
+          arms.push(when(dispatchMatcher(e), one(e.parser), e.caseInsensitive ? { caseInsensitive: true } : undefined))
+        }
+        if (d.otherwise) arms.push(otherwise(one(d.otherwise), d.otherwiseCommit === false ? { commit: false } : undefined))
+        const unsafeDispatch = dispatch as (selector: Combinator<string>, ...items: DispatchArm<unknown>[]) => Combinator<unknown>
+        out = unsafeDispatch(one(d.selector) as Combinator<string>, ...arms)
+        break
+      }
+      case 'withCtx': out = withCtx(d.extra, one(d.parser)); break
+      case 'routed': out = d.fallback === undefined ? routed() : routed(one(d.fallback)); break
+      case 'guard': out = p; break
       case 'grammar': {
         const inner = one(d.parser)
         out = parser({
@@ -273,13 +323,15 @@ export function rebuildCombinator(
         const key = JSON.stringify([here, (d as { tag: string }).tag])
         if (!frozenKeys.has(key)) { frozenKeys.add(key); frozen.push({ tag: (d as { tag: string }).tag, rule: here }) }
         for (const t of subtreeContains(p, targets)) if (!unapplied.includes(t)) unapplied.push(t)
-        memo.set(p, p)
-        return p
+        const kept = after(p, p)
+        memo.set(p, kept)
+        return kept
       }
     }
     carryStamps(p, out)
-    memo.set(p, out)
-    return out
+    const done = after(p, out)
+    memo.set(p, done)
+    return done
   }
 
   const rebuilt = go(root, (root as { _ruleName?: string })._ruleName ?? '<entry>')

package/src/combinators/dispatch.ts

@@ -180,6 +180,15 @@ export function matches(pattern: RegExp): DispatchStringMatcher {
   return { kind: 'matches', value: pattern.source, flags: pattern.flags }
 }
 
+/** The matcher a `dispatch()` def records for a case, as the builder that made it. */
+export function dispatchMatcher(entry: DispatchMatcherCase): DispatchStringMatcher {
+  switch (entry.kind) {
+    case 'startsWith': return startsWith(entry.value)
+    case 'endsWith': return endsWith(entry.value)
+    case 'matches': return matches(new RegExp(entry.value, entry.flags))
+  }
+}
+
 export function when<T>(
   key: DispatchWhenKey,
   parser: Combinator<T>,

package/src/coverage.ts

@@ -2,9 +2,9 @@ import { lines } from './combinators/lines.ts'
 import { run, type RunOptions, type RunResult, type Runnable } from './functional/run.ts'
 import { GRAMMAR_COVERAGE_DEFINITIONS } from './grammar-metadata.ts'
 export { GRAMMAR_COVERAGE_DEFINITIONS } from './grammar-metadata.ts'
-import type { Combinator, DispatchCase, DispatchMatcherCase, ParseContext, ParseResult } from './types.ts'
+import type { Combinator, DispatchCase, ParseContext, ParseResult } from './types.ts'
 import { choice } from './combinators/choice.ts'
-import { dispatch, endsWith, matches, otherwise, startsWith, when, type DispatchArm, type DispatchStringMatcher } from './combinators/dispatch.ts'
+import { dispatch, dispatchMatcher, otherwise, when, type DispatchArm } from './combinators/dispatch.ts'
 import { attempt } from './combinators/attempt.ts'
 import { getCoreLiteralValue } from './combinators/choice.ts'
 import { not } from './combinators/not.ts'
@@ -24,18 +24,6 @@ import { buildGrammarPlan, type GrammarCoverageDefinition, type GrammarCoverageP
 
 export type { GrammarCoverageDefinition } from './compiler/grammar-coverage-ids.ts'
 
-function dispatchMatcher(entry: DispatchMatcherCase): DispatchStringMatcher {
-  switch (entry.kind) {
-    case 'startsWith':
-      return startsWith(entry.value)
-    case 'endsWith':
-      return endsWith(entry.value)
-    case 'matches':
-      return matches(new RegExp(entry.value, entry.flags))
-  }
-}
-
-
 export type GrammarCoverageSnapshot = {
   definitions: readonly GrammarCoverageDefinition[]
   hits: readonly string[]