parseman 0.52.0 → 0.54.0

package/src/coverage.ts

@@ -18,7 +18,7 @@ import { scanTo } from './combinators/scanTo.ts'
 import { sequence } from './combinators/sequence.ts'
 import { token, leaf, sourceLeaf } from './combinators/token.ts'
 import { withCtx } from './combinators/withCtx.ts'
-import { composedCoverageRules } from './compiler/linker.ts'
+import { composedCoverageRules, linkedWinnersOf } from './compiler/linker.ts'
 import { buildGrammarPlan, type GrammarCoverageDefinition, type GrammarCoveragePlan } from './compiler/grammar-coverage-ids.ts'
 
 export type { GrammarCoverageDefinition } from './compiler/grammar-coverage-ids.ts'
@@ -156,11 +156,11 @@ export function grammarCoverageDefinitions(entry: Combinator<unknown>, winners?:
 }
 
 /** Definitions for a runtime `compose()` result, normalized through its final
- * IR winner map. Opaque precompiled pieces intentionally fail rather than
+ * winner map. Anything else — a compiled table, a plain map — fails rather than
  * reporting source-piece identities as though they were final grammar IDs. */
 export function composedGrammarCoverageDefinitions(grammar: Record<string, unknown>, startRule: string): readonly GrammarCoverageDefinition[] {
   const winners = composedCoverageRules(grammar)
-  if (!winners) throw new TypeError('semantic coverage needs re-lowerable composed IR; this composition contains an opaque artifact')
+  if (!winners) throw new TypeError('semantic coverage needs a runtime compose() result (a linked interpreter grammar); this is not one')
   const entry = winners[startRule]
   if (!entry) throw new TypeError(`semantic coverage start rule ${JSON.stringify(startRule)} is not a final winner`)
   return grammarCoverageDefinitions(entry, winners)
@@ -343,7 +343,7 @@ function coverageEntry(entry: Combinator<unknown>, collector: GrammarCoverageCol
         case 'trivia': return trivia(build(def.parser))
         case 'token': return token(build(def.parser))
         case 'leaf': return def.cstValue === 'source' ? sourceLeaf(build(def.parser), def.fn) : leaf(build(def.parser), def.fn)
-        case 'field': return field(def.name, build(def.parser))
+        case 'field': return field(def.name, build(def.parser), { leadingTrivia: def.leadingTrivia === true, trailingTrivia: def.trailingTrivia === true })
         case 'grammar': return grammarParser({
           ...(def.triviaParser === undefined ? (def.clearTrivia ? { trivia: null } : {}) : { trivia: build(def.triviaParser) }),
           ...(def.captureTrivia ? { captureTrivia: true } : {}),
@@ -380,7 +380,9 @@ function coverageEntry(entry: Combinator<unknown>, collector: GrammarCoverageCol
 
 export function runWithGrammarCoverage(entry: Runnable, input: string, options: RunOptions & { collector?: GrammarCoverageCollector; trace?: GrammarTraceSink } = {}): { result: RunResult; coverage: GrammarCoverageSnapshot } {
   if (typeof entry === 'function') throw new TypeError('runWithGrammarCoverage currently requires an interpreter combinator entry')
-  const plan = buildGrammarPlan(entry)
+  // A runtime composition's rule is planned through its winner map, so its IDs are
+  // the ones `composedGrammarCoverageDefinitions` reports.
+  const plan = buildGrammarPlan(entry, linkedWinnersOf(entry))
   const definitions = plan.definitions
   const collector = options.collector ?? createGrammarCoverageCollector(definitions)
   const { collector: _collector, trace, ...runOptions } = options

package/src/cst/reflection.ts

@@ -115,6 +115,11 @@ export function collectGrammarReflection(
 ): GrammarReflection {
   const byType = new Map<string, string[]>()
   const followLazy = opts?.followLazy !== false
+  // ONE walk over the whole map. A node already visited from an earlier rule had
+  // its whole subgraph visited then, and `addNode` is idempotent, so a fresh set
+  // per rule only re-walked shared structure — O(rules × graph), and every unbound
+  // cross-piece hole threw (a stack-capturing Error) once per rule reaching it.
+  const seen = new Set<Combinator<unknown>>()
   for (const [, rule] of ruleMap) {
     if (!followLazy && rule._def.tag === 'lazy') {
       try {
@@ -123,7 +128,7 @@ export function collectGrammarReflection(
       } catch {}
       continue
     }
-    collectFromCombinator(rule, byType, new Set(), followLazy)
+    collectFromCombinator(rule, byType, seen, followLazy)
   }
   return { nodes: [...byType].map(([type, tags]) => ({ type, tags })) }
 }

package/src/cst/trivia-charscan.ts

@@ -226,74 +226,35 @@ function plainRegexSource(p: Combinator<unknown>): string | null {
   return d.tag === 'regex' && d.flags === '' ? d.source : null
 }
 
-let visitSpec0: LabeledTriviaSpec | undefined
-let visitSpec1: LabeledTriviaSpec | undefined
-let visitSpec2: LabeledTriviaSpec | undefined
-let visitSpec3: LabeledTriviaSpec | undefined
-let visitValue0: CssTriviaVisitor | null | undefined
-let visitValue1: CssTriviaVisitor | null | undefined
-let visitValue2: CssTriviaVisitor | null | undefined
-let visitValue3: CssTriviaVisitor | null | undefined
+/** The fields of a spec the char-level lowerings below are computed from. */
+type TriviaArmsSpec = Pick<LabeledTriviaSpec, 'arms' | 'minRepeats'>
 
-/** A direct classified scanner for the four canonical CSS/Less trivia tuples. */
-export function commonLabeledTriviaVisitor(spec: LabeledTriviaSpec): CssTriviaVisitor | null {
-  if (spec === visitSpec0) return visitValue0!
-  if (spec === visitSpec1) return visitValue1!
-  if (spec === visitSpec2) return visitValue2!
-  if (spec === visitSpec3) return visitValue3!
-  const sources = spec.minRepeats <= 1 ? spec.arms.map(arm => plainRegexSource(arm.parser)) : []
-  const visitor = commonCssTriviaVisitor(sources)
-  visitSpec3 = visitSpec2
-  visitValue3 = visitValue2
-  visitSpec2 = visitSpec1
-  visitValue2 = visitValue1
-  visitSpec1 = visitSpec0
-  visitValue1 = visitValue0
-  visitSpec0 = spec
-  visitValue0 = visitor
-  return visitor
+/**
+ * A direct classified scanner for the four canonical CSS/Less trivia tuples.
+ * Computed once per spec, by `analyzeLabeledTrivia`, and read off it as
+ * `spec.visitor`: a scan never asks again.
+ */
+export function commonLabeledTriviaVisitor(spec: TriviaArmsSpec): CssTriviaVisitor | null {
+  return commonCssTriviaVisitor(spec.minRepeats <= 1 ? spec.arms.map(arm => plainRegexSource(arm.parser)) : [])
 }
 
-let armSpec0: LabeledTriviaSpec | undefined
-let armSpec1: LabeledTriviaSpec | undefined
-let armSpec2: LabeledTriviaSpec | undefined
-let armSpec3: LabeledTriviaSpec | undefined
-let armValue0: readonly TriviaArmMatcher[] | null | undefined
-let armValue1: readonly TriviaArmMatcher[] | null | undefined
-let armValue2: readonly TriviaArmMatcher[] | null | undefined
-let armValue3: readonly TriviaArmMatcher[] | null | undefined
-
 /**
  * The character-level matchers for a spec's arms — index-parallel to `spec.arms`,
  * so a matcher's index IS its kind index — or `null` when any arm has no lowering.
+ * Computed once per spec, as `spec.charArms`.
  *
  * `spec.minRepeats > 1` is excluded: the loops below report only an END, and a run
  * that fails a minimum of two must report NOTHING. `classifiedTrivia()` never
  * produces one, and the combinator path still handles it if a host writes one.
  */
-export function charArmsFor(spec: LabeledTriviaSpec): readonly TriviaArmMatcher[] | null {
-  if (spec === armSpec0) return armValue0!
-  if (spec === armSpec1) return armValue1!
-  if (spec === armSpec2) return armValue2!
-  if (spec === armSpec3) return armValue3!
-  let arms: TriviaArmMatcher[] | null = []
-  if (spec.minRepeats > 1) {
-    arms = null
-  } else {
-    for (const arm of spec.arms) {
-      const c = classifyParser(arm.parser)
-      if (!c) { arms = null; break }
-      arms.push(c)
-    }
+export function charArmsFor(spec: TriviaArmsSpec): readonly TriviaArmMatcher[] | null {
+  if (spec.minRepeats > 1) return null
+  const arms: TriviaArmMatcher[] = []
+  for (const arm of spec.arms) {
+    const c = classifyParser(arm.parser)
+    if (!c) return null
+    arms.push(c)
   }
-  armSpec3 = armSpec2
-  armValue3 = armValue2
-  armSpec2 = armSpec1
-  armValue2 = armValue1
-  armSpec1 = armSpec0
-  armValue1 = armValue0
-  armSpec0 = spec
-  armValue0 = arms
   return arms
 }
 

package/src/cst/trivia-kinds.ts

@@ -2,7 +2,10 @@ import type { Combinator, ParseContext } from '../types.ts'
 import { pushCstTriviaEntry, pushTriviaLogEntry } from './capture-buffer.ts'
 import { startsFirstSet } from '../combinators/first-set.ts'
 import { createDetachedParseContext } from '../parse-context.ts'
-import { charArmsFor, charTriviaEnd, charTriviaKindMask, charTriviaVisit, commonLabeledTriviaVisitor } from './trivia-charscan.ts'
+import {
+  charArmsFor, charTriviaEnd, charTriviaKindMask, charTriviaVisit, commonLabeledTriviaVisitor, type TriviaArmMatcher,
+} from './trivia-charscan.ts'
+import type { CssTriviaVisitor } from './trivia-css-scanner.ts'
 
 export type TriviaChunk = { start: number; end: number; kindIndex: number }
 
@@ -10,6 +13,13 @@ export type LabeledTriviaSpec = {
   readonly labels: readonly string[]
   readonly arms: ReadonlyArray<{ label: string; kindIndex: number; parser: Combinator<unknown> }>
   readonly minRepeats: number
+  /**
+   * The spec's char-level scanners, built with it. A scan reads them here: the
+   * four-slot memo they used to sit behind thrashed on a grammar with more trivia
+   * tables than slots, and rebuilt one on every miss.
+   */
+  readonly visitor: CssTriviaVisitor | null
+  readonly charArms: readonly TriviaArmMatcher[] | null
 }
 
 /** See `trivia-skip.ts`: bounded scalar memoisation avoids a WeakMap in the
@@ -93,12 +103,18 @@ export function analyzeLabeledTrivia(trivia: Combinator<unknown>): LabeledTrivia
     arms.push({ label: peeled.label, kindIndex: 0, parser: peeled.parser })
   }
 
-  const spec = {
+  return cacheLabeledTrivia(trivia, labeledTriviaSpec(arms, minRepeats))
+}
+
+/** A spec, with the char-level scanners every scan of it reads. */
+export function labeledTriviaSpec(arms: LabeledTriviaSpec['arms'], minRepeats: number): LabeledTriviaSpec {
+  return {
     labels: arms.map(a => a.label),
     arms,
     minRepeats,
+    visitor: commonLabeledTriviaVisitor({ arms, minRepeats }),
+    charArms: charArmsFor({ arms, minRepeats }),
   }
-  return cacheLabeledTrivia(trivia, spec)
 }
 
 /** Label table on a `trivia()` combinator, if all arms are labeled. */
@@ -169,9 +185,9 @@ export function scanLabeledTriviaEnd(
   spec: LabeledTriviaSpec,
   state?: unknown,
 ): number {
-  // `charArmsFor` only lowers `minRepeats <= 1`, where "no chunk matched" and
+  // `charArms` only lowers `minRepeats <= 1`, where "no chunk matched" and
   // "the minimum was not met" are the same position — so an end IS the answer.
-  const chars = charArmsFor(spec)
+  const chars = spec.charArms
   if (chars !== null) return charTriviaEnd(input, cur, chars)
 
   let pos = cur
@@ -205,9 +221,9 @@ export function visitLabeledTrivia(
   visit: (start: number, end: number, kindIndex: number) => void,
 ): number | undefined {
   if (spec.minRepeats > 1) return undefined
-  const common = commonLabeledTriviaVisitor(spec)
+  const common = spec.visitor
   if (common !== null) return common(input, cur, visit)
-  const chars = charArmsFor(spec)
+  const chars = spec.charArms
   if (chars !== null) return charTriviaVisit(input, cur, chars, visit)
 
   let pos = cur
@@ -331,7 +347,7 @@ export function triviaKindMaskAt(
   spec: LabeledTriviaSpec,
   state?: unknown,
 ): number {
-  const chars = charArmsFor(spec)
+  const chars = spec.charArms
   if (chars !== null) return charTriviaKindMask(input, pos, chars)
 
   let cur = pos

package/src/index.ts

@@ -30,6 +30,7 @@ export { node } from './combinators/node.ts'
 export type { BuildNode, NodeCombinator, NodeOptions, NodeProjectOptions } from './combinators/node.ts'
 // lazy() is intentionally NOT exported.
 export { transform, trivia, classifiedTrivia, label, field } from './combinators/map.ts'
+export type { FieldOptions } from './combinators/map.ts'
 export { parse, parser, noTrivia } from './combinators/grammar.ts'
 export type { ParseOptions, ParserOptions, ParsemanParser } from './combinators/grammar.ts'
 export { token, leaf, sourceLeaf } from './combinators/token.ts'

package/src/plugin/evaluator.ts

@@ -154,6 +154,69 @@ export type ScopeEntry = {
 }
 export type Scope = Map<string, ScopeEntry>
 
+/** How a {@link ModuleScope} answers a name it does not hold: a value, the reason a
+ * binding the module DOES have cannot be known at build time, or `null` — not a
+ * binding this scope can speak for (an ordinary miss). */
+export type FreeNameResolution = { value: unknown } | { unresolved: string } | null
+
+/**
+ * A module's scope whose IMPORTED (and, for a foreign module, top-level) bindings
+ * resolve on first read instead of being absent.
+ *
+ * Absent was the defect. `anyValue` answered an imported name with `null`, so
+ * `rules({ scanSkip: [importedUnit] })` evaluated to nothing, the option quietly
+ * became `undefined`, and the build succeeded with no ambient skip — and a terminal
+ * imported into a rule body could not be lowered at all. Lazy, because a grammar
+ * module imports far more than combinators (AST classes, helpers) and resolving each
+ * one eagerly would read every one of those modules for nothing.
+ *
+ * A binding that exists but cannot be resolved is RECORDED in `unresolved` (shared by
+ * every scope of one transform) as "`name` <reason>", so the plugin can fail the build
+ * naming it rather than warn about an anonymous "isn't statically evaluable". Only a
+ * VALUE read records: `has()` is how the evaluator probes a name it may legitimately
+ * leave to runtime (a named reducer), so a probe that misses is not a failure.
+ */
+export class ModuleScope extends Map<string, unknown> {
+  private readonly tried = new Map<string, string | undefined>()
+  private readonly resolveFree: (name: string) => FreeNameResolution
+  readonly unresolved: Set<string>
+  constructor(resolveFree: (name: string) => FreeNameResolution, unresolved: Set<string>) {
+    super()
+    this.resolveFree = resolveFree
+    this.unresolved = unresolved
+  }
+
+  override has(name: string): boolean {
+    return super.has(name) || this.resolve(name)
+  }
+
+  override get(name: string): unknown {
+    if (super.has(name) || this.resolve(name)) return super.get(name)
+    const why = this.tried.get(name)
+    if (why !== undefined) this.unresolved.add(`\`${name}\` ${why}`)
+    return undefined
+  }
+
+  /** A child scope (a factory body) that keeps resolving this module's free names. */
+  fork(): ModuleScope {
+    const child = new ModuleScope(this.resolveFree, this.unresolved)
+    for (const [k, v] of super.entries()) child.set(k, v)
+    return child
+  }
+
+  private resolve(name: string): boolean {
+    if (this.tried.has(name)) return false
+    const r = this.resolveFree(name)
+    if (r !== null && 'value' in r) {
+      super.set(name, r.value)
+      return true
+    }
+    // A name nothing binds YET is not remembered: a later declaration may bind it.
+    if (r !== null) this.tried.set(name, r.unresolved)
+    return false
+  }
+}
+
 // Internal XScope also holds non-Combinator values (g proxy objects etc.)
 type XScopeVal = ScopeEntry | unknown
 type XScope = Map<string, XScopeVal>
@@ -251,7 +314,7 @@ const SUPPORTED: Record<string, (...args: unknown[]) => Combinator<unknown>> = {
       a[0] as Readonly<Record<string, Combinator<unknown>>>,
     ),
   label:     (...a) => parseman.label(a[0] as string, a[1] as Combinator<unknown>),
-  field:     (...a) => parseman.field(a[0] as string, a[1] as Combinator<unknown>),
+  field:     (...a) => parseman.field(a[0] as string, a[1] as Combinator<unknown>, a[2] as parseman.FieldOptions | undefined),
   noTrivia:  (...a) => parseman.noTrivia(a[0] as Combinator<unknown>),
   token:     (...a) => parseman.token(a[0] as Combinator<unknown>),
   // `routed(fallback)` — the fallback must be forwarded. A zero-arg entry here read
@@ -556,6 +619,20 @@ function staticNodeOptions(expr: Expression, scope: XScope): StaticNodeOptions {
   return opts.unwrap || opts.collapse || opts.project !== undefined || opts.captureTrivia || opts.trailingTrivia || opts.tags !== undefined || opts.buildArity !== undefined ? opts : undefined
 }
 
+/**
+ * `scanTo`/`balanced` options: `undefined` when absent, `UNKNOWN_OPTIONS` when present
+ * but not statically known. Passing the unknown value on as `null` is what built a
+ * region with NO skip units out of `balanced('(', ')', importedOpts)` — the options
+ * silently vanished.
+ */
+const UNKNOWN_OPTIONS = Symbol('parseman.unknownOptions')
+function scanOptions(optsArg: Expression | { type: 'SpreadElement' } | undefined, scope: XScope, code?: string): unknown {
+  if (optsArg === undefined) return undefined
+  if (optsArg.type === 'SpreadElement') return UNKNOWN_OPTIONS
+  const opts = anyValue(optsArg as Expression, scope, code, [])
+  return isHole(optsArg as Expression, opts, scope) ? UNKNOWN_OPTIONS : opts
+}
+
 /**
  * Evaluate a call expression to a Combinator.
  * `mfs` accumulates mapFn source texts in depth-first order — must match
@@ -893,9 +970,8 @@ function exprToCombi(node: Expression, scope: XScope, code?: string, mfs?: strin
     const open = anyValue(openArg as Expression, scope, code, [])
     const close = anyValue(closeArg as Expression, scope, code, [])
     if (typeof open !== 'string' || typeof close !== 'string') return null
-    const opts = optsArg && optsArg.type !== 'SpreadElement'
-      ? anyValue(optsArg as Expression, scope, code, [])
-      : undefined
+    const opts = scanOptions(optsArg, scope, code)
+    if (opts === UNKNOWN_OPTIONS) return null
     try { return parseman.balanced(open, close, opts as parseman.ScanToOptions | undefined) } catch { return null }
   }
 
@@ -906,9 +982,8 @@ function exprToCombi(node: Expression, scope: XScope, code?: string, mfs?: strin
     if (!sentinelArg || sentinelArg.type === 'SpreadElement') return null
     const sentinel = anyValue(sentinelArg as Expression, scope, code, [])
     if (!isCombinator(sentinel)) return null
-    const opts = optsArg && optsArg.type !== 'SpreadElement'
-      ? anyValue(optsArg as Expression, scope, code, [])
-      : undefined
+    const opts = scanOptions(optsArg, scope, code)
+    if (opts === UNKNOWN_OPTIONS) return null
     try { return parseman.scanTo(sentinel, opts as parseman.ScanToOptions | undefined) } catch { return null }
   }
 
@@ -1039,6 +1114,22 @@ function exprToCombi(node: Expression, scope: XScope, code?: string, mfs?: strin
   } catch { return null }
 }
 
+/**
+ * An element that evaluated to `null` without BEING `null` — an unresolved name, a
+ * call the evaluator could not build. Kept as a `null` slot, it made the container
+ * look evaluated: `{ skip: [unknownUnit] }` became `{ skip: [null] }`, and an options
+ * object went missing piece by piece with no warning. So a hole fails the whole
+ * container. A function literal is exempt: it evaluates to `null` by design, its
+ * source being captured by the caller that needs it.
+ */
+function isHole(node: Expression, value: unknown, scope: XScope): boolean {
+  if (value !== null) return false
+  const n = unwrapStaticExpr(node) as { type: string; value?: unknown; name?: string }
+  if (n.type === 'Identifier') return !scope.has(n.name!) // a bound nullish const is a value
+  return !(n.type === 'Literal' && n.value === null)
+    && n.type !== 'ArrowFunctionExpression' && n.type !== 'FunctionExpression'
+}
+
 /** Evaluate any expression to its JS value (not necessarily a Combinator). */
 function anyValue(node: Expression, scope: XScope, code?: string, mfs?: string[]): unknown {
   if (node.type === 'TSAsExpression'
@@ -1064,7 +1155,9 @@ function anyValue(node: Expression, scope: XScope, code?: string, mfs?: string[]
     for (const el of arr.elements) {
       if (el === null) { out.push(null); continue }
       if ((el as { type: string }).type === 'SpreadElement') return null
-      out.push(anyValue(el as Expression, scope, code, mfs))
+      const v = anyValue(el as Expression, scope, code, mfs)
+      if (isHole(el as Expression, v, scope)) return null
+      out.push(v)
     }
     return out
   }
@@ -1074,7 +1167,10 @@ function anyValue(node: Expression, scope: XScope, code?: string, mfs?: string[]
     for (const prop of node.properties) {
       const key = propName(prop as never)
       if (key === null) return null
-      obj[key] = anyValue((prop as unknown as ObjectProperty).value as Expression, scope, code, mfs)
+      const valueNode = (prop as unknown as ObjectProperty).value as Expression
+      const v = anyValue(valueNode, scope, code, mfs)
+      if (isHole(valueNode, v, scope)) return null
+      obj[key] = v
     }
     return obj
   }
@@ -1184,15 +1280,16 @@ export function evaluateExpr(
  * Evaluate a `const X = [combinator, …]` array literal into an array of
  * Combinators. Lets a shared option array (e.g. a `skip` set reused across
  * `scanTo`/`balanced` calls) be referenced by name — `{ skip: X }` — instead of
- * inlining the array at every call site. Returns null when `node` isn't an array
- * literal of statically-resolvable combinators.
+ * inlining the array at every call site. Returns null when `node` doesn't evaluate
+ * to an array of statically-resolvable combinators.
  */
 export function evaluateCombinatorArray(
   node: Expression,
   scope: Scope,
   code?: string,
 ): Combinator<unknown>[] | null {
-  if (node.type !== 'ArrayExpression') return null
+  // Any expression, not just an array literal: `rules({ scanSkip: sharedSkip })`
+  // names the array, and requiring a literal here is what dropped it.
   const val = anyValue(node, scope as XScope, code, [])
   if (!Array.isArray(val) || val.length === 0) return null
   if (!val.every(isCombinator)) return null
@@ -1453,7 +1550,7 @@ export function evaluateParserFactory(
     built = rules(options ?? {}, (g: Record<string, Combinator<unknown>>) => {
       // Outer ScopeEntry values carry their mfSrcs and are replayed by scopeGet()
       // when body statements or return expressions reference them.
-      const localScope: XScope = new Map(scope as XScope)
+      const localScope: XScope = scope instanceof ModuleScope ? scope.fork() : new Map(scope as XScope)
       localScope.set(proxyName, g)
 
       // ── Phase 1: the factory's own body statements ────────────────────────
@@ -1564,7 +1661,10 @@ export function applyDefineStatement(
 /** Check if an AST node references any name from the given scope or names set. */
 export function referencesAny(node: Node, names: Set<string>, scope: Scope): boolean {
   if (node.type === 'Identifier') {
-    return names.has(node.name) || scope.has(node.name)
+    // The names the module has BOUND so far, not every import it could resolve: this
+    // decides whether a declaration is macro code at all, and an ordinary
+    // `const x = importedThing` must not start reading modules to find out.
+    return names.has(node.name) || Map.prototype.has.call(scope, node.name)
   }
   for (const key of Object.keys(node) as (keyof typeof node)[]) {
     const child = node[key]