parseman 0.51.2 → 0.53.0

package/src/coverage.ts

@@ -18,7 +18,7 @@ import { scanTo } from './combinators/scanTo.ts'
 import { sequence } from './combinators/sequence.ts'
 import { token, leaf, sourceLeaf } from './combinators/token.ts'
 import { withCtx } from './combinators/withCtx.ts'
-import { composedCoverageRules } from './compiler/linker.ts'
+import { composedCoverageRules, linkedWinnersOf } from './compiler/linker.ts'
 import { buildGrammarPlan, type GrammarCoverageDefinition, type GrammarCoveragePlan } from './compiler/grammar-coverage-ids.ts'
 
 export type { GrammarCoverageDefinition } from './compiler/grammar-coverage-ids.ts'
@@ -156,11 +156,11 @@ export function grammarCoverageDefinitions(entry: Combinator<unknown>, winners?:
 }
 
 /** Definitions for a runtime `compose()` result, normalized through its final
- * IR winner map. Opaque precompiled pieces intentionally fail rather than
+ * winner map. Anything else — a compiled table, a plain map — fails rather than
  * reporting source-piece identities as though they were final grammar IDs. */
 export function composedGrammarCoverageDefinitions(grammar: Record<string, unknown>, startRule: string): readonly GrammarCoverageDefinition[] {
   const winners = composedCoverageRules(grammar)
-  if (!winners) throw new TypeError('semantic coverage needs re-lowerable composed IR; this composition contains an opaque artifact')
+  if (!winners) throw new TypeError('semantic coverage needs a runtime compose() result (a linked interpreter grammar); this is not one')
   const entry = winners[startRule]
   if (!entry) throw new TypeError(`semantic coverage start rule ${JSON.stringify(startRule)} is not a final winner`)
   return grammarCoverageDefinitions(entry, winners)
@@ -327,7 +327,7 @@ function coverageEntry(entry: Combinator<unknown>, collector: GrammarCoverageCol
         case 'oneOrMore': return oneOrMore(build(def.parser))
         case 'optional': return optional(build(def.parser))
         case 'attempt': {
-          const base = attempt(build(def.parser))
+          const base = attempt(build(def.parser), def.contain === true ? { contain: true } : undefined)
           const id = maps.attempts.get(parser)
           return id === undefined ? base : {
             ...base,
@@ -380,7 +380,9 @@ function coverageEntry(entry: Combinator<unknown>, collector: GrammarCoverageCol
 
 export function runWithGrammarCoverage(entry: Runnable, input: string, options: RunOptions & { collector?: GrammarCoverageCollector; trace?: GrammarTraceSink } = {}): { result: RunResult; coverage: GrammarCoverageSnapshot } {
   if (typeof entry === 'function') throw new TypeError('runWithGrammarCoverage currently requires an interpreter combinator entry')
-  const plan = buildGrammarPlan(entry)
+  // A runtime composition's rule is planned through its winner map, so its IDs are
+  // the ones `composedGrammarCoverageDefinitions` reports.
+  const plan = buildGrammarPlan(entry, linkedWinnersOf(entry))
   const definitions = plan.definitions
   const collector = options.collector ?? createGrammarCoverageCollector(definitions)
   const { collector: _collector, trace, ...runOptions } = options

package/src/cst/reflection.ts

@@ -115,6 +115,11 @@ export function collectGrammarReflection(
 ): GrammarReflection {
   const byType = new Map<string, string[]>()
   const followLazy = opts?.followLazy !== false
+  // ONE walk over the whole map. A node already visited from an earlier rule had
+  // its whole subgraph visited then, and `addNode` is idempotent, so a fresh set
+  // per rule only re-walked shared structure — O(rules × graph), and every unbound
+  // cross-piece hole threw (a stack-capturing Error) once per rule reaching it.
+  const seen = new Set<Combinator<unknown>>()
   for (const [, rule] of ruleMap) {
     if (!followLazy && rule._def.tag === 'lazy') {
       try {
@@ -123,7 +128,7 @@ export function collectGrammarReflection(
       } catch {}
       continue
     }
-    collectFromCombinator(rule, byType, new Set(), followLazy)
+    collectFromCombinator(rule, byType, seen, followLazy)
   }
   return { nodes: [...byType].map(([type, tags]) => ({ type, tags })) }
 }

package/src/cst/trivia-kinds.ts

@@ -52,6 +52,12 @@ export function peelLabel(p: Combinator<unknown>): { label: string; parser: Comb
  * (or a single labeled arm), return the label table and matchers.
  */
 export function analyzeLabeledTrivia(trivia: Combinator<unknown>): LabeledTriviaSpec | null {
+  // A `trivia()` combinator carries its own spec, built once at construction —
+  // the hot path. The four slots below serve every other caller.
+  if (trivia._def.tag === 'trivia') {
+    const own = trivia._meta.labeledTriviaSpec
+    if (own !== undefined) return own
+  }
   if (trivia === labeledTrivia0) return labeledSpec0!
   if (trivia === labeledTrivia1) return labeledSpec1!
   if (trivia === labeledTrivia2) return labeledSpec2!

package/src/index.ts

@@ -19,6 +19,7 @@ export { choice } from './combinators/choice.ts'
 export { dispatch, endsWith, makeWhen, matches, otherwise, routed, startsWith, when } from './combinators/dispatch.ts'
 export type { DispatchArm, DispatchOtherwise, DispatchStringMatcher, DispatchWhen, DispatchWhenFactory, DispatchWhenOptions, DispatchWhenMatcher } from './combinators/dispatch.ts'
 export { attempt } from './combinators/attempt.ts'
+export type { AttemptOptions } from './combinators/attempt.ts'
 export { many, oneOrMore, optional, sepBy, oneOrMoreSep, keepSeparator } from './combinators/repeat.ts'
 export type { RepeatOptions, SepByOptions, TrailingSeparator, KeptSeparator } from './combinators/repeat.ts'
 export { rules } from './combinators/parser.ts'

package/src/plugin/evaluator.ts

@@ -154,6 +154,69 @@ export type ScopeEntry = {
 }
 export type Scope = Map<string, ScopeEntry>
 
+/** How a {@link ModuleScope} answers a name it does not hold: a value, the reason a
+ * binding the module DOES have cannot be known at build time, or `null` — not a
+ * binding this scope can speak for (an ordinary miss). */
+export type FreeNameResolution = { value: unknown } | { unresolved: string } | null
+
+/**
+ * A module's scope whose IMPORTED (and, for a foreign module, top-level) bindings
+ * resolve on first read instead of being absent.
+ *
+ * Absent was the defect. `anyValue` answered an imported name with `null`, so
+ * `rules({ scanSkip: [importedUnit] })` evaluated to nothing, the option quietly
+ * became `undefined`, and the build succeeded with no ambient skip — and a terminal
+ * imported into a rule body could not be lowered at all. Lazy, because a grammar
+ * module imports far more than combinators (AST classes, helpers) and resolving each
+ * one eagerly would read every one of those modules for nothing.
+ *
+ * A binding that exists but cannot be resolved is RECORDED in `unresolved` (shared by
+ * every scope of one transform) as "`name` <reason>", so the plugin can fail the build
+ * naming it rather than warn about an anonymous "isn't statically evaluable". Only a
+ * VALUE read records: `has()` is how the evaluator probes a name it may legitimately
+ * leave to runtime (a named reducer), so a probe that misses is not a failure.
+ */
+export class ModuleScope extends Map<string, unknown> {
+  private readonly tried = new Map<string, string | undefined>()
+  private readonly resolveFree: (name: string) => FreeNameResolution
+  readonly unresolved: Set<string>
+  constructor(resolveFree: (name: string) => FreeNameResolution, unresolved: Set<string>) {
+    super()
+    this.resolveFree = resolveFree
+    this.unresolved = unresolved
+  }
+
+  override has(name: string): boolean {
+    return super.has(name) || this.resolve(name)
+  }
+
+  override get(name: string): unknown {
+    if (super.has(name) || this.resolve(name)) return super.get(name)
+    const why = this.tried.get(name)
+    if (why !== undefined) this.unresolved.add(`\`${name}\` ${why}`)
+    return undefined
+  }
+
+  /** A child scope (a factory body) that keeps resolving this module's free names. */
+  fork(): ModuleScope {
+    const child = new ModuleScope(this.resolveFree, this.unresolved)
+    for (const [k, v] of super.entries()) child.set(k, v)
+    return child
+  }
+
+  private resolve(name: string): boolean {
+    if (this.tried.has(name)) return false
+    const r = this.resolveFree(name)
+    if (r !== null && 'value' in r) {
+      super.set(name, r.value)
+      return true
+    }
+    // A name nothing binds YET is not remembered: a later declaration may bind it.
+    if (r !== null) this.tried.set(name, r.unresolved)
+    return false
+  }
+}
+
 // Internal XScope also holds non-Combinator values (g proxy objects etc.)
 type XScopeVal = ScopeEntry | unknown
 type XScope = Map<string, XScopeVal>
@@ -243,7 +306,7 @@ const SUPPORTED: Record<string, (...args: unknown[]) => Combinator<unknown>> = {
   word:      (...a) => parseman.word(a[0] as string, a[1] as string | undefined, a[2] as Omit<parseman.KeywordsOptions, 'boundary'> | undefined),
   sequence:  (...a) => (parseman.sequence as (...p: Combinator<unknown>[]) => Combinator<unknown[]>)(...(a as Combinator<unknown>[])),
   choice:    (...a) => (parseman.choice as (...p: Combinator<unknown>[]) => Combinator<unknown>)(...(a as Combinator<unknown>[])),
-  attempt:   (...a) => parseman.attempt(a[0] as Combinator<unknown>),
+  attempt:   (...a) => parseman.attempt(a[0] as Combinator<unknown>, a[1] as parseman.AttemptOptions | undefined),
   optional:  (...a) => parseman.optional(a[0] as Combinator<unknown>),
   trivia:    (...a) => parseman.trivia(a[0] as Combinator<unknown>),
   classifiedTrivia: (...a) =>
@@ -556,6 +619,20 @@ function staticNodeOptions(expr: Expression, scope: XScope): StaticNodeOptions {
   return opts.unwrap || opts.collapse || opts.project !== undefined || opts.captureTrivia || opts.trailingTrivia || opts.tags !== undefined || opts.buildArity !== undefined ? opts : undefined
 }
 
+/**
+ * `scanTo`/`balanced` options: `undefined` when absent, `UNKNOWN_OPTIONS` when present
+ * but not statically known. Passing the unknown value on as `null` is what built a
+ * region with NO skip units out of `balanced('(', ')', importedOpts)` — the options
+ * silently vanished.
+ */
+const UNKNOWN_OPTIONS = Symbol('parseman.unknownOptions')
+function scanOptions(optsArg: Expression | { type: 'SpreadElement' } | undefined, scope: XScope, code?: string): unknown {
+  if (optsArg === undefined) return undefined
+  if (optsArg.type === 'SpreadElement') return UNKNOWN_OPTIONS
+  const opts = anyValue(optsArg as Expression, scope, code, [])
+  return isHole(optsArg as Expression, opts, scope) ? UNKNOWN_OPTIONS : opts
+}
+
 /**
  * Evaluate a call expression to a Combinator.
  * `mfs` accumulates mapFn source texts in depth-first order — must match
@@ -893,9 +970,8 @@ function exprToCombi(node: Expression, scope: XScope, code?: string, mfs?: strin
     const open = anyValue(openArg as Expression, scope, code, [])
     const close = anyValue(closeArg as Expression, scope, code, [])
     if (typeof open !== 'string' || typeof close !== 'string') return null
-    const opts = optsArg && optsArg.type !== 'SpreadElement'
-      ? anyValue(optsArg as Expression, scope, code, [])
-      : undefined
+    const opts = scanOptions(optsArg, scope, code)
+    if (opts === UNKNOWN_OPTIONS) return null
     try { return parseman.balanced(open, close, opts as parseman.ScanToOptions | undefined) } catch { return null }
   }
 
@@ -906,9 +982,8 @@ function exprToCombi(node: Expression, scope: XScope, code?: string, mfs?: strin
     if (!sentinelArg || sentinelArg.type === 'SpreadElement') return null
     const sentinel = anyValue(sentinelArg as Expression, scope, code, [])
     if (!isCombinator(sentinel)) return null
-    const opts = optsArg && optsArg.type !== 'SpreadElement'
-      ? anyValue(optsArg as Expression, scope, code, [])
-      : undefined
+    const opts = scanOptions(optsArg, scope, code)
+    if (opts === UNKNOWN_OPTIONS) return null
     try { return parseman.scanTo(sentinel, opts as parseman.ScanToOptions | undefined) } catch { return null }
   }
 
@@ -1039,6 +1114,22 @@ function exprToCombi(node: Expression, scope: XScope, code?: string, mfs?: strin
   } catch { return null }
 }
 
+/**
+ * An element that evaluated to `null` without BEING `null` — an unresolved name, a
+ * call the evaluator could not build. Kept as a `null` slot, it made the container
+ * look evaluated: `{ skip: [unknownUnit] }` became `{ skip: [null] }`, and an options
+ * object went missing piece by piece with no warning. So a hole fails the whole
+ * container. A function literal is exempt: it evaluates to `null` by design, its
+ * source being captured by the caller that needs it.
+ */
+function isHole(node: Expression, value: unknown, scope: XScope): boolean {
+  if (value !== null) return false
+  const n = unwrapStaticExpr(node) as { type: string; value?: unknown; name?: string }
+  if (n.type === 'Identifier') return !scope.has(n.name!) // a bound nullish const is a value
+  return !(n.type === 'Literal' && n.value === null)
+    && n.type !== 'ArrowFunctionExpression' && n.type !== 'FunctionExpression'
+}
+
 /** Evaluate any expression to its JS value (not necessarily a Combinator). */
 function anyValue(node: Expression, scope: XScope, code?: string, mfs?: string[]): unknown {
   if (node.type === 'TSAsExpression'
@@ -1064,7 +1155,9 @@ function anyValue(node: Expression, scope: XScope, code?: string, mfs?: string[]
     for (const el of arr.elements) {
       if (el === null) { out.push(null); continue }
       if ((el as { type: string }).type === 'SpreadElement') return null
-      out.push(anyValue(el as Expression, scope, code, mfs))
+      const v = anyValue(el as Expression, scope, code, mfs)
+      if (isHole(el as Expression, v, scope)) return null
+      out.push(v)
     }
     return out
   }
@@ -1074,7 +1167,10 @@ function anyValue(node: Expression, scope: XScope, code?: string, mfs?: string[]
     for (const prop of node.properties) {
       const key = propName(prop as never)
       if (key === null) return null
-      obj[key] = anyValue((prop as unknown as ObjectProperty).value as Expression, scope, code, mfs)
+      const valueNode = (prop as unknown as ObjectProperty).value as Expression
+      const v = anyValue(valueNode, scope, code, mfs)
+      if (isHole(valueNode, v, scope)) return null
+      obj[key] = v
     }
     return obj
   }
@@ -1184,15 +1280,16 @@ export function evaluateExpr(
  * Evaluate a `const X = [combinator, …]` array literal into an array of
  * Combinators. Lets a shared option array (e.g. a `skip` set reused across
  * `scanTo`/`balanced` calls) be referenced by name — `{ skip: X }` — instead of
- * inlining the array at every call site. Returns null when `node` isn't an array
- * literal of statically-resolvable combinators.
+ * inlining the array at every call site. Returns null when `node` doesn't evaluate
+ * to an array of statically-resolvable combinators.
  */
 export function evaluateCombinatorArray(
   node: Expression,
   scope: Scope,
   code?: string,
 ): Combinator<unknown>[] | null {
-  if (node.type !== 'ArrayExpression') return null
+  // Any expression, not just an array literal: `rules({ scanSkip: sharedSkip })`
+  // names the array, and requiring a literal here is what dropped it.
   const val = anyValue(node, scope as XScope, code, [])
   if (!Array.isArray(val) || val.length === 0) return null
   if (!val.every(isCombinator)) return null
@@ -1453,7 +1550,7 @@ export function evaluateParserFactory(
     built = rules(options ?? {}, (g: Record<string, Combinator<unknown>>) => {
       // Outer ScopeEntry values carry their mfSrcs and are replayed by scopeGet()
       // when body statements or return expressions reference them.
-      const localScope: XScope = new Map(scope as XScope)
+      const localScope: XScope = scope instanceof ModuleScope ? scope.fork() : new Map(scope as XScope)
       localScope.set(proxyName, g)
 
       // ── Phase 1: the factory's own body statements ────────────────────────
@@ -1564,7 +1661,10 @@ export function applyDefineStatement(
 /** Check if an AST node references any name from the given scope or names set. */
 export function referencesAny(node: Node, names: Set<string>, scope: Scope): boolean {
   if (node.type === 'Identifier') {
-    return names.has(node.name) || scope.has(node.name)
+    // The names the module has BOUND so far, not every import it could resolve: this
+    // decides whether a declaration is macro code at all, and an ordinary
+    // `const x = importedThing` must not start reading modules to find out.
+    return names.has(node.name) || Map.prototype.has.call(scope, node.name)
   }
   for (const key of Object.keys(node) as (keyof typeof node)[]) {
     const child = node[key]