parseman 0.46.0 → 0.47.1

package/src/combinators/dispatch.ts

@@ -0,0 +1,358 @@
+import type { Combinator, DispatchCase, DispatchMatcherCase, ParseContext, ParseResult, ParserMeta } from '../types.ts'
+import { cstCaptureActive, pushCstLeaf } from '../cst/capture-buffer.ts'
+import { rollbackTrivia, saveTriviaMark } from './trivia-skip.ts'
+
+export type DispatchWhen<T> = {
+  readonly kind: 'when'
+  readonly keys: readonly string[]
+  readonly parser: Combinator<T>
+  readonly caseInsensitive: boolean
+  readonly usesRouted?: boolean
+}
+
+export type DispatchWhenMatcher<T> = {
+  readonly kind: 'whenMatcher'
+  readonly matcher: DispatchStringMatcher
+  readonly parser: Combinator<T>
+  readonly caseInsensitive: boolean
+  readonly usesRouted?: boolean
+}
+
+export type DispatchStringMatcher =
+  | { readonly kind: 'startsWith'; readonly value: string }
+  | { readonly kind: 'endsWith'; readonly value: string }
+  | { readonly kind: 'matches'; readonly value: string; readonly flags: string }
+
+export type DispatchOtherwise<T> = {
+  readonly kind: 'otherwise'
+  readonly parser: Combinator<T>
+  readonly usesRouted?: boolean
+}
+
+export type DispatchArm<T = unknown> = DispatchWhen<T> | DispatchWhenMatcher<T> | DispatchOtherwise<T>
+export type DispatchWhenOptions = {
+  caseInsensitive?: boolean
+}
+export type DispatchWhenFactory = <T>(
+  key: string | readonly string[],
+  parser: Combinator<T>,
+) => DispatchWhen<T>
+export type DispatchWhenKey = string | readonly string[]
+export type DispatchWhenSelector = DispatchWhenKey | DispatchStringMatcher
+
+type ArmValue<T> = T extends DispatchWhen<infer U>
+  ? U
+  : T extends DispatchWhenMatcher<infer U>
+  ? U
+  : T extends DispatchOtherwise<infer U>
+  ? U
+  : never
+type UnionArms<T extends readonly DispatchArm<unknown>[]> = {
+  [K in keyof T]: ArmValue<T[K]>
+}[number]
+
+export function parserUsesRouted(parser: Combinator<unknown>, seen: Set<Combinator<unknown>> = new Set()): boolean {
+  if (seen.has(parser)) return false
+  seen.add(parser)
+  const def = parser._def
+  switch (def.tag) {
+    case 'routed':
+      return true
+    case 'sequence':
+    case 'choice':
+      return def.parsers.some(entry => parserUsesRouted(entry, seen))
+    case 'dispatch':
+      return false
+    case 'sepBy':
+      return parserUsesRouted(def.parser, seen) || parserUsesRouted(def.separator, seen)
+    case 'grammar':
+      return parserUsesRouted(def.parser, seen) || (def.triviaParser ? parserUsesRouted(def.triviaParser, seen) : false)
+    case 'scanTo':
+      return parserUsesRouted(def.sentinel, seen) || def.skip.some(entry => parserUsesRouted(entry, seen))
+    case 'recover':
+      return parserUsesRouted(def.parser, seen) || parserUsesRouted(def.sentinel, seen)
+    case 'lazy':
+      try { return parserUsesRouted(def.thunk(), seen) } catch { return false }
+    case 'many':
+    case 'oneOrMore':
+    case 'optional':
+    case 'attempt':
+    case 'transform':
+    case 'trivia':
+    case 'token':
+    case 'leaf':
+    case 'label':
+    case 'field':
+    case 'not':
+    case 'peek':
+    case 'node':
+    case 'withCtx':
+    case 'expect':
+      return parserUsesRouted(def.parser, seen)
+    default:
+      return false
+  }
+}
+
+export function branchUsesRouted(branch: { parser: Combinator<unknown>; usesRouted?: boolean | undefined }): boolean {
+  return branch.usesRouted === true || parserUsesRouted(branch.parser)
+}
+
+export function asciiFoldKey(key: string): string {
+  let out = ''
+  for (let i = 0; i < key.length; i++) {
+    const c = key.charCodeAt(i)
+    out += String.fromCharCode(c >= 65 && c <= 90 ? c + 32 : c)
+  }
+  return out
+}
+
+export function matchesDispatchMatcher(value: string, matcher: DispatchMatcherCase): boolean {
+  const candidate = matcher.caseInsensitive ? asciiFoldKey(value) : value
+  const expected = matcher.caseInsensitive ? asciiFoldKey(matcher.value) : matcher.value
+  switch (matcher.kind) {
+    case 'startsWith':
+      return candidate.startsWith(expected)
+    case 'endsWith':
+      return candidate.endsWith(expected)
+    case 'matches': {
+      const flags = matcher.caseInsensitive && !matcher.flags?.includes('i')
+        ? `${matcher.flags ?? ''}i`
+        : matcher.flags ?? ''
+      return new RegExp(matcher.value, flags).test(value)
+    }
+  }
+}
+
+function isStringMatcher(value: unknown): value is DispatchStringMatcher {
+  if (typeof value !== 'object' || value === null) return false
+  const kind = (value as { kind?: unknown }).kind
+  return kind === 'startsWith' || kind === 'endsWith' || kind === 'matches'
+}
+
+function fixedMatcher(kind: 'startsWith' | 'endsWith', value: string): DispatchStringMatcher {
+  if (typeof value !== 'string') throw new TypeError(`parseman: ${kind}() value must be a string`)
+  if (value.length === 0) throw new RangeError(`parseman: ${kind}() requires a non-empty value`)
+  return { kind, value }
+}
+
+export function startsWith(prefix: string): DispatchStringMatcher {
+  return fixedMatcher('startsWith', prefix)
+}
+
+export function endsWith(suffix: string): DispatchStringMatcher {
+  return fixedMatcher('endsWith', suffix)
+}
+
+export function matches(pattern: RegExp): DispatchStringMatcher {
+  if (!(pattern instanceof RegExp)) throw new TypeError('parseman: matches() requires a RegExp')
+  if (pattern.flags.includes('g') || pattern.flags.includes('y')) {
+    throw new TypeError('parseman: matches() does not accept global or sticky regex flags')
+  }
+  return { kind: 'matches', value: pattern.source, flags: pattern.flags }
+}
+
+export function when<T>(
+  key: DispatchWhenKey,
+  parser: Combinator<T>,
+  opts?: DispatchWhenOptions,
+): DispatchWhen<T>
+export function when<T>(
+  matcher: DispatchStringMatcher,
+  parser: Combinator<T>,
+  opts?: DispatchWhenOptions,
+): DispatchWhenMatcher<T>
+export function when<T>(
+  key: DispatchWhenSelector,
+  parser: Combinator<T>,
+  opts: DispatchWhenOptions = {},
+): DispatchWhen<T> | DispatchWhenMatcher<T> {
+  const usesRouted = parserUsesRouted(parser as Combinator<unknown>)
+  if (isStringMatcher(key)) {
+    return { kind: 'whenMatcher', matcher: key, parser, caseInsensitive: opts.caseInsensitive ?? false, usesRouted }
+  }
+  const keys = Array.isArray(key) ? [...key] : [key]
+  if (keys.length === 0) throw new RangeError('parseman: when() requires at least one key')
+  for (const item of keys) {
+    if (typeof item !== 'string') throw new TypeError('parseman: when() keys must be strings')
+  }
+  return { kind: 'when', keys, parser, caseInsensitive: opts.caseInsensitive ?? false, usesRouted }
+}
+
+export function makeWhen(opts: DispatchWhenOptions = {}): DispatchWhenFactory {
+  return (key, parser) => when(key, parser, opts)
+}
+
+export function otherwise<T>(parser: Combinator<T>): DispatchOtherwise<T> {
+  return { kind: 'otherwise', parser, usesRouted: parserUsesRouted(parser as Combinator<unknown>) }
+}
+
+/**
+ * Reuse the token the enclosing `dispatch()` already consumed to select this branch,
+ * instead of re-recognizing it.
+ *
+ * With no argument this is dispatch-only: outside a dispatch branch (or at a position
+ * other than the selector's) it fails with `expected: ['routed()']`.
+ *
+ * `routed(fallback)` makes the SAME production usable in both contexts: inside a
+ * dispatch branch it reuses the routed token; anywhere else it parses `fallback` in
+ * place. That is what collapses a `Routed<X>` twin production into its original —
+ * `sequence(routed(Name), Prelude, ';')` replaces the pair
+ * `sequence(Name, Prelude, ';')` / `sequence(routed(), Prelude, ';')`, which
+ * otherwise differ by exactly one element and duplicate a whole production (and a
+ * whole compiled emission) for a one-token difference.
+ *
+ * A `routed()` in a dispatch SELECTOR is still an error with or without a fallback:
+ * the selector is what produces the routed token, so reading it there is misuse.
+ */
+export function routed<T = string>(fallback?: Combinator<T>): Combinator<T> {
+  return {
+    _tag: 'routed',
+    _meta: {
+      // The routed token is whatever the selector matched, so the first set stays
+      // `any` even with a fallback — `any` already subsumes the fallback's.
+      firstSet: { kind: 'any' },
+      canMatchNewline: fallback?._meta.canMatchNewline ?? false,
+      isTrivia: false,
+    },
+    _def: fallback === undefined
+      ? { tag: 'routed' }
+      : { tag: 'routed', fallback: fallback as Combinator<unknown> },
+    parse(_input: string, pos: number, ctx: ParseContext): ParseResult<T> {
+      const item = ctx._routed
+      if (item === undefined || pos !== item.span.start) {
+        if (fallback !== undefined) return fallback.parse(_input, pos, ctx)
+        return { ok: false, expected: ['routed()'], span: { start: pos, end: pos } }
+      }
+      if (cstCaptureActive(ctx)) {
+        pushCstLeaf(ctx, { _tag: 'leaf', value: item.value, span: item.span })
+      }
+      return { ok: true, value: item.value as T, span: item.span }
+    },
+  }
+}
+
+export function dispatch<S extends string, T extends readonly DispatchArm<unknown>[]>(
+  selector: Combinator<S>,
+  ...arms: T
+): Combinator<[S, UnionArms<T>]> {
+  if (parserUsesRouted(selector as Combinator<unknown>)) {
+    throw new Error('parseman: routed() can only appear inside a dispatch() branch')
+  }
+  let fallback: Combinator<unknown> | undefined
+  let fallbackUsesRouted = false
+  const matchers: DispatchMatcherCase[] = []
+  const cases: DispatchCase[] = []
+  const seen: Array<{ raw: string; folded: string; caseInsensitive: boolean }> = []
+
+  for (let i = 0; i < arms.length; i++) {
+    const arm = arms[i]!
+    if (arm.kind === 'otherwise') {
+      if (fallback !== undefined) throw new Error('parseman: dispatch() accepts at most one otherwise() arm')
+      if (i !== arms.length - 1) throw new Error('parseman: otherwise() must be the last dispatch() arm')
+      fallback = arm.parser
+      fallbackUsesRouted = arm.usesRouted === true
+      continue
+    }
+    if (arm.kind === 'whenMatcher') {
+      const matcher = arm.matcher
+      matchers.push({
+        kind: matcher.kind,
+        value: matcher.value,
+        ...(matcher.kind === 'matches' ? { flags: matcher.flags } : {}),
+        parser: arm.parser as Combinator<unknown>,
+        caseInsensitive: arm.caseInsensitive,
+        ...(arm.usesRouted === true ? { usesRouted: true } : {}),
+      })
+      continue
+    }
+    for (const key of arm.keys) {
+      const folded = asciiFoldKey(key)
+      for (const prior of seen) {
+        const overlaps = arm.caseInsensitive || prior.caseInsensitive
+          ? folded === prior.folded
+          : key === prior.raw
+        if (overlaps) throw new Error(`parseman: duplicate dispatch key ${JSON.stringify(key)}`)
+      }
+      seen.push({ raw: key, folded, caseInsensitive: arm.caseInsensitive })
+    }
+    cases.push({
+      keys: arm.keys,
+      parser: arm.parser as Combinator<unknown>,
+      caseInsensitive: arm.caseInsensitive,
+      ...(arm.usesRouted === true ? { usesRouted: true } : {}),
+    })
+  }
+
+  const meta: ParserMeta = {
+    firstSet: selector._meta.firstSet,
+    canMatchNewline: selector._meta.canMatchNewline ||
+      cases.some(entry => entry.parser._meta.canMatchNewline) ||
+      matchers.some(entry => entry.parser._meta.canMatchNewline) ||
+      (fallback?._meta.canMatchNewline ?? false),
+    isTrivia: false,
+  }
+
+  const byKey = new Map<string, DispatchCase>()
+  const byFoldedKey = new Map<string, DispatchCase>()
+  for (const entry of cases) {
+    for (const key of entry.keys) {
+      if (entry.caseInsensitive) byFoldedKey.set(asciiFoldKey(key), entry)
+      else byKey.set(key, entry)
+    }
+  }
+
+  return {
+    _tag: 'dispatch',
+    _meta: meta,
+    _def: {
+      tag: 'dispatch',
+      selector: selector as Combinator<string>,
+      cases,
+      ...(matchers.length === 0 ? {} : { matchers }),
+      ...(fallback === undefined ? {} : { otherwise: fallback, ...(fallbackUsesRouted ? { otherwiseUsesRouted: true } : {}) }),
+    },
+    parse(input: string, pos: number, ctx: ParseContext): ParseResult<[S, UnionArms<T>]> {
+      if (parserUsesRouted(selector as Combinator<unknown>)) {
+        throw new Error('parseman: routed() can only appear inside a dispatch() branch')
+      }
+      const selectorMark = saveTriviaMark(ctx)
+      const selected = selector.parse(input, pos, ctx)
+      if (!selected.ok) return selected
+
+      const branch = byKey.get(selected.value) ??
+        (byFoldedKey.size === 0 ? undefined : byFoldedKey.get(asciiFoldKey(selected.value))) ??
+        matchers.find(matcher => matchesDispatchMatcher(selected.value, matcher)) ??
+        (fallback === undefined ? undefined : { keys: [], parser: fallback, caseInsensitive: false, usesRouted: fallbackUsesRouted })
+      if (branch === undefined) {
+        const expected = cases.flatMap(entry => entry.keys.map(key => JSON.stringify(key)))
+        return { ok: false, expected, span: { start: selected.span.end, end: selected.span.end } }
+      }
+
+      const savedRouted = ctx._routed
+      const usesRouted = branchUsesRouted(branch)
+      let mark = saveTriviaMark(ctx)
+      if (usesRouted) {
+        rollbackTrivia(ctx, selectorMark)
+        mark = saveTriviaMark(ctx)
+        ctx._routed = { value: selected.value, span: selected.span }
+      }
+      let result: ParseResult<unknown>
+      try {
+        result = branch.parser.parse(input, usesRouted ? pos : selected.span.end, ctx)
+      } finally {
+        if (usesRouted) ctx._routed = savedRouted
+      }
+      if (!result.ok) {
+        rollbackTrivia(ctx, mark)
+        return { ...result, committed: true }
+      }
+
+      return {
+        ok: true,
+        value: [selected.value, result.value as UnionArms<T>],
+        span: { start: pos, end: result.span.end },
+      }
+    },
+  }
+}

package/src/combinators/expect.ts

@@ -0,0 +1,159 @@
+import type { Combinator, ParseContext, ParseResult, ParserMeta, ParseError } from '../types.ts'
+import { captureError } from '../recovery/scan.ts'
+import { annotateSpanFromLineContext } from '../line-index.ts'
+import { matchesEmpty, isZeroWidthAssertion } from './first-set.ts'
+
+export type { ParseError }
+
+/**
+ * Statically derive the "expected" token set from a combinator's structure,
+ * so a failure can name what it wanted without running anything:
+ *
+ *   literal('}')        -> ['"}"']
+ *   keywords(['a','b']) -> ['"a"', '"b"']
+ *   choice(x, y)        -> expected(x) ++ expected(y)   (all alternatives)
+ *   sequence(a, b, …)   -> expected(a) ++ … through the NULLABLE prefix
+ *   label('s', x)       -> ['s']                        (an explicit name wins)
+ *
+ * This is what lets expect() report a meaningful, IDENTICAL expectation in both
+ * the interpreter and the compiled output (the compiled path does not rebuild the
+ * runtime `expected` array, so it reads this precomputed set instead).
+ *
+ * The result is DEDUPLICATED. Deriving through a nullable prefix re-reaches the
+ * same token from several terms — a value grammar whose leading terms are all
+ * optional names its value-start set once per term — and an expectation list that
+ * offers `"@{"` five times is wrong for a reader and expensive for a parser: a
+ * choice that loses every arm materializes `[...arm0, ...arm1, …]` at runtime, and
+ * those arrays feed the enclosing choice's concat in turn, so duplicates compound
+ * multiplicatively up the nesting. On jess's Less grammar the un-deduped sets cost
+ * 32% of parse time on `benchmark.less` (medians 20.7 → 14.2 ms over 100
+ * interleaved pairs, 99/100 wins, min 19.1 → 13.0), which is the whole of the Less
+ * regression 0.35.0 shipped. Dedup is not an optimisation bolted onto the
+ * derivation — a SET is what the derivation always meant.
+ */
+export function deriveExpected(c: Combinator<unknown>): string[] {
+  return [...new Set(derive(c, new Set()))]
+}
+
+function derive(c: Combinator<unknown>, seen: Set<Combinator<unknown>>): string[] {
+  const deriveExpected = (p: Combinator<unknown>): string[] => derive(p, seen)
+  const def = c._def
+  switch (def.tag) {
+    case 'literal':   return [JSON.stringify(def.value)]
+    case 'regex':     return [`/${def.source}/`]
+    case 'keywords':  return def.words.map(w => JSON.stringify(w))
+    case 'label':     return [def.label]
+    case 'choice':    return def.parsers.flatMap(deriveExpected)
+    case 'dispatch':  return deriveExpected(def.selector)
+    // Union through the NULLABLE prefix, mirroring `sequenceFirstSet`. A leading
+    // `optional(…)`/`many(…)` can match nothing, so the term after it is equally
+    // able to fail first — deriving from term 0 alone named a token the parse
+    // never actually required. A leading `not(…)` is zero-width and contributes
+    // nothing (its own expectation is about what must NOT be here), but it is
+    // nullable, so keep scanning past it. Stop at the first term that must match.
+    case 'sequence': {
+      const out: string[] = []
+      for (const term of def.parsers) {
+        if (!isZeroWidthAssertion(term)) out.push(...deriveExpected(term))
+        if (!matchesEmpty(term)) break
+      }
+      return out
+    }
+    case 'attempt':   return deriveExpected(def.parser)
+    // Delegating wrappers whose first-token failure is their inner parser's — must
+    // pass through so a start-failure (and the first-set-miss fast-path in attempt/
+    // node, which reads this) names the real expected token, not the wrapper tag.
+    case 'field':
+    case 'withCtx':
+    case 'recover':   return deriveExpected(def.parser)
+    case 'expect':    return def.expected
+    case 'node':
+    case 'grammar':
+    case 'trivia':
+    case 'token':
+    case 'leaf':
+    case 'optional':
+    case 'many':
+    case 'oneOrMore':
+    case 'transform':
+    case 'not':
+    case 'peek':     return deriveExpected(def.parser)
+    // A bare routed() can only fail as "there is no routed token here". With a
+    // fallback, THAT is the branch that can fail, so its expectations are the useful
+    // ones — the routed path either succeeds or is not taken.
+    case 'routed':   return def.fallback === undefined ? ['routed()'] : deriveExpected(def.fallback)
+    // A bare adjacency assertion names the JOIN it requires. Inside a sequence it is
+    // skipped entirely (isZeroWidthAssertion, above), which is the normal path.
+    case 'adjacency': return [def.polarity]
+    case 'lazy': {
+      // An EXTERNAL ref (a rule from a composed base grammar) has no local
+      // definition yet — its `thunk()` throws until fusion supplies it. Fall back
+      // to the rule name as the expected label instead of descending.
+      const name = (c as { _ruleName?: string })._ruleName
+      // Cycle guard: a RECURSIVE rule re-enters its own definition forever. Reaching
+      // one is newly possible now that `sequence` derives through a nullable prefix
+      // (`list = choice('end', sequence(optional(item), list))` — term-0-only
+      // derivation used to stop short of the self-reference). Falling back to the rule
+      // name, exactly as for an unresolved external ref, cuts the cycle. Removed again
+      // on the way out, so a rule referenced twice NON-cyclically still derives fully.
+      if (seen.has(c)) return name ? [name] : []
+      seen.add(c)
+      try { return deriveExpected(def.thunk()) }
+      catch { return name ? [name] : [] }
+      finally { seen.delete(c) }
+    }
+    default:          return []
+  }
+}
+
+/**
+ * Required-token combinator. Tries `combinator`; on success returns it verbatim.
+ * On failure it does NOT fail — it records a {@link ParseError} (the statically
+ * derived expected set, or `label` when given) into `ctx._errors` and RECOVERS IN
+ * PLACE, returning a zero-width success so the enclosing sequence continues as if
+ * the token were present.
+ *
+ * Use it to mark required delimiters/terminators (`}` `)` `;`, a selector before a
+ * block) so a missing one is reported with position + expectation rather than
+ * aborting the whole parse or being silently swallowed by a catch-all.
+ *
+ *   sequence(literal('{'), declList, expect(literal('}')))
+ *
+ * Errors are only collected when the parse runs with `{ recover: true }`; without
+ * it, expect() still recovers in place but records nothing (zero overhead beyond
+ * the inner attempt).
+ */
+export function expect<T>(combinator: Combinator<T>, label?: string): Combinator<T | ParseError> {
+  const expected = label !== undefined ? [label] : deriveExpected(combinator)
+  const meta: ParserMeta = {
+    firstSet: combinator._meta.firstSet,
+    canMatchNewline: combinator._meta.canMatchNewline,
+    isTrivia: false,
+  }
+  return {
+    _tag: 'expect',
+    _meta: meta,
+    _def: { tag: 'expect', parser: combinator as Combinator<unknown>, label, expected },
+    parse(input: string, pos: number, ctx: ParseContext): ParseResult<T | ParseError> {
+      const result = combinator.parse(input, pos, ctx)
+      if (result.ok) return result as ParseResult<T | ParseError>
+      const span = { start: pos, end: pos }
+      const error: ParseError = { _tag: 'parseError', span: ctx.trackLines ? annotateSpanFromLineContext(span, ctx) : span, expected }
+      ctx._errors?.push(error)
+      // Embed the missing-token error as a `parseError` CST child (no-op when CST
+      // capture is off), so it rides the tree exactly like list-recovery errors —
+      // a tree walk finds every diagnostic and the error survives incremental
+      // reuse, rather than living only in the flat ctx._errors side-channel. Gated
+      // on tolerant like the list-recovery embed sites, so a strict CST build is
+      // byte-identical to before (and stays in parity with the compiled path,
+      // which can only embed when the recovery bundle `_rec` is installed).
+      if (ctx._tolerant) captureError(ctx, error)
+      return { ok: true, value: error, span: { start: pos, end: pos } }
+    },
+  }
+}
+
+/** True when `value` is a recovery {@link ParseError} node. */
+export function isParseError(value: unknown): value is ParseError {
+  return typeof value === 'object' && value !== null && (value as ParseError)._tag === 'parseError'
+}