parseman 0.45.0 → 0.47.0

package/src/combinators/peek.ts

@@ -0,0 +1,63 @@
+import type { Combinator, ParseContext, ParseResult, ParserMeta } from '../types.ts'
+import { any, matchesEmpty } from './first-set.ts'
+import { saveLookaheadMark, rollbackLookahead } from './trivia-skip.ts'
+
+/**
+ * Positive lookahead. Succeeds (consuming nothing) when `combinator` matches at
+ * this position; fails when it doesn't. The mirror of `not()`.
+ *
+ *   // "a mixin reference, not a class selector": require the punctuation ahead,
+ *   // then let the real production consume it.
+ *   const mixinRef = sequence(peek(regex(/[.#]/)), mixinCall)
+ *
+ * WHY THIS EXISTS AS A PRIMITIVE (not `not(not(X))`):
+ *
+ * `not(not(X))` is behaviourally a positive lookahead, but `not()` reports
+ * `firstSet: any()` — it cannot know what it forbids — so `not(not(X))` reports
+ * `any()` too. An arm leading with it therefore POISONS its choice's first-char
+ * dispatch (`analyzeGating` flags it as the `double-not` anti-pattern), and among
+ * sibling arms that share a first char the hand-rolled gate miscompiles.
+ *
+ * `peek(X)` carries X's first-set instead, so a leading `peek()` GATES its arm:
+ * `choice(sequence(peek(regex(/[.#]/)), …), …)` still emits O(1) dispatch.
+ *
+ * The first-set is exact only when X is NON-NULLABLE. A nullable `peek(X)`
+ * succeeds on the empty string, so it constrains no first character at all — it
+ * reports `any()`, the sound over-approximation. (Same rule the sequence
+ * first-set uses; see `isPositiveLookahead` in `first-set.ts`.)
+ */
+export function peek(combinator: Combinator<unknown>): Combinator<null> {
+  const meta: ParserMeta = {
+    // A nullable body imposes no first-char constraint — `any()`, not its set.
+    firstSet: matchesEmpty(combinator) ? any() : combinator._meta.firstSet,
+    canMatchNewline: false,
+    isTrivia: false,
+  }
+
+  return {
+    _tag: 'peek',
+    _meta: meta,
+    _def: { tag: 'peek', parser: combinator },
+    parse(input: string, pos: number, ctx: ParseContext): ParseResult<null> {
+      // Zero-width on BOTH outcomes, so the probe must leave NO observable trace:
+      // whatever the inner attempt captured (CST leaves/rawChildren/per-node
+      // trivia/fields), recovered (`_errors`), or committed to the GLOBAL
+      // `_triviaLog` is rolled back either way.
+      //
+      // `rollbackTrivia`, not `rollbackCstCapture`: the latter's mark omits
+      // `_triviaLog`, which is a SEPARATE sink `scanTrivia().commit()` writes when
+      // the body skips ambient trivia between terms. Missing it duplicates every
+      // probed trivia span once the region is parsed for real — `_triviaLog` has no
+      // dedup anywhere (`triviaEntries()` is a positional view over the flat array),
+      // so the duplicate reaches output. Same failure mode as the one recorded in
+      // `test/parity/trivia-log-regression.test.ts`. The compiled `peek` emits its
+      // body under a non-capturing ctx and so never writes these sinks at all; this
+      // is what keeps the two engines at parity.
+      const mark = saveLookaheadMark(ctx)
+      const result = combinator.parse(input, pos, ctx)
+      rollbackLookahead(ctx, mark)
+      if (result.ok) return { ok: true, value: null, span: { start: pos, end: pos } }
+      return { ok: false, expected: [`peek(${combinator._tag})`], span: { start: pos, end: pos } }
+    },
+  }
+}

package/src/combinators/probe.ts

@@ -0,0 +1,25 @@
+import type { ParseContext, ParseFail } from '../types.ts'
+
+/**
+ * Creates a ParseFail and, if a completions probe is active, records it.
+ *
+ * Rules:
+ * - Only failures at or before the probe's target offset are eligible.
+ * - A deeper failure (higher pos) replaces the current best.
+ * - A tie (same pos) merges expected arrays — this is what makes choice arms
+ *   aggregate their alternatives rather than each overwriting the last.
+ * - Shallower failures are ignored.
+ */
+export function failAt(ctx: ParseContext, expected: string[], pos: number): ParseFail {
+  const r: ParseFail = { ok: false, expected, span: { start: pos, end: pos } }
+  const probe = ctx._probe
+  if (probe !== undefined && pos <= probe.offset) {
+    const best = probe.best
+    if (best === null || pos > best.span.start) {
+      probe.best = r
+    } else if (pos === best.span.start) {
+      probe.best = { ...best, expected: [...best.expected, ...expected] }
+    }
+  }
+  return r
+}

package/src/combinators/ref.ts

@@ -0,0 +1,109 @@
+import type { Combinator, ParseContext, ParseResult, ParserMeta } from '../types.ts'
+import { any } from './first-set.ts'
+import { hasOwnTriviaBoundary } from './trivia-boundary.ts'
+
+/**
+ * Create a forward-declared parser slot for mutually recursive grammars.
+ *
+ * Because JS evaluates arguments eagerly, you can't reference a variable
+ * before it's declared. ref() creates a placeholder you fill in later:
+ *
+ *   const value = ref<JSONValue>()
+ *   const array  = transform(sequence(literal('['), sepBy(value, literal(',')), literal(']')), ...)
+ *   const object = transform(sequence(literal('{'), sepBy(pair, literal(',')), literal('}')), ...)
+ *   value.define(choice(object, array, string, number, bool, nullVal))
+ *
+ * Unlike lazy(() => x), you use the ref directly — no wrapping at each call site.
+ */
+export function ref<T>(): Combinator<T> & { define(p: Combinator<T>): void } {
+  let resolved: Combinator<T> | null = null
+  /** `hasOwnTriviaBoundary(resolved)`, resolved at `define()`. See `parse`. */
+  let ownBoundary = false
+
+  const meta: ParserMeta = {
+    firstSet: any(),
+    canMatchNewline: true,
+    isTrivia: false,
+  }
+
+  const slot = {
+    _tag: 'lazy' as const,
+    _meta: meta,
+    _def: {
+      tag: 'lazy' as const,
+      thunk: () => {
+        if (!resolved) throw new Error('ref<T>() used before .define() was called')
+        return resolved as Combinator<unknown>
+      },
+    },
+    parse(input: string, pos: number, ctx: ParseContext): ParseResult<T> {
+      if (!resolved) throw new Error('ref<T>() used before .define() was called')
+      // A RULE CARRIES ITS OWN AMBIENT TRIVIA — see `rules()` in parser.ts, which
+      // stamps `grammarTrivia` on EVERY rule precisely so that entering any one of
+      // them installs it. Reading it only at the parse ENTRY (grammar.ts) made the
+      // interpreter scope trivia DYNAMICALLY: a rule referenced from underneath a
+      // `noTrivia(...)` ran without trivia, so its meaning depended on its caller.
+      // Codegen never had that behaviour — it binds each rule's trivia scanner at
+      // COMPILE time (the emitted `_tf0(input, cur, ctx, 1)` before every sequence
+      // boundary), so a `g.Foo` reference always runs under Foo's own scope.
+      //
+      // The two engines therefore parsed DIFFERENT LANGUAGES, and jess's Less
+      // grammar sits exactly on the seam: `StandardDeclaration` wraps its value in
+      // `noTrivia(...)` and the `!important` tail lives in the referenced rule
+      // `ValueListWithPriority`, so the interpreter and both table drivers could not
+      // cross the space in `color: red !important`. That one construct truncated a
+      // 107 KB stylesheet at 68.5% while reporting `ok: true` with no errors.
+      // Codegen shipped through 0.44-0.46 and was right; 0.47 made the table the
+      // shipping engine and the divergence became the product.
+      //
+      // ONLY WHERE THE SCOPE WAS CLEARED, not wherever it merely differs. The
+      // broader form (`ctx.trivia !== gt`) also OVERRIDES a caller that set a
+      // different live trivia, and that measurably broke recognition: over jess's
+      // four corpora it regressed `@less/test-data`'s `selectors.less` from 3791
+      // bytes to 1784. A caller running under its own live trivia is making a
+      // deliberate choice; a caller that CLEARED it is the `noTrivia(...)` case this
+      // fixes. Restricting to the cleared case keeps every repair and drops that
+      // regression.
+      //
+      // ONLY WHERE THE RULE HAS A BOUNDARY TO REPAIR. `ownBoundary` is resolved
+      // once in `define()` (see `hasOwnTriviaBoundary`): a rule whose body is a
+      // bare ALTERNATION, a dispatch, or a single terminal has no position of its
+      // own at which ambient trivia is consulted, so re-establishing the scope
+      // there cannot repair anything — it can only leak past the arms into
+      // whatever they delegate to.
+      //
+      // That leak is what jess's SCSS value grammar hit. `ValueTerm` clears, and
+      // `MathUnary` — `choice(noTrivia(…), noTrivia(…), g.ValueAtom)` — restored
+      // for a body that never uses it, handing its third arm a live scope.
+      // `ValueAtom` → `KeywordOrInterpolatedValue`, whose `many()` concatenates
+      // identifier chunks, then skipped the space between its terms:
+      // `gen-workload.scss` stopped at byte 218 of 287543, and `a{b: c d}` silently
+      // produced the ONE keyword `bc` with `ok: true` and no errors.
+      //
+      // Hot path is one boolean check.
+      const gt = meta.grammarTrivia
+      if (gt !== undefined && ownBoundary && ctx.trivia === undefined) {
+        const savedLabels = ctx.triviaKindLabels
+        ctx.trivia = gt
+        ctx.triviaKindLabels = gt._meta.triviaKindLabels
+        try { return resolved.parse(input, pos, ctx) }
+        finally { ctx.trivia = undefined; ctx.triviaKindLabels = savedLabels }
+      }
+      return resolved.parse(input, pos, ctx)
+    },
+    define(p: Combinator<T>): void {
+      if (resolved) throw new Error('ref<T>() already defined')
+      resolved = p
+      ownBoundary = hasOwnTriviaBoundary(p as Combinator<unknown>)
+      meta.firstSet = p._meta.firstSet
+      meta.canMatchNewline = p._meta.canMatchNewline
+      meta.isTrivia = p._meta.isTrivia
+      if (p._meta.triviaKindLabels !== undefined) meta.triviaKindLabels = p._meta.triviaKindLabels
+      else delete meta.triviaKindLabels
+      if (p._meta.disjoint !== undefined) meta.disjoint = p._meta.disjoint
+      else delete meta.disjoint
+    },
+  }
+
+  return slot
+}

package/src/combinators/regex.ts

@@ -0,0 +1,170 @@
+import type { Combinator, ParseContext, ParseResult, ParserMeta, FirstSet } from '../types.ts'
+import { any } from './first-set.ts'
+import { failAt } from './probe.ts'
+import { pushCstLeaf, cstCaptureActive } from '../cst/capture-buffer.ts'
+import { recordLineRangeFromContext } from '../line-index.ts'
+import { shorthandRanges, parseClassRanges } from '../regex/classes.ts'
+import { firstSetFromRegex } from '../regex/first-set.ts'
+
+/**
+ * A regex terminal's first-set (for choice-dispatch fast paths) is derived by
+ * `firstSetFromRegex` in `../regex/first-set.ts` — a small, dependency-free
+ * hand-rolled regex parser, imported DIRECTLY and unconditionally.
+ *
+ * ── WHY NOT INJECTED ────────────────────────────────────────────────────────────
+ * It used to be registered at run time by the library entry (`index.ts`) via a
+ * `registerRegexAnalyzer` seam, so that a bundle holding `regex()` without the
+ * library entry never pulled the analyzer in. That saved ~3 KB and cost a shipped
+ * export: `parseman/table` is its OWN module graph, so its private copy of this
+ * module never saw the registration, `regex()` there returned the permissive
+ * `any()` first-set, and `classifiedTrivia()` — which REQUIRES a concrete finite
+ * first set per arm — rejected every arm of every table-lowered grammar
+ * (`buildTrivia` in `../table/program.ts`). The failure was invisible from source,
+ * where `src/index.ts` is always in the graph.
+ *
+ * A mutable module-global that a *different* entry has to remember to write makes
+ * `regex()`'s result depend on which bundle it landed in. Deriving the first set
+ * intrinsically makes every module graph agree by construction. The analyzer adds
+ * no new leaf modules — it imports only `./first-set.ts` and `../regex/classes.ts`,
+ * both already reachable from here — so the cost is the analyzer's own bytes.
+ */
+const SCAN_BAIL_AT = 64
+type ShortScanner = (input: string, pos: number) => number | null | undefined
+
+function inRanges(cp: number, ranges: Array<[number, number]>): boolean {
+  for (let i = 0; i < ranges.length; i++) {
+    const [lo, hi] = ranges[i]!
+    if (cp >= lo && cp <= hi) return true
+  }
+  return false
+}
+
+/**
+ * The positive char-class body (chars between `[` and `]`) as ranges for the
+ * short-scan fast path, or null if it can't be lowered — including a NEGATED
+ * class (`[^…]`), which the scan can't express. The class parsing itself is the
+ * shared `parseClassRanges` (see `../regex/classes.ts`).
+ */
+function readClassRanges(body: string): Array<[number, number]> | null {
+  if (body.startsWith('^')) return null
+  return parseClassRanges(body)
+}
+
+function shortScanner(source: string, flags: string): ShortScanner | null {
+  if (/[imsuvy]/.test(flags)) return null
+  let ranges: Array<[number, number]> | null = null
+  let quant = ''
+  if (source[0] === '[') {
+    let end = 1
+    while (end < source.length && source[end] !== ']') {
+      if (source[end] === '\\') end += 2
+      else end++
+    }
+    if (source[end] !== ']') return null
+    ranges = readClassRanges(source.slice(1, end))
+    quant = source.slice(end + 1)
+  } else if (source[0] === '\\' && (source[1] === 'd' || source[1] === 'w' || source[1] === 's')) {
+    ranges = shorthandRanges(source[1])
+    quant = source.slice(2)
+  }
+  if (!ranges || (quant !== '+' && quant !== '*')) return null
+  const minOne = quant === '+'
+  return (input, pos) => {
+    let end = pos
+    while (end < input.length && inRanges(input.charCodeAt(end), ranges)) {
+      end++
+      if (end - pos >= SCAN_BAIL_AT) return undefined
+    }
+    return minOne && end === pos ? null : end
+  }
+}
+
+/**
+ * ASCII-case-fold a first-set: for every ASCII letter reachable as a leading char,
+ * also admit its opposite-case twin (A–Z ↔ a–z). Non-letter code points and `any`/
+ * `empty` are unchanged. Only widens the set (sound superset) — used to correct a
+ * case-insensitive regex's flag-agnostic first-set for first-char dispatch. Unicode
+ * case-folding beyond ASCII is deliberately NOT applied (parseman first-sets are
+ * ASCII-BMP for dispatch); a non-ASCII letter keeps its own code point only, which
+ * stays sound (the interpreter's real `/i` match still runs).
+ */
+function asciiCaseFold(fs: FirstSet): FirstSet {
+  if (fs.kind !== 'ranges') return fs
+  const ranges: { lo: number; hi: number }[] = fs.ranges.map(r => ({ lo: r.lo, hi: r.hi }))
+  const add = (lo: number, hi: number): void => { if (lo <= hi) ranges.push({ lo, hi }) }
+  for (const r of fs.ranges) {
+    // Uppercase portion [A–Z] → lowercase twin (+32).
+    const uLo = Math.max(r.lo, 65), uHi = Math.min(r.hi, 90)
+    add(uLo + 32, uHi + 32)
+    // Lowercase portion [a–z] → uppercase twin (−32).
+    const lLo = Math.max(r.lo, 97), lHi = Math.min(r.hi, 122)
+    add(lLo - 32, lHi - 32)
+  }
+  // Normalize: sort + coalesce overlapping/adjacent ranges.
+  ranges.sort((a, b) => a.lo - b.lo)
+  const merged: { lo: number; hi: number }[] = []
+  for (const r of ranges) {
+    const last = merged[merged.length - 1]
+    if (last && r.lo <= last.hi + 1) last.hi = Math.max(last.hi, r.hi)
+    else merged.push({ lo: r.lo, hi: r.hi })
+  }
+  return { kind: 'ranges', ranges: merged }
+}
+
+export function regex(pattern: string | RegExp, flags = ''): Combinator<string> {
+  const source = typeof pattern === 'string' ? pattern : pattern.source
+  const resolvedFlags = typeof pattern === 'string' ? flags : pattern.flags
+
+  const anchored = new RegExp(source, 'y' + resolvedFlags.replace(/[gy]/g, ''))
+  const scan = shortScanner(source, resolvedFlags)
+
+  const raw = firstSetFromRegex(source)
+  // The first-set analyzer is flag-agnostic, so for a CASE-INSENSITIVE pattern it
+  // returns only the literal-case leading chars — e.g. `/red|blue/i` → `{r,b}`, NOT
+  // `{r,R,b,B}`. Using that narrow set for first-char DISPATCH would false-EXCLUDE the
+  // opposite-case input (`ReD` gated out of a `/(?:red|…)/i` arm), so widen it under
+  // `i`. The widening is FLAG-AWARE because `i` alone and `i`+`u` fold different sets:
+  //   - `/i` WITHOUT `u`: ECMAScript case-insensitive matching folds ONLY the ASCII
+  //     Basic-Latin case pairs (`a`↔`A` … `z`↔`Z`) — a plain `/[a-z]/i` does NOT match
+  //     `ſ`(U+017F) or `K`(U+212A). So `asciiCaseFold` is a SOUND, tight superset, and
+  //     it preserves at-keyword dispatch (`/@media…/i` still gates on `@`).
+  //   - `/ui` or `/iv` (Unicode mode — the `u` OR the ES2024 `v` Unicode-sets flag):
+  //     matching uses Unicode *simple case folding*, so `/[a-z]/ui` (and `/[a-z]/iv`)
+  //     ALSO match `ſ`→s and `K`→k. ASCII-folding can't enumerate those astral/BMP case
+  //     pairs, so gating on the ASCII set would false-exclude them. Fall back to `any()`
+  //     (always-try) — sound, and only forfeits gating for the rare Unicode-mode `/i`
+  //     recognizer, never the ASCII-only `/i` at-keywords the fix targets.
+  const firstSet = !resolvedFlags.includes('i')
+    ? raw.firstSet
+    : (resolvedFlags.includes('u') || resolvedFlags.includes('v'))
+      ? any()
+      : asciiCaseFold(raw.firstSet)
+  const canMatchNewline = raw.canMatchNewline
+  const meta: ParserMeta = { firstSet, canMatchNewline, isTrivia: false }
+
+  return {
+    _tag: 'regex',
+    _meta: meta,
+    _def: { tag: 'regex', source, flags: resolvedFlags },
+    parse(input: string, pos: number, ctx: ParseContext): ParseResult<string> {
+      const scanEnd = scan?.(input, pos)
+      if (scanEnd !== undefined) {
+        if (scanEnd === null) return failAt(ctx, [`/${source}/`], pos)
+        const value = input.slice(pos, scanEnd)
+        const span = { start: pos, end: scanEnd }
+        if (canMatchNewline && ctx.trackLines) recordLineRangeFromContext(ctx, input, pos, scanEnd)
+        if (cstCaptureActive(ctx)) pushCstLeaf(ctx, { _tag: 'leaf', value, span })
+        return { ok: true, value, span }
+      }
+      anchored.lastIndex = pos
+      const m = anchored.exec(input)
+      if (m === null) {
+        return failAt(ctx, [`/${source}/`], pos)
+      }
+      const span = { start: pos, end: pos + m[0]!.length }
+      if (canMatchNewline && ctx.trackLines) recordLineRangeFromContext(ctx, input, pos, span.end)
+      if (cstCaptureActive(ctx)) pushCstLeaf(ctx, { _tag: 'leaf', value: m[0]!, span })
+      return { ok: true, value: m[0]!, span }
+    },
+  }
+}