parseman 0.46.0 → 0.47.0

package/src/combinators/keywords.ts

@@ -0,0 +1,145 @@
+import type { Combinator, ParseContext, ParseResult, ParserMeta, FirstSet } from '../types.ts'
+import { fromChar, union, empty } from './first-set.ts'
+import { caseFoldVariants } from './case-fold.ts'
+import { failAt } from './probe.ts'
+import { pushCstLeaf, cstCaptureActive } from '../cst/capture-buffer.ts'
+
+export type KeywordsOptions = {
+  /** Match case-insensitively. */
+  caseInsensitive?: boolean
+  /**
+   * Character class (regex source, e.g. `'A-Za-z0-9_-'`) that must NOT follow a
+   * match — a word boundary. Prevents matching `red` inside `redish`.
+   */
+  boundary?: string
+}
+
+export type WordOptions = Omit<KeywordsOptions, 'boundary'>
+
+/**
+ * Match a single keyword with an automatic word-boundary guard. Prevents
+ * matching `true` inside `trueish`. The boundary defaults to `_0-9A-Za-z`,
+ * which covers most programming-language identifiers.
+ *
+ *   word('true')                                    // matches "true" but not "trueish"
+ *   word('color', 'A-Za-z-')                        // CSS-style identifier boundary
+ *   word('media', 'A-Za-z0-9_-', { caseInsensitive: true })
+ *   word('media', { caseInsensitive: true })        // default boundary + options
+ *
+ * `caseInsensitive` matches `keywords()` and exists because it is REQUIRED, not a
+ * nicety: CSS at-keywords, function names and units are ASCII case-insensitive per
+ * spec (CSS Syntax §3). Without it the only conforming spellings were
+ * `regex(/media/i)` — which `analyzeGating` correctly flags as the `keyword-regex`
+ * anti-pattern — or `keywords(['media'], …)` for a single word.
+ */
+export function word(str: string, boundary?: string, opts?: WordOptions): Combinator<string>
+export function word(str: string, opts: WordOptions): Combinator<string>
+export function word(
+  str: string,
+  boundaryOrOpts?: string | WordOptions,
+  opts?: WordOptions,
+): Combinator<string> {
+  const boundary = typeof boundaryOrOpts === 'string' ? boundaryOrOpts : '_0-9A-Za-z'
+  const rest = typeof boundaryOrOpts === 'object' && boundaryOrOpts !== null ? boundaryOrOpts : opts
+  return keywords([str], { ...rest, boundary })
+}
+
+/**
+ * Create a keyword factory with a fixed word-boundary class. Use when many
+ * keywords share the same boundary (e.g. CSS identifiers). For a single keyword,
+ * `word(str, boundary?)` is enough.
+ *
+ *   const kw = makeWord()                                      // default boundary
+ *   const cssKw = makeWord('A-Za-z0-9_-', { caseInsensitive: true })
+ *
+ *   const query = kw('query')
+ *   const color = cssKw('color')                               // matches "COLOR" but not "color-scheme"
+ */
+export function makeWord(boundary?: string, opts?: WordOptions): (str: string) => Combinator<string>
+export function makeWord(opts: WordOptions): (str: string) => Combinator<string>
+export function makeWord(
+  boundaryOrOpts: string | WordOptions = '_0-9A-Za-z',
+  opts?: WordOptions,
+): (str: string) => Combinator<string> {
+  const boundary = typeof boundaryOrOpts === 'string' ? boundaryOrOpts : '_0-9A-Za-z'
+  const rest = typeof boundaryOrOpts === 'object' && boundaryOrOpts !== null ? boundaryOrOpts : opts
+  return (str: string) => keywords([str], { ...rest, boundary })
+}
+
+function escapeRe(s: string): string {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+}
+
+/**
+ * Match any one of a set of fixed keywords, longest-first (so `border` wins over
+ * `bord`), with an optional trailing word-boundary guard. Compiles to a single
+ * sticky regex — far less error-prone than a hand-maintained alternation, and
+ * the common case (keyword sets: CSS colors, units, at-rule names, HTML tags).
+ *
+ *   keywords(CSS_COLOR_NAMES, { caseInsensitive: true, boundary: 'A-Za-z0-9_-' })
+ *
+ * On success the matched text is recorded as a CSTLeaf (like literal()/regex()).
+ */
+export function keywords(words: readonly string[], opts: KeywordsOptions = {}): Combinator<string> {
+  // Longest-first keeps the alternation greedy-correct (regex alternation is
+  // first-match, not longest-match).
+  const sorted = [...new Set(words)].sort((a, b) => b.length - a.length)
+  const alt = sorted.map(escapeRe).join('|')
+  const boundary = opts.boundary ? `(?![${opts.boundary}])` : ''
+  // Case-INSENSITIVE drops `u` deliberately, so that MATCHING and the first-set
+  // below fold the SAME set of characters — the invariant `regex()` established in
+  // 0.32.0 (see the flag-aware widening in `src/combinators/regex.ts`):
+  //   - `/i` WITHOUT `u` folds only pairs that stay on the SAME side of the ASCII
+  //     boundary — every Basic-Latin pair (a↔A … z↔Z), and also the non-ASCII ones
+  //     (ä↔Ä, σ↔Σ↔ς), but never a pair that crosses it. `caseFoldVariants` below
+  //     enumerates exactly that relation → the first-set is a tight, SOUND superset
+  //     and dispatch stays exact. (Widening by toUpperCase/toLowerCase alone would
+  //     NOT be: 67 BMP code points sit in fold classes those two miss.)
+  //   - `/iu` folds by Unicode *simple case folding*, so `/(?:stroke)/iu` also
+  //     matches `ſtroke` (U+017F → s). An ASCII-only first-set would then dispatch
+  //     that input AWAY from this arm — an unsound gate. `regex()` answers that by
+  //     falling back to `any()` (forfeiting the gate); `keywords()` answers it by
+  //     not entering Unicode mode at all, which keeps the gate.
+  // Every keyword is fully escaped by `escapeRe`, so `u` mode changes nothing about
+  // what a case-SENSITIVE set matches; that path keeps `u` unchanged.
+  const flags = opts.caseInsensitive ? 'iy' : 'uy'
+  const re = new RegExp(`(?:${alt})${boundary}`, flags)
+
+  // First-set: the set of first code points across all keywords (and their
+  // case-folded variants when case-insensitive), for choice() dispatch.
+  //
+  // The fold must cover the SAME pairs the `iy` matcher above accepts, or the gate
+  // dispatches valid input away from this arm. `/i` folds non-ASCII pairs too
+  // (`/ärger/i` matches `Ärger`, `/σ/i` matches both `Σ` and `ς`) — it only refuses
+  // folds that would CROSS the ASCII boundary. `caseFoldVariants` is that exact
+  // relation, so the widening stays a tight, sound superset for every first char,
+  // not just the Basic-Latin ones.
+  let firstSet: FirstSet = empty()
+  for (const w of sorted) {
+    if (w.length === 0) continue
+    const cp = w.codePointAt(0)!
+    firstSet = union(firstSet, fromChar(cp))
+    if (opts.caseInsensitive) {
+      for (const v of caseFoldVariants(cp)) firstSet = union(firstSet, fromChar(v))
+    }
+  }
+
+  const meta: ParserMeta = { firstSet, canMatchNewline: false, isTrivia: false }
+
+  return {
+    _tag: 'keywords',
+    _meta: meta,
+    _def: { tag: 'keywords', words: sorted, caseInsensitive: opts.caseInsensitive ?? false, boundary: opts.boundary },
+    parse(input: string, pos: number, ctx: ParseContext): ParseResult<string> {
+      re.lastIndex = pos
+      const m = re.exec(input)
+      if (m === null || m.index !== pos) {
+        return failAt(ctx, ['keyword'], pos)
+      }
+      const value = m[0]!
+      const span = { start: pos, end: pos + value.length }
+      if (cstCaptureActive(ctx)) pushCstLeaf(ctx, { _tag: 'leaf', value, span })
+      return { ok: true, value, span }
+    },
+  }
+}

package/src/combinators/literal.ts

@@ -0,0 +1,102 @@
+import type { Combinator, ParseContext, ParseResult, ParserMeta } from '../types.ts'
+import { fromChar, empty } from './first-set.ts'
+import { failAt } from './probe.ts'
+import { pushCstLeaf, cstCaptureActive } from '../cst/capture-buffer.ts'
+import { recordLineRangeFromContext } from '../line-index.ts'
+
+/**
+ * ASCII case-fold equality — the interpreter twin of the compiler's `foldEq`
+ * (and `/i`-flag regex lowering). Folds ASCII letters via bit 0x20, exact-matches
+ * everything else. Deliberately NOT `Intl.Collator` (measured ~9× slower, and its
+ * Unicode accent-folding is the wrong semantic for a parser). Keeps interpreter /
+ * compiled / macro output identical for case-insensitive `literal()`.
+ */
+export function asciiFoldEq(a: string, b: string): boolean {
+  if (a.length !== b.length) return false
+  for (let i = 0; i < a.length; i++) {
+    let ca = a.charCodeAt(i)
+    let cb = b.charCodeAt(i)
+    if (ca >= 65 && ca <= 90) ca += 32
+    if (cb >= 65 && cb <= 90) cb += 32
+    if (ca !== cb) return false
+  }
+  return true
+}
+
+export type LiteralOptions = {
+  caseInsensitive?: boolean
+}
+
+export function literal(value: string, opts: LiteralOptions = {}): Combinator<string> {
+  const caseInsensitive = opts.caseInsensitive ?? false
+
+  const firstSet = value.length > 0
+    ? fromChar(value.codePointAt(0)!)
+    : empty()
+
+  const meta: ParserMeta = {
+    firstSet,
+    canMatchNewline: value.includes('\n'),
+    isTrivia: false,
+  }
+  const canMatchNewline = meta.canMatchNewline
+
+  if (caseInsensitive) {
+    const upper = value.toUpperCase()
+    const lower = value.toLowerCase()
+    const firstUpper = upper.codePointAt(0)
+    const firstLower = lower.codePointAt(0)
+    // Two discrete points, not one spanning range — `asciiFoldEq` accepts only
+    // the char and its fold twin, so `[min, max]` would over-approximate (e.g.
+    // 'a'/'A' → the whole 65..97 span, pulling B–Z/`[\]^_\`` into dispatch).
+    meta.firstSet = firstLower !== undefined && firstUpper !== undefined
+      ? firstLower === firstUpper
+        ? { kind: 'ranges', ranges: [{ lo: firstLower, hi: firstLower }] }
+        : { kind: 'ranges', ranges: [
+            { lo: firstLower, hi: firstLower },
+            { lo: firstUpper, hi: firstUpper },
+          ]}
+      : firstSet
+  }
+
+  const expected = [JSON.stringify(value)]
+
+  // Single-char case-sensitive literals are the bulk of punctuation-heavy
+  // grammars (GraphQL `{ } ( ) : $ @ [ ] ! =`, JSON, CSS). A bare `charCodeAt`
+  // compare beats the generic `startsWith` builtin call, and an out-of-bounds
+  // `charCodeAt` returns NaN — so the length check folds into the compare.
+  const parse = !caseInsensitive && value.length === 1
+    ? (() => {
+        const code = value.charCodeAt(0)
+        return function parse(input: string, pos: number, ctx: ParseContext): ParseResult<string> {
+          if (input.charCodeAt(pos) === code) {
+            const span = { start: pos, end: pos + 1 }
+            if (canMatchNewline && ctx.trackLines) recordLineRangeFromContext(ctx, input, pos, span.end)
+            if (cstCaptureActive(ctx)) pushCstLeaf(ctx, { _tag: 'leaf', value, span })
+            return { ok: true, value, span }
+          }
+          return failAt(ctx, expected, pos)
+        }
+      })()
+    : function parse(input: string, pos: number, ctx: ParseContext): ParseResult<string> {
+        const end = pos + value.length
+        if (end > input.length) {
+          return failAt(ctx, expected, pos)
+        }
+        const matchedValue = caseInsensitive ? input.slice(pos, end) : value
+        if (caseInsensitive ? asciiFoldEq(matchedValue, value) : input.startsWith(value, pos)) {
+          const span = { start: pos, end }
+          if (canMatchNewline && ctx.trackLines) recordLineRangeFromContext(ctx, input, pos, end)
+          if (cstCaptureActive(ctx)) pushCstLeaf(ctx, { _tag: 'leaf', value: matchedValue, span })
+          return { ok: true, value: matchedValue, span }
+        }
+        return failAt(ctx, expected, pos)
+      }
+
+  return {
+    _tag: 'literal',
+    _meta: meta,
+    _def: { tag: 'literal', value, caseInsensitive },
+    parse,
+  }
+}

package/src/combinators/map.ts

@@ -0,0 +1,106 @@
+import type { Combinator, ParseContext, ParseResult } from '../types.ts'
+import { analyzeLabeledTrivia } from '../cst/trivia-kinds.ts'
+import { choice } from './choice.ts'
+import { matchesEmpty } from './first-set.ts'
+import { oneOrMore } from './repeat.ts'
+
+export function transform<T, U>(
+  combinator: Combinator<T>,
+  fn: (value: T, span: { start: number; end: number }) => U
+): Combinator<U> {
+  return {
+    _tag: 'transform',
+    _meta: combinator._meta,
+    _def: { tag: 'transform', parser: combinator as Combinator<unknown>, fn: fn as (v: unknown, span: { start: number; end: number }) => unknown },
+    parse(input: string, pos: number, ctx: ParseContext): ParseResult<U> {
+      const result = combinator.parse(input, pos, ctx)
+      if (!result.ok) return result
+      return { ok: true, value: fn(result.value, result.span), span: result.span }
+    },
+  }
+}
+
+export function trivia<T>(combinator: Combinator<T>): Combinator<T> {
+  const kindLabels = analyzeLabeledTrivia(combinator as Combinator<unknown>)?.labels
+  return {
+    _tag: combinator._tag,
+    _meta: {
+      ...combinator._meta,
+      isTrivia: true,
+      ...(kindLabels ? { triviaKindLabels: kindLabels } : {}),
+    },
+    _def: { tag: 'trivia', parser: combinator as Combinator<unknown> },
+    parse: combinator.parse.bind(combinator),
+  }
+}
+
+/**
+ * Build trivia whose category labels are part of its recognition structure.
+ *
+ * Use this for a grammar that exposes selected root trivia. In contrast to
+ * `trivia(label('categoryA', broadRegex))`, every category here owns one arm,
+ * so one selected category cannot be silently consumed under another category's
+ * broad matcher.
+ * `rootTrivia: { select }` rejects ordinary scoped trivia
+ * unless that scope explicitly declares itself opaque.
+ */
+export function classifiedTrivia(
+  arms: Readonly<Record<string, Combinator<unknown>>>,
+): Combinator<unknown> {
+  const entries = Object.entries(arms)
+  if (entries.length === 0) {
+    throw new TypeError('classifiedTrivia() requires at least one named trivia arm.')
+  }
+  for (let i = 0; i < entries.length; i++) {
+    const [name, arm] = entries[i]!
+    const first = arm._meta.firstSet
+    if (matchesEmpty(arm) || first.kind !== 'ranges' || first.ranges.length === 0) {
+      throw new TypeError(
+        `classifiedTrivia(): ${JSON.stringify(name)} must be non-nullable with a concrete finite first set.`,
+      )
+    }
+  }
+  const labeledArms = entries.map(([name, arm]) => label(name, arm)) as [
+    Combinator<unknown>,
+    ...Combinator<unknown>[],
+  ]
+  const result = trivia(oneOrMore(choice(...labeledArms)))
+  result._meta.rootTriviaClassified = true
+  return result
+}
+
+/**
+ * Attach a string label to a parser arm (e.g. trivia `choice` branches).
+ * Parse behavior is unchanged; the label is metadata for tooling and future
+ * trivia-kind capture (`'whitespace'`, `'blockComment'`, …).
+ */
+export function label<T>(name: string, combinator: Combinator<T>): Combinator<T> {
+  return {
+    _tag: combinator._tag,
+    _meta: combinator._meta,
+    _def: { tag: 'label', label: name, parser: combinator as Combinator<unknown> },
+    parse(input: string, pos: number, ctx: ParseContext): ParseResult<T> {
+      const result = combinator.parse(input, pos, ctx)
+      if (!result.ok) return { ok: false, expected: [name], span: result.span }
+      return result
+    },
+  }
+}
+
+/**
+ * Capture a named parse result for the nearest enclosing node() builder.
+ * Parse behavior and the normal returned value are unchanged.
+ */
+export function field<T>(name: string, combinator: Combinator<T>): Combinator<T> {
+  return {
+    _tag: combinator._tag,
+    _meta: combinator._meta,
+    _def: { tag: 'field', name, parser: combinator as Combinator<unknown> },
+    parse(input: string, pos: number, ctx: ParseContext): ParseResult<T> {
+      const result = combinator.parse(input, pos, ctx)
+      if (!result.ok) return result
+      ctx._fields?.push({ name, value: result.value, span: result.span })
+      return result
+    },
+  }
+}