parseman 0.47.1 → 0.48.0

package/src/combinators/keywords.ts

@@ -3,6 +3,7 @@ import { fromChar, union, empty } from './first-set.ts'
 import { caseFoldVariants } from './case-fold.ts'
 import { failAt } from './probe.ts'
 import { pushCstLeaf, cstCaptureActive } from '../cst/capture-buffer.ts'
+import { directTerminalFailureExpected } from './expected.ts'
 
 export type KeywordsOptions = {
   /** Match case-insensitively. */
@@ -70,6 +71,19 @@ function escapeRe(s: string): string {
   return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
 }
 
+/** The terminal's one canonical recognizer spelling. Table-selected lexical
+ * bodies call this at construction time so the authored parser, CHARACTER row,
+ * and TOKEN replacement cannot drift on alternation order, boundary, or flags. */
+export function keywordsRegExp(
+  words: readonly string[],
+  boundary: string | undefined,
+  caseInsensitive: boolean,
+): RegExp {
+  const alt = words.map(escapeRe).join('|')
+  const boundarySource = boundary ? `(?![${boundary}])` : ''
+  return new RegExp(`(?:${alt})${boundarySource}`, caseInsensitive ? 'iy' : 'uy')
+}
+
 /**
  * Match any one of a set of fixed keywords, longest-first (so `border` wins over
  * `bord`), with an optional trailing word-boundary guard. Compiles to a single
@@ -84,8 +98,6 @@ export function keywords(words: readonly string[], opts: KeywordsOptions = {}):
   // Longest-first keeps the alternation greedy-correct (regex alternation is
   // first-match, not longest-match).
   const sorted = [...new Set(words)].sort((a, b) => b.length - a.length)
-  const alt = sorted.map(escapeRe).join('|')
-  const boundary = opts.boundary ? `(?![${opts.boundary}])` : ''
   // Case-INSENSITIVE drops `u` deliberately, so that MATCHING and the first-set
   // below fold the SAME set of characters — the invariant `regex()` established in
   // 0.32.0 (see the flag-aware widening in `src/combinators/regex.ts`):
@@ -102,8 +114,7 @@ export function keywords(words: readonly string[], opts: KeywordsOptions = {}):
   //     not entering Unicode mode at all, which keeps the gate.
   // Every keyword is fully escaped by `escapeRe`, so `u` mode changes nothing about
   // what a case-SENSITIVE set matches; that path keeps `u` unchanged.
-  const flags = opts.caseInsensitive ? 'iy' : 'uy'
-  const re = new RegExp(`(?:${alt})${boundary}`, flags)
+  const re = keywordsRegExp(sorted, opts.boundary, opts.caseInsensitive ?? false)
 
   // First-set: the set of first code points across all keywords (and their
   // case-folded variants when case-insensitive), for choice() dispatch.
@@ -125,16 +136,21 @@ export function keywords(words: readonly string[], opts: KeywordsOptions = {}):
   }
 
   const meta: ParserMeta = { firstSet, canMatchNewline: false, isTrivia: false }
+  const def = {
+    tag: 'keywords', words: sorted,
+    caseInsensitive: opts.caseInsensitive ?? false,
+    boundary: opts.boundary,
+  } as const
 
   return {
     _tag: 'keywords',
     _meta: meta,
-    _def: { tag: 'keywords', words: sorted, caseInsensitive: opts.caseInsensitive ?? false, boundary: opts.boundary },
+    _def: def,
     parse(input: string, pos: number, ctx: ParseContext): ParseResult<string> {
       re.lastIndex = pos
       const m = re.exec(input)
       if (m === null || m.index !== pos) {
-        return failAt(ctx, ['keyword'], pos)
+        return failAt(ctx, directTerminalFailureExpected(def), pos)
       }
       const value = m[0]!
       const span = { start: pos, end: pos + value.length }

package/src/combinators/literal.ts

@@ -3,6 +3,7 @@ import { fromChar, empty } from './first-set.ts'
 import { failAt } from './probe.ts'
 import { pushCstLeaf, cstCaptureActive } from '../cst/capture-buffer.ts'
 import { recordLineRangeFromContext } from '../line-index.ts'
+import { directTerminalFailureExpected } from './expected.ts'
 
 /**
  * ASCII case-fold equality — the interpreter twin of the compiler's `foldEq`
@@ -59,7 +60,7 @@ export function literal(value: string, opts: LiteralOptions = {}): Combinator<st
       : firstSet
   }
 
-  const expected = [JSON.stringify(value)]
+  const expected = directTerminalFailureExpected({ tag: 'literal', value, caseInsensitive })
 
   // Single-char case-sensitive literals are the bulk of punctuation-heavy
   // grammars (GraphQL `{ } ( ) : $ @ [ ] ! =`, JSON, CSS). A bare `charCodeAt`

package/src/combinators/not.ts

@@ -1,6 +1,7 @@
 import type { Combinator, ParseContext, ParseResult, ParserMeta } from '../types.ts'
 import { any } from './first-set.ts'
 import { saveLookaheadMark, rollbackLookahead } from './trivia-skip.ts'
+import { assertionFailureExpected } from './expected.ts'
 
 /**
  * Negative lookahead. Succeeds (consuming nothing) when `combinator` fails;
@@ -47,7 +48,10 @@ export function not(combinator: Combinator<unknown>): Combinator<null> {
       const result = combinator.parse(input, pos, ctx)
       rollbackLookahead(ctx, mark)
       if (result.ok) {
-        return { ok: false, expected: [`not(${combinator._tag})`], span: { start: pos, end: pos } }
+        return {
+          ok: false, expected: assertionFailureExpected(false, combinator._tag),
+          span: { start: pos, end: pos },
+        }
       }
       return { ok: true, value: null, span: { start: pos, end: pos } }
     },

package/src/combinators/peek.ts

@@ -1,6 +1,7 @@
 import type { Combinator, ParseContext, ParseResult, ParserMeta } from '../types.ts'
 import { any, matchesEmpty } from './first-set.ts'
 import { saveLookaheadMark, rollbackLookahead } from './trivia-skip.ts'
+import { assertionFailureExpected } from './expected.ts'
 
 /**
  * Positive lookahead. Succeeds (consuming nothing) when `combinator` matches at
@@ -57,7 +58,10 @@ export function peek(combinator: Combinator<unknown>): Combinator<null> {
       const result = combinator.parse(input, pos, ctx)
       rollbackLookahead(ctx, mark)
       if (result.ok) return { ok: true, value: null, span: { start: pos, end: pos } }
-      return { ok: false, expected: [`peek(${combinator._tag})`], span: { start: pos, end: pos } }
+      return {
+        ok: false, expected: assertionFailureExpected(true, combinator._tag),
+        span: { start: pos, end: pos },
+      }
     },
   }
 }

package/src/combinators/regex.ts

@@ -5,6 +5,7 @@ import { pushCstLeaf, cstCaptureActive } from '../cst/capture-buffer.ts'
 import { recordLineRangeFromContext } from '../line-index.ts'
 import { shorthandRanges, parseClassRanges } from '../regex/classes.ts'
 import { firstSetFromRegex } from '../regex/first-set.ts'
+import { directTerminalFailureExpected } from './expected.ts'
 
 /**
  * A regex terminal's first-set (for choice-dispatch fast paths) is derived by
@@ -141,15 +142,16 @@ export function regex(pattern: string | RegExp, flags = ''): Combinator<string>
       : asciiCaseFold(raw.firstSet)
   const canMatchNewline = raw.canMatchNewline
   const meta: ParserMeta = { firstSet, canMatchNewline, isTrivia: false }
+  const def = { tag: 'regex', source, flags: resolvedFlags } as const
 
   return {
     _tag: 'regex',
     _meta: meta,
-    _def: { tag: 'regex', source, flags: resolvedFlags },
+    _def: def,
     parse(input: string, pos: number, ctx: ParseContext): ParseResult<string> {
       const scanEnd = scan?.(input, pos)
       if (scanEnd !== undefined) {
-        if (scanEnd === null) return failAt(ctx, [`/${source}/`], pos)
+        if (scanEnd === null) return failAt(ctx, directTerminalFailureExpected(def), pos)
         const value = input.slice(pos, scanEnd)
         const span = { start: pos, end: scanEnd }
         if (canMatchNewline && ctx.trackLines) recordLineRangeFromContext(ctx, input, pos, scanEnd)
@@ -159,7 +161,7 @@ export function regex(pattern: string | RegExp, flags = ''): Combinator<string>
       anchored.lastIndex = pos
       const m = anchored.exec(input)
       if (m === null) {
-        return failAt(ctx, [`/${source}/`], pos)
+        return failAt(ctx, directTerminalFailureExpected(def), pos)
       }
       const span = { start: pos, end: pos + m[0]!.length }
       if (canMatchNewline && ctx.trackLines) recordLineRangeFromContext(ctx, input, pos, span.end)

package/src/combinators/repeat.ts

@@ -1,5 +1,5 @@
 import type { Combinator, ParseContext, ParseResult, ParserMeta } from '../types.ts'
-import { advanceTrivia, needsDeferredTriviaCommit, rollbackTrivia, saveTriviaMark, scanTrivia } from './trivia-skip.ts'
+import { advanceTrivia, commitTriviaScan, needsDeferredTriviaCommit, rollbackTrivia, saveTriviaMark, scanTriviaCompact } from './trivia-skip.ts'
 import { matchesEmpty, startsFirstSet } from './first-set.ts'
 import { deriveExpected } from './expect.ts'
 import { matchesAt, orSentinel, recoverScan, captureError } from '../recovery/scan.ts'
@@ -36,9 +36,7 @@ function repItem<T>(
   let pos = cur
   if (ctx.trivia) {
     if (needsDeferredTriviaCommit(ctx)) {
-      const scan = scanTrivia(input, cur, ctx)
-      scan.commit()
-      pos = scan.end
+      pos = commitTriviaScan(scanTriviaCompact(input, cur, ctx))
     } else {
       pos = advanceTrivia(input, cur, ctx)
     }
@@ -478,9 +476,7 @@ export function sepBy<T, S>(
         let sepPos = cur
         if (ctx.trivia) {
           if (needsDeferredTriviaCommit(ctx)) {
-            const scan = scanTrivia(input, cur, ctx)
-            scan.commit()
-            sepPos = scan.end
+            sepPos = commitTriviaScan(scanTriviaCompact(input, cur, ctx))
           } else {
             sepPos = advanceTrivia(input, cur, ctx)
           }
@@ -503,9 +499,7 @@ export function sepBy<T, S>(
         let nextPos = sep.span.end
         if (ctx.trivia) {
           if (needsDeferredTriviaCommit(ctx)) {
-            const scan = scanTrivia(input, sep.span.end, ctx)
-            scan.commit()
-            nextPos = scan.end
+            nextPos = commitTriviaScan(scanTriviaCompact(input, sep.span.end, ctx))
           } else {
             nextPos = advanceTrivia(input, sep.span.end, ctx)
           }

package/src/combinators/sequence.ts

@@ -1,6 +1,9 @@
 import type { Combinator, ParseContext, ParseResult, ParserMeta, ParseFail } from '../types.ts'
 import { sequenceFirstSet, firstSetOf, union } from './first-set.ts'
-import { advanceTrivia, needsDeferredTriviaCommit, rollbackScannedTriviaAt, scanTrivia } from './trivia-skip.ts'
+import {
+  advanceTrivia, commitTriviaScan, needsDeferredTriviaCommit, rollbackScannedTriviaAt,
+  scanTriviaCompact,
+} from './trivia-skip.ts'
 import { cstTlLen } from '../cst/capture-buffer.ts'
 import { firstSetSentinel } from '../recovery/scan.ts'
 import { adjacencyFail, adjacencyOf, holdsAdjacency, type AdjacencyDef } from './adjacency.ts'
@@ -86,9 +89,7 @@ export function sequence<T extends [Combinator<unknown>, ...Combinator<unknown>[
           const mRootLog = ctx._rootTriviaLog?.length ?? 0
           let scanEnd: number
           if (needsDeferredTriviaCommit(ctx)) {
-            const scan = scanTrivia(input, cur, ctx)
-            scan.commit()
-            scanEnd = scan.end
+            scanEnd = commitTriviaScan(scanTriviaCompact(input, cur, ctx))
           } else {
             scanEnd = advanceTrivia(input, cur, ctx)
           }
@@ -98,7 +99,9 @@ export function sequence<T extends [Combinator<unknown>, ...Combinator<unknown>[
           const result = parsers[i]!.parse(input, scanEnd, ctx)
           if (!result.ok) return result as ParseFail
           if (result.span.end > scanEnd) cur = result.span.end
-          else rollbackScannedTriviaAt(ctx, mTlog, scanTlog, mLog, scanLog, mRootLog, scanRootLog)
+          else rollbackScannedTriviaAt(
+            ctx, mTlog, scanTlog, mLog, scanLog, mRootLog, scanRootLog,
+          )
           if (values !== undefined) values.push(result.value)
           continue
         }
@@ -141,9 +144,7 @@ export function sequence<T extends [Combinator<unknown>, ...Combinator<unknown>[
         const mLog = ctx._triviaLog?.length ?? 0
         const mRootLog = ctx._rootTriviaLog?.length ?? 0
         if (needsDeferredTriviaCommit(ctx)) {
-          const scan = scanTrivia(input, cur, ctx)
-          scan.commit()
-          scanEnd = scan.end
+          scanEnd = commitTriviaScan(scanTriviaCompact(input, cur, ctx))
         } else {
           scanEnd = advanceTrivia(input, cur, ctx)
         }
@@ -153,7 +154,9 @@ export function sequence<T extends [Combinator<unknown>, ...Combinator<unknown>[
         const result = parsers[i]!.parse(input, scanEnd, ctx)
         if (!result.ok) return result as ParseFail
         if (result.span.end > scanEnd) cur = result.span.end
-        else rollbackScannedTriviaAt(ctx, mTlog, scanTlog, mLog, scanLog, mRootLog, scanRootLog)
+        else rollbackScannedTriviaAt(
+          ctx, mTlog, scanTlog, mLog, scanLog, mRootLog, scanRootLog,
+        )
         if (values !== undefined) values.push(result.value)
         continue
       }
@@ -199,22 +202,22 @@ export function sequence<T extends [Combinator<unknown>, ...Combinator<unknown>[
           const mRootLog = ctx._rootTriviaLog?.length ?? 0
 
           if (needsDeferredTriviaCommit(ctx)) {
-            const scan = scanTrivia(input, cur, ctx)
-            scan.commit()
-            scanEnd = scan.end
+            scanEnd = commitTriviaScan(scanTriviaCompact(input, cur, ctx))
           } else {
             scanEnd = advanceTrivia(input, cur, ctx)
           }
-
           const scanTlog = cstTlLen(ctx)
           const scanLog = ctx._triviaLog?.length ?? 0
           const scanRootLog = ctx._rootTriviaLog?.length ?? 0
+
           const result = parsers[i]!.parse(input, scanEnd, ctx)
           if (!result.ok) return result as ParseFail
           if (result.span.end > scanEnd) {
             cur = result.span.end
           } else {
-            rollbackScannedTriviaAt(ctx, mTlog, scanTlog, mLog, scanLog, mRootLog, scanRootLog)
+            rollbackScannedTriviaAt(
+              ctx, mTlog, scanTlog, mLog, scanLog, mRootLog, scanRootLog,
+            )
           }
           if (values !== undefined) values.push(result.value)
           continue

package/src/combinators/trivia-skip.ts

@@ -16,6 +16,7 @@ import {
 } from '../cst/capture-buffer.ts'
 import { recordLineRangeFromContext } from '../line-index.ts'
 import { createDetachedParseContext } from '../parse-context.ts'
+import { commonCssTriviaScanner } from '../cst/trivia-css-scanner.ts'
 
 /**
  * Result of scanning trivia: the position after it, plus a `commit()` that
@@ -25,11 +26,22 @@ import { createDetachedParseContext } from '../parse-context.ts'
  * trivia that doesn't actually sit between two accepted items.
  */
 export type TriviaScan = { end: number; commit: () => void }
+export type CompactTriviaScan = number | TriviaScan
 
 /** Saved lengths for rolling back speculative trivia commits. */
 export type TriviaRollbackMark = { raw: number; tlog: number; leaves: number; fields: number; errors: number; log: number; rootLog: number }
 
 const NOOP_COMMIT = () => {}
+
+export function triviaScanEnd(scan: CompactTriviaScan): number {
+  return typeof scan === 'number' ? scan : scan.end
+}
+
+export function commitTriviaScan(scan: CompactTriviaScan): number {
+  if (typeof scan === 'number') return scan
+  scan.commit()
+  return scan.end
+}
 export type FastTriviaScanner = (input: string, cur: number) => number
 /**
  * Bounded structural cache for the handful of trivia grammars a process actively
@@ -102,8 +114,14 @@ function removeNumberRange(values: number[] | undefined, start: number, end: num
 }
 
 /**
- * Remove only the trivia rows appended by an ambient scan, preserving effects
- * appended later by a successful zero-width child.
+ * Remove only the trivia rows appended by an ambient scan, preserving rows a
+ * successful zero-width child appended after that scan.
+ *
+ * A sequence cannot use `rollbackTriviaAt` for this case: its pre-scan mark also
+ * precedes the child, so truncating to it erases the child's nodes, fields,
+ * recovery errors, and trivia. The scan's contribution is instead the half-open
+ * range between the pre-scan and post-scan lengths. Compact that range in place;
+ * this is the cold zero-width-success path and allocates no replacement arrays.
  */
 export function rollbackScannedTriviaAt(
   ctx: ParseContext,
@@ -169,10 +187,10 @@ export function rollbackLookahead(ctx: ParseContext, mark: LookaheadMark): void
   if (ctx._probe) ctx._probe.best = mark.probeBest
 }
 
-function scanWithLabels(input: string, cur: number, ctx: ParseContext): TriviaScan {
+function scanWithLabels(input: string, cur: number, ctx: ParseContext): CompactTriviaScan {
   const triviaP = ctx.trivia!
   const spec = analyzeLabeledTrivia(triviaP)
-  if (!spec) return { end: cur, commit: NOOP_COMMIT }
+  if (!spec) return cur
 
   const log = ctx._triviaLog
   const rootLog = ctx._rootTriviaLog
@@ -193,7 +211,8 @@ function scanWithLabels(input: string, cur: number, ctx: ParseContext): TriviaSc
     }
   })
   if (visitedEnd !== undefined) {
-    if (visitedEnd === cur) return { end: cur, commit: NOOP_COMMIT }
+    if (visitedEnd === cur) return cur
+    if (fullRows === undefined && rootRows === undefined && cstRows === undefined) return visitedEnd
     return {
       end: visitedEnd,
       commit: () => {
@@ -213,7 +232,7 @@ function scanWithLabels(input: string, cur: number, ctx: ParseContext): TriviaSc
   }
 
   const { end, chunks } = scanLabeledTriviaChunks(input, cur, spec, ctx.state)
-  if (end === cur) return { end: cur, commit: NOOP_COMMIT }
+  if (end === cur) return cur
 
   return {
     end,
@@ -273,9 +292,11 @@ export function advanceTrivia(input: string, cur: number, ctx: ParseContext): nu
 
 /**
  * Scan trivia at `cur` using `ctx.trivia`, WITHOUT recording it. Returns the
- * position after the trivia (or `cur` if none) and a `commit()` to record it.
+ * position directly when a classified scan retained no rows; otherwise returns
+ * that position with a deferred `commit()` that records the retained rows.
+ * Use `triviaScanEnd()` / `commitTriviaScan()` rather than inspecting the shape.
  */
-export function scanTrivia(input: string, cur: number, ctx: ParseContext): TriviaScan {
+export function scanTriviaCompact(input: string, cur: number, ctx: ParseContext): CompactTriviaScan {
   const triviaP = ctx.trivia
   if (!triviaP) return { end: cur, commit: NOOP_COMMIT }
 
@@ -337,7 +358,7 @@ export function scanTrivia(input: string, cur: number, ctx: ParseContext): Trivi
 
   if (ctx.triviaKindLabels && (log !== undefined || rootLog !== undefined || captureTl)) {
     const scan = scanWithLabels(input, cur, ctx)
-    if (trackTriviaLines) recordLineRangeFromContext(ctx, input, cur, scan.end)
+    if (trackTriviaLines) recordLineRangeFromContext(ctx, input, cur, triviaScanEnd(scan))
     return scan
   }
 
@@ -377,6 +398,12 @@ export function scanTrivia(input: string, cur: number, ctx: ParseContext): Trivi
   return { end: tr.ok ? tr.span.end : cur, commit: NOOP_COMMIT }
 }
 
+/** Legacy object-shaped contract used by previously emitted assembly factories. */
+export function scanTrivia(input: string, cur: number, ctx: ParseContext): TriviaScan {
+  const scan = scanTriviaCompact(input, cur, ctx)
+  return typeof scan === 'number' ? { end: scan, commit: NOOP_COMMIT } : scan
+}
+
 /**
  * Skip trivia at `cur` through an INSTALLED scanner, recording it immediately.
  *
@@ -410,9 +437,7 @@ export function skipTriviaScanned(
  */
 export function consumeTrivia(input: string, cur: number, ctx: ParseContext): number {
   if (!needsDeferredTriviaCommit(ctx)) return advanceTrivia(input, cur, ctx)
-  const scan = scanTrivia(input, cur, ctx)
-  scan.commit()
-  return scan.end
+  return commitTriviaScan(scanTriviaCompact(input, cur, ctx))
 }
 
 /**
@@ -436,7 +461,7 @@ export function probeTriviaEnd(input: string, cur: number, ctx: ParseContext): n
   // actually need to find the end (`trivia`, `state`, the deferred-commit sinks)
   // is carried through unchanged.
   const probeCtx: ParseContext = { ...ctx, trackLines: false }
-  if (needsDeferredTriviaCommit(probeCtx)) return scanTrivia(input, cur, probeCtx).end
+  if (needsDeferredTriviaCommit(probeCtx)) return triviaScanEnd(scanTriviaCompact(input, cur, probeCtx))
   return advanceTrivia(input, cur, probeCtx)
 }
 
@@ -480,11 +505,19 @@ function buildFastTriviaScanner(trivia: Combinator<unknown>): FastTriviaScanner
   if (one) return loopScanner([one])
 
   if (repeat._def.tag !== 'choice') return null
+  const fused = commonCssTriviaScanner(repeat._def.parsers.map(triviaRegexSource))
+  if (fused !== null) return fused
   const arms = repeat._def.parsers.map(regexTriviaScanner)
   if (arms.some(s => s === null)) return null
   return loopScanner(arms as FastTriviaScanner[])
 }
 
+/** The regex source beneath a classified-trivia `label()`, when it is plain. */
+function triviaRegexSource(parser: Combinator<unknown>): string | null {
+  const d = parser._def.tag === 'label' ? parser._def.parser._def : parser._def
+  return d.tag === 'regex' && d.flags === '' ? d.source : null
+}
+
 function loopScanner(arms: FastTriviaScanner[]): FastTriviaScanner {
   return (input, cur) => {
     let pos = cur

package/src/compiler/direct-projection.ts

@@ -0,0 +1,43 @@
+/**
+ * Recognise the deliberately tiny transform shape whose result is already one
+ * of a sequence's values: `([, value]) => value`.
+ *
+ * This is a descriptor extraction, not callback source execution.  Anything
+ * with defaults, rest, nesting, a block body, or an expression is left as an
+ * ordinary callback.  `fnSrc` is preferred because serialized grammars carry
+ * the author's stable source; live combinators fall back to the function's own
+ * source only for the same exact syntax.
+ */
+export function directArrayProjection(
+  fn: (...args: never[]) => unknown,
+  fnSrc: string | undefined,
+  arity: number,
+): number | null {
+  let source = fnSrc
+  if (source === undefined) {
+    source = Function.prototype.toString.call(fn)
+    if (source.includes('[native code]')) return null
+  }
+
+  const match = /^\s*\(\s*\[([^\]]*)\]\s*\)\s*=>\s*([A-Za-z_$][\w$]*)\s*$/.exec(source)
+  if (match === null) return null
+
+  const slots = match[1]!.split(',').map(slot => slot.trim())
+  // Array destructuring may omit trailing elements (`([key]) => key`) while
+  // the sequence still has more children.  Only an explicit slot can be the
+  // result, and an over-long pattern cannot describe this sequence.
+  if (slots.length > arity) return null
+
+  const result = match[2]!
+  let selected = -1
+  for (let i = 0; i < slots.length; i++) {
+    const slot = slots[i]!
+    if (slot === '') continue
+    if (!/^[A-Za-z_$][\w$]*$/.test(slot)) return null
+    if (slot === result) {
+      if (selected >= 0) return null
+      selected = i
+    }
+  }
+  return selected >= 0 ? selected : null
+}