parseman 0.46.0 → 0.47.0

package/src/combinators/token.ts

@@ -0,0 +1,120 @@
+import type { Combinator, ParseContext, ParseResult, ParserMeta } from '../types.ts'
+import { pushCstLeaf, cstCaptureActive } from '../cst/capture-buffer.ts'
+
+/**
+ * Treat a parser as one source token.
+ *
+ * The wrapped parser runs with trivia cleared, so its parts must be contiguous.
+ * On success the token returns the matched source text and captures one CST leaf
+ * for the full span, suppressing any internal terminal leaves.
+ */
+export function token(root: Combinator<unknown>): Combinator<string> {
+  const meta: ParserMeta = {
+    firstSet: root._meta.firstSet,
+    canMatchNewline: root._meta.canMatchNewline,
+    isTrivia: false,
+  }
+  return {
+    _tag: 'token',
+    _meta: meta,
+    _def: { tag: 'token', parser: root },
+    parse(input: string, pos: number, ctx: ParseContext): ParseResult<string> {
+      const savedTrivia = ctx.trivia
+      const savedKinds = ctx.triviaKindLabels
+      const savedBuf = ctx._cstBuf
+      const savedChildren = ctx._cstChildren
+      const savedLeaves = ctx._cstLeaves
+      const savedRaw = ctx._cstRawChildren
+      const savedTriviaLog = ctx._cstTriviaLog
+      const savedOuterTriviaLog = ctx._triviaLog
+      const savedRootTriviaLog = ctx._rootTriviaLog
+      const wasCapturing = cstCaptureActive(ctx)
+
+      ctx.trivia = undefined
+      ctx.triviaKindLabels = undefined
+      ctx._cstBuf = undefined
+      ctx._cstChildren = undefined
+      ctx._cstLeaves = undefined
+      ctx._cstRawChildren = undefined
+      ctx._cstTriviaLog = undefined
+      ctx._triviaLog = undefined
+      ctx._rootTriviaLog = undefined
+
+      let result: ParseResult<unknown>
+      try {
+        result = root.parse(input, pos, ctx)
+      } finally {
+        ctx.trivia = savedTrivia
+        ctx.triviaKindLabels = savedKinds
+        ctx._cstBuf = savedBuf
+        ctx._cstChildren = savedChildren
+        ctx._cstLeaves = savedLeaves
+        ctx._cstRawChildren = savedRaw
+        ctx._cstTriviaLog = savedTriviaLog
+        ctx._triviaLog = savedOuterTriviaLog
+        ctx._rootTriviaLog = savedRootTriviaLog
+      }
+
+      if (!result.ok) return result
+
+      const span = result.span
+      const value = input.slice(pos, span.end)
+      if (wasCapturing) pushCstLeaf(ctx, { _tag: 'leaf', value, span })
+      return { ok: true, value, span }
+    },
+  }
+}
+
+/**
+ * Reduce a structural parser to one semantic leaf.
+ *
+ * Unlike `token()`, `leaf()` does not change trivia policy: put `noTrivia()` or
+ * a scoped `parser({ trivia })` inside the supplied grammar when that region has
+ * a local spacing rule. Internal CST captures are suppressed, while selected
+ * root-source trivia remains visible, and one leaf with the reducer's value and
+ * the complete consumed span is exposed to the parent.
+ */
+export function leaf<T, U>(
+  root: Combinator<T>,
+  fn: (value: T, span: { start: number; end: number }) => U,
+): Combinator<U> {
+  const meta: ParserMeta = {
+    firstSet: root._meta.firstSet,
+    canMatchNewline: root._meta.canMatchNewline,
+    isTrivia: false,
+  }
+  return {
+    _tag: 'leaf',
+    _meta: meta,
+    _def: { tag: 'leaf', parser: root as Combinator<unknown>, fn: fn as (v: unknown, span: { start: number; end: number }) => unknown },
+    parse(input: string, pos: number, ctx: ParseContext): ParseResult<U> {
+      const savedBuf = ctx._cstBuf
+      const savedChildren = ctx._cstChildren
+      const savedLeaves = ctx._cstLeaves
+      const savedRaw = ctx._cstRawChildren
+      const savedTriviaLog = ctx._cstTriviaLog
+      const savedOuterTriviaLog = ctx._triviaLog
+      const wasCapturing = cstCaptureActive(ctx)
+      ctx._cstBuf = undefined
+      ctx._cstChildren = undefined
+      ctx._cstLeaves = undefined
+      ctx._cstRawChildren = undefined
+      ctx._cstTriviaLog = undefined
+      ctx._triviaLog = undefined
+      let result: ParseResult<T>
+      try { result = root.parse(input, pos, ctx) }
+      finally {
+        ctx._cstBuf = savedBuf
+        ctx._cstChildren = savedChildren
+        ctx._cstLeaves = savedLeaves
+        ctx._cstRawChildren = savedRaw
+        ctx._cstTriviaLog = savedTriviaLog
+        ctx._triviaLog = savedOuterTriviaLog
+      }
+      if (!result.ok) return result
+      const value = fn(result.value, result.span)
+      if (wasCapturing) pushCstLeaf(ctx, { _tag: 'leaf', value, span: result.span })
+      return { ok: true, value, span: result.span }
+    },
+  }
+}

package/src/combinators/trivia-boundary.ts

@@ -0,0 +1,48 @@
+import type { Combinator, ParserDef } from '../types.ts'
+
+/** Wrapper nesting this walk will look through before giving up and saying yes. */
+const MAX_DEPTH = 12
+
+/**
+ * Does this rule body have a position OF ITS OWN at which ambient trivia is
+ * consulted?
+ *
+ * This is the gate on re-establishing a rule's ambient trivia when it is
+ * referenced from a region that CLEARED it (`ref.ts` in the interpreter,
+ * `Encoder.scopedRef` in the table — both must ask the same question, or the two
+ * engines parse different languages again).
+ *
+ * A body with no such position — a bare alternation, a dispatch, a single
+ * terminal — cannot be repaired by installing a scanner, because it never
+ * consults one. Installing it there changes nothing about the rule and
+ * everything about what its arms delegate to, which is how a `noTrivia(...)`
+ * region stopped meaning anything two references down.
+ *
+ * `lazy` is deliberately false and is never followed: a reference is the
+ * delegation this question is about, so following it would both loop and answer
+ * for a different rule. `grammar` is false because it declares its own scope, so
+ * an outer install never reaches its body. `dispatch` is false because it is a
+ * ROUTE: its arms re-enter from the routed token and spell their own padding.
+ *
+ * Structural and pure — callers resolve it ONCE, not per parse.
+ */
+export function hasOwnTriviaBoundary(p: Combinator<unknown>, depth: number = MAX_DEPTH): boolean {
+  // Out of budget: assume yes. The gate exists to REMOVE a scope install that
+  // cannot help, and a wrapper stack this deep is not a case it was measured on.
+  if (depth === 0) return true
+  const d = p._def as ParserDef
+  switch (d.tag) {
+    // Multi-term forms: the gaps BETWEEN terms are the ambient-trivia positions.
+    case 'sequence': return d.parsers.length > 1
+    case 'many': case 'oneOrMore': case 'sepBy': return true
+    // Transparent wrappers — they add no position, so ask what they wrap.
+    case 'node': case 'transform': case 'leaf': case 'label': case 'field':
+    case 'token': case 'attempt': case 'optional': case 'expect': case 'withCtx':
+    case 'recover': case 'trivia':
+      return hasOwnTriviaBoundary(d.parser, depth - 1)
+    // An alternation adds no position either, but ONE arm that has one is enough
+    // to make the scope meaningful for this rule.
+    case 'choice': return d.parsers.some(a => hasOwnTriviaBoundary(a, depth - 1))
+    default: return false
+  }
+}