parseman 0.56.0 → 0.58.0

package/src/combinators/trivia-skip.ts

@@ -570,11 +570,54 @@ function regexTriviaScanner(parser: Combinator<unknown>): FastTriviaScanner | nu
   // and every trivia gap in every dialect went through the labelled char scanner.
   const d = parser._def.tag === 'label' ? parser._def.parser._def : parser._def
   if (d.tag !== 'regex' || d.flags) return null
-  const source = d.source
+  return sourceScanner(d.source)
+}
+
+function sourceScanner(source: string): FastTriviaScanner | null {
   return classRunSource(source)
     ?? altStarSource(source)
     ?? prefixRunSource(source)
     ?? delimitedSource(source)
+    ?? sharedPrefixAltSource(source)
+}
+
+/**
+ * `<literal>(?:<alt>|<alt>…)`: a literal every alternative shares, as in jess's
+ * one comment arm `\/(?:\*(?:[^*]|\*(?!\/))*\*\/|\/[^\n\r]*)` for both `/*…*\/`
+ * and `//…`. Each `<literal><alt>` is lowered on its own, and the scanner takes
+ * the first that matches, as the regex does. Without it that arm had no
+ * scanner, so neither did the trivia it sits in.
+ */
+function sharedPrefixAltSource(source: string): FastTriviaScanner | null {
+  const m = /^((?:\\.|[^\\[\]()*+?|^$.{}])+)\(\?:(.*)\)$/.exec(source)
+  if (!m) return null
+  const alts: string[] = []
+  const body = m[2]!
+  let depth = 0
+  let start = 0
+  for (let i = 0; i < body.length; i++) {
+    const c = body[i]
+    if (c === '\\') { i++; continue }
+    if (c === '[') { while (i < body.length && body[i] !== ']') i += body[i] === '\\' ? 2 : 1; continue }
+    if (c === '(') depth++
+    else if (c === ')' && --depth < 0) return null
+    else if (c === '|' && depth === 0) { alts.push(body.slice(start, i)); start = i + 1 }
+  }
+  if (depth !== 0 || alts.length === 0) return null
+  alts.push(body.slice(start))
+  const scanners: FastTriviaScanner[] = []
+  for (const alt of alts) {
+    const scanner = sourceScanner(m[1]! + alt)
+    if (scanner === null) return null
+    scanners.push(scanner)
+  }
+  return (input, cur) => {
+    for (const scanner of scanners) {
+      const end = scanner(input, cur)
+      if (end > cur) return end
+    }
+    return cur
+  }
 }
 
 /**

package/src/compiler/build-arity.ts

@@ -338,7 +338,7 @@ export function confirmedArityForDef(def: NodeDef): number | null {
     where: `node("${def.type ?? '<inferred>'}")`,
     subject,
     fellBackTo: `could not confirm its declared arity — ${unconfirmableReason(def, src)} — `
-      + 'so this node captures children, fields and raw children, logs trivia, and clones `_ctx.state` on every match',
+      + 'so this node captures children, fields and raw children, logs trivia, and hands it `_ctx.state` on every match',
     otherwise: 'only the tiers the reducer actually declares would be captured '
       + '(arity >= 1 children, >= 2 fields, >= 4 raw, >= 5 trivia, >= 6 state). '
       + 'Declare it with `node(..., { buildArity: n })` if it cannot be derived',
@@ -371,9 +371,9 @@ export function buildReadsTrivia(def: NodeDef): boolean {
   return arity >= 5
 }
 
-/** Build reads the 6th (state) arg? Unknown/unparseable → true (keep state clone). */
+/** Build reads the 6th (state) arg? Unknown/unparseable → true (hand it the state). */
 export function buildReadsState(def: NodeDef): boolean {
-  if (!def.build) return true // structural node: host may read state — keep clone
+  if (!def.build) return true // structural node: host may read state
   const arity = confirmedArityForDef(def)
   if (arity === null) return true
   return arity >= 6

package/src/compiler/token-alphabet.ts

@@ -37,6 +37,7 @@ import type { BalancedSpec } from '../combinators/scanTo.ts'
 import { deriveExpected } from '../combinators/expect.ts'
 import { assertionFailureExpected, directTerminalFailureExpected } from '../combinators/expected.ts'
 import { keywordsRegExp } from '../combinators/keywords.ts'
+import { hasOwnTriviaBoundary } from '../combinators/trivia-boundary.ts'
 
 /** A terminal in the derived alphabet, with its globally assigned id. */
 export type TokenTerminal =
@@ -135,12 +136,17 @@ export type ExecutableLexNode =
   | {
     readonly kind: 'choice'
     readonly arms: readonly ExecutableLexNode[]
-    readonly firstSets: readonly FirstSet[]
+    /** The authored arms: the encoder classifies them as the character choice would. */
+    readonly parsers: readonly Combinator<unknown>[]
+    /** Inside a no-trivia scope of the token, where the encoder's trivia is none. */
+    readonly cleared: boolean
     readonly armExpected: readonly (readonly string[])[]
     readonly expected: readonly string[]
   }
   | { readonly kind: 'not'; readonly body: ExecutableLexNode; readonly expected: readonly string[] }
   | { readonly kind: 'optional'; readonly body: ExecutableLexNode }
+  /** A separator-less `many`/`oneOrMore`; `max` −1 is unbounded. */
+  | { readonly kind: 'repeat'; readonly body: ExecutableLexNode; readonly min: number; readonly max: number }
 
 export type ExecutableFixedLexBody = {
   readonly kind: 'fixed-tree'
@@ -204,9 +210,22 @@ export function selectOptionalSuffixLexBody(ir: LexicalIr): LexBodyCandidate {
 /** Unwrap only scopes that token() has already installed identically. The
  * returned parser is still the authored terminal/sequence; no effectful wrapper
  * or final named winner is guessed from construction metadata. */
+/**
+ * A rule reference made with no trivia in force reinstalls the rule's own
+ * ambient trivia when the rule has a position that reads it (`ref.ts`, the
+ * encoder's `scopedRef`), and inside a token none is in force. Such a rule's
+ * terms are then trivia-separated, which a lexical body does not model, so the
+ * reference is not lowered.
+ */
+function reinstallsRuleTrivia(reference: Combinator<unknown>, target: Combinator<unknown>): boolean {
+  return (reference._meta.grammarTrivia ?? target._meta.grammarTrivia) !== undefined
+    && hasOwnTriviaBoundary(target)
+}
+
 function directTokenBodyParser(
   parser: Combinator<unknown>,
   resolve?: (name: string) => Combinator<unknown> | undefined,
+  scope?: { cleared: boolean },
 ): Combinator<unknown> | undefined {
   const outer = parser._def
   if (outer.tag !== 'token') return undefined
@@ -220,12 +239,13 @@ function directTokenBodyParser(
         || wrapper.captureTrivia === true || wrapper.rootCapture === 'opaque'
         || wrapper.constructionTrackLines !== 'inherit'
         || wrapper.constructionCaptureTriviaKinds !== undefined) return undefined
+      if (scope !== undefined) scope.cleared = true
       body = wrapper.parser
       continue
     }
     if (wrapper.tag === 'lazy') {
       const target = resolvedLazyTarget(body, resolve)
-      if (target === undefined) return undefined
+      if (target === undefined || reinstallsRuleTrivia(body, target)) return undefined
       body = target
       continue
     }
@@ -436,6 +456,7 @@ function directFixedLexNode(
   resolve: ((name: string) => Combinator<unknown> | undefined) | undefined,
   active: Set<Combinator<unknown>>,
   pureOnly = false,
+  cleared = false,
 ): ExecutableLexNode | undefined {
   let current = parser
   const wrappers = new Set<Combinator<unknown>>()
@@ -444,7 +465,7 @@ function directFixedLexNode(
     const def = current._def
     if (def.tag === 'lazy') {
       const target = resolvedLazyTarget(current, resolve)
-      if (target === undefined) return undefined
+      if (target === undefined || reinstallsRuleTrivia(current, target)) return undefined
       current = target
       continue
     }
@@ -459,43 +480,57 @@ function directFixedLexNode(
     if (terminal !== undefined) return { kind: 'terminal', terminal }
     const def = current._def
     if (def.tag === 'grammar') {
+      // A scope with no trivia of its own writes nothing under a token that is
+      // not already in force: a cleared one runs as its child in the table
+      // (`scopeIsIdentity`) and an inheriting one encodes no row. Its line
+      // policy is the table's own (`hasScopedTrackLines`). So it is its child
+      // at any position.
+      if (def.triviaParser === undefined && def.captureTrivia !== true
+        && def.rootCapture !== 'opaque' && def.constructionCaptureTriviaKinds === undefined) {
+        return directFixedLexNode(def.parser, resolve, active, pureOnly, cleared || def.clearTrivia === true)
+      }
       if (!pureOnly || (def.parser._def.tag !== 'token'
         && (def.clearTrivia !== true || def.triviaParser !== undefined
           || def.captureTrivia === true || def.rootCapture === 'opaque'))) return undefined
-      return directFixedLexNode(def.parser, resolve, active, true)
+      return directFixedLexNode(def.parser, resolve, active, true, cleared)
     }
     if (def.tag === 'sequence') {
       if (def.parsers.length < 2 || def.parsers.length > 5) return undefined
-      const parts = def.parsers.map(part => directFixedLexNode(part, resolve, active, pureOnly))
+      const parts = def.parsers.map(part => directFixedLexNode(part, resolve, active, pureOnly, cleared))
       if (parts.some(part => part === undefined)) return undefined
       return { kind: 'sequence', parts: parts as ExecutableLexNode[] }
     }
     if (def.tag === 'choice') {
-      if ((def.parsers.length !== 2 && def.parsers.length !== 4 && def.parsers.length !== 8)
+      if (def.parsers.length < 2
         || def.strategy.tag !== 'firstMatch'
         || def.gates.some(gate => gate !== null)
         || def.autoNot.some(check => check !== null)) return undefined
-      const arms = def.parsers.map(arm => directFixedLexNode(arm, resolve, active, pureOnly))
+      const arms = def.parsers.map(arm => directFixedLexNode(arm, resolve, active, pureOnly, cleared))
       if (arms.some(arm => arm === undefined)) return undefined
-      const final = classifyFinalChoice(def.parsers, resolve)
       return {
         kind: 'choice', arms: arms as ExecutableLexNode[],
-        firstSets: final.firstSets,
+        parsers: def.parsers, cleared,
         armExpected: def.parsers.map(deriveExpected),
         expected: def.parsers.flatMap(deriveExpected),
       }
     }
     if (def.tag === 'not') {
-      const body = directFixedLexNode(def.parser, resolve, active, true)
+      const body = directFixedLexNode(def.parser, resolve, active, true, cleared)
       return body === undefined ? undefined : {
         kind: 'not', body,
         expected: assertionFailureExpected(false, def.parser._tag),
       }
     }
     if (def.tag === 'optional') {
-      const body = directFixedLexNode(def.parser, resolve, active, pureOnly)
+      const body = directFixedLexNode(def.parser, resolve, active, pureOnly, cleared)
       return body === undefined ? undefined : { kind: 'optional', body }
     }
+    if (def.tag === 'many' || def.tag === 'oneOrMore') {
+      const body = directFixedLexNode(def.parser, resolve, active, pureOnly, cleared)
+      return body === undefined ? undefined : {
+        kind: 'repeat', body, min: def.tag === 'many' ? 0 : def.min, max: def.max ?? -1,
+      }
+    }
     return undefined
   } finally {
     active.delete(current)
@@ -506,9 +541,10 @@ export function directFixedTokenBody(
   parser: Combinator<unknown>,
   resolve?: (name: string) => Combinator<unknown> | undefined,
 ): ExecutableFixedLexBody | undefined {
-  const body = directTokenBodyParser(parser, resolve)
+  const scope = { cleared: false }
+  const body = directTokenBodyParser(parser, resolve, scope)
   if (body === undefined) return undefined
-  const root = directFixedLexNode(body, resolve, new Set())
+  const root = directFixedLexNode(body, resolve, new Set(), false, scope.cleared)
   return root === undefined ? undefined : { kind: 'fixed-tree', root }
 }
 
@@ -1556,8 +1592,11 @@ function normalizeLexical(
   if (stack.has(parser)) return { refusal: 'recursive token body' }
   stack.add(parser)
   try {
-    return withLexicalControl(session, parser._def.tag, () =>
-      normalizeLexicalBody(parser, resolve, stack, session))
+    // A nested `balanced()` is its scanner machine, not the transform that
+    // builds its value — `sequence('@', balanced('{', '}'))` as a skipper.
+    return normalizeBalancedLexical(parser, resolve, session)
+      ?? withLexicalControl(session, parser._def.tag, () =>
+        normalizeLexicalBody(parser, resolve, stack, session))
   } finally {
     stack.delete(parser)
   }
@@ -1841,8 +1880,9 @@ function fixedBindingProjection(
   // and the emitted adapter captures that site's named body after linking every
   // scan root. Only ambient trivia/scanSkip remain live runtime callbacks; they
   // are scanner POLICY, never binding authority for these authored own children.
-  // Recovery/withCtx still have no equivalent named template.
-  if (edge.parentTag === 'recover' || edge.parentTag === 'withCtx') readerMask = 0b011
+  // Recovery still has no equivalent named template. `withCtx` has one: the
+  // emitter's OP_WITHCTX body calls its child by the name `link()` minted.
+  if (edge.parentTag === 'recover') readerMask = 0b011
   // Sequence strict, adjacency, and recovery bodies now all use scalar block
   // chains, so no supported variant is withheld here.
   const words = [

package/src/cst/capture-buffer.ts

@@ -111,6 +111,31 @@ export function pushCstChild(ctx: ParseContext, built: unknown, rawEntry: unknow
   if (ctx._cstRawChildren) ctx._cstRawChildren.push(rawEntry)
 }
 
+/**
+ * Undo the LAST `pushCstChild` into the active collector, whichever form it took:
+ * a child the lazy buffer's empty sentinel dropped (`undefined` first) leaves no
+ * semantic entry to remove, only its raw one.
+ */
+export function popCstChild(ctx: ParseContext): void {
+  const b = ctx._cstBuf
+  if (b) {
+    const ch = b.ch
+    if (b.rawOnly !== true && ch) {
+      ch.pop()
+      if (ch.length === 1) { b.single = ch[0]; b.ch = undefined }
+    }
+    if (b.noRaw === true) b.rawLen = (b.rawLen ?? 1) - 1
+    else if (b.raw) {
+      b.raw.pop()
+      if (b.raw.length === 1) { b.rawSingle = b.raw[0]; b.raw = undefined }
+    } else b.rawSingle = undefined
+    return
+  }
+  if (ctx._cstChildren) ctx._cstChildren.pop()
+  else if (ctx._cstLeaves) ctx._cstLeaves.pop()
+  if (ctx._cstRawChildren) ctx._cstRawChildren.pop()
+}
+
 export function cstRawLen(ctx: ParseContext): number {
   const b = ctx._cstBuf
   if (b) {

package/src/cst/trivia-kinds.ts

@@ -1,4 +1,4 @@
-import type { Combinator, ParseContext } from '../types.ts'
+import type { CharRange, Combinator, ParseContext } from '../types.ts'
 import { pushCstTriviaEntry, pushTriviaLogEntry } from './capture-buffer.ts'
 import { startsFirstSet } from '../combinators/first-set.ts'
 import { createDetachedParseContext } from '../parse-context.ts'
@@ -20,6 +20,14 @@ export type LabeledTriviaSpec = {
    */
   readonly visitor: CssTriviaVisitor | null
   readonly charArms: readonly TriviaArmMatcher[] | null
+  /**
+   * `table/trivia-gap.ts`'s root-selection fast path, for the selection it was
+   * last primed with: the selected arms' first characters, and the unlabelled
+   * scanner that answers a run free of them (`null`: no fast path).
+   */
+  rootKinds: Readonly<Record<string, number>> | undefined
+  rootLeaders: readonly CharRange[]
+  rootScan: ((input: string, cur: number) => number) | null
 }
 
 /** See `trivia-skip.ts`: bounded scalar memoisation avoids a WeakMap in the
@@ -114,6 +122,9 @@ export function labeledTriviaSpec(arms: LabeledTriviaSpec['arms'], minRepeats: n
     minRepeats,
     visitor: commonLabeledTriviaVisitor({ arms, minRepeats }),
     charArms: charArmsFor({ arms, minRepeats }),
+    rootKinds: undefined,
+    rootLeaders: [],
+    rootScan: null,
   }
 }
 

package/src/parse-context.ts

@@ -68,6 +68,7 @@ export function createParseContext(): ParseContext {
     _rootTriviaKindIndex: undefined,
     _rootTriviaStrictScopes: undefined,
     _rootTriviaCapture: undefined,
+    _gap: undefined,
     _cstTriviaLog: undefined,
     _fields: undefined,
     _cstBuf: undefined,

package/src/plugin/entry-supercompiler.ts

@@ -11,7 +11,7 @@ import { parseSync } from 'oxc-parser'
 import { OP_ATTEMPT, OP_NODE, OP_NODE_TRACK, OP_REPV, OP_SCAN } from '../table/ops.ts'
 import { emitAssemblySource, type EmitResult } from '../table/emit-assembly.ts'
 import { defaultAssemblyCfgs } from '../table/emit.ts'
-import { resolveTable, type TableProgram } from '../table/program.ts'
+import { assemblyExtraIps, resolveTable, type TableProgram } from '../table/program.ts'
 
 type Ast = { type: string; start?: number; end?: number; [key: string]: unknown }
 
@@ -229,20 +229,19 @@ export function supercompileEntryAssembly(
  * then finds the precompiled assembly through the table object's AST shape and
  * replaces only that factory body. Any mismatch returns the canonical source.
  */
-export function supercompileRuleMapReplacement(prog: TableProgram, replacement: string): string {
+export function supercompileRuleMapReplacement(
+  prog: TableProgram,
+  replacement: string,
+  select?: ReadonlySet<number>,
+): string {
   const cfg = defaultAssemblyCfgs(prog)[0]
   if (cfg === undefined) return replacement
-  const extraIps: number[] = []
-  for (const scan of prog.scans ?? []) {
-    for (const ref of scan.skip) extraIps.push(ref[0])
-    if (scan.sentinel !== undefined) extraIps.push(scan.sentinel[0])
-  }
-  for (const set of prog.scanSkip ?? []) for (const ref of set) extraIps.push(ref[0])
+  const extraIps = assemblyExtraIps(prog)
 
   let canonical: EmitResult
   let candidate: EntrySupercompileResult
   try {
-    canonical = emitAssemblySource(resolveTable(prog), prog, cfg, extraIps, true)
+    canonical = emitAssemblySource(resolveTable(prog), prog, cfg, extraIps, true, select)
     candidate = supercompileEmittedAssembly(prog, cfg, extraIps, canonical)
   } catch {
     return replacement