parseman 0.47.1 → 0.48.1

package/src/table/exec.ts

@@ -6,7 +6,10 @@ import { projectChild, unwrapChild } from '../combinators/node.ts'
 import { asciiFoldEq } from '../combinators/literal.ts'
 import { cstOutputHost } from '../compiler/build-arity.ts'
 import { consumeTrivia } from '../combinators/trivia-skip.ts'
-import { advanceTrivia, needsDeferredTriviaCommit, rollbackScannedTriviaAt, rollbackTrivia, rollbackTriviaAt, saveTriviaMark, scanTrivia, skipTriviaScanned, type FastTriviaScanner } from '../combinators/trivia-skip.ts'
+import {
+  advanceTrivia, commitTriviaScan, needsDeferredTriviaCommit, rollbackScannedTriviaAt, rollbackTrivia,
+  rollbackTriviaAt, saveTriviaMark, scanTriviaCompact, skipTriviaScanned, type FastTriviaScanner,
+} from '../combinators/trivia-skip.ts'
 import {
   beginCstNodeCapture, cstCaptureActive, cstLeavesLen, cstRawLen, cstTlLen,
   demoteCapturedToRaw, endCstNodeCapture, pushCstChild, pushCstLeaf,
@@ -19,6 +22,7 @@ import {
   OP_FIELD, OP_DISPATCH, OP_ROUTED, OP_LIT_CI, OP_LIT_CI_TRACK, OP_TOKEN, OP_WITHCTX, OP_GUARD, OP_ATTEMPT, OP_LABEL,
   OP_COV,
   OP_ADJ, OP_GREEDY, OP_REJECT, OP_ARMGATE,
+  OP_LEX_BODY, OP_LEX_PROGRAM,
 } from './ops.ts'
 import { adjacencyHolds, adjacencyMisuse } from '../combinators/adjacency.ts'
 import { failAt } from '../combinators/probe.ts'
@@ -41,9 +45,10 @@ import { refuseUnclassifiedRootScope } from '../cst/root-trivia-scope.ts'
 import { captureError, firstSetSentinel, matchesAt, orSentinel, recoverScan } from '../recovery/scan.ts'
 import {
   decodeClassSpec, expandCompact, resolveTable,
-  type CompactProgram, type ResolvedClass, type ResolvedDispatch, type ResolvedDispatchSpec,
+  type CompactProgram, type ResolvedClass, type ResolvedDispatch, type ResolvedDispatchSpec, type ResolvedTable,
   type SubtreeRef, type TableProgram, type TableRule,
 } from './program.ts'
+import { scalarTerminalNotChild } from './scalar-terminal.ts'
 
 /**
  * THE SHARED DRIVER.
@@ -199,6 +204,8 @@ function makeDriver(
   fx: readonly (readonly string[])[],
   disp: readonly ResolvedDispatch[],
   dsp: readonly ResolvedDispatchSpec[],
+  lex: readonly ((input: string, pos: number) => number)[],
+  lexPrograms: ResolvedTable['lexPrograms'],
   trivia: readonly unknown[],
   triviaScan: readonly (FastTriviaScanner | null)[],
   triviaLabelled: readonly boolean[],
@@ -353,9 +360,7 @@ function makeDriver(
     // grammar that already had a scanner — keeps the exact branch it had.
     if (s !== null) return skipTriviaScanned(s, input, cur, ctx)
     if (needsDeferredTriviaCommit(ctx)) {
-      const scan = scanTrivia(input, cur, ctx)
-      scan.commit()
-      return scan.end
+      return commitTriviaScan(scanTriviaCompact(input, cur, ctx))
     }
     return advanceTrivia(input, cur, ctx)
   }
@@ -622,6 +627,8 @@ function makeDriver(
             ctx._routed = savedRouted
           }
         } else {
+          // The reference engine mirrors dispatch.parse(): plain arms do not
+          // install routed state and stay outside the exception guard.
           v = exec(target, input, selEnd, ctx)
         }
         if (v === FAIL) {
@@ -692,6 +699,84 @@ function makeDriver(
         return value
       }
 
+      case OP_LEX_BODY: {
+        const recognized = lex[code[ip + 1]!]!(input, pos)
+        if (recognized < 0) {
+          ctx._fe = pos
+          ctx._fx = fx[code[ip + 2]!] as string[]
+          if (ctx._probe !== undefined) failAt(ctx, ctx._fx, pos)
+          return FAIL
+        }
+        const lineFlags = code[ip + 4]!
+        const hasSuffix = (lineFlags & 4) !== 0
+        const suffixMatched = hasSuffix && recognized % 2 === 1
+        const end = (recognized - (suffixMatched ? 1 : 0)) / 2
+        if ((lineFlags & 1) !== 0) trackLines(ctx, input, suffixMatched ? end - 1 : end)
+        if (hasSuffix) ctx._fc = false
+        if (hasSuffix && suffixMatched && (lineFlags & 2) !== 0) trackLines(ctx, input, end)
+        if (hasSuffix && !suffixMatched) {
+          const suffixExpected = fx[code[ip + 3]!] as string[]
+          ctx._fe = end
+          ctx._fx = suffixExpected
+          if (ctx._probe !== undefined) failAt(ctx, suffixExpected, end)
+        }
+        const value = input.slice(pos, end)
+        if (cstCaptureActive(ctx)) pushCstLeaf(ctx, { _tag: 'leaf', value, span: { start: pos, end } })
+        EC.e = end
+        return value
+      }
+
+      case OP_LEX_PROGRAM: {
+        const run = lexPrograms[code[ip + 1]!]!
+        const scanId = run.scan
+        if (scanId === undefined) {
+          const end = run(input, pos, ctx)
+          if (end < 0) return FAIL
+          const value = input.slice(pos, end)
+          if (cstCaptureActive(ctx)) pushCstLeaf(ctx, { _tag: 'leaf', value, span: { start: pos, end } })
+          EC.e = end
+          return value
+        }
+        // The recognizer is the canonical balanced() pool entry. Reproduce the
+        // enclosing token boundary here so its internal token leaf and skippers
+        // cannot publish into the outer collector; this row publishes one leaf.
+        const sTrivia = ctx.trivia, sKinds = ctx.triviaKindLabels
+        const sBuf = ctx._cstBuf, sChildren = ctx._cstChildren, sLeaves = ctx._cstLeaves
+        const sRaw = ctx._cstRawChildren, sTl = ctx._cstTriviaLog
+        const sOuterTl = ctx._triviaLog, sRootTl = ctx._rootTriviaLog
+        const wasCapturing = cstCaptureActive(ctx)
+        const sScan = SCAN
+        SCAN = null
+        ctx.trivia = undefined
+        ctx.triviaKindLabels = undefined
+        ctx._cstBuf = undefined
+        ctx._cstChildren = undefined
+        ctx._cstLeaves = undefined
+        ctx._cstRawChildren = undefined
+        ctx._cstTriviaLog = undefined
+        ctx._triviaLog = undefined
+        ctx._rootTriviaLog = undefined
+        let end: number
+        try { end = run(input, pos, ctx, scans[scanId]) }
+        finally {
+          SCAN = sScan
+          ctx.trivia = sTrivia
+          ctx.triviaKindLabels = sKinds
+          ctx._cstBuf = sBuf
+          ctx._cstChildren = sChildren
+          ctx._cstLeaves = sLeaves
+          ctx._cstRawChildren = sRaw
+          ctx._cstTriviaLog = sTl
+          ctx._triviaLog = sOuterTl
+          ctx._rootTriviaLog = sRootTl
+        }
+        if (end < 0) return FAIL
+        const value = input.slice(pos, end)
+        if (wasCapturing) pushCstLeaf(ctx, { _tag: 'leaf', value, span: { start: pos, end } })
+        EC.e = end
+        return value
+      }
+
       case OP_SCAN: {
         const c = scans[code[ip + 1]!]!
         const r = c.parse(input, pos, ctx)
@@ -854,7 +939,9 @@ function makeDriver(
         if (REC) ctx._sync = inheritedSync
         EC.e = cur
         if (fused) {
-          const fn = fns[code[ip + 1]!] as (value: unknown, span: { start: number; end: number }) => unknown
+          const reducer = code[ip + 1]!
+          if (reducer < 0) return values![~reducer]
+          const fn = fns[reducer] as (value: unknown, span: { start: number; end: number }) => unknown
           if (COUNT) siteFn('SEQX fn()', fn)
           return fn(values, { start: pos, end: cur })
         }
@@ -865,9 +952,8 @@ function makeDriver(
         const d = code[ip + 1]!
         const base = ip + 4
         const choiceFx = fx[code[ip + 3]!] as string[]
-        // Report the union, at the choice's own position, on every exit that
-        // fails — matching both engines and, more importantly, never handing a
-        // user an empty expected set.
+        // A pure dispatch miss has no attempted arm to rank. Report the static
+        // opener union at the choice's own position so the result is meaningful.
         const failChoice = (): typeof FAIL => {
           ctx._fe = pos
           ctx._fx = choiceFx
@@ -916,12 +1002,9 @@ function makeDriver(
           // union IS that flatMap's answer, without running seven parsers to
           // rediscover it.
           //
-          // NOT furthest-failure merging, which no engine on the `expected` path
-          // performs. `failAt` (combinators/probe.ts:13-24) and its compiled mirror
-          // `probeUpdate` (codegen.ts:692) are the only positional merges in the
-          // library, they are gated on `_ctx._probe` and they surface as
-          // `RunResult.furthestFail` under `{recover:true}` — a different field.
-          // Every `_fe`/`_fx` site in all three drivers is last-write-wins.
+          // There is only one selected arm on this path, so ordered-arm depth
+          // ranking does not apply. `furthestFail` under `{recover:true}` remains
+          // a separate probe result.
           //
           // A residual JSON divergence survives this and is NOT a failure-reporting
           // bug: on `{"a":]` this driver reports `["}"]` (the Obj arm's own
@@ -947,18 +1030,15 @@ function makeDriver(
         const mEr = need ? ctx._errors?.length ?? 0 : 0
         const mLog = need ? ctx._triviaLog?.length ?? 0 : 0
         const mRoot = need ? ctx._rootTriviaLog?.length ?? 0 : 0
-        // THE ORDERED PATH REPORTS THE ARMS THAT ACTUALLY RAN, accumulated in
-        // order — `choice.ts:167`'s `expected.push(...result.expected)` per failed
-        // arm, reported at the choice's own position. The static `choiceFx` union
-        // is a set of OPENERS, so a left-factored or shared-prefix choice whose
-        // arms all failed PAST their common opener reported that opener instead of
-        // the tokens the arms were actually waiting for: `['/::?/']` where the
-        // interpreter says `['"before"', '"hover"']`.
+        // The ordered path keeps only arms at the deepest failure offset. Exact
+        // ties merge in source order. The final failure is still anchored at the
+        // choice's own position; the deeper offset ranks diagnostics only.
         //
         // `choiceFx` is still the answer when nothing ran or every arm reported
         // nothing — the char gate can skip arms the interpreter would have entered,
         // and the union is what those arms' start-failures would have contributed.
         let acc: string[] | undefined
+        let best = pos
         const push = (ax: readonly string[] | undefined): void => {
           if (ax === undefined || ax.length === 0) return
           if (acc === undefined) acc = ax.slice()
@@ -973,7 +1053,7 @@ function makeDriver(
             // enters the arm, the arm's opener fails, and its static set is what
             // lands in the accumulation. Declining to enter must not also decline
             // to say what the arm wanted.
-            push(fx[code[armFx + i]!] as string[])
+            if (best === pos) push(fx[code[armFx + i]!] as string[])
             continue
           }
           ctx._fc = false
@@ -982,7 +1062,9 @@ function makeDriver(
           const v = exec(code[base + i]!, input, pos, ctx)
           if (v !== FAIL) return v
           if (COUNT) { tableCounters.ungatedFails++; tableCounters.ungatedFailRows += tableCounters.rows - rows0 }
-          push(ctx._fx)
+          const at = ctx._fe ?? pos
+          if (at > best) { best = at; acc = undefined }
+          if (at === best) push(ctx._fx)
           // A committed arm cuts, and the interpreter reports the accumulation at
           // the arm's OWN span (`choice.ts:170`), not at the choice's position.
           if (committed(ctx)) { if (acc !== undefined) ctx._fx = acc; return FAIL }
@@ -1116,6 +1198,11 @@ function makeDriver(
         const mySync = REC ? ctx._sync : undefined
         const recFx = REC ? fx[code[ip + 6]!] as string[] : undefined
         const sepSent = REC ? sentinelFor(code[ip + 7]!) : undefined
+        const itemClassIndex = sep < 0 ? code[ip + 7]! : -1
+        const itemClass = REC && itemClassIndex >= 0 ? cc[itemClassIndex]! : undefined
+        // `completionsAt` observes failures swallowed by an optional iteration;
+        // tolerant recovery does too. Both keep the ordinary child path.
+        const gateItems = itemClass !== undefined && ctx._tolerant !== true && ctx._probe === undefined
         let cur = pos
         let count = 0
         for (;;) {
@@ -1128,6 +1215,8 @@ function makeDriver(
           // must attempt the separator so its failure sets the expected set, which
           // is the only thing an under-`min` list has to report.
           if (sep >= 0 && count > 0 && count >= min && cur >= input.length) break
+          if (count >= min && sep < 0 && !hasTrivia
+            && gateItems && !classHas(itemClass!, lead(input, cur))) break
           // One mark pair for the whole loop when a rollback is even possible,
           // refreshed per iteration rather than reallocated.
           // SCALAR MARKS. This loop took TWO allocations per item (a CST mark and
@@ -1199,6 +1288,11 @@ function makeDriver(
             if (trailingAllowed && sepEnd >= 0) cur = sepEnd
             break
           }
+          if (count >= min && gateItems && !classHas(itemClass!, lead(input, itemStart))) {
+            if (needMark) rollbackTriviaAt(ctx, mRaw, mTl, mLv, mFl, mEr, mLog, mRoot)
+            if (trailingAllowed && sepEnd >= 0) cur = sepEnd
+            break
+          }
           ctx._fc = false
           const v = exec(child, input, itemStart, ctx)
           if (v === FAIL) {
@@ -1454,6 +1548,15 @@ function makeDriver(
       }
 
       case OP_NOT: {
+        // The shipping generic closure/emitted NOT bodies predate pure-global
+        // lookahead rollback. Keep the reference driver aligned with them for
+        // every generic child; only the direct-terminal shape this lane replaces
+        // needs the reference oracle to hide the terminal's diagnostic writes.
+        const scalarChild = scalarTerminalNotChild(code, ip) >= 0
+        const savedFc = scalarChild ? ctx._fc : false
+        const savedFe = scalarChild ? ctx._fe : -1
+        const savedFx = scalarChild ? ctx._fx : undefined
+        const savedProbe = scalarChild ? ctx._probe?.best : undefined
         const need = rollbackNeeded(ctx)
         const mRaw = need ? cstRawLen(ctx) : 0
         const mTl = need ? cstTlLen(ctx) : 0
@@ -1464,6 +1567,12 @@ function makeDriver(
         const mRoot = need ? ctx._rootTriviaLog?.length ?? 0 : 0
         const v = exec(code[ip + 1]!, input, pos, ctx)
         if (need) rollbackTriviaAt(ctx, mRaw, mTl, mLv, mFl, mEr, mLog, mRoot)
+        if (scalarChild) {
+          ctx._fc = savedFc
+          ctx._fe = savedFe
+          ctx._fx = savedFx
+          if (ctx._probe !== undefined) ctx._probe.best = savedProbe ?? null
+        }
         if (v === FAIL) { EC.e = pos; return null }
         // `not.ts:50` — the ASSERTION's own set, at the assertion's position.
         ctx._fe = pos
@@ -1651,7 +1760,7 @@ export function execRules(
   const prog = expandCompact(source)
   const t = resolveTable(prog)
   const scan = opts.leafSwap === false ? t.triviaScan.map(() => null) : t.triviaScan
-  const d = makeDriver(t.code, t.k, t.fns, t.cc, t.fx, t.disp, t.dsp, t.trivia, scan, t.triviaLabelled, prog, newEndCell())
+  const d = makeDriver(t.code, t.k, t.fns, t.cc, t.fx, t.disp, t.dsp, t.lex, t.lexPrograms, t.trivia, scan, t.triviaLabelled, prog, newEndCell())
   const names = Object.keys(prog.rules)
   const entries = names.map(n => prog.rules[n]!)
   let last: unknown

package/src/table/lex-body.ts

@@ -0,0 +1,30 @@
+import type { LexBodySpec } from './program.ts'
+
+/** -1 fails. A success is `end * 2 + suffixMatched`, so the reader can
+ * reproduce optional()'s swallowed suffix failure without allocating a tuple. */
+export type LexBodyRecognizer = (input: string, pos: number) => number
+
+/** Build one selected lexical body from its numeric table row. Recognition is
+ * pure; diagnostics, token boundary state, source value and CST publication
+ * remain owned by the reader's OP_LEX_BODY shell. */
+export function buildLexBodyRecognizer(
+  spec: LexBodySpec,
+  constants: readonly unknown[],
+): LexBodyRecognizer {
+  const raw = constants[spec[0]]
+  if (!(raw instanceof RegExp)) {
+    throw new TypeError('table lexical body does not reference a RegExp constant')
+  }
+  if (!Number.isInteger(spec[1]) || spec[1] < -1 || spec[1] > 0xFFFF) {
+    throw new TypeError('table lexical body suffix is not absent or one UTF-16 code unit')
+  }
+  const re = raw.sticky ? new RegExp(raw.source, raw.flags) : new RegExp(raw.source, `${raw.flags.replace(/[gy]/g, '')}y`)
+  const suffix = spec[1]
+  return (input, pos) => {
+    re.lastIndex = pos
+    if (!re.test(input)) return -1
+    const end = re.lastIndex
+    if (suffix < 0) return end * 2
+    return input.charCodeAt(end) === suffix ? (end + 1) * 2 + 1 : end * 2
+  }
+}