parseman 0.47.0 → 0.48.0

package/src/table/exec.ts

@@ -6,7 +6,10 @@ import { projectChild, unwrapChild } from '../combinators/node.ts'
 import { asciiFoldEq } from '../combinators/literal.ts'
 import { cstOutputHost } from '../compiler/build-arity.ts'
 import { consumeTrivia } from '../combinators/trivia-skip.ts'
-import { advanceTrivia, needsDeferredTriviaCommit, rollbackTrivia, rollbackTriviaAt, saveTriviaMark, scanTrivia, skipTriviaScanned, type FastTriviaScanner } from '../combinators/trivia-skip.ts'
+import {
+  advanceTrivia, commitTriviaScan, needsDeferredTriviaCommit, rollbackScannedTriviaAt, rollbackTrivia,
+  rollbackTriviaAt, saveTriviaMark, scanTriviaCompact, skipTriviaScanned, type FastTriviaScanner,
+} from '../combinators/trivia-skip.ts'
 import {
   beginCstNodeCapture, cstCaptureActive, cstLeavesLen, cstRawLen, cstTlLen,
   demoteCapturedToRaw, endCstNodeCapture, pushCstChild, pushCstLeaf,
@@ -14,11 +17,12 @@ import {
 import {
   OP_CHOICE, OP_EMPTY, OP_GATE, OP_LEAF, OP_LIT, OP_NODE, OP_NOT, OP_OPT,
   OP_PEEK, OP_REP, OP_REPV, OP_RULE, OP_RX, OP_SEQ, OP_SEQV, OP_XFORM,
-  OP_LIT_TRACK, OP_RX_TRACK, OP_NODE_TRACK, OP_SCOPE, OP_SCOPE_CAP, OP_EXPECT, OP_SEQX, OP_SCAN,
+  OP_LIT_TRACK, OP_RX_TRACK, OP_NODE_TRACK, OP_SCOPE, OP_SCOPE_CAP, OP_SCOPE_PLAIN, OP_EXPECT, OP_SEQX, OP_SCAN,
   OP_LIVE,
   OP_FIELD, OP_DISPATCH, OP_ROUTED, OP_LIT_CI, OP_LIT_CI_TRACK, OP_TOKEN, OP_WITHCTX, OP_GUARD, OP_ATTEMPT, OP_LABEL,
   OP_COV,
   OP_ADJ, OP_GREEDY, OP_REJECT, OP_ARMGATE,
+  OP_LEX_BODY, OP_LEX_PROGRAM,
 } from './ops.ts'
 import { adjacencyHolds, adjacencyMisuse } from '../combinators/adjacency.ts'
 import { failAt } from '../combinators/probe.ts'
@@ -41,9 +45,10 @@ import { refuseUnclassifiedRootScope } from '../cst/root-trivia-scope.ts'
 import { captureError, firstSetSentinel, matchesAt, orSentinel, recoverScan } from '../recovery/scan.ts'
 import {
   decodeClassSpec, expandCompact, resolveTable,
-  type CompactProgram, type ResolvedClass, type ResolvedDispatch, type ResolvedDispatchSpec,
+  type CompactProgram, type ResolvedClass, type ResolvedDispatch, type ResolvedDispatchSpec, type ResolvedTable,
   type SubtreeRef, type TableProgram, type TableRule,
 } from './program.ts'
+import { scalarTerminalNotChild } from './scalar-terminal.ts'
 
 /**
  * THE SHARED DRIVER.
@@ -199,6 +204,8 @@ function makeDriver(
   fx: readonly (readonly string[])[],
   disp: readonly ResolvedDispatch[],
   dsp: readonly ResolvedDispatchSpec[],
+  lex: readonly ((input: string, pos: number) => number)[],
+  lexPrograms: ResolvedTable['lexPrograms'],
   trivia: readonly unknown[],
   triviaScan: readonly (FastTriviaScanner | null)[],
   triviaLabelled: readonly boolean[],
@@ -353,9 +360,7 @@ function makeDriver(
     // grammar that already had a scanner — keeps the exact branch it had.
     if (s !== null) return skipTriviaScanned(s, input, cur, ctx)
     if (needsDeferredTriviaCommit(ctx)) {
-      const scan = scanTrivia(input, cur, ctx)
-      scan.commit()
-      return scan.end
+      return commitTriviaScan(scanTriviaCompact(input, cur, ctx))
     }
     return advanceTrivia(input, cur, ctx)
   }
@@ -609,15 +614,23 @@ function makeDriver(
           usesRouted = spec.routed[arm] === 1
         }
 
-        const savedRouted = ctx._routed
         let mark = saveTriviaMark(ctx)
+        let v: unknown
         if (usesRouted) {
+          const savedRouted = ctx._routed
           rollbackTrivia(ctx, selectorMark)
           mark = saveTriviaMark(ctx)
           ctx._routed = { value: key, span: { start: pos, end: selEnd } }
+          try {
+            v = exec(target, input, pos, ctx)
+          } finally {
+            ctx._routed = savedRouted
+          }
+        } else {
+          // The reference engine mirrors dispatch.parse(): plain arms do not
+          // install routed state and stay outside the exception guard.
+          v = exec(target, input, selEnd, ctx)
         }
-        const v = exec(target, input, usesRouted ? pos : selEnd, ctx)
-        if (usesRouted) ctx._routed = savedRouted
         if (v === FAIL) {
           rollbackTrivia(ctx, mark)
           // The interpreter marks a failed dispatch branch COMMITTED: the
@@ -686,6 +699,84 @@ function makeDriver(
         return value
       }
 
+      case OP_LEX_BODY: {
+        const recognized = lex[code[ip + 1]!]!(input, pos)
+        if (recognized < 0) {
+          ctx._fe = pos
+          ctx._fx = fx[code[ip + 2]!] as string[]
+          if (ctx._probe !== undefined) failAt(ctx, ctx._fx, pos)
+          return FAIL
+        }
+        const lineFlags = code[ip + 4]!
+        const hasSuffix = (lineFlags & 4) !== 0
+        const suffixMatched = hasSuffix && recognized % 2 === 1
+        const end = (recognized - (suffixMatched ? 1 : 0)) / 2
+        if ((lineFlags & 1) !== 0) trackLines(ctx, input, suffixMatched ? end - 1 : end)
+        if (hasSuffix) ctx._fc = false
+        if (hasSuffix && suffixMatched && (lineFlags & 2) !== 0) trackLines(ctx, input, end)
+        if (hasSuffix && !suffixMatched) {
+          const suffixExpected = fx[code[ip + 3]!] as string[]
+          ctx._fe = end
+          ctx._fx = suffixExpected
+          if (ctx._probe !== undefined) failAt(ctx, suffixExpected, end)
+        }
+        const value = input.slice(pos, end)
+        if (cstCaptureActive(ctx)) pushCstLeaf(ctx, { _tag: 'leaf', value, span: { start: pos, end } })
+        EC.e = end
+        return value
+      }
+
+      case OP_LEX_PROGRAM: {
+        const run = lexPrograms[code[ip + 1]!]!
+        const scanId = run.scan
+        if (scanId === undefined) {
+          const end = run(input, pos, ctx)
+          if (end < 0) return FAIL
+          const value = input.slice(pos, end)
+          if (cstCaptureActive(ctx)) pushCstLeaf(ctx, { _tag: 'leaf', value, span: { start: pos, end } })
+          EC.e = end
+          return value
+        }
+        // The recognizer is the canonical balanced() pool entry. Reproduce the
+        // enclosing token boundary here so its internal token leaf and skippers
+        // cannot publish into the outer collector; this row publishes one leaf.
+        const sTrivia = ctx.trivia, sKinds = ctx.triviaKindLabels
+        const sBuf = ctx._cstBuf, sChildren = ctx._cstChildren, sLeaves = ctx._cstLeaves
+        const sRaw = ctx._cstRawChildren, sTl = ctx._cstTriviaLog
+        const sOuterTl = ctx._triviaLog, sRootTl = ctx._rootTriviaLog
+        const wasCapturing = cstCaptureActive(ctx)
+        const sScan = SCAN
+        SCAN = null
+        ctx.trivia = undefined
+        ctx.triviaKindLabels = undefined
+        ctx._cstBuf = undefined
+        ctx._cstChildren = undefined
+        ctx._cstLeaves = undefined
+        ctx._cstRawChildren = undefined
+        ctx._cstTriviaLog = undefined
+        ctx._triviaLog = undefined
+        ctx._rootTriviaLog = undefined
+        let end: number
+        try { end = run(input, pos, ctx, scans[scanId]) }
+        finally {
+          SCAN = sScan
+          ctx.trivia = sTrivia
+          ctx.triviaKindLabels = sKinds
+          ctx._cstBuf = sBuf
+          ctx._cstChildren = sChildren
+          ctx._cstLeaves = sLeaves
+          ctx._cstRawChildren = sRaw
+          ctx._cstTriviaLog = sTl
+          ctx._triviaLog = sOuterTl
+          ctx._rootTriviaLog = sRootTl
+        }
+        if (end < 0) return FAIL
+        const value = input.slice(pos, end)
+        if (wasCapturing) pushCstLeaf(ctx, { _tag: 'leaf', value, span: { start: pos, end } })
+        EC.e = end
+        return value
+      }
+
       case OP_SCAN: {
         const c = scans[code[ip + 1]!]!
         const r = c.parse(input, pos, ctx)
@@ -706,7 +797,8 @@ function makeDriver(
       }
 
       case OP_SCOPE:
-      case OP_SCOPE_CAP: {
+      case OP_SCOPE_CAP:
+      case OP_SCOPE_PLAIN: {
         const ki = code[ip + 1]!
         const saved = ctx.trivia
         const savedLabels = ctx.triviaKindLabels
@@ -735,7 +827,7 @@ function makeDriver(
         // (`grammar.ts:141`), and the flag is restored here because the table
         // shares one ctx where `parser()` copies it. Bit 2 = the unclassified-scope
         // refusal `grammar.ts:98` raises.
-        const policy = code[ip + 3]!
+        const policy = code[ip] === OP_SCOPE_PLAIN ? 0 : code[ip + 3]!
         if ((policy & 2) !== 0) refuseUnclassifiedRootScope(ctx._rootTriviaStrictScopes)
         const savedRootCap = ctx._rootTriviaCapture
         if ((policy & 1) !== 0) ctx._rootTriviaCapture = false
@@ -786,18 +878,19 @@ function makeDriver(
             // SCALAR MARKS — `saveTriviaMark` allocated TWICE per term (its own
             // seven-field object plus the five-field CST mark it delegates to).
             const need = rollbackNeeded(ctx)
-            const mRaw = need ? cstRawLen(ctx) : 0
             const mTl = need ? cstTlLen(ctx) : 0
-            const mLv = need ? cstLeavesLen(ctx) : 0
-            const mFl = need ? ctx._fields?.length ?? 0 : 0
-            const mEr = need ? ctx._errors?.length ?? 0 : 0
             const mLog = need ? ctx._triviaLog?.length ?? 0 : 0
             const mRoot = need ? ctx._rootTriviaLog?.length ?? 0 : 0
             const scanEnd = skipTrivia(input, cur, ctx)
+            const scanTl = need ? cstTlLen(ctx) : 0
+            const scanLog = need ? ctx._triviaLog?.length ?? 0 : 0
+            const scanRoot = need ? ctx._rootTriviaLog?.length ?? 0 : 0
             const v = exec(child, input, scanEnd, ctx)
             if (v === FAIL) { if (REC) ctx._sync = inheritedSync; return FAIL }
             if (EC.e > scanEnd) cur = EC.e
-            else if (need) rollbackTriviaAt(ctx, mRaw, mTl, mLv, mFl, mEr, mLog, mRoot)
+            else if (need) rollbackScannedTriviaAt(
+              ctx, mTl, scanTl, mLog, scanLog, mRoot, scanRoot,
+            )
             if (values !== undefined) values.push(v)
             continue
           }
@@ -846,7 +939,9 @@ function makeDriver(
         if (REC) ctx._sync = inheritedSync
         EC.e = cur
         if (fused) {
-          const fn = fns[code[ip + 1]!] as (value: unknown, span: { start: number; end: number }) => unknown
+          const reducer = code[ip + 1]!
+          if (reducer < 0) return values![~reducer]
+          const fn = fns[reducer] as (value: unknown, span: { start: number; end: number }) => unknown
           if (COUNT) siteFn('SEQX fn()', fn)
           return fn(values, { start: pos, end: cur })
         }
@@ -1108,6 +1203,11 @@ function makeDriver(
         const mySync = REC ? ctx._sync : undefined
         const recFx = REC ? fx[code[ip + 6]!] as string[] : undefined
         const sepSent = REC ? sentinelFor(code[ip + 7]!) : undefined
+        const itemClassIndex = sep < 0 ? code[ip + 7]! : -1
+        const itemClass = REC && itemClassIndex >= 0 ? cc[itemClassIndex]! : undefined
+        // `completionsAt` observes failures swallowed by an optional iteration;
+        // tolerant recovery does too. Both keep the ordinary child path.
+        const gateItems = itemClass !== undefined && ctx._tolerant !== true && ctx._probe === undefined
         let cur = pos
         let count = 0
         for (;;) {
@@ -1120,6 +1220,8 @@ function makeDriver(
           // must attempt the separator so its failure sets the expected set, which
           // is the only thing an under-`min` list has to report.
           if (sep >= 0 && count > 0 && count >= min && cur >= input.length) break
+          if (count >= min && sep < 0 && !hasTrivia
+            && gateItems && !classHas(itemClass!, lead(input, cur))) break
           // One mark pair for the whole loop when a rollback is even possible,
           // refreshed per iteration rather than reallocated.
           // SCALAR MARKS. This loop took TWO allocations per item (a CST mark and
@@ -1191,6 +1293,11 @@ function makeDriver(
             if (trailingAllowed && sepEnd >= 0) cur = sepEnd
             break
           }
+          if (count >= min && gateItems && !classHas(itemClass!, lead(input, itemStart))) {
+            if (needMark) rollbackTriviaAt(ctx, mRaw, mTl, mLv, mFl, mEr, mLog, mRoot)
+            if (trailingAllowed && sepEnd >= 0) cur = sepEnd
+            break
+          }
           ctx._fc = false
           const v = exec(child, input, itemStart, ctx)
           if (v === FAIL) {
@@ -1446,6 +1553,15 @@ function makeDriver(
       }
 
       case OP_NOT: {
+        // The shipping generic closure/emitted NOT bodies predate pure-global
+        // lookahead rollback. Keep the reference driver aligned with them for
+        // every generic child; only the direct-terminal shape this lane replaces
+        // needs the reference oracle to hide the terminal's diagnostic writes.
+        const scalarChild = scalarTerminalNotChild(code, ip) >= 0
+        const savedFc = scalarChild ? ctx._fc : false
+        const savedFe = scalarChild ? ctx._fe : -1
+        const savedFx = scalarChild ? ctx._fx : undefined
+        const savedProbe = scalarChild ? ctx._probe?.best : undefined
         const need = rollbackNeeded(ctx)
         const mRaw = need ? cstRawLen(ctx) : 0
         const mTl = need ? cstTlLen(ctx) : 0
@@ -1456,6 +1572,12 @@ function makeDriver(
         const mRoot = need ? ctx._rootTriviaLog?.length ?? 0 : 0
         const v = exec(code[ip + 1]!, input, pos, ctx)
         if (need) rollbackTriviaAt(ctx, mRaw, mTl, mLv, mFl, mEr, mLog, mRoot)
+        if (scalarChild) {
+          ctx._fc = savedFc
+          ctx._fe = savedFe
+          ctx._fx = savedFx
+          if (ctx._probe !== undefined) ctx._probe.best = savedProbe ?? null
+        }
         if (v === FAIL) { EC.e = pos; return null }
         // `not.ts:50` — the ASSERTION's own set, at the assertion's position.
         ctx._fe = pos
@@ -1643,7 +1765,7 @@ export function execRules(
   const prog = expandCompact(source)
   const t = resolveTable(prog)
   const scan = opts.leafSwap === false ? t.triviaScan.map(() => null) : t.triviaScan
-  const d = makeDriver(t.code, t.k, t.fns, t.cc, t.fx, t.disp, t.dsp, t.trivia, scan, t.triviaLabelled, prog, newEndCell())
+  const d = makeDriver(t.code, t.k, t.fns, t.cc, t.fx, t.disp, t.dsp, t.lex, t.lexPrograms, t.trivia, scan, t.triviaLabelled, prog, newEndCell())
   const names = Object.keys(prog.rules)
   const entries = names.map(n => prog.rules[n]!)
   let last: unknown

package/src/table/lex-body.ts

@@ -0,0 +1,30 @@
+import type { LexBodySpec } from './program.ts'
+
+/** -1 fails. A success is `end * 2 + suffixMatched`, so the reader can
+ * reproduce optional()'s swallowed suffix failure without allocating a tuple. */
+export type LexBodyRecognizer = (input: string, pos: number) => number
+
+/** Build one selected lexical body from its numeric table row. Recognition is
+ * pure; diagnostics, token boundary state, source value and CST publication
+ * remain owned by the reader's OP_LEX_BODY shell. */
+export function buildLexBodyRecognizer(
+  spec: LexBodySpec,
+  constants: readonly unknown[],
+): LexBodyRecognizer {
+  const raw = constants[spec[0]]
+  if (!(raw instanceof RegExp)) {
+    throw new TypeError('table lexical body does not reference a RegExp constant')
+  }
+  if (!Number.isInteger(spec[1]) || spec[1] < -1 || spec[1] > 0xFFFF) {
+    throw new TypeError('table lexical body suffix is not absent or one UTF-16 code unit')
+  }
+  const re = raw.sticky ? new RegExp(raw.source, raw.flags) : new RegExp(raw.source, `${raw.flags.replace(/[gy]/g, '')}y`)
+  const suffix = spec[1]
+  return (input, pos) => {
+    re.lastIndex = pos
+    if (!re.test(input)) return -1
+    const end = re.lastIndex
+    if (suffix < 0) return end * 2
+    return input.charCodeAt(end) === suffix ? (end + 1) * 2 + 1 : end * 2
+  }
+}