parseman 0.58.2 → 0.59.0

package/src/table/assemble.ts

@@ -69,7 +69,7 @@ import type { Combinator, FieldMap, FirstSet, ParseContext, ParseResult, ParserD
 import { balanced, scanTo } from '../combinators/scanTo.ts'
 import { buildFieldMap } from '../compiler/fields.ts'
 import { FIELD_LEADING_TRIVIA, FIELD_TRAILING_TRIVIA, LEAD, triviaFieldCapture } from '../combinators/field-trivia.ts'
-import { skipDeferredTrivia } from './trivia-gap.ts'
+import { skipDeferredTrivia, triviaLeadTable } from './trivia-gap.ts'
 import { asciiFoldKey } from '../combinators/dispatch.ts'
 import { projectChild, unwrapChild } from '../combinators/node.ts'
 import { cstOutputHost } from '../compiler/build-arity.ts'
@@ -83,6 +83,7 @@ import {
   demoteCapturedToRaw, pushCstChild, pushCstLeaf,
   type CstCaptureBuf,
 } from '../cst/capture-buffer.ts'
+import { PlainBuf, PlainField, PlainLeaf, PlainNode, PlainSpan } from '../cst/plain.js'
 import {
   OP_CHOICE, OP_EMPTY, OP_GATE, OP_LEAF, OP_SOURCE_LEAF, OP_LIT, OP_NODE, OP_NOT, OP_OPT,
   OP_PEEK, OP_REP, OP_REPV, OP_RULE, OP_RX, OP_SEQ, OP_SEQV, OP_XFORM,
@@ -197,7 +198,6 @@ function linkMatcher(m: readonly [number, string, string, number]): (key: string
   }
 }
 
-type Leaf = { _tag: 'leaf'; value: string; span: { start: number; end: number } }
 
 /**
  * The `COV` slot's value when a parse installs no collector. ONE shared function
@@ -542,7 +542,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
   const CLEAN = REC ? (): boolean => false : failCleanAnalysis(code)
 
   function pushLeaf(ctx: ParseContext, value: string, s: number, e: number): void {
-    const lf: Leaf = { _tag: 'leaf', value, span: { start: s, end: e } }
+    const lf = new PlainLeaf(value, new PlainSpan(s, e))
     pushCstLeaf(ctx, lf)
   }
 
@@ -592,7 +592,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
       ctx.captureTrivia = sCap
       if (value === FAIL) return FAIL
       const end = EC.e
-      const span = { start: pos, end }
+      const span = new PlainSpan(pos, end)
       const node = build(kids, undefined, span, rawKids, EMPTY_TL, undefined)
       if (sBuf !== undefined || sCh !== undefined) {
         pushCstChild(ctx, node, rawEntry(node, input, pos, end))
@@ -646,7 +646,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
       ctx.captureTrivia = sCap
       if (value === FAIL) return FAIL
       const end = EC.e
-      const node = build(captured, undefined, { start: pos, end }, EMPTY_CH, EMPTY_TL, undefined)
+      const node = build(captured, undefined, new PlainSpan(pos, end), EMPTY_CH, EMPTY_TL, undefined)
       if (sBuf !== undefined || sCh !== undefined) {
         pushCstChild(ctx, node, sBuf !== undefined || sRaw !== undefined ? rawEntry(node, input, pos, end) : undefined)
       }
@@ -684,7 +684,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
       ctx.captureTrivia = sCap
       if (value === FAIL) return FAIL
       const end = EC.e
-      const node = build(captured, fieldMap, { start: pos, end }, EMPTY_CH, EMPTY_TL, undefined)
+      const node = build(captured, fieldMap, new PlainSpan(pos, end), EMPTY_CH, EMPTY_TL, undefined)
       if (sBuf !== undefined || sCh !== undefined) {
         pushCstChild(ctx, node, sBuf !== undefined || sRaw !== undefined ? rawEntry(node, input, pos, end) : undefined)
       }
@@ -722,7 +722,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
       ctx.captureTrivia = sCap
       if (value === FAIL) return FAIL
       const end = EC.e
-      const span = { start: pos, end }
+      const span = new PlainSpan(pos, end)
       const node = kids.length === 1
         ? kids[0]
         : build(kids, undefined, span, rawKids, EMPTY_TL, undefined)
@@ -764,7 +764,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
       const end = EC.e
       const node = captured.length === 1
         ? captured[0]
-        : build(captured, undefined, { start: pos, end }, EMPTY_CH, EMPTY_TL, undefined)
+        : build(captured, undefined, new PlainSpan(pos, end), EMPTY_CH, EMPTY_TL, undefined)
       if (sBuf !== undefined || sCh !== undefined) {
         pushCstChild(ctx, node, sBuf !== undefined || sRaw !== undefined ? rawEntry(node, input, pos, end) : undefined)
       }
@@ -959,7 +959,31 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
     return out
   }
 
+  /**
+   * NO TRIVIA CAN START AT `cur` — `triviaLeadTable` for the installed trivia. A
+   * gap there is empty: nothing to scan, nothing recorded, so nothing for a
+   * sequence term to mark. One table per trivia slot, built with the assembly;
+   * the installed trivia is matched to its slot only when it changes (a trivia
+   * the caller passed that is no slot gets its own table then).
+   */
+  const LEAD_TABLES = trivia.map(tp => tp === undefined ? null : triviaLeadTable(tp))
+  let LEAD_OF: Combinator<unknown> | undefined
+  let LEAD_TABLE: Uint8Array | null = null
+  function cannotStartTrivia(input: string, cur: number, triviaP: Combinator<unknown>): boolean {
+    if (triviaP !== LEAD_OF) {
+      LEAD_OF = triviaP
+      const slot = trivia.indexOf(triviaP)
+      LEAD_TABLE = slot >= 0 ? LEAD_TABLES[slot]! : triviaLeadTable(triviaP)
+    }
+    const table = LEAD_TABLE
+    if (table === null) return false
+    const c = input.charCodeAt(cur)
+    return c < 128 && table[c] === 0
+  }
+
   function skipTrivia(input: string, cur: number, ctx: ParseContext): number {
+    const t = ctx.trivia
+    if (t !== undefined && cannotStartTrivia(input, cur, t)) return cur
     const s = SCAN
     if (s !== null
       && ctx._triviaLog === undefined
@@ -1041,7 +1065,8 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
    * into them.
    */
   function nextTerm(child: Piece, input: string, cur: number, ctx: ParseContext): number {
-    if (ctx.trivia === undefined) {
+    const t = ctx.trivia
+    if (t === undefined || cannotStartTrivia(input, cur, t)) {
       const v = child(input, cur, ctx)
       if (v === FAIL) return -1
       TERMV = v
@@ -1089,7 +1114,8 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
 
   /** A non-first term at a site proven to run under an open node capture buffer. */
   function nextTermBuffered(child: Piece, input: string, cur: number, ctx: ParseContext): number {
-    if (ctx.trivia === undefined) {
+    const t = ctx.trivia
+    if (t === undefined || cannotStartTrivia(input, cur, t)) {
       const v = child(input, cur, ctx)
       if (v === FAIL) return -1
       TERMV = v
@@ -1354,7 +1380,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
     return (input, pos, selEnd, key, ctx, need, mRaw, mTl, mLv, mFl, mEr, mLog, mRoot) => {
       const savedRouted = ctx._routed
       if (need) rollbackTriviaAt(ctx, mRaw, mTl, mLv, mFl, mEr, mLog, mRoot)
-      ctx._routed = { value: key, span: { start: pos, end: selEnd } }
+      ctx._routed = { value: key, span: new PlainSpan(pos, selEnd) }
       let v: unknown
       try {
         v = child(input, pos, ctx)
@@ -1886,7 +1912,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
         // table engines. Do not use the incoming ctx bit: a grammar scope can
         // declare tracking even when the outer run context begins untracked.
         const expectSpan = (ctx: ParseContext, pos: number) =>
-          prog.lines === 1 ? spanLines(ctx, pos, pos) : { start: pos, end: pos }
+          prog.lines === 1 ? spanLines(ctx, pos, pos) : new PlainSpan(pos, pos)
         // TOLERANT `expect()` EMBEDS ITS ERROR IN THE TREE, not only in the flat
         // `_errors` side-channel (`combinators/expect.ts:150`, and codegen's
         // `_ctx._rec.capture` at codegen.ts:4470) — so a tree walk finds every
@@ -1933,7 +1959,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
           return (input, pos, ctx) => {
             const v = child(input, pos, ctx)
             if (v === FAIL) return FAIL
-            ctx._fields?.push({ name, value: v, span: { start: pos, end: EC.e } })
+            ctx._fields?.push(new PlainField(name, v, new PlainSpan(pos, EC.e)))
             return v
           }
         }
@@ -1958,7 +1984,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
         return (input, pos, ctx) => {
           const v = child(input, pos, ctx)
           if (v === FAIL) return FAIL
-          return fn(v, { start: pos, end: EC.e })
+          return fn(v, new PlainSpan(pos, EC.e))
         }
       }
 
@@ -2016,6 +2042,9 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
         // known here, so the scope piece holds the scanner it installs as a `const`.
         const scanFor: FastTriviaScanner | null =
           swapLegal && ki >= 0 && !triviaLabelled[ki]! ? triviaScan[ki]! : null
+        // The scope installs its slot's lead table with its trivia, so the gate
+        // (`cannotStartTrivia`) never has to look the slot up.
+        const scopeLead = ki < 0 ? null : LEAD_TABLES[ki]!
         const child = link(code[ip + 2]!)
         // TWO PIECES, SELECTED HERE — not one piece that tests the opcode. Capture
         // is fixed for the whole parse, so `code[ip] === OP_SCOPE_CAP` is an
@@ -2032,6 +2061,9 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
               const savedScan = SCAN
               const savedCap = ctx.captureTrivia
               SCAN = scanFor
+              const savedLeadOf = LEAD_OF, savedLead = LEAD_TABLE
+              LEAD_OF = scopeTrivia
+              LEAD_TABLE = scopeLead
               ctx.trivia = scopeTrivia
               ctx.triviaKindLabels = scopeLabels
               ctx.captureTrivia = true
@@ -2040,6 +2072,8 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
               ctx.trivia = saved
               ctx.triviaKindLabels = savedLabels
               SCAN = savedScan
+              LEAD_OF = savedLeadOf
+              LEAD_TABLE = savedLead
               return v
             }
           : (input, pos, ctx) => {
@@ -2047,12 +2081,17 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
               const savedLabels = ctx.triviaKindLabels
               const savedScan = SCAN
               SCAN = scanFor
+              const savedLeadOf = LEAD_OF, savedLead = LEAD_TABLE
+              LEAD_OF = scopeTrivia
+              LEAD_TABLE = scopeLead
               ctx.trivia = scopeTrivia
               ctx.triviaKindLabels = scopeLabels
               const v = child(input, pos, ctx)
               ctx.trivia = saved
               ctx.triviaKindLabels = savedLabels
               SCAN = savedScan
+              LEAD_OF = savedLeadOf
+              LEAD_TABLE = savedLead
               return v
             }
         return scopeRootPolicy(scopePiece, code[ip] === OP_SCOPE_PLAIN ? 0 : code[ip + 3]!)
@@ -2068,7 +2107,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
             ctx._fe = pos; ctx._fx = ROUTED_FX
             return FAIL
           }
-          if (cstCaptureActive(ctx)) pushCstLeaf(ctx, { _tag: 'leaf', value: item.value, span: item.span })
+          if (cstCaptureActive(ctx)) pushCstLeaf(ctx, new PlainLeaf(item.value, item.span))
           EC.e = item.span.end
           return item.value
         }
@@ -2256,7 +2295,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
           if (v === FAIL) return FAIL
           const end = EC.e
           const value = input.slice(pos, end)
-          if (wasCapturing) pushCstLeaf(ctx, { _tag: 'leaf', value, span: { start: pos, end } })
+          if (wasCapturing) pushCstLeaf(ctx, new PlainLeaf(value, new PlainSpan(pos, end)))
           EC.e = end
           return value
         }
@@ -2291,7 +2330,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
             if (ctx._probe !== undefined) failAt(ctx, suffixExpected, end)
           }
           const value = input.slice(pos, end)
-          if (cstCaptureActive(ctx)) pushCstLeaf(ctx, { _tag: 'leaf', value, span: { start: pos, end } })
+          if (cstCaptureActive(ctx)) pushCstLeaf(ctx, new PlainLeaf(value, new PlainSpan(pos, end)))
           EC.e = end
           return value
         }
@@ -2334,7 +2373,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
             }
             if (end < 0) return FAIL
             const value = input.slice(pos, end)
-            if (wasCapturing) pushCstLeaf(ctx, { _tag: 'leaf', value, span: { start: pos, end } })
+            if (wasCapturing) pushCstLeaf(ctx, new PlainLeaf(value, new PlainSpan(pos, end)))
             EC.e = end
             return value
           }
@@ -2343,7 +2382,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
           const end = run(input, pos, ctx)
           if (end < 0) return FAIL
           const value = input.slice(pos, end)
-          if (cstCaptureActive(ctx)) pushCstLeaf(ctx, { _tag: 'leaf', value, span: { start: pos, end } })
+          if (cstCaptureActive(ctx)) pushCstLeaf(ctx, new PlainLeaf(value, new PlainSpan(pos, end)))
           EC.e = end
           return value
         }
@@ -2379,12 +2418,8 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
           }
           if (v === FAIL) return FAIL
           const end = EC.e
-          const out = fn(v, { start: pos, end })
-          if (wasCapturing) pushCstLeaf(ctx, {
-            _tag: 'leaf',
-            value: op === OP_SOURCE_LEAF && hostCst ? input.slice(pos, end) : out,
-            span: { start: pos, end },
-          })
+          const out = fn(v, new PlainSpan(pos, end))
+          if (wasCapturing) pushCstLeaf(ctx, new PlainLeaf(op === OP_SOURCE_LEAF && hostCst ? input.slice(pos, end) : out, new PlainSpan(pos, end)))
           EC.e = end
           return out
         }
@@ -2566,7 +2601,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
               const cur = runSyncTerms(input, pos, ctx, values)
               if (cur < 0) return FAIL
               EC.e = cur
-              return projection >= 0 ? values[projection] : fn!(values, { start: pos, end: cur })
+              return projection >= 0 ? values[projection] : fn!(values, new PlainSpan(pos, cur))
             }
           }
           if (wantValues) {
@@ -2661,7 +2696,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
               const cur = runAdjTerms(input, pos, ctx, values)
               if (cur < 0) return FAIL
               EC.e = cur
-              return projection >= 0 ? values[projection] : fn!(values, { start: pos, end: cur })
+              return projection >= 0 ? values[projection] : fn!(values, new PlainSpan(pos, cur))
             }
           }
           if (wantValues) {
@@ -2704,7 +2739,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
             return (input, pos, ctx) => {
               const v = k0(input, pos, ctx)
               if (v === FAIL) return FAIL
-              return projection === 0 ? v : fn!([v], { start: pos, end: EC.e })
+              return projection === 0 ? v : fn!([v], new PlainSpan(pos, EC.e))
             }
           }
           if (wantValues) {
@@ -2732,7 +2767,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
               EC.e = cur
               return projection === 0 ? v0
                 : projection === 1 ? TERMV
-                : fn!([v0, TERMV], { start: pos, end: cur })
+                : fn!([v0, TERMV], new PlainSpan(pos, cur))
             }
           }
           if (wantValues) {
@@ -2770,7 +2805,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
               return projection === 0 ? v0
                 : projection === 1 ? v1
                 : projection === 2 ? TERMV
-                : fn!([v0, v1, TERMV], { start: pos, end: cur })
+                : fn!([v0, v1, TERMV], new PlainSpan(pos, cur))
             }
           }
           if (wantValues) {
@@ -2875,7 +2910,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
             const cur = runDirectTerms(input, pos, ctx, values)
             if (cur < 0) return FAIL
             EC.e = cur
-            return projection >= 0 ? values[projection] : fn!(values, { start: pos, end: cur })
+            return projection >= 0 ? values[projection] : fn!(values, new PlainSpan(pos, cur))
           }
         }
         if (wantValues) {
@@ -3742,10 +3777,10 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
               return FAIL
             }
             const value = regexValue ? input.slice(pos, end) : spec as string
-            const leaf = { _tag: 'leaf', value, span: { start: pos, end } }
+            const leaf = new PlainLeaf(value, new PlainSpan(pos, end))
             const kids = [leaf]
             const rawKids = [leaf]
-            const span = { start: pos, end }
+            const span = new PlainSpan(pos, end)
             EC.e = end
             const nd = build(kids, undefined, span, rawKids, EMPTY_TL, undefined)
             if (ctx._cstBuf !== undefined || ctx._cstChildren !== undefined) {
@@ -3858,8 +3893,8 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
           const sCap = ctx.captureTrivia
           const sBuf = ctx._cstBuf
           const buf: CstCaptureBuf = rawOnly
-            ? { rawOnly: true }
-            : omitsRaw ? { noRaw: true, rawLen: 0 } : {}
+            ? new PlainBuf(true, undefined, 0)
+            : omitsRaw ? new PlainBuf(undefined, true, 0) : new PlainBuf(undefined, undefined, 0)
           ctx._cstBuf = buf
           // `begin` sets this true and the caller immediately cleared it when the
           // node does not capture wide. Net: the flag IS `captureWide`, a const.
@@ -3890,7 +3925,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
           ctx.captureTrivia = sCap
           if (v === FAIL) return FAIL
           const end = EC.e
-          const span = tracked ? spanLines(ctx, pos, end) : { start: pos, end }
+          const span = tracked ? spanLines(ctx, pos, end) : new PlainSpan(pos, end)
           const st = readsState ? ctx.state : undefined
 
           let nd: unknown
@@ -3923,7 +3958,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
           } else if (host !== undefined) {
             nd = host(type, hostKids, fieldMap, span, rawKids, tlog, st, tags)
           } else {
-            nd = { _tag: 'node', type, span, state: st ?? null, children: kids }
+            nd = new PlainNode(type, span, st ?? null, kids === EMPTY_CH ? kids : [...kids])
           }
           // A ROOT NODE IS NOT A CHILD. `node()` pushes only when it was entered
           // from an enclosing collector — `saved.buf !== undefined || saved.ch !==
@@ -3964,7 +3999,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
           const at = fe === undefined || fe < 0 ? pos : fe
           return { ok: false, expected: (ctx._fx ?? EMPTY_FX) as string[], span: { start: at, end: at } }
         }
-        return { ok: true, value: v, span: { start: pos, end: EC.e } }
+        return { ok: true, value: v, span: new PlainSpan(pos, EC.e) }
       },
     }
   }
@@ -4080,6 +4115,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
         scalarRecognizers, commitTriviaScan, scanTriviaCompact, t.lex, adjacencyHolds, t.lexPrograms,
         LEAD, triviaFieldCapture, skipDeferredTrivia,
         collapseEmpty,
+        PlainSpan, PlainLeaf, PlainNode, PlainField, PlainBuf,
       )
       emitReached = new Set(pre.reached)
       if (emitted.ext !== undefined) HYBRID = emitted
@@ -4164,6 +4200,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
         scalarRecognizers, commitTriviaScan, scanTriviaCompact, t.lex, adjacencyHolds, t.lexPrograms,
         LEAD, triviaFieldCapture, skipDeferredTrivia,
         collapseEmpty,
+        PlainSpan, PlainLeaf, PlainNode, PlainField, PlainBuf,
       )
       emitReached = em.reached
     } catch (e) {

package/src/table/compile-rule-map.ts

@@ -355,10 +355,12 @@ export function compileRuleMap(
     const roots = assemblyExtraIps(artifact)
     let siteBytes, fixed
     try {
-      siteBytes = emitAssemblySource(table, artifact, cfg, roots, true).siteBytes
+      // Priced without the `_errors` marks a lexical program over a scan adds
+      // (`scanSinkNeutral`), so they move no site in or out of the budget.
+      siteBytes = emitAssemblySource(table, artifact, cfg, roots, true, undefined, true).siteBytes
       // What any selective assembly costs before its first site body: the shared
       // prelude and the factory's tail. A budget that cannot cover it buys nothing.
-      fixed = emitAssemblySource(table, artifact, cfg, roots, true, new Set()).source.length
+      fixed = emitAssemblySource(table, artifact, cfg, roots, true, new Set(), true).source.length
     } catch (e) {
       if (e instanceof Unemittable) return { ...none, refusal: e.construct }
       throw e
@@ -384,7 +386,7 @@ export function compileRuleMap(
           refusal: `a site within its ${Math.floor(ceiling)} B selection ceiling (the prelude alone is ${fixed} B)`,
         }
       }
-      const bytes = emitAssemblySource(table, artifact, cfg, roots, true, sites).source.length
+      const bytes = emitAssemblySource(table, artifact, cfg, roots, true, sites, true).source.length
       if (bytes <= ceiling) return { sites, bytes: Math.ceil(bytes * SUPERCOMPILE_GROWTH) }
       // Scale by the overshoot, and by at least 1%: near the ceiling a site is
       // the unit, and the ratio alone can stall a few bytes over it.