parseman 0.58.2 → 0.58.3

package/src/table/assemble.ts

@@ -69,7 +69,7 @@ import type { Combinator, FieldMap, FirstSet, ParseContext, ParseResult, ParserD
 import { balanced, scanTo } from '../combinators/scanTo.ts'
 import { buildFieldMap } from '../compiler/fields.ts'
 import { FIELD_LEADING_TRIVIA, FIELD_TRAILING_TRIVIA, LEAD, triviaFieldCapture } from '../combinators/field-trivia.ts'
-import { skipDeferredTrivia } from './trivia-gap.ts'
+import { skipDeferredTrivia, triviaLeadTable } from './trivia-gap.ts'
 import { asciiFoldKey } from '../combinators/dispatch.ts'
 import { projectChild, unwrapChild } from '../combinators/node.ts'
 import { cstOutputHost } from '../compiler/build-arity.ts'
@@ -959,7 +959,31 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
     return out
   }
 
+  /**
+   * NO TRIVIA CAN START AT `cur` — `triviaLeadTable` for the installed trivia. A
+   * gap there is empty: nothing to scan, nothing recorded, so nothing for a
+   * sequence term to mark. One table per trivia slot, built with the assembly;
+   * the installed trivia is matched to its slot only when it changes (a trivia
+   * the caller passed that is no slot gets its own table then).
+   */
+  const LEAD_TABLES = trivia.map(tp => tp === undefined ? null : triviaLeadTable(tp))
+  let LEAD_OF: Combinator<unknown> | undefined
+  let LEAD_TABLE: Uint8Array | null = null
+  function cannotStartTrivia(input: string, cur: number, triviaP: Combinator<unknown>): boolean {
+    if (triviaP !== LEAD_OF) {
+      LEAD_OF = triviaP
+      const slot = trivia.indexOf(triviaP)
+      LEAD_TABLE = slot >= 0 ? LEAD_TABLES[slot]! : triviaLeadTable(triviaP)
+    }
+    const table = LEAD_TABLE
+    if (table === null) return false
+    const c = input.charCodeAt(cur)
+    return c < 128 && table[c] === 0
+  }
+
   function skipTrivia(input: string, cur: number, ctx: ParseContext): number {
+    const t = ctx.trivia
+    if (t !== undefined && cannotStartTrivia(input, cur, t)) return cur
     const s = SCAN
     if (s !== null
       && ctx._triviaLog === undefined
@@ -1041,7 +1065,8 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
    * into them.
    */
   function nextTerm(child: Piece, input: string, cur: number, ctx: ParseContext): number {
-    if (ctx.trivia === undefined) {
+    const t = ctx.trivia
+    if (t === undefined || cannotStartTrivia(input, cur, t)) {
       const v = child(input, cur, ctx)
       if (v === FAIL) return -1
       TERMV = v
@@ -1089,7 +1114,8 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
 
   /** A non-first term at a site proven to run under an open node capture buffer. */
   function nextTermBuffered(child: Piece, input: string, cur: number, ctx: ParseContext): number {
-    if (ctx.trivia === undefined) {
+    const t = ctx.trivia
+    if (t === undefined || cannotStartTrivia(input, cur, t)) {
       const v = child(input, cur, ctx)
       if (v === FAIL) return -1
       TERMV = v
@@ -2016,6 +2042,9 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
         // known here, so the scope piece holds the scanner it installs as a `const`.
         const scanFor: FastTriviaScanner | null =
           swapLegal && ki >= 0 && !triviaLabelled[ki]! ? triviaScan[ki]! : null
+        // The scope installs its slot's lead table with its trivia, so the gate
+        // (`cannotStartTrivia`) never has to look the slot up.
+        const scopeLead = ki < 0 ? null : LEAD_TABLES[ki]!
         const child = link(code[ip + 2]!)
         // TWO PIECES, SELECTED HERE — not one piece that tests the opcode. Capture
         // is fixed for the whole parse, so `code[ip] === OP_SCOPE_CAP` is an
@@ -2032,6 +2061,9 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
               const savedScan = SCAN
               const savedCap = ctx.captureTrivia
               SCAN = scanFor
+              const savedLeadOf = LEAD_OF, savedLead = LEAD_TABLE
+              LEAD_OF = scopeTrivia
+              LEAD_TABLE = scopeLead
               ctx.trivia = scopeTrivia
               ctx.triviaKindLabels = scopeLabels
               ctx.captureTrivia = true
@@ -2040,6 +2072,8 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
               ctx.trivia = saved
               ctx.triviaKindLabels = savedLabels
               SCAN = savedScan
+              LEAD_OF = savedLeadOf
+              LEAD_TABLE = savedLead
               return v
             }
           : (input, pos, ctx) => {
@@ -2047,12 +2081,17 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
               const savedLabels = ctx.triviaKindLabels
               const savedScan = SCAN
               SCAN = scanFor
+              const savedLeadOf = LEAD_OF, savedLead = LEAD_TABLE
+              LEAD_OF = scopeTrivia
+              LEAD_TABLE = scopeLead
               ctx.trivia = scopeTrivia
               ctx.triviaKindLabels = scopeLabels
               const v = child(input, pos, ctx)
               ctx.trivia = saved
               ctx.triviaKindLabels = savedLabels
               SCAN = savedScan
+              LEAD_OF = savedLeadOf
+              LEAD_TABLE = savedLead
               return v
             }
         return scopeRootPolicy(scopePiece, code[ip] === OP_SCOPE_PLAIN ? 0 : code[ip + 3]!)

package/src/table/compile-rule-map.ts

@@ -355,10 +355,12 @@ export function compileRuleMap(
     const roots = assemblyExtraIps(artifact)
     let siteBytes, fixed
     try {
-      siteBytes = emitAssemblySource(table, artifact, cfg, roots, true).siteBytes
+      // Priced without the `_errors` marks a lexical program over a scan adds
+      // (`scanSinkNeutral`), so they move no site in or out of the budget.
+      siteBytes = emitAssemblySource(table, artifact, cfg, roots, true, undefined, true).siteBytes
       // What any selective assembly costs before its first site body: the shared
       // prelude and the factory's tail. A budget that cannot cover it buys nothing.
-      fixed = emitAssemblySource(table, artifact, cfg, roots, true, new Set()).source.length
+      fixed = emitAssemblySource(table, artifact, cfg, roots, true, new Set(), true).source.length
     } catch (e) {
       if (e instanceof Unemittable) return { ...none, refusal: e.construct }
       throw e
@@ -384,7 +386,7 @@ export function compileRuleMap(
           refusal: `a site within its ${Math.floor(ceiling)} B selection ceiling (the prelude alone is ${fixed} B)`,
         }
       }
-      const bytes = emitAssemblySource(table, artifact, cfg, roots, true, sites).source.length
+      const bytes = emitAssemblySource(table, artifact, cfg, roots, true, sites, true).source.length
       if (bytes <= ceiling) return { sites, bytes: Math.ceil(bytes * SUPERCOMPILE_GROWTH) }
       // Scale by the overshoot, and by at least 1%: near the ceiling a site is
       // the unit, and the ratio alone can stall a few bytes over it.

package/src/table/emit-assembly.ts

@@ -414,7 +414,7 @@ function emitRollback(t: string, buf: boolean, rawMode: number, s: Sinks = NO_SI
  * only the first branch, a known slot keeps only the second (with the scope's own
  * scanner already bound into `skip`), and `TRI_UNKNOWN` keeps both.
  */
-function emitTerm(callee: string, dst: string, t: string, l: SiteLabel, skip: string): string {
+function emitTerm(callee: string, dst: string, t: string, l: SiteLabel, skip: string, miss?: (c: string) => string): string {
   const fast = `const ${t}v=${callee}(input,cur,ctx)
 if(${t}v===FAIL)return FAIL
 ${dst}=${t}v
@@ -432,7 +432,14 @@ if(${t}v===FAIL)return FAIL
 ${dst}=${t}v
 if(EC.e>${t}s)cur=EC.e
 else{rollbackScannedTriviaAt(ctx,${t}tl,${t}stl,${t}lg,${t}slg,${t}rt,${t}srt)}`
-  if (l.tri !== TRI_UNKNOWN) return scanned
+  // A character no trivia of this slot can start opens the empty gap, which
+  // records nothing, so the term runs at `cur` with no marks around a scan.
+  if (l.tri !== TRI_UNKNOWN) return miss === undefined ? scanned : `const ${t}c=input.charCodeAt(cur)
+if(${miss(`${t}c`)}){
+${fast}
+}else{
+${scanned}
+}`
   return `if(ctx.trivia===undefined){
 ${fast}
 }else{
@@ -588,6 +595,13 @@ export function emitAssemblySource(
   extraIps: readonly number[] = [],
   staticBuild = false,
   select?: ReadonlySet<number>,
+  /**
+   * Leave the `_errors` bit of a lexical program over a scan out of the side-sink
+   * census below. A budgeted selection is priced and chosen on this lowering, so
+   * the marks that bit adds move no site in or out of a budget; the artifact is
+   * then emitted with them.
+   */
+  scanSinkNeutral = false,
 ): EmitResult {
   const { code, k, fx, disp, dsp, triviaLabelled } = t
   const swapLegal = !cfg.trackLines
@@ -733,13 +747,15 @@ export function emitAssemblySource(
   // that otherwise matched byte for byte.
   //
   // Any future row whose child is not an offset in `code` belongs here for the
-  // same reason. `OP_LIVE` is the other such row, and it is unemittable.
+  // same reason. `OP_LIVE` is the other such row, and it is unemittable. An
+  // `OP_LEX_PROGRAM` over a scan (`t.lexPrograms[id].scan`, a whole `balanced()`
+  // lowered as one recognizer) runs that same interpreted scan, so it is one too.
   const sinkSites = reachable
   const sinkBits = new Map<number, number>()
   for (const ip of sinkSites) {
     const op = code[ip]
     sinkBits.set(ip,
-      op === OP_SCAN ? 3
+      op === OP_SCAN || (!scanSinkNeutral && op === OP_LEX_PROGRAM && t.lexPrograms[code[ip + 1]!]!.scan !== undefined) ? 3
         : (op === OP_FIELD ? 1 : 0)
           | (op === OP_EXPECT || (REC && (op === OP_REP || op === OP_REPV)) ? 2 : 0))
   }
@@ -974,6 +990,19 @@ return 1
     return `if(${gateMissTest(child)}){${miss}}\n`
   }
 
+  /**
+   * The test that no trivia of slot `ki` can start at the code unit `c`: the
+   * slot's first characters, as code points, decide every code unit below the
+   * surrogates; a surrogate, or `NaN` past the end of input, is left to the scan.
+   * `undefined` where the first set is unbounded or too wide to print.
+   */
+  function triviaMiss(ki: number): ((c: string) => string) | undefined {
+    const leads = triviaLeadRanges(ki, t.trivia)
+    if (leads === undefined || leads.length > 16) return undefined
+    if (leads.length === 0) return c => `${c}<55296`
+    return c => `${c}<55296&&!(${leads.map(r => r.lo === r.hi ? `${c}===${r.lo}` : `(${c}>=${r.lo}&&${c}<=${r.hi})`).join('||')})`
+  }
+
   function skipFor(l: SiteLabel): string {
     if (l.tri < 0) return '_skipTrivia'
     const ki = l.tri
@@ -991,11 +1020,16 @@ return 1
     if (!hasScan) {
       // No installed scanner: the labelled/`trackLines` path, which records
       // through `scanTrivia`. `needsDeferredTriviaCommit` is implied by an open
-      // buffer, so an in-node site skips the call that asks.
+      // buffer, so an in-node site skips the call that asks. A character no gap
+      // can start answers first, with no call (`triviaMiss`).
+      const miss = triviaMiss(ki)
+      const pre = miss === undefined ? '' : `const c=input.charCodeAt(cur)
+if(${miss('c')})return cur
+`
       skipDefs.push(l.buf
-        ? `function ${nm}(input,cur,ctx){return skipDeferredTrivia(input,cur,ctx)}`
+        ? `function ${nm}(input,cur,ctx){${pre === '' ? '' : `\n${pre}`}return skipDeferredTrivia(input,cur,ctx)}`
         : `function ${nm}(input,cur,ctx){
-if(needsDeferredTriviaCommit(ctx))return skipDeferredTrivia(input,cur,ctx)
+${pre}if(needsDeferredTriviaCommit(ctx))return skipDeferredTrivia(input,cur,ctx)
 return advanceTrivia(input,cur,ctx)}`)
       return nm
     }
@@ -1830,7 +1864,7 @@ return v
             const callee = leadingTriviaFieldAt(code, k, childIp) ? `(LEAD.at=cur,${kids[i]!})` : kids[i]!
             parts.push(holdsGap(code, k, base, n, i, nullable)
               ? emitHoldTerm(callee, vn, tmp(), L, skipFor(L))
-              : emitTerm(callee, vn, tmp(), L, skipFor(L)))
+              : emitTerm(callee, vn, tmp(), L, skipFor(L), L.tri >= 0 ? triviaMiss(L.tri) : undefined))
           }
           names.push(vn)
         }

package/src/table/encode-baseline.ts

@@ -1,5 +1,6 @@
 import type { Combinator, FirstSet, ParserDef } from '../types.ts'
 import { firstSetOf, matchesEmpty } from '../combinators/first-set.ts'
+import { leadRanges } from '../lead-ranges.ts'
 import { deriveExpected } from '../combinators/expect.ts'
 import { buildReadsState, buildReadsTrivia } from '../compiler/build-arity.ts'
 import {
@@ -34,7 +35,7 @@ function disjointSets(sets: readonly FirstSet[]): boolean {
   const seen: Array<{ lo: number; hi: number }> = []
   for (const fs of sets) {
     if (fs.kind !== 'ranges' || fs.ranges.length === 0) return false
-    for (const r of fs.ranges) {
+    for (const r of leadRanges(fs.ranges)) {
       for (const p of seen) if (r.lo <= p.hi && p.lo <= r.hi) return false
       seen.push(r)
     }

package/src/table/program.ts

@@ -2,6 +2,7 @@ import type { Combinator, ParseContext, ParseResult } from '../types.ts'
 import { classifiedTrivia, trivia as triviaOf } from '../combinators/map.ts'
 import { fastTriviaScanner, type FastTriviaScanner } from '../combinators/trivia-skip.ts'
 import { regex } from '../combinators/regex.ts'
+import { leadRanges } from '../lead-ranges.ts'
 import { choice } from '../combinators/choice.ts'
 import { many } from '../combinators/repeat.ts'
 import type { EmittedFactory, PoolPlan } from './emit-assembly.ts'
@@ -642,7 +643,7 @@ export function decodeClassSpec(spec: string): Array<{ lo: number; hi: number }>
 export function resolveClass(spec: string): ResolvedClass {
   const ascii = new Uint8Array(128)
   const hi: number[] = []
-  for (const { lo, hi: up } of decodeClassSpec(spec)) {
+  for (const { lo, hi: up } of leadRanges(decodeClassSpec(spec))) {
     if (lo < 128) for (let c = lo; c <= Math.min(up, 127); c++) ascii[c] = 1
     if (up >= 128) hi.push(Math.max(lo, 128), up)
   }

package/src/table/scan-shapes.ts

@@ -235,6 +235,9 @@ function parseSeqParts(source: string): SeqPart[] | null {
       let cp: number
       if (uni) { cp = uni.cp; i = uni.next }
       else if (e === undefined) return null
+      // A digit escape is a backreference or a legacy octal escape (`\012`); only a
+      // lone `\0` is NUL.
+      else if (e >= '0' && e <= '9' && (e !== '0' || (source[i + 2] !== undefined && source[i + 2]! >= '0' && source[i + 2]! <= '9'))) return null
       else if (e in CLASS_ESCAPES) { cp = CLASS_ESCAPES[e]!; i += 2 }
       else if ((e >= 'a' && e <= 'z') || (e >= 'A' && e <= 'Z')) return null
       else { cp = e.codePointAt(0)!; i += 2 }
@@ -1078,11 +1081,11 @@ function parseScanShapeCore(source: string): ScanShape | null {
 /**
  * Flag-aware wrapper around `parseScanShape`. Lowering to a raw code-point scan
  * assumes default regex semantics, so any flag that changes matching (`i` case
- * folding, `u` surrogate handling, `m`/`s` anchor/dot behavior) disables it.
+ * folding, `u`/`v` code points and sets, `m`/`s` anchor/dot behavior) disables it.
  * `g`/`y` are stickiness-only and safe.
  */
 export function scanShapeFromRegex(source: string, flags: string): ScanShape | null {
-  if (/[msu]/.test(flags)) return null
+  if (/[msuv]/.test(flags)) return null
   if (/i/.test(flags)) return caseFoldShape(source)
   return parseScanShape(source)
 }

package/src/table/site-labels.ts

@@ -65,6 +65,7 @@ import {
   OP_SCAN, OP_SCOPE, OP_SCOPE_CAP, OP_SCOPE_PLAIN, OP_SEQ, OP_SEQV, OP_SEQX, OP_TOKEN, OP_XFORM,
 } from './ops.ts'
 import { childSlots } from './child-slots.ts'
+import { leadRanges } from '../lead-ranges.ts'
 
 /** No scope on this path installs a known trivia — the lattice's top element. */
 export const TRI_UNKNOWN = -2
@@ -380,5 +381,5 @@ export function triviaLeadRanges(
   const fs = trivia[tri]?._meta.firstSet
   if (fs === undefined) return undefined
   if (fs.kind === 'empty') return []
-  return fs.kind === 'ranges' && fs.ranges !== undefined ? fs.ranges : undefined
+  return fs.kind === 'ranges' && fs.ranges !== undefined ? leadRanges(fs.ranges) : undefined
 }

package/src/table/trivia-gap.ts

@@ -48,6 +48,25 @@ export function skipDeferredTrivia(input: string, cur: number, ctx: ParseContext
   return scan
 }
 
+/**
+ * THE CHARACTERS A TRIVIA CAN START WITH, as a table over ASCII: `1` where a run
+ * may start, `0` where none can. `null` when the trivia's first set is unbounded.
+ *
+ * A gap at a character outside the table is the empty gap. It consumes nothing
+ * and records nothing in any sink (trivia log, root log, capture buffer, line
+ * table), so a skip there can answer `cur` without scanning. A character at or
+ * above 128, or past the end of input, is not decided by the table.
+ */
+export function triviaLeadTable(triviaP: Combinator<unknown>): Uint8Array | null {
+  const fs = triviaP._meta.firstSet
+  if (fs.kind === 'any') return null
+  const table = new Uint8Array(128)
+  if (fs.kind === 'ranges') {
+    for (const r of fs.ranges) for (let c = Math.max(0, r.lo); c <= Math.min(127, r.hi); c++) table[c] = 1
+  }
+  return table
+}
+
 /** The end of the run at `cur` when nothing root-selected lies in it, else −1. */
 function clearRunEnd(input: string, cur: number, ctx: ParseContext, triviaP: Combinator<unknown> | undefined, plain: boolean): number {
   const rootKinds = ctx._rootTriviaKindIndex

package/src/version.ts

@@ -13,4 +13,4 @@
  * Kept in sync with `package.json` by `test/unit/version-sync.test.ts`, which fails
  * the build if this constant drifts from the published version.
  */
-export const PARSEMAN_VERSION = '0.58.2'
+export const PARSEMAN_VERSION = '0.58.3'