parseman 0.47.1 → 0.48.0

package/src/table/ops.ts

@@ -37,8 +37,11 @@ export const OP_SEQV = 4
  */
 export const OP_CHOICE = 5
 /**
- * `REP c min max sep flags` — `sep` is a child offset or −1; `max` −1 = ∞.
- * `flags` bit 0 = trailing separator allowed, bit 1 = `keepSeparators`.
+ * `REP c min max sep flags fx sepClass` — `sep` is a child offset or −1; `max`
+ * −1 = ∞. `fx` is recovery metadata. The last class is the separator sentinel
+ * for a separated list, otherwise the finite/non-nullable optional-item class
+ * (or −1). `flags` bit 0 = trailing separator allowed, bit 1 =
+ * `keepSeparators`, bit 2 = item expected reporting.
  */
 export const OP_REP = 6
 /** `REPV …` — `REP` with `valueUnused`. */
@@ -57,7 +60,8 @@ export const OP_XFORM = 9
  * none), `type` and `tags` index the const pool (`tags` −1 = none).
  *
  * `flags` is a BIT FIELD, not a boolean: bit 0 (`&1`) = grammar-owned explicit
- * `captureTrivia`, bit 2 (`&4`) = the builder reads
+ * `captureTrivia`, bit 1 (`&2`) = a direct builder is proven not to read
+ * `rawChildren`, bit 2 (`&4`) = the builder reads
  * `triviaLog`, bit 3 (`&8`) = it reads `ctx.state`, bit 4 (`&16`) = the node has
  * read fields, bit 5 (`&32`) = `collapse`, bit 6 (`&64`) = `unwrap`, bit 7
  * (`&128`) = `trailingTrivia`. Bit 0 distinguishes capture a host may not
@@ -110,8 +114,12 @@ export const OP_NODE_TRACK = 19
  * on `ctx.trivia` for the duration and restores the outer one after, which is
  * how the runtime's own `advanceTrivia` fast scanner gets reached — the same
  * shared machinery the interpreter uses, not a second copy of it.
- * Synthetic rule-entry/restoration scopes use `OP_SCOPE_PLAIN` so a zero
- * policy stays a three-word row without changing this opcode's ABI.
+ *
+ * `policy` bit 0 suppresses selected root capture for an opaque scope; bit 1
+ * refuses an unclassified local scope while selected root capture is active.
+ * Synthetic rule-entry and cross-rule restoration scopes have no policy and use
+ * the three-word `OP_SCOPE_PLAIN` row below. Keeping the opcodes distinct means
+ * a zero policy costs no extra program word without changing this opcode's ABI.
  */
 export const OP_SCOPE = 20
 /**
@@ -122,6 +130,11 @@ export const OP_EXPECT = 21
 /**
  * `SEQX f n c1 … cn` — a `transform()` whose child is a `sequence()`.
  *
+ * `f >= 0` indexes the reducer in `prog.fns`. `f < 0` is the descriptor
+ * `~childIndex`: the transform is the exact direct projection
+ * `([…, value, …]) => value`, so the row returns that already-parsed child and
+ * carries no reducer.  Both forms have the same row width.
+ *
  * That pair is the dominant shape in every grammar here (json is nine of them),
  * and running it as two rows costs two switch dispatches and two JS call frames
  * per rule invocation where the emitted code pays neither. Fusing them into one
@@ -428,10 +441,33 @@ export const OP_LABEL = 39
  */
 export const OP_COV = 40
 /**
- * `SCOPE_PLAIN k c` — a synthetic ambient-trivia scope with policy fixed to zero.
- * A distinct opcode prevents the next row's opcode from being read as policy.
+ * `SCOPE_PLAIN k c` — a synthetic zero-policy ambient-trivia scope.
+ *
+ * This is the same scanner/context swap as `OP_SCOPE`, with policy fixed to
+ * zero at assembly. `encodeRule()` uses it for a rule entry and `scopedRef()`
+ * for the lexical-trivia restoration around a reference reached from
+ * `noTrivia()`. Neither row represents an authored `parser()` scope, so neither
+ * can be opaque or require the unclassified-scope refusal.
+ *
+ * A separate opcode is load-bearing: treating a three-word synthetic row as the
+ * four-word `OP_SCOPE` makes the following row's opcode become the policy bits.
  */
 export const OP_SCOPE_PLAIN = 41
+/**
+ * `LEX_BODY b fx suffixFx` — one compiler-selected childless lexical replacement
+ * body. `b` indexes `prog.lex`; `fx` is the exact base-recognizer failure set;
+ * `suffixFx` is optional()'s swallowed literal failure publication.
+ *
+ * There is deliberately no child operand and no mode bit. The complete
+ * CHARACTER and TOKEN candidates were compared before serialization; this row
+ * is only the winning TOKEN body. A program without a selected TOKEN body has
+ * no row or lexical pool.
+ */
+/** `LEX_BODY body fx suffixFx lineFlags` — selected childless lexical body.
+ * lineFlags bit 0 publishes the regex range; bit 1 publishes a matched suffix. */
+export const OP_LEX_BODY = 42
+/** `LEX_PROGRAM p` — selected fixed composite lexical body, childless. */
+export const OP_LEX_PROGRAM = 43
 /**
  * `DISPATCH sel d other otherRouted n a1 … an` — `dispatch()`.
  *
@@ -464,4 +500,6 @@ export const OP_NAMES: Record<number, string> = {
   [OP_GREEDY]: 'GREEDY', [OP_REJECT]: 'REJECT', [OP_ARMGATE]: 'ARMGATE',
   [OP_LIVE]: 'LIVE', [OP_ATTEMPT]: 'ATTEMPT', [OP_LABEL]: 'LABEL', [OP_COV]: 'COV',
   [OP_SCOPE_PLAIN]: 'SCOPE_PLAIN',
+  [OP_LEX_BODY]: 'LEX_BODY',
+  [OP_LEX_PROGRAM]: 'LEX_PROGRAM',
 }

package/src/table/program.ts

@@ -5,6 +5,8 @@ import { regex } from '../combinators/regex.ts'
 import { choice } from '../combinators/choice.ts'
 import { many } from '../combinators/repeat.ts'
 import type { EmittedFactory, PoolPlan } from './emit-assembly.ts'
+import { buildLexBodyRecognizer, type LexBodyRecognizer } from './lex-body.ts'
+import { buildLexProgramRunner, type LexProgramRunner } from './lex-program.ts'
 
 /**
  * One assembly the BUILD already compiled, so the run does not have to.
@@ -57,6 +59,11 @@ export type TableProgram = {
    * at `resolveTable`, exactly like the char-class ASCII lookups.
    */
   readonly dsp: readonly DispatchSpec[]
+  /** Selected childless lexical bodies. Losing compiler candidates never enter
+   * this pool. The first row is `[regexConst, suffixCodeUnit]`. */
+  readonly lex?: readonly LexBodySpec[]
+  /** Selected composite lexical programs. Losing candidates never enter it. */
+  readonly lexPrograms?: readonly LexProgramSpec[]
   /** Rule name → entry offset in `code`. */
   readonly rules: Readonly<Record<string, number>>
   /**
@@ -291,6 +298,8 @@ export type CompactProgram = {
   readonly f: readonly unknown[]
   readonly l?: 0 | 1
   readonly p?: readonly DispatchSpec[]
+  readonly lx?: readonly LexBodySpec[]
+  readonly lp?: readonly LexProgramSpec[]
   readonly lb?: readonly string[]
   readonly rc?: 0 | 1
   readonly h?: 'ast' | 'cst'
@@ -309,6 +318,8 @@ export function expandCompact(p: TableProgram | CompactProgram): TableProgram {
   return ownTableProgram({
     code: p.c, k: p.k, cc: p.x, fx: p.e, disp: p.d, rules: p.r, fns: p.f,
     lines: p.l ?? 0, dsp: p.p ?? [],
+    ...(p.lx === undefined ? {} : { lex: p.lx }),
+    ...(p.lp === undefined ? {} : { lexPrograms: p.lp }),
     ...(p.lb === undefined ? {} : { labels: p.lb }),
     ...(p.rc === undefined ? {} : { classified: p.rc }),
     ...(p.h === undefined ? {} : { hostMode: p.h }),
@@ -322,6 +333,28 @@ export function expandCompact(p: TableProgram | CompactProgram): TableProgram {
   })
 }
 
+/** Selected lexical body. `suffixCodeUnit === -1` is a direct regex terminal;
+ * 0..0xFFFF is the legacy regex + optional(one code unit) spelling. */
+export type LexBodySpec = readonly [regexConst: number, suffixCodeUnit: number]
+
+/** `kind=0`: fixed ordered four-terminal selected body. */
+export type LexProgramSpec = readonly [
+  kind: 0, digest: number, commonClass: number, choiceFx: number,
+  arm0Matcher: number, arm0Fx: number,
+  arm1Matcher: number, arm1Fx: number,
+  arm2Matcher: number, arm2Fx: number,
+  arm3Matcher: number, arm3Fx: number,
+  trackLines: 0 | 1,
+] | readonly [
+  kind: 1, digest: number,
+  guard0Arm0: number, guard0Arm1: number, guard0Arm2: number, guard0Arm3: number,
+  guard0Fx: number,
+  guard1Matcher: number, guard1Fx: number,
+  terminalMatcher: number, terminalFx: number,
+  trackLines: 0 | 1,
+] | readonly [kind: 2, digest: number, ...fixedTreeWords: number[]]
+  | readonly [kind: 3, digest: number, scan: number]
+
 /** One `dispatch()`'s routing tables, in serialisable form. */
 export type DispatchSpec = {
   /** Exact keys, parallel to `keyArm`. */
@@ -353,7 +386,12 @@ export type ResolvedDispatchSpec = {
   readonly expected: readonly string[]
 }
 
-/** Refuse malformed dispatch arm references before closure/emitted linking. */
+/**
+ * Refuse malformed fixed arm references before closure or emitted linking.
+ *
+ * This is deliberately an assembly-time trust boundary. `execRules()` stays an
+ * independent semantic oracle and never pays artifact validation per parse.
+ */
 export function validateDispatchSpec(
   spec: ResolvedDispatchSpec | undefined,
   arity: number,
@@ -428,6 +466,8 @@ export type ResolvedTable = {
   readonly fx: readonly (readonly string[])[]
   readonly disp: readonly ResolvedDispatch[]
   readonly dsp: readonly ResolvedDispatchSpec[]
+  readonly lex: readonly LexBodyRecognizer[]
+  readonly lexPrograms: readonly LexProgramRunner[]
   /** Trivia combinators rebuilt from `triviaSpecs`, once per table. */
   readonly trivia: readonly Combinator<unknown>[]
   /**
@@ -605,6 +645,7 @@ export function resolveTable(prog: TableProgram): ResolvedTable {
   if (hit !== undefined) return hit
   const cc = prog.cc.map(resolveClass)
   const triviaBuilt = (prog.triviaSpecs ?? []).map(buildTrivia)
+  const lex = (prog.lex ?? []).map(spec => buildLexBodyRecognizer(spec, prog.k))
   const built: ResolvedTable = {
     prog,
     code: Int32Array.from(prog.code),
@@ -623,6 +664,9 @@ export function resolveTable(prog: TableProgram): ResolvedTable {
       routed: d.routed,
       expected: d.expected,
     })),
+    lex,
+    lexPrograms: (prog.lexPrograms ?? []).map(spec =>
+      buildLexProgramRunner(spec, lex, cc, prog.fx)),
     rules: prog.rules,
   }
   if (owner !== undefined) owner.resolved = built
@@ -687,7 +731,7 @@ export type CompactFolded = {
  * Named rather than derived so the refusal can say which one broke.
  */
 const SHARED_FIELDS = [
-  'k', 'cc', 'fx', 'disp', 'dsp', 'rules', 'labels', 'classified',
+  'k', 'cc', 'fx', 'disp', 'dsp', 'lex', 'lexPrograms', 'rules', 'labels', 'classified',
   'scanSkip', 'scanSkipOf', 'scans', 'triviaSpecs', 'runtimeOnly',
   // `rec` is a property of the GRAMMAR BUILD, not of the trackLines/hostMode
   // axis a fold varies — every variant of one export is encoded with the same

package/src/table/scalar-terminal.ts

@@ -0,0 +1,51 @@
+import { OP_LIT, OP_NOT, OP_RX } from './ops.ts'
+
+/** Pure recognition only. Consumption owns value, CST, line, failure and cursor effects. */
+export type ScalarRecognizer = (input: string, pos: number, seed?: number) => number
+
+/**
+ * One recognizer per existing terminal constant. This is the common raw/pending
+ * seam: a later token cursor may provide a cached end for the same (pos, spec)
+ * without changing either consumer's materialization protocol.
+ */
+export function makeScalarRecognizer(op: number, spec: unknown): ScalarRecognizer | undefined {
+  if (op === OP_RX && spec instanceof RegExp) {
+    return (input, pos) => {
+      spec.lastIndex = pos
+      return spec.test(input) ? spec.lastIndex : -1
+    }
+  }
+  if (op !== OP_LIT || typeof spec !== 'string') return undefined
+  const len = spec.length
+  if (len === 1) {
+    const c0 = spec.charCodeAt(0)
+    return (input, pos) => input.charCodeAt(pos) === c0 ? pos + 1 : -1
+  }
+  if (len === 2) {
+    const c0 = spec.charCodeAt(0), c1 = spec.charCodeAt(1)
+    return (input, pos) => input.charCodeAt(pos) === c0 && input.charCodeAt(pos + 1) === c1 ? pos + 2 : -1
+  }
+  if (len === 3) {
+    const c0 = spec.charCodeAt(0), c1 = spec.charCodeAt(1), c2 = spec.charCodeAt(2)
+    return (input, pos) => input.charCodeAt(pos) === c0
+      && input.charCodeAt(pos + 1) === c1
+      && input.charCodeAt(pos + 2) === c2 ? pos + 3 : -1
+  }
+  return (input, pos) => input.startsWith(spec, pos) ? pos + len : -1
+}
+
+/** Exact bounded shape whose terminal value can be materialized without a node capture frame. */
+export function scalarTerminalNodeChild(code: ArrayLike<number>, ip: number): number {
+  if (code[ip + 1]! < 0 || code[ip + 3] !== 0 || code[ip + 4] !== -1) return -1
+  const child = code[ip + 2]!
+  const op = code[child]
+  return op === OP_RX || op === OP_LIT ? child : -1
+}
+
+/** Direct untracked terminal whose recognition `not()` can inspect without materializing it. */
+export function scalarTerminalNotChild(code: ArrayLike<number>, ip: number): number {
+  if (code[ip] !== OP_NOT) return -1
+  const child = code[ip + 1]!
+  const op = code[child]
+  return op === OP_RX || op === OP_LIT ? child : -1
+}

package/src/table/site-labels.ts

@@ -148,10 +148,20 @@ function transfer(code: Int32Array, ip: number, at: SiteLabel, hostCst: boolean)
     return { tri, buf: at.buf, cap }
   }
   if (op === OP_NODE || op === OP_NODE_TRACK) {
+    const flags = code[ip + 3]!
+    const directChildren = !hostCst && op === OP_NODE && code[ip + 1]! >= 0 && code[ip + 4]! < 0
+      && (flags === 2 || flags === 18 || flags === 34)
+    if (directChildren) {
+      // The confirmed low-arity builder projection installs split child/leaf
+      // arrays instead of a CstCaptureBuf. Descendants still capture, but they
+      // must take the generic collector branch rather than dereference `_cstBuf`.
+      // False means UNKNOWN in this lattice, which is exactly the safe answer.
+      return { tri: at.tri, buf: false, cap: CAP_OFF }
+    }
     // `ctx._cstBuf = buf` and `ctx.captureTrivia = <literal>`, both unconditional
     // — `emit-assembly.ts`'s `OP_NODE` body opens the buffer before it descends
     // and closes it after, whatever the host mode is.
-    const cap = ((code[ip + 3]! & 4) !== 0 || hostCst) ? CAP_ON : CAP_OFF
+    const cap = ((flags & 4) !== 0 || hostCst) ? CAP_ON : CAP_OFF
     if (at.buf && cap === at.cap) return at
     return { tri: at.tri, buf: true, cap }
   }

package/src/table/stamp.ts

@@ -98,7 +98,7 @@ export function stampRuleMap(
   // up as `ok: true` with bytes missing, never as an error.
   //
   // Read from the ENTRY ROW, not from one program-wide slot: `encodeRule` wraps each
-  // rule's entry in `OP_SCOPE <triviaSlot> <body>` iff THAT rule has ambient trivia,
+  // rule's entry in `OP_SCOPE_PLAIN <triviaSlot> <body>` iff THAT rule has ambient trivia,
   // and a `composeLeaf` grammar's pieces do not agree (the same disagreement
   // `scanSkipOf` exists for). The owning driver passes its resolved table, so
   // this is the exact trivia array it already built.

package/src/types.ts

@@ -82,7 +82,7 @@ export type ParserDef =
   | { tag: 'leaf';      parser: Combinator<unknown>; fn: (v: unknown, span: { start: number; end: number }) => unknown; fnSrc?: string }
   | { tag: 'label';     label: string; parser: Combinator<unknown> }
   | { tag: 'field';     name: string; parser: Combinator<unknown> }
-  | { tag: 'grammar';   parser: Combinator<unknown>; triviaParser: Combinator<unknown> | undefined; clearTrivia?: boolean; captureTrivia?: boolean; rootCapture?: 'opaque'; trackLines: boolean }
+  | { tag: 'grammar';   parser: Combinator<unknown>; triviaParser: Combinator<unknown> | undefined; clearTrivia?: boolean; captureTrivia?: boolean; rootCapture?: 'opaque'; trackLines: boolean; constructionTrackLines?: 'on' | 'off' | 'inherit'; constructionCaptureTriviaKinds?: readonly string[] }
   | { tag: 'lazy';     thunk: () => Combinator<unknown> }
   | { tag: 'not';      parser: Combinator<unknown> }
   // Positive lookahead. Zero-width like `not`, but — unlike `not` — it KNOWS what

package/src/version.ts

@@ -13,4 +13,4 @@
  * Kept in sync with `package.json` by `test/unit/version-sync.test.ts`, which fails
  * the build if this constant drifts from the published version.
  */
-export const PARSEMAN_VERSION = '0.47.1'
+export const PARSEMAN_VERSION = '0.48.0'