parseman 0.53.0 → 0.55.0

package/src/table/emit.ts

@@ -352,12 +352,16 @@ export type ExpressionEmitOptions = EmitOptions & {
    *
    * `null` is what a `rules()` call site needs: its initialiser evaluates to the
    * `{ name: fn, … }` object, exactly what `tableRules(…)` already returns, so
-   * the map form is the expression WITHOUT the trailing index — not a second
+   * the map form is the expression WITHOUT the entry argument — not a second
    * emitter. `compileRuleMap`'s replacement has the same shape and the same
    * splice point, which is what makes the two lowerings interchangeable there.
    */
   readonly entry?: string | null
-  /** Static metadata object passed to `tableRules` as its second argument. */
+  /**
+   * Static metadata object passed to `tableRules` as its second argument. Only
+   * the rule-map form (`entry: null`) takes it; with an entry, that argument is
+   * the entry's name.
+   */
   readonly metadataSource?: string
 }
 
@@ -379,11 +383,18 @@ export function emitTableExpression(prog: TableProgram, opts: ExpressionEmitOpti
   const entry = opts.entry === undefined ? 'grammar' : opts.entry
   const ref = opts.runtimeRef ?? 'tableRules'
   const fns = opts.fnSources ?? prog.fns.map(() => '() => {}')
-  const close = `}${opts.metadataSource === undefined ? '' : `,${opts.metadataSource}`})`
+  // The entry is an ARGUMENT, never a trailing `[entry]`: `@__PURE__` covers the
+  // call and not a property read after it, so `tableRules({…})["Entry"]` left an
+  // unreferenced terminal's whole table in every bundle.
+  const meta = opts.metadataSource
+  if (entry !== null && meta !== undefined) {
+    throw new Error('emitTableExpression: metadataSource is for the rule-map form (entry: null); a single entry carries none')
+  }
+  const tail = entry !== null ? `,${jsString(entry)}` : meta === undefined ? '' : `,${meta}`
   return [
     `/* @__PURE__ */ ${ref}({`,
     ...programFields(prog, fns, opts),
-    entry === null ? close : `${close}[${jsString(entry)}]`,
+    `}${tail})`,
   ].join('\n')
 }
 

package/src/table/encode.ts

@@ -7,6 +7,7 @@ import { deriveExpected } from '../combinators/expect.ts'
 import { assertionFailureExpected, directTerminalFailureExpected } from '../combinators/expected.ts'
 import { buildReadsRaw, buildReadsState, buildReadsTrivia } from '../compiler/build-arity.ts'
 import { buildReadsFields, parserHasOwnFields } from '../compiler/fields.ts'
+import { FIELD_LEADING_TRIVIA, FIELD_TRAILING_TRIVIA } from '../combinators/field-trivia.ts'
 import { asciiFoldKey, branchUsesRouted, parserUsesRouted } from '../combinators/dispatch.ts'
 import {
   OP_CHOICE, OP_EMPTY, OP_GATE, OP_LEAF, OP_SOURCE_LEAF, OP_LIT, OP_NODE, OP_NOT, OP_OPT,
@@ -555,6 +556,17 @@ class Encoder {
     return i
   }
 
+  private fieldSpecIndex = new Map<string, number>()
+  private fieldSpecSlot(name: string, bits: number): number {
+    const key = JSON.stringify([name, bits])
+    const hit = this.fieldSpecIndex.get(key)
+    if (hit !== undefined) return hit
+    const i = this.k.length
+    this.k.push([name, bits])
+    this.fieldSpecIndex.set(key, i)
+    return i
+  }
+
   /**
    * A `notAdjacent({ kinds })` filter, pooled by its CONTENTS.
    *
@@ -1518,8 +1530,12 @@ class Encoder {
         for (let i = 0; i < arms.length; i++) this.code[head + 6 + i] = arms[i]!
         return head
       }
-      case 'field':
-        return this.emit(OP_FIELD, this.constant(d.name), this.node(d.parser).ip)
+      case 'field': {
+        // A trivia-attributing field carries `[name, bits]` in the name slot
+        // (`FIELD_LEADING_TRIVIA`/`FIELD_TRAILING_TRIVIA`); a plain one the name.
+        const bits = (d.leadingTrivia === true ? FIELD_LEADING_TRIVIA : 0) | (d.trailingTrivia === true ? FIELD_TRAILING_TRIVIA : 0)
+        return this.emit(OP_FIELD, bits === 0 ? this.constant(d.name) : this.fieldSpecSlot(d.name, bits), this.node(d.parser).ip)
+      }
       case 'lazy': {
         // A named reference is not a hop: it resolves to the target's row.
         // Emitting a trampoline here cost one dispatch per reference for nothing.
@@ -1576,6 +1592,11 @@ class Encoder {
           this.expected([adjacencyExpected(d)]),
         )
       case 'withCtx': {
+        // The DERIVE form is an author callback, parked in `fns` like a gate
+        // predicate and told apart from a const-pool index by its sign.
+        if (typeof d.extra === 'function') {
+          return this.emit(OP_WITHCTX, ~this.fn(d.extra, d.extraSrc ?? null), this.node(d.parser).ip)
+        }
         // `extra` is arbitrary user data in the const pool. `emitConst` takes
         // scalars, arrays of scalars, and plain objects of those — anything
         // richer (a class instance, a function value) RUNS fine but cannot be

package/src/table/exec.ts

@@ -72,6 +72,8 @@ import { scalarTerminalNotChild } from './scalar-terminal.ts'
  * a successful match to the other engine.
  */
 import { FAIL, newEndCell, type EndCell } from './cell.ts'
+import { fieldSpec, leadingTriviaFieldAt } from './field-spec.ts'
+import { FIELD_LEADING_TRIVIA, FIELD_TRAILING_TRIVIA, LEAD, triviaFieldCapture } from '../combinators/field-trivia.ts'
 
 /**
  * DIAGNOSTIC ROW COUNTER — off unless `PM_TABLE_COUNT=1` at process start.
@@ -645,11 +647,21 @@ function makeDriver(
       }
 
       case OP_FIELD: {
+        const spec = fieldSpec(k[code[ip + 1]!])
+        const leadFrom = LEAD.at
+        if (spec.bits !== 0) LEAD.at = -1
         const v = exec(code[ip + 2]!, input, pos, ctx)
         if (v === FAIL) return FAIL
         // Conditional on a live sink, exactly as src/combinators/map.ts has it:
         // a `field()` outside any field-reading node costs nothing.
-        ctx._fields?.push({ name: k[code[ip + 1]!] as string, value: v, span: { start: pos, end: EC.e } })
+        if (spec.bits === 0) {
+          ctx._fields?.push({ name: spec.name, value: v, span: { start: pos, end: EC.e } })
+          return v
+        }
+        const end = EC.e
+        const trailEnd = (spec.bits & FIELD_TRAILING_TRIVIA) !== 0 && ctx.trivia !== undefined ? consumeTrivia(input, end, ctx) : end
+        ctx._fields?.push(triviaFieldCapture(spec.name, v, input, pos, end, (spec.bits & FIELD_LEADING_TRIVIA) !== 0 ? leadFrom : -1, trailEnd))
+        EC.e = trailEnd
         return v
       }
 
@@ -791,7 +803,8 @@ function makeDriver(
 
       case OP_WITHCTX: {
         const saved = ctx.state
-        ctx.state = k[code[ip + 1]!]
+        const ki = code[ip + 1]!
+        ctx.state = ki >= 0 ? k[ki] : (fns[~ki] as (s: unknown) => unknown)(saved)
         try { return exec(code[ip + 2]!, input, pos, ctx) }
         finally { ctx.state = saved }
       }
@@ -885,6 +898,7 @@ function makeDriver(
             const scanTl = need ? cstTlLen(ctx) : 0
             const scanLog = need ? ctx._triviaLog?.length ?? 0 : 0
             const scanRoot = need ? ctx._rootTriviaLog?.length ?? 0 : 0
+            if (leadingTriviaFieldAt(code, k, child)) LEAD.at = cur
             const v = exec(child, input, scanEnd, ctx)
             if (v === FAIL) { if (REC) ctx._sync = inheritedSync; return FAIL }
             if (EC.e > scanEnd) cur = EC.e
@@ -1256,6 +1270,7 @@ function makeDriver(
             let sp = cur
             if (hasTrivia) sp = skipTrivia(input, sp, ctx)
             ctx._fc = false
+            if (leadingTriviaFieldAt(code, k, sep)) LEAD.at = cur
             const sv = exec(sep, input, sp, ctx)
             if (sv === FAIL) {
               if (needMark) rollbackTriviaAt(ctx, mRaw, mTl, mLv, mFl, mEr, mLog, mRoot)

package/src/table/field-spec.ts

@@ -0,0 +1,17 @@
+import { OP_FIELD } from './ops.ts'
+import { FIELD_LEADING_TRIVIA } from '../combinators/field-trivia.ts'
+
+/**
+ * An `OP_FIELD` name slot: the name, or `[name, bits]` for a field that
+ * attributes trivia (`FIELD_LEADING_TRIVIA` / `FIELD_TRAILING_TRIVIA`). A
+ * program printed before those options existed carries only names, and reads
+ * as bits 0.
+ */
+export function fieldSpec(slot: unknown): { readonly name: string; readonly bits: number } {
+  return Array.isArray(slot) ? { name: slot[0] as string, bits: slot[1] as number } : { name: slot as string, bits: 0 }
+}
+
+/** Is the instruction at `ip` a `field(…, { leadingTrivia: true })`? */
+export function leadingTriviaFieldAt(code: ArrayLike<number>, k: readonly unknown[], ip: number): boolean {
+  return code[ip] === OP_FIELD && (fieldSpec(k[code[ip + 1]!]).bits & FIELD_LEADING_TRIVIA) !== 0
+}

package/src/table/ops.ts

@@ -217,7 +217,9 @@ export const OP_TOKEN = 29
  */
 export const OP_SCOPE_CAP = 30
 /**
- * `WITHCTX k c` — `withCtx(extra, c)`. `k` is `extra` in the const pool.
+ * `WITHCTX k c` — `withCtx(extra, c)`. `k >= 0` is `extra` in the const pool
+ * (the replace form); `k < 0` is the derive form, `~k` indexing the author's
+ * `(state) => state` in `prog.fns`, called once per entry with the outer state.
  *
  * SAVE / RESTORE, matching `withCtx.ts`. Both used to CLONE the context, which
  * scoped far more than the state: every scalar the child wrote on `ctx` landed

package/src/table/scalar-terminal.ts

@@ -261,3 +261,28 @@ export function leadingLiteralFamily(
 
   return walk(ip, 0)
 }
+
+/**
+ * The literal terminals an arm of a LARGE ordered choice (more than three arms)
+ * may be pretested on before it is entered, or `undefined` when it has none.
+ *
+ * A pretest runs again when the arm then matches, so a large choice only takes
+ * literals of two or more characters: one `startsWith` is the price of skipping an
+ * arm that first-char dispatch could not exclude (`@{` and `@@` beside `@name`).
+ * A one-character literal is exactly what the dispatch mask already decided.
+ * The emitted and the closure engines both select through this function.
+ */
+export function largeChoiceLiteralPretest(
+  code: ArrayLike<number>, constants: readonly unknown[], armIp: number,
+): readonly number[] | undefined {
+  const child = leadingScalarTerminal(code, armIp, 2, true, true)
+  if (child >= 0) {
+    const spec = constants[code[child + 1]!]
+    if (code[child] === OP_LIT && typeof spec === 'string' && spec.length >= 2) return [child]
+  }
+  const family = leadingLiteralFamily(code, constants, armIp)
+  return family !== undefined && family.length >= 2 && family.every(terminal => {
+    const value = constants[code[terminal + 1]!]
+    return typeof value === 'string' && value.length >= 2
+  }) ? family : undefined
+}

package/src/types.ts

@@ -83,7 +83,7 @@ export type ParserDef =
   | { tag: 'routed';    fallback?: Combinator<unknown> }
   | { tag: 'leaf';      parser: Combinator<unknown>; fn: (v: unknown, span: { start: number; end: number }) => unknown; fnSrc?: string; cstValue?: 'source' }
   | { tag: 'label';     label: string; parser: Combinator<unknown> }
-  | { tag: 'field';     name: string; parser: Combinator<unknown> }
+  | { tag: 'field';     name: string; parser: Combinator<unknown>; leadingTrivia?: true; trailingTrivia?: true }
   | { tag: 'grammar';   parser: Combinator<unknown>; triviaParser: Combinator<unknown> | undefined; clearTrivia?: boolean; captureTrivia?: boolean; rootCapture?: 'opaque'; trackLines: boolean; constructionTrackLines?: 'on' | 'off' | 'inherit'; constructionCaptureTriviaKinds?: readonly string[] }
   | { tag: 'lazy';     thunk: () => Combinator<unknown> }
   | { tag: 'not';      parser: Combinator<unknown> }
@@ -102,6 +102,8 @@ export type ParserDef =
   // asserts that something did. `kinds` (notAdjacent only) narrows the assertion to
   // trivia CATEGORIES from `classifiedTrivia({...})`. See combinators/adjacency.ts.
   | { tag: 'adjacency'; polarity: 'adjacent' | 'notAdjacent'; kinds?: readonly string[] }
+  // `extra` is the subtree's state (replace form) or, when it is a function, the
+  // `(state) => state` that derives it from the state in force at entry.
   | { tag: 'withCtx';  extra: unknown; parser: Combinator<unknown>; extraSrc?: string }
   | { tag: 'recover';  parser: Combinator<unknown>; sentinel: Combinator<unknown> }
   | { tag: 'expect';   parser: Combinator<unknown>; label: string | undefined; expected: string[] }
@@ -203,6 +205,16 @@ export type BuildHost = ((
 export type FieldCapture<T = unknown> = {
   value: T
   span: Span
+  /**
+   * `field(…, { leadingTrivia: true })` only: the source text of the trivia the
+   * enclosing sequence or separated list skipped right before this field, or `''`.
+   */
+  leading?: string
+  /**
+   * `field(…, { trailingTrivia: true })` only: the source text of the trivia the
+   * field consumed after its match, or `''`.
+   */
+  trailing?: string
 }
 
 export type FieldMap = Record<string, FieldCapture | FieldCapture[]>
@@ -382,7 +394,7 @@ export type ParseContext = {
    */
   _cstTriviaLog?: number[] | undefined
   /** Framework-internal: active node() field captures, enabled only when needed. */
-  _fields?: Array<{ name: string; value: unknown; span: Span }> | undefined
+  _fields?: Array<{ name: string; value: unknown; span: Span; leading?: string; trailing?: string }> | undefined
   /** Framework-internal: lazy capture buffer for active node() parse. */
   _cstBuf?: CstCaptureBuf | undefined
   /** Framework-internal: value/span already consumed by an enclosing dispatch(). */

package/src/version.ts

@@ -13,4 +13,4 @@
  * Kept in sync with `package.json` by `test/unit/version-sync.test.ts`, which fails
  * the build if this constant drifts from the published version.
  */
-export const PARSEMAN_VERSION = '0.53.0'
+export const PARSEMAN_VERSION = '0.55.0'