parseman 0.55.4 → 0.57.0

package/src/plugin/index.ts

@@ -26,7 +26,7 @@ import MagicString from 'magic-string'
 import { evaluateExpr, evaluateCombinatorArray, evaluateParserFactory, evaluateStaticValue, evaluateWordFactory, evaluateWhenFactory, evaluateRefDeclaration, applyDefineStatement, referencesAny, setReducerResolver, setBuilderImportResolver, propName, ModuleScope, type FreeNameResolution, type Scope, type ScopeEntry } from './evaluator.ts'
 import { classifyRuleMap } from '../analysis/commitment.ts'
 import { compile } from '../table/compile.ts'
-import { compileRuleMap } from '../table/compile-rule-map.ts'
+import { compileRuleMap, type CompiledRuleMapTable } from '../table/compile-rule-map.ts'
 import { compileLinkableTable } from '../compiler/compile-linkable-table.ts'
 import { createReducerResolver, topLevelIsStable } from './reducer-resolver.ts'
 import { findFreeIdentifiers } from './free-identifiers.ts'
@@ -38,7 +38,9 @@ import {
 import type { HostMode } from '../cst/host-mode.ts'
 import { COMPOSED_PIECES } from '../compiler/linker.ts'
 import { serializeCombinator, serializeRuleMap } from '../compiler/ir-serialize.ts'
-import { evalCombinatorIR, evalRulesIR } from './ir-eval.ts'
+import { evalCombinatorIR, evalRulesIR, INERT_BUILDER } from './ir-eval.ts'
+import { countSiteVisits } from '../table/assemble.ts'
+import { run } from '../functional/run.ts'
 import { linkRules, RULES_RECIPE, type LinkOptions, type RulesRecipe } from '../combinators/parser.ts'
 import { PARSEMAN_VERSION } from '../version.ts'
 import { grammarReflectionSource, type GrammarReflection } from '../cst/reflection.ts'
@@ -87,10 +89,59 @@ export type ParsecraftPluginOptions = {
   /** Emit grammar-coverage hooks in macro output. Off by default, so ordinary
    * macro output remains byte-identical. */
   grammarCoverage?: boolean
+  /**
+   * OPTIONAL TUNING for selective assemblies. **Off by default.**
+   *
+   * A grammar gets generated code only for its hottest sites, within a byte budget.
+   * Without this option the build ranks sites by a static estimate. With it, the
+   * build parses `files` from rule `entry` on the closure engine (nothing is
+   * generated to measure) and ranks sites by measured calls per byte instead. It
+   * changes which sites are generated, never what a parse returns.
+   */
+  assemblyProfile?: AssemblyProfile
+  /**
+   * Generated assembly bytes each module may spend, as a multiple of its own source
+   * length. **Default `0.25`.** `0` ships no assembly. The right number is
+   * per grammar: a larger one parses faster in steady state and costs import and
+   * first-parse time, so measure both, and keep the artifact within 5× its source
+   * (docs/design/runtime-and-size-contract.md, rules 4 and 6).
+   */
+  assemblyBudget?: number
+}
+
+/** A corpus a build parses to rank selective-assembly sites — see `assemblyProfile`. */
+export type AssemblyProfile = {
+  /** The rule each corpus file is parsed from, e.g. `'Stylesheet'`. A grammar
+   * without this rule keeps the static ranking. */
+  readonly entry: string
+  /** Paths of representative inputs. */
+  readonly files: readonly string[]
+}
+
+/** Corpus file contents, re-read when a file's mtime moves, so a watch build that
+ * re-lowers after a corpus edit ranks by the edited corpus. */
+const corpusCache = new Map<string, { readonly mtimeMs: number; readonly text: string }>()
+function corpusTexts(profile: AssemblyProfile): readonly string[] {
+  return profile.files.map(file => {
+    const mtimeMs = fs.statSync(file).mtimeMs
+    const hit = corpusCache.get(file)
+    if (hit !== undefined && hit.mtimeMs === mtimeMs) return hit.text
+    const text = fs.readFileSync(file, 'utf8')
+    corpusCache.set(file, { mtimeMs, text })
+    return text
+  })
 }
 
 const PARSEMAN_MODULE = 'parseman'
 
+/**
+ * The default `assemblyBudget`: generated assembly bytes a module may spend, as a
+ * multiple of its own source length. A quarter of the source keeps every jess
+ * grammar far under the 5x ceiling (contract rule 4); the 0.57.0 CHANGELOG has the
+ * measured curve.
+ */
+const DEFAULT_ASSEMBLY_BUDGET = 0.25
+
 /**
  * The REASON a lowering refused, appended to the warning that reports it.
  *
@@ -280,11 +331,14 @@ export default createUnplugin((opts: ParsecraftPluginOptions = {}) => ({
     return /\.[jt]sx?$/.test(id) && !id.includes('node_modules')
   },
 
-  transform(this: { warn?: (msg: string) => void }, code: string, id: string) {
+  transform(this: { warn?: (msg: string) => void; addWatchFile?: (file: string) => void }, code: string, id: string) {
     if (!code.includes('parseman')) return null
     if (!code.includes('macro')) return null
+    // A corpus edit changes which sites a profiled build generates, so a watch
+    // build re-lowers when one does.
+    for (const file of opts.assemblyProfile?.files ?? []) this?.addWatchFile?.(file)
     const moduleAliases = new Set([PARSEMAN_MODULE, ...(opts.moduleAliases ?? [])])
-    const result = transformMacro(code, id, moduleAliases, opts.warnUnloweredRegex === true, opts.recovery === true, opts.grammarCoverage === true)
+    const result = transformMacro(code, id, moduleAliases, opts.warnUnloweredRegex === true, opts.recovery === true, opts.grammarCoverage === true, opts.assemblyProfile, opts.assemblyBudget)
     if (result?.warnings.length) {
       for (const w of result.warnings) {
         if (typeof this?.warn === 'function') this.warn(`[parseman] ${w}`)
@@ -544,10 +598,12 @@ export function transformMacro(
   warnUnloweredRegex = false,
   recovery = false,
   grammarCoverage = false,
+  assemblyProfile?: AssemblyProfile,
+  assemblyBudget = DEFAULT_ASSEMBLY_BUDGET,
 ): TransformMacroResult | null {
   const depth = degradationCaptureDepth()
   try {
-    return transformMacroImpl(code, id, moduleAliases, warnUnloweredRegex, recovery, grammarCoverage)
+    return transformMacroImpl(code, id, moduleAliases, warnUnloweredRegex, recovery, grammarCoverage, assemblyProfile, assemblyBudget)
   } finally {
     setReducerResolver(null)
     setBuilderImportResolver(null)
@@ -564,6 +620,8 @@ function transformMacroImpl(
   warnUnloweredRegex: boolean,
   recovery: boolean,
   grammarCoverage: boolean,
+  assemblyProfile: AssemblyProfile | undefined,
+  assemblyBudgetShare: number,
 ): TransformMacroResult | null {
   let result: ReturnType<typeof parseSync>
   try {
@@ -1984,9 +2042,10 @@ function transformMacroImpl(
           carried,
           reflection: compiled.reflection,
         })
+        const select = selectAssembly(init.start, 'compose()', compiled, !declaresLeaf())
         return {
-          replacement: compiled.replacementWithMetadata(reflectionMetadata),
-          exportedReplacement: compiled.replacementWithMetadata(exportedMetadata),
+          replacement: assembledReplacement(compiled, reflectionMetadata, select),
+          exportedReplacement: assembledReplacement(compiled, exportedMetadata, select),
           carried,
           ...(composing ? { trivia: composing } : {}),
           ...(importedFactories.length ? { importedFactories } : {}),
@@ -2006,6 +2065,74 @@ function transformMacroImpl(
     return null
   }
 
+  /**
+   * SELECTIVE ASSEMBLY, BUDGETED PER MODULE. A fused grammar's default AST/no-lines
+   * table carries generated code for its hottest sites only (`select-hot.ts`); the
+   * closure engine runs the rest. The site bodies of every fused grammar in this
+   * module together may not exceed `assemblyBudget` times the module's own source.
+   *
+   * A module that declares a `composeLeaf` has named its terminal parser, so its
+   * `compose()` results are composable PIECES: another build re-lowers them and
+   * never parses through their table, so they spend nothing. Without a
+   * `composeLeaf`, a `compose()` result is the module's parser.
+   */
+  let assemblyBudget = Math.floor(code.length * assemblyBudgetShare)
+  let leafDeclared: boolean | undefined
+  const declaresLeaf = (): boolean => leafDeclared ??= (body as Statement[]).some(stmt => {
+    const vd = unwrapVd(stmt)
+    return vd !== null && vd.declarations.some(d => d.init != null && isComposeLeafCall(d.init as Expression))
+  })
+  const selectAssembly = (
+    pos: number,
+    label: string,
+    compiled: CompiledRuleMapTable,
+    terminal: boolean,
+  ): ReadonlySet<number> => {
+    // A coverage build instruments the table; its counters are not the shipping
+    // parser, so it carries no assembly.
+    if (!terminal || grammarCoverage) return new Set()
+    const { sites, bytes, refusal } = compiled.selectAssembly(assemblyBudget, profileOf(compiled))
+    // The artifact still parses, on the closure engine, but a build that would
+    // have generated code says which construct stopped it rather than fall back
+    // silently. Not `warn()`: every declaration still lowers, so the module stays
+    // fully resolved.
+    if (refusal !== undefined) {
+      warnings.push(`${id}:${lineOf(pos)} — ${label}: its precompiled assembly cannot emit ${refusal}; the artifact runs on the closure engine`)
+    }
+    assemblyBudget -= bytes
+    return sites
+  }
+  /** Measured site calls over the optional corpus, or `undefined` for the static
+   * ranking. A file whose parse throws still contributes the calls it made. */
+  const profileOf = (compiled: CompiledRuleMapTable): Int32Array | undefined => {
+    // Only the default AST/no-lines table carries an assembly, so only it is profiled.
+    if (assemblyProfile === undefined || compiled.hostMode !== 'ast' || compiled.prog.lines === 1
+      || !(assemblyProfile.entry in compiled.prog.rules)) return undefined
+    let parsed = 0
+    // A builder re-lowered from carried IR is inert at build time; for COUNTING
+    // only, a plain node stands in for it, so the corpus can run past it.
+    const fns = compiled.prog.fns.map(fn => {
+      const type = (fn as { [INERT_BUILDER]?: string } | null)?.[INERT_BUILDER]
+      return type === undefined ? fn
+        : (children: unknown, _fields: unknown, span: unknown) => ({ type, span, children })
+    })
+    const counts = countSiteVisits(compiled.prog, rules => {
+      const entry = rules[assemblyProfile.entry]!
+      for (const text of corpusTexts(assemblyProfile)) {
+        try { if (run(entry, text).ok) parsed++ } catch { /* the calls it made still count */ }
+      }
+    }, fns)
+    if (parsed > 0) return counts
+    // Not `warn()`: nothing fell back to the interpreter — the build is only untuned.
+    warnings.push(`${id} — assemblyProfile: no corpus file parsed from rule "${assemblyProfile.entry}"; using the static ranking`)
+    return undefined
+  }
+  /** The fused table's replacement, with its selective default assembly if any. */
+  const assembledReplacement = (compiled: CompiledRuleMapTable, metadata: string, select: ReadonlySet<number>): string =>
+    select.size === 0
+      ? compiled.replacementWithMetadata(metadata)
+      : supercompileRuleMapReplacement(compiled.prog, compiled.replacementWithMetadata(metadata, { select }), select)
+
   /**
    * Compile a terminal composition. Imported/base pieces still travel as normal
    * re-lowerable IR, but the final local rules map is compiled directly in this
@@ -2136,12 +2263,10 @@ function transformMacroImpl(
         if (compiled !== null) {
           usedTableRuntime = true
           return {
-            replacement: supercompileRuleMapReplacement(
-              compiled.prog,
-              compiled.replacementWithMetadata(
-                staticTableMetadataSource({ reflection: compiled.reflection, leaf: true }),
-                { precompileDefault: true },
-              ),
+            replacement: assembledReplacement(
+              compiled,
+              staticTableMetadataSource({ reflection: compiled.reflection, leaf: true }),
+              selectAssembly(init.start, 'composeLeaf()', compiled, true),
             ),
             ...(importedFactories.length ? { importedFactories } : {}),
           }

package/src/plugin/ir-eval.ts

@@ -2,6 +2,7 @@
  * The macro plugin's IR evaluator: turns a composed grammar's carried IR back
  * into combinators so the build can re-lower it. Build time only — see below.
  */
+import { columns, lines } from '../combinators/lines.ts'
 import type { Combinator } from '../types.ts'
 import { rules } from '../combinators/parser.ts'
 import { ref } from '../combinators/ref.ts'
@@ -26,6 +27,9 @@ import { adjacent, notAdjacent } from '../combinators/adjacency.ts'
 
 type Comb = Combinator<unknown>
 
+/** Marks a carried direct builder that cannot run at build time; its value is the node type. */
+export const INERT_BUILDER: unique symbol = Symbol('parseman.inertBuilder')
+
 /** Reconstruct a rule map from serialized IR (the inverse of `serializeRuleMap`) —
  * evaluate the combinator-construction expression with every constructor in scope.
  *
@@ -87,7 +91,10 @@ function evalIR(ir: string, rulesImpl: unknown): unknown {
     // carries the callback source. The compiler is the only consumer that may
     // materialize `buildSrc`; raw IR interpretation deliberately rejects direct
     // builders rather than evaluating arbitrary captured source at runtime.
-    const n = node(type, child as never, (() => { throw new Error('IR node build requires static re-lowering') }) as never, opts as never)
+    const inert = (() => { throw new Error('IR node build requires static re-lowering') }) as (() => never) & { [INERT_BUILDER]?: string }
+    // Named, so a build-time profile run can stand a plain node in for it.
+    inert[INERT_BUILDER] = type
+    const n = node(type, child as never, inert as never, opts as never)
     ;(n._def as { buildSrc?: string; buildStaticError?: readonly string[] }).buildSrc = src
     if (staticError !== undefined) (n._def as { buildStaticError?: readonly string[] }).buildStaticError = staticError
     // Analysis-only resolved reducer signature (see `buildAnalysisSrc`). Without it a
@@ -155,11 +162,13 @@ function evalIR(ir: string, rulesImpl: unknown): unknown {
     'rules', 'ref', 'regex', 'literal', 'keywords', 'sequence', 'choice', 'dispatch', 'when', 'startsWith', 'endsWith', 'matches', 'otherwise', 'routed', 'attempt',
     'many', 'oneOrMore', 'optional', 'sepBy', 'keepSeparator', 'not', 'peek', 'node', 'parser',
     'scanTo', 'balanced', 'token', 'leaf', 'transform', 'trivia', 'classifiedTrivia', 'label', 'field', 'expect', 'adjacent', 'notAdjacent', '_tf', '_lf', '_nd', '_gch', '_wc',
+    'lines', 'columns',
     `return (${ir})`,
   )
   return fn(
     rulesImpl, ref, regex, literal, keywords, sequence, choice, dispatch, when, startsWith, endsWith, matches, otherwise, routed, attempt,
     many, oneOrMore, optional, sepBy, keepSeparator, not, peek, node, parser,
     scanTo, balanced, token, leaf, transform, trivia, classifiedTrivia, label, field, expectC, adjacent, notAdjacent, _tf, _lf, _nd, _gch, _wc,
+    lines, columns,
   )
 }

package/src/regex/classes.ts

@@ -63,6 +63,9 @@ export function literalCodePoints(frag: string): number[] | null {
     if (ch === '\\') {
       const e = frag[i + 1]
       if (e === undefined) return null
+      // Any other word-character escape is a class (`\d`), an assertion (`\b`), a
+      // backreference or a hex, unicode or control escape, never the character itself.
+      if (!(e in CLASS_ESCAPES) && /\w/.test(e)) return null
       out.push(e in CLASS_ESCAPES ? CLASS_ESCAPES[e]! : e.codePointAt(0)!)
       i += 2
       continue

package/src/spec/model.ts

@@ -477,6 +477,21 @@ class Builder {
         return this.walk(def.parser)
       case 'scanTo':
         return { kind: 'annotation', text: '…' }
+      // A run of lines: each line opens with the prefix, the first with `first`
+      // when given, and the body reads the lines' contents.
+      case 'lines':
+        return {
+          kind: 'seq',
+          items: [
+            { kind: 'annotation', text: 'each line begins with' },
+            this.walk(def.prefix),
+            ...(def.first === undefined ? [] : [{ kind: 'annotation', text: 'the first with' } as SpecNode, this.walk(def.first)]),
+            ...(def.lazy === undefined ? [] : [{ kind: 'annotation', text: 'or continues where' } as SpecNode, this.walk(def.lazy)]),
+            this.walk(def.body),
+          ],
+        }
+      case 'columns':
+        return { kind: 'annotation', text: `${def.n} columns` }
 
       // Trivia — elided by default; included as an annotation when asked.
       case 'trivia':

package/src/table/assemble.ts

@@ -63,10 +63,13 @@
  * Semantics are `exec.ts`'s, case for case. That file remains the reference and
  * the three-way identity sweep gates this against it.
  */
+import { rebuildLineScan } from './line-scan.ts'
+import { assemblyExtraIps } from './program.ts'
 import type { Combinator, FieldMap, FirstSet, ParseContext, ParseResult, ParserDef } from '../types.ts'
 import { balanced, scanTo } from '../combinators/scanTo.ts'
 import { buildFieldMap } from '../compiler/fields.ts'
 import { FIELD_LEADING_TRIVIA, FIELD_TRAILING_TRIVIA, LEAD, triviaFieldCapture } from '../combinators/field-trivia.ts'
+import { skipDeferredTrivia } from './trivia-gap.ts'
 import { asciiFoldKey } from '../combinators/dispatch.ts'
 import { projectChild, unwrapChild } from '../combinators/node.ts'
 import { cstOutputHost } from '../compiler/build-arity.ts'
@@ -120,7 +123,7 @@ import { lead, rawEntry, spanLines } from './run-support.ts'
  * property read per terminal miss — the same price codegen pays.
  */
 import {
-  classHas, decodeClassSpec, expandCompact, resolveTable,
+  classHas, decodeClassSpec, expandCompact, ownTableProgram, resolveTable,
   validateDispatchSpec,
   type CompactProgram, type ResolvedClass, type ResolvedTable,
   type SubtreeRef, type TableProgram, type TableRule,
@@ -475,6 +478,15 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
   // Value/CST/failure/cursor effects stay in those consumers.
   const scalarRecognizers: Array<ScalarRecognizer | undefined> = []
   const rawScalarSpecs = new Set<number>()
+  /**
+   * The class of an `optional`/`not`/`peek` child's first-character gate
+   * (`encode.ts` `lookaheadGate`), tested before the wrapper takes its mark.
+   * `null` where there is none, or where a probe or tolerant parse forwards
+   * through the gate (`OP_GATE` below).
+   */
+  function lookaheadClass(child: number): ResolvedClass | null {
+    return code[child] === OP_GATE && !cfg.tolerant && !cfg.probe ? cc[code[child + 1]!]! : null
+  }
   function scalarFor(child: number): ScalarRecognizer | undefined {
     const spec = code[child + 1]!
     let recognize = scalarRecognizers[spec]
@@ -485,14 +497,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
     return recognize
   }
 
-  const labelExtraIps: number[] = []
-  for (const s of prog.scans ?? []) {
-    for (const r of s.skip) labelExtraIps.push(r[0])
-    if (s.sentinel !== undefined) labelExtraIps.push(s.sentinel[0])
-    if (s.recoverAt !== undefined) labelExtraIps.push(s.recoverAt[0])
-    if (s.stopAt !== undefined) labelExtraIps.push(s.stopAt[0])
-  }
-  for (const set of prog.scanSkip ?? []) for (const r of set) labelExtraIps.push(r[0])
+  const labelExtraIps = assemblyExtraIps(prog)
   const closureLabels = computeSiteLabels(
     code, [...Object.values(prog.rules), ...labelExtraIps], hostCst,
   )
@@ -507,6 +512,23 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
    */
   let SCAN: FastTriviaScanner | null = null
 
+  /**
+   * A SELECTIVE build-time assembly: emitted bodies for its selected sites, this
+   * engine for the rest. `link` hands out its exported sites, wrapped so the
+   * callee sees this engine's installed scanner — the one slot both engines keep
+   * separately (the end cell `EC` is already shared).
+   */
+  let HYBRID: EmittedAssembly | undefined
+  function hybridPiece(piece: Piece): Piece {
+    const setScan = HYBRID!.scan!
+    return (input, pos, ctx) => {
+      const saved = setScan(SCAN)
+      const value = piece(input, pos, ctx)
+      setScan(saved)
+      return value
+    }
+  }
+
   const memo = new Map<number, Piece>()
   const reached = new Set<number>()
 
@@ -909,9 +931,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
     // Appended rather than folded into the test above so the non-capturing path
     // keeps the exact branch it had.
     if (s !== null) return skipTriviaScanned(s, input, cur, ctx)
-    if (needsDeferredTriviaCommit(ctx)) {
-      return commitTriviaScan(scanTriviaCompact(input, cur, ctx))
-    }
+    if (needsDeferredTriviaCommit(ctx)) return skipDeferredTrivia(input, cur, ctx)
     return advanceTrivia(input, cur, ctx)
   }
 
@@ -1162,7 +1182,9 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
       if (next !== undefined) {
         return next(input, pos, ctx, bits, need, mRaw, mTl, mLv, mFl, mEr, mLog, mRoot, acc, prev, best)
       }
-      if (best === pos) acc = through(total, prev, acc)
+      // No arm entered (`prev` 0, `acc` unset): the catch-up would rebuild
+      // `choiceFx`, the arms' flatMap, set by set. It IS the answer.
+      if (best === pos && prev !== 0) acc = through(total, prev, acc)
       ctx._fe = pos; ctx._fx = acc ?? choiceFx
       return FAIL
     }
@@ -1231,7 +1253,9 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
       if (next !== undefined) {
         return next(input, pos, ctx, c, need, mRaw, mTl, mLv, mFl, mEr, mLog, mRoot, acc, prev, best)
       }
-      if (best === pos) acc = through(total, prev, acc)
+      // No arm entered (`prev` 0, `acc` unset): the catch-up would rebuild
+      // `choiceFx`, the arms' flatMap, set by set. It IS the answer.
+      if (best === pos && prev !== 0) acc = through(total, prev, acc)
       ctx._fe = pos; ctx._fx = acc ?? choiceFx
       return FAIL
     }
@@ -1342,6 +1366,13 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
     if (done !== undefined) return done
     const flight = inFlight.get(ip)
     if (flight !== undefined) return flight.fwd
+    const selected = HYBRID?.byIp[ip]
+    if (selected !== undefined) {
+      const piece = hybridPiece(selected as Piece)
+      memo.set(ip, piece)
+      reached.add(ip)
+      return piece
+    }
 
     let target: Piece | undefined
     const fwd: Piece = (input, pos, ctx) => target!(input, pos, ctx)
@@ -1357,7 +1388,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
       && code[ip + 1]! >= 0
       && closureLabels.at(ip).tri === code[ip + 1]!
       ? link(code[ip + 2]!)
-      : lower(ip)
+      : countedPiece(lower(ip), ip)
 
     inFlight.delete(ip)
     holder.set(piece)
@@ -1999,7 +2030,9 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
         }
         const child = link(code[ip + 1]!)
         const xf = fx[code[ip + 2]!] as string[]
+        const gate = lookaheadClass(code[ip + 1]!)
         return (input, pos, ctx) => {
+          if (gate !== null && !classHas(gate, lead(input, pos))) { EC.e = pos; return null }
           const need = markCst(ctx)
           const mRaw = MRAW
           const mTl = MTL
@@ -2021,7 +2054,9 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
       case OP_PEEK: {
         const child = link(code[ip + 1]!)
         const xf = fx[code[ip + 2]!] as string[]
+        const gate = lookaheadClass(code[ip + 1]!)
         return (input, pos, ctx) => {
+          if (gate !== null && !classHas(gate, lead(input, pos))) { ctx._fe = pos; ctx._fx = xf; return FAIL }
           const need = markCst(ctx)
           const mRaw = MRAW
           const mTl = MTL
@@ -2041,7 +2076,9 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
 
       case OP_OPT: {
         const child = link(code[ip + 1]!)
+        const gate = lookaheadClass(code[ip + 1]!)
         return (input, pos, ctx) => {
+          if (gate !== null && !classHas(gate, lead(input, pos))) { EC.e = pos; return null }
           const need = markCst(ctx)
           const mRaw = MRAW
           const mTl = MTL
@@ -2918,6 +2955,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
                 if (committed(ctx)) { if (acc !== undefined) ctx._fx = acc; return FAIL }
                 if (need) rollbackTriviaAt(ctx, mRaw, mTl, mLv, mFl, mEr, mLog, mRoot)
               }
+              if (prev === 0) { ctx._fe = pos; ctx._fx = choiceFx; return FAIL }
               if (best === pos && prev < 1) acc = accSet(e0, acc)
               if (best === pos && prev < 2) acc = accSet(e1, acc)
               if (best === pos && n === 3 && prev < 3) acc = accSet(e2, acc)
@@ -3137,9 +3175,10 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
         return (input, pos, ctx) => {
           if (!pred(ctx.state)) {
             // A `continue` in the interpreter (choice.ts:150), not a failure —
-            // so no cut survives to the enclosing choice.
+            // so no cut survives to the enclosing choice, and no failure position
+            // ranks it on the ordered path. A dispatched choice reads `_fx` alone.
             ctx._fc = false
-            ctx._fe = pos
+            ctx._fe = -1
             ctx._fx = xf
             return FAIL
           }
@@ -3666,9 +3705,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
           if (v === FAIL) return FAIL
           const end = EC.e
           const span = tracked ? spanLines(ctx, pos, end) : { start: pos, end }
-          const st = readsState && ctx.state !== undefined
-            ? Object.assign({}, ctx.state as Record<string, unknown>)
-            : undefined
+          const st = readsState ? ctx.state : undefined
 
           let nd: unknown
           if (unwrap && kids.length === 1) {
@@ -3688,12 +3725,12 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
             nd = kids[0]
           } else if (proj >= 0) {
             nd = hostCst && host !== undefined
-              ? host(type, kids, fieldMap, span, rawKids, tlog, readsState ? st : ctx.state !== undefined ? Object.assign({}, ctx.state as Record<string, unknown>) : undefined, tags)
+              ? host(type, kids, fieldMap, span, rawKids, tlog, ctx.state, tags)
               : projectChild(kids, proj, type)
           } else if (build !== undefined) {
             if (hostCst && host !== undefined) {
               // A direct builder is bypassed under a CST host.
-              nd = host(type, kids, fieldMap, span, rawKids, tlog, readsState ? st : ctx.state !== undefined ? Object.assign({}, ctx.state as Record<string, unknown>) : undefined, tags)
+              nd = host(type, kids, fieldMap, span, rawKids, tlog, ctx.state, tags)
             } else {
               nd = build(kids, fieldMap, span, rawKids, captureWide ? tlog : EMPTY_TL, st)
             }
@@ -3780,14 +3817,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
     // The scan pool and the scan-skip sets are built FROM subtrees, so their
     // sites need emitted names too or `subtreeComb` would have to fall back to
     // a closure for them and the two engines would both be live in one parse.
-    const extraIps: number[] = []
-    for (const s of prog.scans ?? []) {
-      for (const r of s.skip) extraIps.push(r[0])
-      if (s.sentinel !== undefined) extraIps.push(s.sentinel[0])
-      if (s.recoverAt !== undefined) extraIps.push(s.recoverAt[0])
-      if (s.stopAt !== undefined) extraIps.push(s.stopAt[0])
-    }
-    for (const set of prog.scanSkip ?? []) for (const r of set) extraIps.push(r[0])
+    const extraIps = assemblyExtraIps(prog)
     const roots = [...Object.values(prog.rules), ...extraIps]
     for (const ip of reachableSites(code, roots)) {
       if (code[ip] === OP_DISPATCH) {
@@ -3860,9 +3890,10 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
         REC ? prog.cc.map((_, i) => sentinelFor(i)) : EMPTY_SENTS,
         matchesAt, recoverScan, orSentinel, captureError,
         scalarRecognizers, commitTriviaScan, scanTriviaCompact, t.lex, adjacencyHolds, t.lexPrograms,
-        LEAD, triviaFieldCapture,
+        LEAD, triviaFieldCapture, skipDeferredTrivia,
       )
       emitReached = new Set(pre.reached)
+      if (emitted.ext !== undefined) HYBRID = emitted
     }
     /**
      * ONCE A PROGRAM CARRIES ASSEMBLIES, THE CONSTRUCTOR IS OFF FOR IT — for
@@ -3942,7 +3973,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
         REC ? prog.cc.map((_, i) => sentinelFor(i)) : EMPTY_SENTS,
         matchesAt, recoverScan, orSentinel, captureError,
         scalarRecognizers, commitTriviaScan, scanTriviaCompact, t.lex, adjacencyHolds, t.lexPrograms,
-        LEAD, triviaFieldCapture,
+        LEAD, triviaFieldCapture, skipDeferredTrivia,
       )
       emitReached = em.reached
     } catch (e) {
@@ -3955,7 +3986,7 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
   /** One piece for a site, from whichever engine this assembly is running. */
   function pieceAt(ip: number): Piece {
     const em = emitted
-    if (em !== undefined) {
+    if (em !== undefined && HYBRID === undefined) {
       const p = em.byIp[ip]
       if (p === undefined) throw new Error(`table emitter: no emitted body for site ${ip}`)
       return p as Piece
@@ -3964,6 +3995,8 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
   }
 
   for (const s of prog.scans ?? []) {
+    const rebuilt = rebuildLineScan(s, subtreeComb)
+    if (rebuilt !== undefined) { scansArr.push(rebuilt); continue }
     const skip = s.skip.map(r => subtreeComb(r))
     const raw = (s.flags & 1) !== 0
     if (s.kind === 1) {
@@ -3988,6 +4021,28 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
 
   /* ── link the rules ──────────────────────────────────────────────────────── */
 
+  if (HYBRID !== undefined) {
+    const em = HYBRID
+    em.bind!(
+      s => { const saved = SCAN; SCAN = s as FastTriviaScanner | null; return saved },
+      em.ext!.map(ip => link(ip)),
+    )
+    const pieces: Record<string, Piece> = {}
+    for (const [name, entryIp] of Object.entries(prog.rules)) {
+      pieces[name] = (em.pieces[name] as Piece | undefined) ?? link(entryIp)
+    }
+    for (const ip of emitReached!) reached.add(ip)
+    return {
+      pieces,
+      end: () => EC.e,
+      begin: ctx => { begin(ctx); em.begin(ctx) },
+      finish: () => { em.finish(); finish() },
+      scanSkip,
+      reached,
+      emitRefusal: undefined,
+    }
+  }
+
   if (emitted !== undefined) {
     const em = emitted
     return {
@@ -4040,6 +4095,43 @@ export function assemble(t: ResolvedTable, prog: TableProgram, cfg: RunCfg): Ass
   return { pieces, end: () => EC.e, begin, finish, scanSkip, reached, emitRefusal }
 }
 
+/**
+ * BUILD-TIME SITE PROFILING (`countSiteVisits`). Set only while that call runs;
+ * `link` reads it when it LOWERS a site, never per parse, so an ordinary assembly
+ * carries no counter and no branch.
+ */
+let SITE_COUNTS: Int32Array | undefined
+
+function countedPiece(piece: Piece, ip: number): Piece {
+  const counts = SITE_COUNTS
+  if (counts === undefined) return piece
+  return (input, pos, ctx) => {
+    counts[ip]!++
+    return piece(input, pos, ctx)
+  }
+}
+
+/**
+ * How many times each table site runs while `parse` drives the program — the
+ * profile an optional corpus gives selective assembly (`select-hot.ts`). Runs the
+ * CLOSURE engine, so it constructs no code; `counts[ip]` is the number of calls
+ * of the piece lowered for site `ip` (an alias site counts at its target).
+ */
+export function countSiteVisits(
+  prog: TableProgram,
+  parse: (rules: Record<string, TableRule>) => void,
+  /** A replacement callback pool (same indices), e.g. stand-ins for inert builders. */
+  fns: readonly unknown[] = prog.fns,
+): Int32Array {
+  const counts = new Int32Array(prog.code.length)
+  // A fresh program, not a spread: a spread keeps the source's cached resolution,
+  // which holds the ORIGINAL callback pool.
+  const profiled = ownTableProgram({ ...prog, fns, asm: [] })
+  SITE_COUNTS = counts
+  try { parse(tableRules(profiled)) } finally { SITE_COUNTS = undefined }
+  return counts
+}
+
 /**
  * Assemblies for one resolved table, one per option set, built on demand.
  *

package/src/table/child-slots.ts

@@ -19,12 +19,10 @@ import {
  * only against it happening WITHIN that file.
  *
  * The subset was never load-bearing. The five opcodes it omitted — COV, GREEDY,
- * REJECT, ARMGATE, WITHCTX — are exactly the five `emit-assembly.ts` has no case
- * for, so any program containing one throws `Unemittable` from the default arm and
- * the whole assembly falls back to closures. Walking into their subtrees can
- * therefore only affect a result that is about to be discarded. (Verified: none of
- * the five is so much as imported by `emit-assembly.ts`. Its one consumer of these
- * edges, the `fd`/`er` sink census, is computed before the lowering that throws.)
+ * REJECT, ARMGATE, WITHCTX — were exactly the five `emit-assembly.ts` had no case
+ * for. ARMGATE and WITHCTX are lowered there now, so their subtrees are walked like
+ * any other; COV, GREEDY and REJECT still throw `Unemittable` from the default arm,
+ * and the whole assembly falls back to closures.
  *
  * `code` is `ArrayLike<number>` because the two callers hold different
  * representations of the same stream — `TableProgram.code` is a `readonly number[]`,