parseman 0.61.1 → 0.61.3

package/dist/table/select-hot.d.ts

@@ -1,6 +1,25 @@
-/** The sites to emit: highest weight per byte first, while they fit in `budget` bytes. */
+/**
+ * The sites to emit: highest weight per byte first, stopping at the first that
+ * does not fit in `budget` bytes. Stopping rather than skipping is what makes a
+ * larger budget a superset: a skip lets a budget that just admits a big site lose
+ * the small ones a smaller budget took after skipping it. Insertion order is rank.
+ */
 export declare function selectHotSites(code: readonly number[], entries: readonly number[], siteBytes: ReadonlyMap<number, number>, budget: number, 
 /** Measured calls per site (`countSiteVisits` over a corpus). Replaces the
  * static estimate when given. */
 profile?: ArrayLike<number>): Set<number>;
+/**
+ * The largest prefix of the ranking whose real source fits `ceiling`. `siteBytes`
+ * leaves out what sites add around their bodies, so the estimate can overshoot:
+ * each overshoot drops the last site and measures again. Walking down the one
+ * order is what keeps a selection a prefix of any larger budget's: that budget has
+ * every candidate this one has, so it can never settle on fewer. `undefined` when
+ * not even the first site fits.
+ */
+export declare function fitHotSites(code: readonly number[], entries: readonly number[], siteBytes: ReadonlyMap<number, number>, 
+/** What any selection costs before its first site body. */
+fixed: number, ceiling: number, sourceBytes: (sites: ReadonlySet<number>) => number, profile?: ArrayLike<number>): {
+    readonly sites: ReadonlySet<number>;
+    readonly bytes: number;
+} | undefined;
 //# sourceMappingURL=select-hot.d.ts.map

package/dist/table/select-hot.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"select-hot.d.ts","sourceRoot":"","sources":["../../src/table/select-hot.ts"],"names":[],"mappings":"AAuBA,0FAA0F;AAC1F,wBAAgB,cAAc,CAC5B,IAAI,EAAE,SAAS,MAAM,EAAE,EACvB,OAAO,EAAE,SAAS,MAAM,EAAE,EAC1B,SAAS,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,EACtC,MAAM,EAAE,MAAM;AACd;iCACiC;AACjC,OAAO,CAAC,EAAE,SAAS,CAAC,MAAM,CAAC,GAC1B,GAAG,CAAC,MAAM,CAAC,CAiBb"}
+{"version":3,"file":"select-hot.d.ts","sourceRoot":"","sources":["../../src/table/select-hot.ts"],"names":[],"mappings":"AAuBA;;;;;GAKG;AACH,wBAAgB,cAAc,CAC5B,IAAI,EAAE,SAAS,MAAM,EAAE,EACvB,OAAO,EAAE,SAAS,MAAM,EAAE,EAC1B,SAAS,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,EACtC,MAAM,EAAE,MAAM;AACd;iCACiC;AACjC,OAAO,CAAC,EAAE,SAAS,CAAC,MAAM,CAAC,GAC1B,GAAG,CAAC,MAAM,CAAC,CAgBb;AAED;;;;;;;GAOG;AACH,wBAAgB,WAAW,CACzB,IAAI,EAAE,SAAS,MAAM,EAAE,EACvB,OAAO,EAAE,SAAS,MAAM,EAAE,EAC1B,SAAS,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC;AACtC,2DAA2D;AAC3D,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,EACf,WAAW,EAAE,CAAC,KAAK,EAAE,WAAW,CAAC,MAAM,CAAC,KAAK,MAAM,EACnD,OAAO,CAAC,EAAE,SAAS,CAAC,MAAM,CAAC,GAC1B;IAAE,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,CAQ7E"}

package/dist/version.d.ts

@@ -13,5 +13,5 @@
  * Kept in sync with `package.json` by `test/unit/version-sync.test.ts`, which fails
  * the build if this constant drifts from the published version.
  */
-export declare const PARSEMAN_VERSION = "0.61.1";
+export declare const PARSEMAN_VERSION = "0.61.3";
 //# sourceMappingURL=version.d.ts.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "parseman",
-  "version": "0.61.1",
+  "version": "0.61.3",
   "packageManager": "pnpm@10.15.1",
   "engines": {
     "node": "^20.19.0 || >=22.13.0"

package/src/plugin/evaluator.ts

@@ -17,6 +17,7 @@ import type { Combinator } from '../types.ts'
 import type { DispatchArm } from '../combinators/dispatch.ts'
 import { ref } from '../combinators/ref.ts'
 import { rules, RULES_RECIPE, type PriorRules, type RulesOptions, type RulesRecipe } from '../combinators/parser.ts'
+import { parseSync } from 'oxc-parser'
 import * as parseman from '../index.ts'
 import { confirmedBuildParamUnused } from '../compiler/build-arity.ts'
 import { directBuilderBindings } from './direct-builder-static.ts'
@@ -72,7 +73,7 @@ function stripTsFromSource(node: Node, code: string): string {
   walk(node)
   const start = (node as { start: number }).start
   const end = (node as { end: number }).end
-  if (cuts.length === 0) return code.slice(start, end)
+  if (cuts.length === 0) return stripComments(code.slice(start, end))
   cuts.sort((a, b) => a[0] - b[0])
   let out = ''
   let cur = start
@@ -81,7 +82,46 @@ function stripTsFromSource(node: Node, code: string): string {
     out += code.slice(cur, s)
     cur = e
   }
-  return out + code.slice(cur, end)
+  return stripComments(out + code.slice(cur, end))
+}
+
+/** The four characters ECMAScript treats as ending a line. */
+const LINE_TERMINATOR = /[\n\r\u2028\u2029]/
+
+/**
+ * A reducer's source without its comments, and without the whitespace before them.
+ * A reducer ships as a string in the carried IR, where a bundler cannot strip it, and
+ * a comment must not change what a module ships. A comment still separates tokens and,
+ * if it crossed or followed a line end, still ends a line: automatic semicolon
+ * insertion and `return` depend on that line end.
+ */
+function stripComments(text: string): string {
+  if (!text.includes('/*') && !text.includes('//')) return text
+  const wrapped = `(${text})`
+  let out = ''
+  let cur = 0
+  let lineEnd = false
+  const comments = parseSync('reducer.ts', wrapped).comments
+  comments.forEach((c, i) => {
+    const before = text.slice(cur, c.start - 1)
+    const kept = before.trimEnd()
+    out += kept
+    lineEnd ||= LINE_TERMINATOR.test(before.slice(kept.length)) || LINE_TERMINATOR.test(c.value)
+    cur = c.end - 1
+    const after = /^\s*/.exec(text.slice(cur))![0]
+    const endsLine = LINE_TERMINATOR.test(after)
+    if (comments[i + 1]?.start === c.end + after.length) {
+      lineEnd ||= endsLine
+      cur += after.length
+      return
+    }
+    if (out !== '' && cur + after.length < text.length && !endsLine) {
+      if (lineEnd) out += '\n'
+      else if (after === '') out += ' '
+    }
+    lineEnd = false
+  })
+  return out + text.slice(cur)
 }
 
 /**

package/src/plugin/index.ts

@@ -170,11 +170,13 @@ export type ParsecraftPluginOptions = {
    */
   assemblyProfile?: AssemblyProfile
   /**
-   * Generated assembly bytes each module may spend, as a multiple of its own source
-   * length. **Default `0.25`.** `0` ships no assembly. The right number is
+   * Generated assembly bytes each module may spend, as a multiple of its own code,
+   * comments and whitespace aside. **Default `0.5`.** `0` ships no assembly. The right number is
    * per grammar: a larger one parses faster in steady state and costs import and
    * first-parse time, so measure both, and keep the artifact within 5× its source
-   * (docs/design/runtime-and-size-contract.md, rules 4 and 6).
+   * (docs/design/runtime-and-size-contract.md, rules 4 and 6). A larger budget never
+   * loses a site, with one limit: a module with two AST grammars spends it in order, so
+   * a larger budget can buy the first a big site and leave the second fewer.
    */
   assemblyBudget?: number
 }
@@ -184,7 +186,9 @@ export type AssemblyProfile = {
   /** The rule each corpus file is parsed from, e.g. `'Stylesheet'`. A grammar
    * without this rule keeps the static ranking. */
   readonly entry: string
-  /** Paths of representative inputs. */
+  /** Paths of representative inputs. At least one must have content, and each that
+   * does has to parse in full without recovery, or the build fails. A file with only
+   * whitespace counts as empty. */
   readonly files: readonly string[]
 }
 
@@ -206,11 +210,26 @@ const PARSEMAN_MODULE = 'parseman'
 
 /**
  * The default `assemblyBudget`: generated assembly bytes a module may spend, as a
- * multiple of its own source length. A quarter of the source keeps every jess
- * grammar far under the 5x ceiling (contract rule 4); the 0.57.0 CHANGELOG has the
- * measured curve.
+ * multiple of its own code (`codeBytes`). Half keeps every jess grammar far under the
+ * 5x ceiling (contract rule 4). A grammar that wants a different trade passes its own.
  */
-const DEFAULT_ASSEMBLY_BUDGET = 0.25
+const DEFAULT_ASSEMBLY_BUDGET = 0.5
+
+/**
+ * A module's code in bytes, without its comments and whitespace. It is what a budget
+ * is a share of, so editing a comment or reflowing a line cannot move a site in or
+ * out of the assembly, and with it the module's size and parse speed.
+ */
+function codeBytes(code: string, comments: readonly { start: number; end: number }[], end = code.length): number {
+  let bytes = 0
+  let at = 0
+  for (const c of comments) {
+    if (c.start >= end) break
+    bytes += code.slice(at, c.start).replace(/\s+/g, '').length
+    at = c.end
+  }
+  return bytes + code.slice(at, end).replace(/\s+/g, '').length
+}
 
 /**
  * The REASON a lowering refused, appended to the warning that reports it.
@@ -767,6 +786,8 @@ export type TransformMacroResult = {
   map: ReturnType<MagicString['generateMap']>
   /** Diagnostics for macro-referencing shapes that fell back to the interpreter. */
   warnings: string[]
+  /** The sites the module's `assemblyBudget` bought, by program offset, in rank order. */
+  assemblySites: number[]
 }
 
 /**
@@ -1749,7 +1770,9 @@ function transformMacroImpl(
 
   // Stable, reproducible per-artifact namespace: hash of module id + a label
   // (binding name / arg position) — never a counter, so rebuilds are byte-stable
-  // and two artifacts never collide when fused.
+  // and two artifacts never collide when fused. A call is labelled by where it sits
+  // in the module's code (`codeBytes`), so a comment above it leaves the name alone.
+  const callLabel = (fn: string, start: number): string => `${fn}${codeBytes(code, result.comments, start)}`
   const nsFor = (label: string): string =>
     `_${createHash('sha1').update(`${id}#${label}`).digest('hex').slice(0, 8)}_`
 
@@ -2253,7 +2276,7 @@ function transformMacroImpl(
     const carried: CarriedItem[] = []       // re-lowerable; also SERIALIZED onto the value
     const importedFactories: string[] = []
     for (let i = 0; i < elements.length; i++) {
-      const r = argPieces(elements[i]!, `compose${init.start}_${i}`, composing)
+      const r = argPieces(elements[i]!, `${callLabel('compose', init.start)}_${i}`, composing)
       if (!r) {
         // A build-compiled argument cannot fall back: runtime `compose()` links live
         // `rules()` grammars only and would throw at import. Fail here instead.
@@ -2342,14 +2365,15 @@ function transformMacroImpl(
    * SELECTIVE ASSEMBLY, BUDGETED PER MODULE. A fused grammar's default AST/no-lines
    * table carries generated code for its hottest sites only (`select-hot.ts`); the
    * closure engine runs the rest. The site bodies of every fused grammar in this
-   * module together may not exceed `assemblyBudget` times the module's own source.
+   * module together may not exceed `assemblyBudget` times the module's own code (`codeBytes`).
    *
    * A module that declares a `composeLeaf` has named its terminal parser, so its
    * `compose()` results are composable PIECES: another build re-lowers them and
    * never parses through their table, so they spend nothing. Without a
    * `composeLeaf`, a `compose()` result is the module's parser.
    */
-  let assemblyBudget = Math.floor(code.length * assemblyBudgetShare)
+  let assemblyBudget = Math.floor(codeBytes(code, result.comments) * assemblyBudgetShare)
+  const assemblySites: number[] = []
   let leafDeclared: boolean | undefined
   const declaresLeaf = (): boolean => leafDeclared ??= (body as Statement[]).some(stmt => {
     const vd = unwrapVd(stmt)
@@ -2373,32 +2397,48 @@ function transformMacroImpl(
       warnings.push(`${id}:${lineOf(pos)} — ${label}: its precompiled assembly cannot emit ${refusal}; the artifact runs on the closure engine`)
     }
     assemblyBudget -= bytes
+    assemblySites.push(...sites)
     return sites
   }
   /** Measured site calls over the optional corpus, or `undefined` for the static
-   * ranking. A file whose parse throws still contributes the calls it made. */
+   * ranking. A corpus that cannot run fails the build: a static ranking standing in
+   * for the measured one configured is a slower parser that nothing reports. */
   const profileOf = (compiled: CompiledRuleMapTable): Int32Array | undefined => {
     // Only the default AST/no-lines table carries an assembly, so only it is profiled.
     if (assemblyProfile === undefined || compiled.hostMode !== 'ast' || compiled.prog.lines === 1
       || !(assemblyProfile.entry in compiled.prog.rules)) return undefined
-    let parsed = 0
+    const refuse = (why: string): never => { throw new Error(`${id} — assemblyProfile: ${why}`) }
+    if (corpusTexts(assemblyProfile).every(text => text.trim() === '')) refuse('`files` has no non-empty file')
+    let thrower: unknown
     // A builder re-lowered from carried IR is inert at build time; for COUNTING
     // only, a plain node stands in for it, so the corpus can run past it.
     const fns = compiled.prog.fns.map(fn => {
       const type = (fn as { [INERT_BUILDER]?: string } | null)?.[INERT_BUILDER]
-      return type === undefined ? fn
-        : (children: unknown, _fields: unknown, span: unknown) => ({ type, span, children })
+      if (type !== undefined) return (children: unknown, _fields: unknown, span: unknown) => ({ type, span, children })
+      return typeof fn !== 'function' ? fn : new Proxy(fn, {
+        apply(target, self, args) {
+          try { return Reflect.apply(target, self, args) as unknown } catch (e) { thrower ??= target; throw e }
+        },
+      })
     })
-    const counts = countSiteVisits(compiled.prog, rules => {
+    return countSiteVisits(compiled.prog, rules => {
       const entry = rules[assemblyProfile.entry]!
-      for (const text of corpusTexts(assemblyProfile)) {
-        try { if (run(entry, text).ok) parsed++ } catch { /* the calls it made still count */ }
-      }
+      corpusTexts(assemblyProfile).forEach((text, i) => {
+        const file = assemblyProfile.files[i]!
+        if (text.trim() === '') return
+        let result
+        try { result = run(entry, text) } catch (e) {
+          const source = String(thrower).replace(/\s+/g, ' ').slice(0, 200)
+          throw new Error(`${id} — assemblyProfile: ${thrower === undefined ? 'the parse' : `gate or reducer \`${source}\``} threw on ${file}: ${String(e)}`, { cause: e })
+        }
+        if (!result.ok || result.unconsumedFrom !== null) {
+          refuse(`${file} does not parse from rule "${assemblyProfile.entry}": it stops at offset ${result.ok ? result.unconsumedFrom : result.span.end}`)
+        }
+        if (result.errors.length > 0) {
+          refuse(`${file} parses only by recovering from ${result.errors.length} error(s), first at offset ${result.errors[0]!.span.start}`)
+        }
+      })
     }, fns)
-    if (parsed > 0) return counts
-    // Not `warn()`: nothing fell back to the interpreter — the build is only untuned.
-    warnings.push(`${id} — assemblyProfile: no corpus file parsed from rule "${assemblyProfile.entry}"; using the static ranking`)
-    return undefined
   }
   /** The fused table's replacement, with its selective default assembly if any. */
   const assembledReplacement = (compiled: CompiledRuleMapTable, metadata: string, select: ReadonlySet<number>): string =>
@@ -2428,7 +2468,7 @@ function transformMacroImpl(
     let localRules: ReadonlyMap<string, Combinator<unknown>> | null = null
     const importedFactories: string[] = []
     if (isRulesCall(localArg)) {
-      const evaluated = evaluateRulesFactory(localArg, `composeLeaf${init.start}`)
+      const evaluated = evaluateRulesFactory(localArg, callLabel('composeLeaf', init.start))
       localRules = evaluated?.ruleMap ?? null
       if (evaluated?.importedFactory) importedFactories.push(evaluated.importedFactory)
     } else if (localArg.type === 'Identifier') {
@@ -2455,7 +2495,7 @@ function transformMacroImpl(
     const composing = composingTrivia(elements)
     const carried: CarriedItem[] = []
     for (let i = 0; i < elements.length - 1; i++) {
-      const r = argPieces(elements[i]!, `composeLeaf${init.start}_${i}`, composing)
+      const r = argPieces(elements[i]!, `${callLabel('composeLeaf', init.start)}_${i}`, composing)
       if (!r) {
         warn(init.start, `composeLeaf(): argument ${i} isn't a build-resolvable recognition grammar`)
         return null
@@ -3277,6 +3317,7 @@ function transformMacroImpl(
   }
 
   return {
+    assemblySites: applied.length > 0 ? assemblySites : [],
     code: ms.toString(),
     map: ms.generateMap({ hires: true }),
     warnings,

package/src/table/compile-rule-map.ts

@@ -5,7 +5,7 @@ import { collectGrammarReflection, GRAMMAR_REFLECTION, type GrammarReflection }
 import { encodeTableProgram, type TableSettings } from './encode.ts'
 import { defaultAssemblyCfgs, emitTableExpression } from './emit.ts'
 import { emitAssemblySource, Unemittable } from './emit-assembly.ts'
-import { selectHotSites } from './select-hot.ts'
+import { fitHotSites } from './select-hot.ts'
 
 /** The most the entry supercompiler may grow an assembly (`entry-supercompiler.ts`, its 5% source cap). */
 const SUPERCOMPILE_GROWTH = 1.05
@@ -372,29 +372,16 @@ export function compileRuleMap(
     // The estimate (prelude plus site bodies) leaves out what selected sites add
     // around their bodies — shared helpers and the `_px` wrappers of the closure
     // sites they call — and the supercompiler may grow the source by up to 5%. So
-    // the selection is judged by the source it actually emits, scaled for that 5%,
-    // and narrowed until that fits.
+    // the selection is judged by the source it actually emits, scaled for that 5%.
     const ceiling = budget / SUPERCOMPILE_GROWTH
     // A budget that cannot cover the prelude buys nothing, by design: no refusal.
     if (ceiling <= fixed) return none
-    // Narrowed until it fits, with no attempt limit: the target shrinks by at
-    // least 1% a round, so the selection either fits or empties. An empty one
-    // says why — a build that asked for an assembly and silently shipped none is
-    // a regression no test sees (the artifact still parses, on the closures).
-    let target = ceiling
-    for (;;) {
-      const sites = selectHotSites(artifact.code, Object.values(artifact.rules), siteBytes, target - fixed, profile)
-      if (sites.size === 0) {
-        return {
-          ...none,
-          refusal: `a site within its ${Math.floor(ceiling)} B selection ceiling (the prelude alone is ${fixed} B)`,
-        }
-      }
-      const bytes = emitAssemblySource(table, artifact, cfg, roots, true, sites, true).source.length
-      if (bytes <= ceiling) return { sites, bytes: Math.ceil(bytes * SUPERCOMPILE_GROWTH) }
-      // Scale by the overshoot, and by at least 1%: near the ceiling a site is
-      // the unit, and the ratio alone can stall a few bytes over it.
-      target *= Math.min(ceiling / bytes, 0.99)
+    const fit = fitHotSites(artifact.code, Object.values(artifact.rules), siteBytes, fixed, ceiling,
+      sites => emitAssemblySource(table, artifact, cfg, roots, true, sites, true).source.length, profile)
+    if (fit !== undefined) return { sites: fit.sites, bytes: Math.ceil(fit.bytes * SUPERCOMPILE_GROWTH) }
+    return {
+      ...none,
+      refusal: `a site within its ${Math.floor(ceiling)} B selection ceiling (the prelude alone is ${fixed} B)`,
     }
   }
 

package/src/table/select-hot.ts

@@ -21,7 +21,12 @@ const LOOP = 8
 const ROUNDS = 60
 const CAP = 1e12
 
-/** The sites to emit: highest weight per byte first, while they fit in `budget` bytes. */
+/**
+ * The sites to emit: highest weight per byte first, stopping at the first that
+ * does not fit in `budget` bytes. Stopping rather than skipping is what makes a
+ * larger budget a superset: a skip lets a budget that just admits a big site lose
+ * the small ones a smaller budget took after skipping it. Insertion order is rank.
+ */
 export function selectHotSites(
   code: readonly number[],
   entries: readonly number[],
@@ -41,14 +46,40 @@ export function selectHotSites(
   let spent = 0
   for (const ip of ranked) {
     if (w(ip) === 0) break
-    const bytes = siteBytes.get(ip)!
-    if (spent + bytes > budget) continue
-    spent += bytes
+    spent += siteBytes.get(ip)!
+    if (spent > budget) break
     selected.add(ip)
   }
   return selected
 }
 
+/**
+ * The largest prefix of the ranking whose real source fits `ceiling`. `siteBytes`
+ * leaves out what sites add around their bodies, so the estimate can overshoot:
+ * each overshoot drops the last site and measures again. Walking down the one
+ * order is what keeps a selection a prefix of any larger budget's: that budget has
+ * every candidate this one has, so it can never settle on fewer. `undefined` when
+ * not even the first site fits.
+ */
+export function fitHotSites(
+  code: readonly number[],
+  entries: readonly number[],
+  siteBytes: ReadonlyMap<number, number>,
+  /** What any selection costs before its first site body. */
+  fixed: number,
+  ceiling: number,
+  sourceBytes: (sites: ReadonlySet<number>) => number,
+  profile?: ArrayLike<number>,
+): { readonly sites: ReadonlySet<number>; readonly bytes: number } | undefined {
+  const ranked = [...selectHotSites(code, entries, siteBytes, ceiling - fixed, profile)]
+  for (; ranked.length > 0; ranked.pop()) {
+    const sites = new Set(ranked)
+    const bytes = sourceBytes(sites)
+    if (bytes <= ceiling) return { sites, bytes }
+  }
+  return undefined
+}
+
 function staticWeights(code: readonly number[], entries: readonly number[]): Map<number, number> {
   let weight = new Map<number, number>()
   const kids: number[] = []

package/src/version.ts

@@ -13,4 +13,4 @@
  * Kept in sync with `package.json` by `test/unit/version-sync.test.ts`, which fails
  * the build if this constant drifts from the published version.
  */
-export const PARSEMAN_VERSION = '0.61.1'
+export const PARSEMAN_VERSION = '0.61.3'