parseman 0.61.5 → 0.61.6

package/dist/table/select-hot.d.ts

@@ -8,6 +8,15 @@ export declare function selectHotSites(code: readonly number[], entries: readonl
 /** Measured calls per site (`countSiteVisits` over a corpus). Replaces the
  * static estimate when given. */
 profile?: ArrayLike<number>): Set<number>;
+/**
+ * The sites the function for `ip` calls, as `emit-assembly.ts` lowers it. An `OP_CONT`
+ * with a body is a late frame, lowered with its wrapper in one function, calling its
+ * corner and its tail. With no body, the emitter can't decide there and it's an alias
+ * to its wrapper, as `OP_SEED` is to its corner. A gap gate has no body either, so its
+ * item counts as the repeat's own child, and adding the row moves no other site's
+ * estimate (`gap-gate.ts`).
+ */
+export declare function edgesAsLowered(code: readonly number[], siteBytes: ReadonlyMap<number, number>, ip: number, out: number[]): void;
 /**
  * The largest prefix of the ranking whose real source fits `ceiling`. `siteBytes`
  * leaves out what sites add around their bodies, so the estimate can overshoot:

package/dist/table/select-hot.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"select-hot.d.ts","sourceRoot":"","sources":["../../src/table/select-hot.ts"],"names":[],"mappings":"AAwBA;;;;;GAKG;AACH,wBAAgB,cAAc,CAC5B,IAAI,EAAE,SAAS,MAAM,EAAE,EACvB,OAAO,EAAE,SAAS,MAAM,EAAE,EAC1B,SAAS,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,EACtC,MAAM,EAAE,MAAM;AACd;iCACiC;AACjC,OAAO,CAAC,EAAE,SAAS,CAAC,MAAM,CAAC,GAC1B,GAAG,CAAC,MAAM,CAAC,CAgBb;AAED;;;;;;;GAOG;AACH,wBAAgB,WAAW,CACzB,IAAI,EAAE,SAAS,MAAM,EAAE,EACvB,OAAO,EAAE,SAAS,MAAM,EAAE,EAC1B,SAAS,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC;AACtC,2DAA2D;AAC3D,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,EACf,WAAW,EAAE,CAAC,KAAK,EAAE,WAAW,CAAC,MAAM,CAAC,KAAK,MAAM,EACnD,OAAO,CAAC,EAAE,SAAS,CAAC,MAAM,CAAC,GAC1B;IAAE,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,CAQ7E"}
+{"version":3,"file":"select-hot.d.ts","sourceRoot":"","sources":["../../src/table/select-hot.ts"],"names":[],"mappings":"AAqCA;;;;;GAKG;AACH,wBAAgB,cAAc,CAC5B,IAAI,EAAE,SAAS,MAAM,EAAE,EACvB,OAAO,EAAE,SAAS,MAAM,EAAE,EAC1B,SAAS,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,EACtC,MAAM,EAAE,MAAM;AACd;iCACiC;AACjC,OAAO,CAAC,EAAE,SAAS,CAAC,MAAM,CAAC,GAC1B,GAAG,CAAC,MAAM,CAAC,CAUb;AAED;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,EAAE,SAAS,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,IAAI,CAe/H;AAqDD;;;;;;;GAOG;AACH,wBAAgB,WAAW,CACzB,IAAI,EAAE,SAAS,MAAM,EAAE,EACvB,OAAO,EAAE,SAAS,MAAM,EAAE,EAC1B,SAAS,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC;AACtC,2DAA2D;AAC3D,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,EACf,WAAW,EAAE,CAAC,KAAK,EAAE,WAAW,CAAC,MAAM,CAAC,KAAK,MAAM,EACnD,OAAO,CAAC,EAAE,SAAS,CAAC,MAAM,CAAC,GAC1B;IAAE,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,CAQ7E"}

package/dist/version.d.ts

@@ -13,5 +13,5 @@
  * Kept in sync with `package.json` by `test/unit/version-sync.test.ts`, which fails
  * the build if this constant drifts from the published version.
  */
-export declare const PARSEMAN_VERSION = "0.61.5";
+export declare const PARSEMAN_VERSION = "0.61.6";
 //# sourceMappingURL=version.d.ts.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "parseman",
-  "version": "0.61.5",
+  "version": "0.61.6",
   "packageManager": "pnpm@10.15.1",
   "engines": {
     "node": "^20.19.0 || >=22.13.0"

package/src/plugin/entry-supercompiler.ts

@@ -65,6 +65,8 @@ type Call = {
 type Edit = { readonly start: number; readonly end: number; readonly text: string }
 
 const PIECE = /^_pf(\d+)$/
+const MERGEABLE = /^_(?:pf|sk)\d+$/
+const TEMP = /^_t\d+_/
 const WRAP = 'function __parseman_supercompile__(){\n'
 const SUFFIX = '\n}'
 
@@ -225,6 +227,113 @@ export function supercompileEntryAssembly(
   return supercompileEmittedAssembly(prog, cfg, extraIps, emitted)
 }
 
+/** A name in the source: a site, or a temporary (`tmp` in `emit-assembly.ts`). */
+type NameUse = { readonly start: number; readonly end: number; readonly name: string }
+
+/** The uses of site and temporary names under `root`, and the names bound there. */
+function namesUnder(root: Ast, uses: NameUse[], bound: Set<string>): boolean {
+  let plain = true
+  const bind = (id: Ast | undefined): void => {
+    const name = nameOf(id)
+    if (name !== undefined) bound.add(name)
+  }
+  const walk = (node: Ast, parent: Ast | undefined): void => {
+    if (node.type === 'VariableDeclarator') bind(ast(node.id))
+    else if (node.type === 'LabeledStatement') bind(ast(node.label))
+    else if (isFunction(node)) {
+      if (node !== root) bind(ast(node.id))
+      nodes(node.params).forEach(bind)
+    }
+    const name = nameOf(node)
+    if (name !== undefined && (MERGEABLE.test(name) || TEMP.test(name))) {
+      const isKey = parent?.type === 'Property' && parent.key === node && parent.computed !== true
+      const isProperty = parent?.type === 'MemberExpression' && parent.property === node && parent.computed !== true
+      if (parent?.type === 'Property' && parent.shorthand === true) plain = false
+      if (!isKey && !isProperty) uses.push({ ...range(node), name })
+    }
+    for (const child of children(node)) walk(child, node)
+  }
+  walk(root, undefined)
+  return plain
+}
+
+/**
+ * Functions that read the same are one function: `optional(x)` written in 25 places
+ * is 25 rows and one body, and a trivia skip can be pooled under two labels that
+ * lower alike. This runs after the join, so a site with one caller has
+ * already gone into it, the selection is untouched and every call a parse makes is
+ * still made. Each pass can make the callers of a merged function read alike, so it
+ * repeats until nothing merges.
+ *
+ * Only identifiers are compared or renamed: a grammar's own text reaches this source
+ * as strings and regexes, and they count as written. A temporary is local by the
+ * emitter's contract and is numbered per assembly, so one this function binds is
+ * compared in order of use. A site name that's also bound somewhere, or abbreviated
+ * in `{_pf1}`, can't be renamed by name, so that source stays as it is.
+ */
+export function mergeSameBodies(source: string): string {
+  const parsed = parseSync('parseman-merge-same-bodies.js', WRAP + source + SUFFIX)
+  if (parsed.errors.length > 0) return source
+  const statements = nodes(ast(nodes((parsed.program as unknown as Ast).body)[0]?.body)?.body)
+  type Body = NameUse & { readonly uses: readonly NameUse[]; readonly local: ReadonlySet<string> }
+  const bodies: Body[] = []
+  const others: NameUse[] = []
+  const declared = new Set<string>()
+  for (const statement of statements) {
+    const name = statement.type === 'FunctionDeclaration' ? nameOf(ast(statement.id)) : undefined
+    const uses: NameUse[] = []
+    const bound = new Set<string>()
+    if (!namesUnder(statement, uses, bound)) return source
+    for (const local of bound) declared.add(local)
+    if (name === undefined || !MERGEABLE.test(name)) others.push(...uses)
+    else bodies.push({ ...range(statement), name, uses: uses.sort((a, b) => a.start - b.start), local: bound })
+  }
+  for (const local of declared) if (MERGEABLE.test(local)) return source
+
+  const merged = new Map<string, string>()
+  const current = (name: string): string => {
+    let at = name
+    for (let next = merged.get(at); next !== undefined; next = merged.get(at)) at = next
+    return at
+  }
+  let live = bodies
+  for (let more = true; more;) {
+    more = false
+    const first = new Map<string, string>()
+    live = live.filter(body => {
+      const temps = new Map<string, number>()
+      const edits = body.uses.map((use): Edit => {
+        if (MERGEABLE.test(use.name)) {
+          const site = current(use.name)
+          return { ...use, text: site === body.name ? '#self' : site }
+        }
+        if (!body.local.has(use.name)) return { ...use, text: use.name }
+        const seen = temps.get(use.name) ?? temps.size
+        temps.set(use.name, seen)
+        return { ...use, text: `#t${seen}` }
+      })
+      const key = render(source, body.start, body.end, edits)
+      const kept = first.get(key)
+      if (kept === undefined) { first.set(key, body.name); return true }
+      merged.set(body.name, kept)
+      more = true
+      return false
+    })
+  }
+  if (merged.size === 0) return source
+
+  const edits: Edit[] = []
+  const rename = (use: NameUse): void => {
+    if (MERGEABLE.test(use.name) && merged.has(use.name)) edits.push({ ...use, text: current(use.name) })
+  }
+  others.forEach(rename)
+  for (const body of bodies) {
+    if (merged.has(body.name)) edits.push({ start: body.start, end: body.end, text: '' })
+    else body.uses.forEach(rename)
+  }
+  return render(source, 0, source.length, edits)
+}
+
 /**
  * Plugin-only final splice for an ordinary macro replacement. The table
  * compiler emits its canonical callback-free public artifact first; this pass
@@ -274,7 +383,7 @@ export function supercompileRuleMapReplacement(
   const start = body.start - prefix.length + 1
   const end = body.end - prefix.length - 1
   if (replacement.slice(start, end) !== FACTORY_ARITY_GUARD + canonical.source + '\n') return replacement
-  const transformed = replacement.slice(0, start) + FACTORY_ARITY_GUARD + candidate.source + '\n' + replacement.slice(end)
+  const transformed = replacement.slice(0, start) + FACTORY_ARITY_GUARD + mergeSameBodies(candidate.source) + '\n' + replacement.slice(end)
   const checked = parseSync('parseman-entry-supercompile-replacement-result.js', prefix + transformed)
   return checked.errors.length === 0 ? transformed : replacement
 }

package/src/table/select-hot.ts

@@ -12,15 +12,28 @@
  * `LOOP`, and a choice or dispatch splits its weight evenly across its arms. The
  * weights are propagated `ROUNDS` times, so a recursive cycle saturates at `CAP`
  * instead of diverging. Sites are then taken greedily by weight per emitted byte.
- * A build given a corpus (`assemblyProfile`) ranks by measured calls instead.
+ *
+ * A build given a corpus (`assemblyProfile`) ranks by measured calls, and takes
+ * sites connected: every call from an emitted site to a closure site, or back,
+ * crosses the engine boundary through a wrapper, so a hot site beside the emitted
+ * ones is worth more than the same site alone. Ranking by calls per byte alone
+ * leaves islands: on Jess's Less grammar at budget 0.5, it made 6.7% to 9.0% more calls
+ * per parse than the static ranking.
  */
 import { childSlots } from './child-slots.ts'
-import { continuationGate } from './late-frame.ts'
-import { OP_CHOICE, OP_CONT, OP_DISPATCH, OP_GAPGATE, OP_REP, OP_REPV } from './ops.ts'
+import { OP_CHOICE, OP_CONT, OP_DISPATCH, OP_GAPGATE, OP_REP, OP_REPV, OP_SEED } from './ops.ts'
 
 const LOOP = 8
 const ROUNDS = 60
 const CAP = 1e12
+/**
+ * What a crossing a site removes counts for, against one call of the site itself.
+ * Better than no weight, on every file and budget measured with real builds of
+ * Jess's grammars, but not tuned: 1 and 16 were only tried on a prototype, 2 and 8
+ * were never tried, and only 4 has been built and measured.
+ */
+const CROSSING_WEIGHT = 4
+const TAKEN = -1
 
 /**
  * The sites to emit: highest weight per byte first, stopping at the first that
@@ -39,14 +52,8 @@ export function selectHotSites(
 ): Set<number> {
   const selected = new Set<number>()
   if (budget <= 0) return selected
-  const weight = profile === undefined ? staticWeights(code, entries) : undefined
-  const w = (ip: number): number => (weight !== undefined ? weight.get(ip) : profile![ip]) ?? 0
-  // Ties (saturated cycles) break by site offset, so a build is deterministic.
-  const ranked = [...siteBytes.keys()].sort((a, b) =>
-    w(b) / siteBytes.get(b)! - w(a) / siteBytes.get(a)! || a - b)
   let spent = 0
-  for (const ip of ranked) {
-    if (w(ip) === 0) break
+  for (const ip of profile === undefined ? staticRanking(code, entries, siteBytes) : connectedRanking(code, siteBytes, profile)) {
     spent += siteBytes.get(ip)!
     if (spent > budget) break
     selected.add(ip)
@@ -54,6 +61,82 @@ export function selectHotSites(
   return selected
 }
 
+/**
+ * The sites the function for `ip` calls, as `emit-assembly.ts` lowers it. An `OP_CONT`
+ * with a body is a late frame, lowered with its wrapper in one function, calling its
+ * corner and its tail. With no body, the emitter can't decide there and it's an alias
+ * to its wrapper, as `OP_SEED` is to its corner. A gap gate has no body either, so its
+ * item counts as the repeat's own child, and adding the row moves no other site's
+ * estimate (`gap-gate.ts`).
+ */
+export function edgesAsLowered(code: readonly number[], siteBytes: ReadonlyMap<number, number>, ip: number, out: number[]): void {
+  out.length = 0
+  if (code[ip] !== OP_CONT) childSlots(code, ip, out)
+  else if (siteBytes.has(ip)) out.push(code[ip + 1]!, code[ip + 3]!)
+  else out.push(code[ip + 2]!)
+  for (let i = 0; i < out.length; i++) {
+    let row = out[i]!
+    for (;;) {
+      const op = code[row]
+      if (op === OP_GAPGATE || op === OP_SEED) row = code[row + 1]!
+      else if (op === OP_CONT && !siteBytes.has(row)) row = code[row + 2]!
+      else break
+    }
+    out[i] = row
+  }
+}
+
+/** The sites by estimated weight per byte. Ties (saturated cycles) break by site
+ * offset, so a build is deterministic. */
+function staticRanking(code: readonly number[], entries: readonly number[], siteBytes: ReadonlyMap<number, number>): number[] {
+  const weight = staticWeights(code, entries, siteBytes)
+  const w = (ip: number): number => weight.get(ip) ?? 0
+  return [...siteBytes.keys()].filter(ip => w(ip) > 0).sort((a, b) => w(b) / siteBytes.get(b)! - w(a) / siteBytes.get(a)! || a - b)
+}
+
+/**
+ * The sites the corpus ran, each next one being the best by calls plus the crossings
+ * it removes, per byte. A crossing with a neighbour is estimated by the calls of
+ * whichever is the callee, since the profile counts sites, not edges. Nothing bounds
+ * the sum: a callee with N emitted callers gains N crossings' worth of its own calls,
+ * and `sequence(g.A, g.A)` counts twice. The order doesn't depend on the budget, so a
+ * larger budget's selection is still a superset.
+ *
+ * ponytail: rescans every site per pick, O(n^2) in sites the corpus ran. 3 ms of a
+ * 50 s build on Jess's Less grammar. A heap if a grammar's ever takes seconds.
+ */
+function* connectedRanking(code: readonly number[], siteBytes: ReadonlyMap<number, number>, calls: ArrayLike<number>): Generator<number> {
+  // Ascending, so the first best a scan meets is the lowest: a tie breaks by site.
+  const ran = [...siteBytes.keys()].filter(ip => calls[ip]! > 0).sort((a, b) => a - b)
+  const slot = new Map(ran.map((ip, i) => [ip, i]))
+  const bytes = ran.map(ip => siteBytes.get(ip)!)
+  const worth = ran.map(ip => calls[ip]!)
+  const score = worth.map((value, i) => value / bytes[i]!)
+  const crossings: [neighbour: number, gain: number][][] = ran.map(() => [])
+  const kids: number[] = []
+  ran.forEach((ip, from) => {
+    edgesAsLowered(code, siteBytes, ip, kids)
+    for (const callee of kids) {
+      const to = slot.get(callee)
+      if (to === undefined) continue
+      const gain = CROSSING_WEIGHT * calls[callee]!
+      crossings[from]!.push([to, gain])
+      crossings[to]!.push([from, gain])
+    }
+  })
+  for (let left = ran.length; left > 0; left--) {
+    let best = 0
+    for (let i = 1; i < score.length; i++) if (score[i]! > score[best]!) best = i
+    score[best] = TAKEN
+    yield ran[best]!
+    for (const [neighbour, gain] of crossings[best]!) {
+      if (score[neighbour] === TAKEN) continue
+      worth[neighbour] = worth[neighbour]! + gain
+      score[neighbour] = worth[neighbour]! / bytes[neighbour]!
+    }
+  }
+}
+
 /**
  * The largest prefix of the ranking whose real source fits `ceiling`. `siteBytes`
  * leaves out what sites add around their bodies, so the estimate can overshoot:
@@ -81,30 +164,19 @@ export function fitHotSites(
   return undefined
 }
 
-function staticWeights(code: readonly number[], entries: readonly number[]): Map<number, number> {
+function staticWeights(code: readonly number[], entries: readonly number[], siteBytes: ReadonlyMap<number, number>): Map<number, number> {
   let weight = new Map<number, number>()
   const kids: number[] = []
   for (let round = 0; round < ROUNDS; round++) {
     const next = new Map<number, number>()
     for (const ip of entries) next.set(ip, 1)
     for (const [ip, w] of weight) {
-      kids.length = 0
-      childSlots(code, ip, kids)
+      edgesAsLowered(code, siteBytes, ip, kids)
       const op = code[ip]
-      // A late frame is lowered with its wrapper in one function, calling these two.
-      if (op === OP_CONT && continuationGate(code, code[ip + 3]!) !== undefined) {
-        kids.length = 0
-        kids.push(code[ip + 1]!, code[ip + 3]!)
-      }
       const scale = op === OP_REP || op === OP_REPV ? LOOP
         : op === OP_CHOICE || op === OP_DISPATCH ? 1 / Math.max(1, kids.length)
         : 1
-      for (const k of kids) {
-        // A gap gate has no body: weigh its item as the repeat's own child, so
-        // adding the row moves no other site's estimate (`gap-gate.ts`).
-        const kid = code[k] === OP_GAPGATE ? code[k + 1]! : k
-        next.set(kid, Math.min(CAP, (next.get(kid) ?? 0) + w * scale))
-      }
+      for (const kid of kids) next.set(kid, Math.min(CAP, (next.get(kid) ?? 0) + w * scale))
     }
     weight = next
   }

package/src/version.ts

@@ -13,4 +13,4 @@
  * Kept in sync with `package.json` by `test/unit/version-sync.test.ts`, which fails
  * the build if this constant drifts from the published version.
  */
-export const PARSEMAN_VERSION = '0.61.5'
+export const PARSEMAN_VERSION = '0.61.6'