parseman 0.57.0 → 0.58.0

package/src/plugin/index.ts

@@ -36,7 +36,7 @@ import {
   resolveDegradationLevel, recordDegradation, degradationCaptureDepth, unwindDegradationCapture,
 } from '../compiler/degradation.ts'
 import type { HostMode } from '../cst/host-mode.ts'
-import { COMPOSED_PIECES } from '../compiler/linker.ts'
+import { COMPOSED_PIECES, type IRPiece } from '../compiler/linker.ts'
 import { serializeCombinator, serializeRuleMap } from '../compiler/ir-serialize.ts'
 import { evalCombinatorIR, evalRulesIR, INERT_BUILDER } from './ir-eval.ts'
 import { countSiteVisits } from '../table/assemble.ts'
@@ -65,6 +65,48 @@ import type {
  */
 const TABLE_RUNTIME_SPECIFIER = 'parseman/table'
 
+/**
+ * BUILD-ONLY CARRIED PIECES.
+ *
+ * Nothing at runtime reads a grammar's carried IR: `compose()` links live `rules()`
+ * maps, and every runtime reader treats an IR piece as opaque
+ * (docs/design/runtime-and-size-contract.md, rules 1 and 2). Only a downstream macro
+ * build reads it. Under a rollup-family bundler the macro therefore leaves the IR out
+ * of the artifact and writes each chunk's carried lists beside it, as
+ * `<chunk>.parseman.json`; `resolveModulePieces` reads that file before it looks for
+ * an inline literal. Inline, the IR is about a quarter of a composed grammar's
+ * artifact, and the runtime spread that forwards an ancestor's pieces keeps that
+ * ancestor's module in the import graph of every parse.
+ *
+ * An item is an IR piece, or a reference to an export of a package module, resolved
+ * from the artifact's own location.
+ */
+export type CarriedSidecarItem = IRPiece | { from: string; name: string }
+/** One module's carried lists, by export name. */
+export type CarriedSink = Map<string, CarriedSidecarItem[]>
+export const CARRIED_SIDECAR_SUFFIX = '.parseman.json'
+type CarriedSidecar = { parseman: string; exports: Record<string, CarriedSidecarItem[]> }
+
+const _sidecarCache = new Map<string, { mtimeMs: number; sidecar: CarriedSidecar | null }>()
+/** The carried-pieces sidecar written beside a compiled artifact, or null. */
+function readCarriedSidecar(artifact: string): CarriedSidecar | null {
+  const file = artifact + CARRIED_SIDECAR_SUFFIX
+  let mtimeMs: number
+  try { mtimeMs = fs.statSync(file).mtimeMs } catch { return null }
+  const hit = _sidecarCache.get(file)
+  if (hit && hit.mtimeMs === mtimeMs) return hit.sidecar
+  let sidecar: CarriedSidecar | null = null
+  try {
+    // A file without an `exports` object is not a sidecar this plugin wrote: treat
+    // it as absent, so the inline literal is looked for and a missing one reports.
+    const parsed: unknown = JSON.parse(fs.readFileSync(file, 'utf8'))
+    const exports = typeof parsed === 'object' && parsed !== null ? (parsed as { exports?: unknown }).exports : undefined
+    if (typeof exports === 'object' && exports !== null) sidecar = parsed as CarriedSidecar
+  } catch { /* unreadable: no sidecar */ }
+  _sidecarCache.set(file, { mtimeMs, sidecar })
+  return sidecar
+}
+
 export type ParsecraftPluginOptions = {
   /** Extra module specifiers to treat as parseman re-exports */
   moduleAliases?: string[]
@@ -320,41 +362,142 @@ function parseModuleCached(filePath: string): ParsedModule | null {
   return parsed
 }
 
-export default createUnplugin((opts: ParsecraftPluginOptions = {}) => ({
-  name: 'parseman',
-  // Run BEFORE the bundler's TS/JS transform — otherwise esbuild (Vite) strips
-  // the `with { type: 'macro' }` import attribute before we ever see it, and the
-  // macro silently never fires.
-  enforce: 'pre' as const,
-
-  transformInclude(id: string) {
-    return /\.[jt]sx?$/.test(id) && !id.includes('node_modules')
-  },
-
-  transform(this: { warn?: (msg: string) => void; addWatchFile?: (file: string) => void }, code: string, id: string) {
-    if (!code.includes('parseman')) return null
-    if (!code.includes('macro')) return null
-    // A corpus edit changes which sites a profiled build generates, so a watch
-    // build re-lowers when one does.
-    for (const file of opts.assemblyProfile?.files ?? []) this?.addWatchFile?.(file)
-    const moduleAliases = new Set([PARSEMAN_MODULE, ...(opts.moduleAliases ?? [])])
-    const result = transformMacro(code, id, moduleAliases, opts.warnUnloweredRegex === true, opts.recovery === true, opts.grammarCoverage === true, opts.assemblyProfile, opts.assemblyBudget)
-    if (result?.warnings.length) {
-      for (const w of result.warnings) {
-        if (typeof this?.warn === 'function') this.warn(`[parseman] ${w}`)
-        else console.warn(`[parseman] ${w}`)
-      }
+/** The rollup-family hook that writes each chunk's carried pieces beside it. */
+type SidecarBundle = Record<string, { type: string; fileName: string; exports?: string[]; moduleIds?: string[]; facadeModuleId?: string | null }>
+type SidecarEmitter = {
+  emitFile: (file: { type: 'asset'; fileName: string; source: string }) => unknown
+  getModuleInfo: (id: string) => { code: string | null } | null
+  resolve: (source: string, importer?: string) => Promise<{ id: string; external?: boolean | string } | null>
+}
+
+/**
+ * Where a module's export `exported` comes from when the module re-exports it,
+ * possibly renamed: `export { base as renamed } from './grammar'`, or an imported
+ * binding exported under another name. Null when the module declares it itself.
+ */
+function reexportSource(body: AnyNode[], exported: string): { source: string; imported: string } | null {
+  for (const st of body) {
+    if (st.type !== 'ExportNamedDeclaration') continue
+    for (const sp of (st.specifiers as AnyNode[] | undefined) ?? []) {
+      const exp = sp.exported as { name?: string; value?: string } | undefined
+      if ((exp?.name ?? exp?.value) !== exported) continue
+      const local = sp.local as { name?: string; value?: string } | undefined
+      const name = local?.name ?? local?.value
+      if (name === undefined) return null
+      const from = (st.source as { value?: string } | null | undefined)?.value
+      return from !== undefined ? { source: from, imported: name } : extractImportBindings(body).get(name) ?? null
     }
-    if (!result) return null
-    return { code: result.code, map: result.map }
-  },
-}))
+  }
+  return null
+}
+
+/**
+ * The carried list behind an entry chunk's export, followed from the entry module
+ * through its re-exports to the module that recorded it, under the name that module
+ * exports it as. A chunk's export names are its entry module's, so an entry that
+ * renames a grammar (`export { base as renamed } from './grammar'`) still finds the
+ * pieces recorded as `base`.
+ */
+async function followCarried(this: SidecarEmitter, sinks: Map<string, CarriedSink>, moduleId: string, name: string): Promise<CarriedSidecarItem[] | undefined> {
+  for (let hop = 0; hop < 64; hop++) {
+    const own = sinks.get(moduleId)?.get(name)
+    if (own !== undefined) return own
+    const code = this.getModuleInfo(moduleId)?.code
+    if (!code) return undefined
+    const parsed = parseSync(moduleId, code)
+    if (parsed.errors.length > 0) return undefined
+    const next = reexportSource(parsed.program.body as unknown as AnyNode[], name)
+    if (next === null) return undefined
+    const resolved = await this.resolve(next.source, moduleId)
+    if (resolved === null || resolved.external) return undefined
+    moduleId = resolved.id
+    name = next.imported
+  }
+  return undefined
+}
+
+/**
+ * Write `<chunk>.parseman.json` for every chunk that exports a grammar with carried
+ * pieces. An entry chunk's export is followed from its entry module to the module
+ * that recorded it (`followCarried`). Any other export is matched by name to the
+ * module that recorded it, preferring a module of the chunk itself; a name two
+ * modules of the bundle both record, and neither of them in the chunk, is ambiguous
+ * and is not written — a downstream compose of it then fails its build naming the
+ * artifact.
+ */
+async function emitCarriedSidecars(this: SidecarEmitter, sinks: Map<string, CarriedSink>, bundle: SidecarBundle): Promise<void> {
+  const recorders = new Map<string, string[]>()
+  for (const [moduleId, sink] of sinks) for (const name of sink.keys()) (recorders.get(name) ?? recorders.set(name, []).get(name)!).push(moduleId)
+  for (const chunk of Object.values(bundle)) {
+    if (chunk.type !== 'chunk') continue
+    const exports: Record<string, CarriedSidecarItem[]> = {}
+    for (const name of chunk.exports ?? []) {
+      const followed = chunk.facadeModuleId ? await followCarried.call(this, sinks, chunk.facadeModuleId, name) : undefined
+      if (followed !== undefined) { exports[name] = followed; continue }
+      const ids = recorders.get(name)
+      if (ids === undefined) continue
+      const own = ids.filter(m => chunk.moduleIds?.includes(m))
+      const moduleId = own.length === 1 ? own[0] : own.length === 0 && ids.length === 1 ? ids[0] : undefined
+      if (moduleId !== undefined) exports[name] = sinks.get(moduleId)!.get(name)!
+    }
+    if (Object.keys(exports).length === 0) continue
+    const sidecar: CarriedSidecar = { parseman: PARSEMAN_VERSION, exports }
+    this.emitFile({ type: 'asset', fileName: chunk.fileName + CARRIED_SIDECAR_SUFFIX, source: JSON.stringify(sidecar) })
+  }
+}
+
+const SIDECAR_FRAMEWORKS = new Set(['rollup', 'rolldown', 'vite'])
+
+export default createUnplugin((opts: ParsecraftPluginOptions = {}, meta) => {
+  // Module id → its carried lists, under a bundler that writes them beside the chunk.
+  // Any other bundler gets them inline, as before.
+  const sinks = SIDECAR_FRAMEWORKS.has(meta.framework) ? new Map<string, CarriedSink>() : undefined
+  const generateBundle = sinks === undefined ? undefined : {
+    generateBundle(this: SidecarEmitter, _options: unknown, bundle: SidecarBundle) { return emitCarriedSidecars.call(this, sinks, bundle) },
+  }
+  return {
+    name: 'parseman',
+    ...(generateBundle === undefined ? {} : { rollup: generateBundle, rolldown: generateBundle, vite: generateBundle }),
+    // Run BEFORE the bundler's TS/JS transform — otherwise esbuild (Vite) strips
+    // the `with { type: 'macro' }` import attribute before we ever see it, and the
+    // macro silently never fires.
+    enforce: 'pre' as const,
+
+    transformInclude(id: string) {
+      return /\.[jt]sx?$/.test(id) && !id.includes('node_modules')
+    },
+
+    transform(this: { warn?: (msg: string) => void; addWatchFile?: (file: string) => void }, code: string, id: string) {
+      if (!code.includes('parseman')) return null
+      if (!code.includes('macro')) return null
+      // A corpus edit changes which sites a profiled build generates, so a watch
+      // build re-lowers when one does.
+      for (const file of opts.assemblyProfile?.files ?? []) this?.addWatchFile?.(file)
+      const moduleAliases = new Set([PARSEMAN_MODULE, ...(opts.moduleAliases ?? [])])
+      const sink: CarriedSink | undefined = sinks && new Map()
+      const result = transformMacro(code, id, moduleAliases, opts.warnUnloweredRegex === true, opts.recovery === true, opts.grammarCoverage === true, opts.assemblyProfile, opts.assemblyBudget, sink)
+      if (sinks && sink) {
+        if (sink.size > 0) sinks.set(id, sink)
+        else sinks.delete(id)
+      }
+      if (result?.warnings.length) {
+        for (const w of result.warnings) {
+          if (typeof this?.warn === 'function') this.warn(`[parseman] ${w}`)
+          else console.warn(`[parseman] ${w}`)
+        }
+      }
+      if (!result) return null
+      return { code: result.code, map: result.map }
+    },
+  }
+})
 
 // ---------------------------------------------------------------------------
 // Reading carried pieces back out of a COMPILED grammar module
 // ---------------------------------------------------------------------------
-// The macro embeds a grammar's linkable pieces inside its exported const's
-// initializer (see `withCarriedPieces`). To compose an imported grammar we parse
+// Under a rollup-family bundler the pieces are in the sidecar beside the module
+// (`readCarriedSidecar`). Otherwise the macro embeds them inside its exported const's
+// initializer (see `withCarriedPieces`), and to compose an imported grammar we parse
 // its compiled module, find that const, and pull the pieces literal back out —
 // these three pure helpers do exactly that (no runtime, no source recompile).
 
@@ -600,10 +743,11 @@ export function transformMacro(
   grammarCoverage = false,
   assemblyProfile?: AssemblyProfile,
   assemblyBudget = DEFAULT_ASSEMBLY_BUDGET,
+  carriedSink?: CarriedSink,
 ): TransformMacroResult | null {
   const depth = degradationCaptureDepth()
   try {
-    return transformMacroImpl(code, id, moduleAliases, warnUnloweredRegex, recovery, grammarCoverage, assemblyProfile, assemblyBudget)
+    return transformMacroImpl(code, id, moduleAliases, warnUnloweredRegex, recovery, grammarCoverage, assemblyProfile, assemblyBudget, carriedSink)
   } finally {
     setReducerResolver(null)
     setBuilderImportResolver(null)
@@ -622,6 +766,7 @@ function transformMacroImpl(
   grammarCoverage: boolean,
   assemblyProfile: AssemblyProfile | undefined,
   assemblyBudgetShare: number,
+  carriedSink: CarriedSink | undefined,
 ): TransformMacroResult | null {
   let result: ReturnType<typeof parseSync>
   try {
@@ -1653,6 +1798,23 @@ function transformMacroImpl(
     local = false,
   ): RawItem[] | null => {
     const { file } = module
+    const fail = (why: string): null => { unresolved.why ??= `${exportName} in ${file}: ${why}`; return null }
+    // A build-only sidecar wins over an inline literal: an artifact that has one
+    // carries no literal (see `CarriedSidecarItem`).
+    const sidecarItems = module.source || local ? undefined : readCarriedSidecar(file)?.exports[exportName]
+    if (sidecarItems !== undefined) {
+      const list: RawItem[] = []
+      for (const item of sidecarItems) {
+        if ('ir' in item) { list.push(item); continue }
+        const sub = resolveGrammarModule(file, item.from)
+        if (sub === null) return fail(`its carried pieces name \`${item.name}\` from '${item.from}', a module that could not be found`)
+        if (seen.has(sub.file)) continue
+        const pieces = resolveModulePieces(sub, item.name, new Set(seen).add(sub.file), unresolved)
+        if (pieces === null) return fail(`its carried pieces name \`${item.name}\` from '${item.from}', which could not be resolved`)
+        list.push(...pieces)
+      }
+      return list
+    }
     const mod = module.source
       ? lowerPrivateSourceModule(file, moduleAliases, warnUnloweredRegex, recovery)
       : parseModuleCached(file)
@@ -1660,9 +1822,15 @@ function transformMacroImpl(
     const localFor = local ? exportName : exportLocalName(mod.body as AnyNode[], exportName)
     const initNode = localFor ? topLevelInit(mod.body as AnyNode[], localFor) : null
     const literalRange = initNode ? findCarriedPiecesLiteral(initNode) : null
-    if (!literalRange) return null
+    if (!literalRange) {
+      // A compiled grammar with neither a literal nor a sidecar entry was built to be
+      // composed or it was not, and a runtime compose() cannot link it either way.
+      return initNode !== null && !module.source && mod.src.slice(initNode.start, initNode.end).includes('tableRules(')
+        ? fail(`is a compiled grammar with no carried pieces — no inline literal, and no entry in ${path.basename(file)}${CARRIED_SIDECAR_SUFFIX}. `
+          + 'Rebuild it with this parseman, and keep that file beside the artifact')
+        : null
+    }
     const literal = mod.src.slice(literalRange.start, literalRange.end)
-    const fail = (why: string): null => { unresolved.why ??= `${exportName} in ${file}: ${why}`; return null }
 
     const imports = extractImportBindings(mod.body as AnyNode[])
     const stubNames: string[] = []
@@ -1733,6 +1901,32 @@ function transformMacroImpl(
     return result
   }
 
+  /**
+   * Record an exported grammar's carried list in the build-only sink rather than
+   * emitting it into the runtime artifact. `false` when there is no sink, or when an
+   * entry cannot be written as data; the caller then emits it inline, as before.
+   *
+   * A package import stays a reference, resolved from the artifact's location just as
+   * the import itself is, so a descendant never re-serializes its ancestor. A relative
+   * import names a module the bundler folds into this chunk, which has no artifact of
+   * its own to point at, so its pieces are written out here.
+   */
+  const sinkCarried = (name: string, list: CarriedItem[]): boolean => {
+    if (carriedSink === undefined) return false
+    const out: CarriedSidecarItem[] = []
+    for (const it of list) {
+      if (!isSpread(it)) { out.push(it); continue }
+      const b = importBindings.get(it.__spreadLocal)
+      if (b === undefined) return false
+      if (!b.source.startsWith('.') && !path.isAbsolute(b.source)) { out.push({ from: b.source, name: b.imported }); continue }
+      const pieces = importedPieces(it.__spreadLocal)
+      if (pieces === null) return false
+      out.push(...pieces)
+    }
+    carriedSink.set(name, out)
+    return true
+  }
+
   /**
    * Evaluate a carried list back to combinators, one entry per piece: its rule map
    * and the `rules()` recipe it was built from. `compose()` links the recipes into
@@ -2493,6 +2687,7 @@ function transformMacroImpl(
             else warn(init.start, `${varName}: exported grammar could not be serialized to IR, so it carries no `
               + 'composable pieces — a downstream compose() of this grammar will fall back to runtime. '
               + 'Re-run with PARSEMAN_IR_DEBUG=1 to print the combinator that blocked serialization.')
+            if (carriedMetadata !== undefined && sinkCarried(varName, carriedMetadata)) carriedMetadata = undefined
           }
           const replacement = compiledRules.replacement !== null
             ? compiledRules.replacementWithMetadata!(staticTableMetadataSource({
@@ -2528,7 +2723,7 @@ function transformMacroImpl(
           // grammar via `import { <name> }` (re-composition, no source) and re-lower it
           // under ITS composing trivia.
           localComposedCarried.set(varName, fused.carried)
-          const replacement = exportPrefix ? fused.exportedReplacement : fused.replacement
+          const replacement = exportPrefix && !sinkCarried(varName, fused.carried) ? fused.exportedReplacement : fused.replacement
           // Coverage definitions already ride in the table program's `cv` pool;
           // `tableRules` exposes that same pool through the metadata prototype.
           replacements.push({ start: init.start, end: init.end, replacement })