tosijs-ui 1.6.21 → 1.6.23

package/dist/doc-system/site/orchestrator.js

@@ -12,32 +12,135 @@ the icon system here would put src/icon-data.ts into `bun --watch`'s graph and
 cause an endless rebuild loop.
 */
 import * as path from 'path';
-import { existsSync, mkdirSync } from 'fs';
-import { gzipSync } from 'zlib';
-import { $ } from 'bun';
+import { existsSync, mkdirSync, unlinkSync } from 'fs';
+import { tmpdir } from 'os';
+import { $, spawn } from 'bun';
 import { extractDocs } from './docs';
-import { checkExamples, formatExampleProblems } from './check-examples';
+import { preflight } from './preflight';
+import { checkExamples, formatExampleProblems, } from './check-examples';
 import { ensureSections } from './sections';
 import { generateLlmsTxt } from './make-llms-txt';
 import { generateSite } from './generate-site';
 import { findOutputDirOverlap } from './output-guard';
-import { buildEpub } from './epub';
+// Module specifiers contain regex metacharacters (`/`, `.`, `@`, …), so escape
+// before interpolating into the require-shim detector below.
+/** Give up on a hung ePub child rather than wedge the dev server's rebuild. */
+const EPUB_TIMEOUT_MS = 120_000;
+/**
+ * Run buildEpub() in a child process.
+ *
+ * buildEpub drives happy-dom (HTML→XHTML for every chapter) and @resvg/resvg-js
+ * (cover raster) — both native, both retaining — and it runs on EVERY dev rebuild,
+ * so in-process it strands memory in a watch process that lives for days. The child
+ * hands it all back on exit. If it hangs we kill it, and if it fails we warn: the
+ * ePub is a side artifact, so neither should block the page you're trying to look at.
+ *
+ * Only the data buildEpub reads is forwarded — SiteConfig also carries functions
+ * (llmsTxt, libraryBuild, prebuild) that don't survive JSON.
+ */
+async function buildEpubInChild(config, opts) {
+    // Resolve the sibling relative to THIS module so it works both in-repo (.ts)
+    // and when shipped (compiled .js) — same trick as generate-css below.
+    const cliTs = `${import.meta.dir}/epub-cli.ts`;
+    const cli = existsSync(cliTs) ? cliTs : `${import.meta.dir}/epub-cli.js`;
+    const payload = {
+        config: {
+            basePath: config.basePath,
+            baseUrl: config.baseUrl,
+            book: config.book,
+            docsJson: config.docsJson,
+            favicon: config.favicon,
+            lang: config.lang,
+            name: config.name,
+            outputDir: config.outputDir,
+        },
+        opts,
+    };
+    const payloadPath = path.join(tmpdir(), `tosijs-epub-${process.pid}.json`);
+    await Bun.write(payloadPath, JSON.stringify(payload));
+    const child = spawn(['bun', cli, payloadPath], {
+        stdout: 'inherit',
+        stderr: 'inherit',
+    });
+    const killer = setTimeout(() => {
+        console.warn(`⚠️  epub build exceeded ${EPUB_TIMEOUT_MS / 1000}s — killing it. The site is\n` +
+            `    fine; the .epub may be stale.`);
+        child.kill();
+    }, EPUB_TIMEOUT_MS);
+    try {
+        const code = await child.exited;
+        if (code !== 0) {
+            console.warn(`⚠️  epub build failed (exit ${code}). The site is fine; the .epub may be stale.`);
+        }
+    }
+    finally {
+        clearTimeout(killer);
+        try {
+            unlinkSync(payloadPath);
+        }
+        catch {
+            // already gone — fine
+        }
+    }
+}
 // Module specifiers contain regex metacharacters (`/`, `.`, `@`, …), so escape
 // before interpolating into the require-shim detector below.
 const escapeRegExp = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
-// Externalize the tjs CodeMirror editor extension ONLY when tjs-lang isn't
-// installed — otherwise it's bundled in so it shares the editor's CodeMirror
-// instance (see the `external` comment in buildSite). Returns [] when present.
-function tjsEditorExternal(root) {
+/**
+ * Run checkExamples() in a child process.
+ *
+ * It compiles every executable block in the corpus with `new AsyncFunction` on every
+ * rebuild — and JSC caches compiled code keyed by source text, so it retains, and
+ * (because a rebuild only happens when a file CHANGED) the source is fresh every time
+ * and nothing dedups. It also constructs a `Bun.Transpiler` per `ts` block (~40KB
+ * stranded per construction — oven-sh/bun#34053). Both accumulate in a watch process
+ * that lives for days; the child gives it all back on exit. Falls back to in-process
+ * if the child can't run — a build must not break because the leak-safe path is
+ * unavailable.
+ */
+async function checkExamplesInChild(docsJson) {
+    const cliTs = `${import.meta.dir}/check-examples-cli.ts`;
+    const cli = existsSync(cliTs)
+        ? cliTs
+        : `${import.meta.dir}/check-examples-cli.js`;
     try {
-        Bun.resolveSync('tjs-lang/editors/codemirror', root);
-        return [];
+        const child = spawn(['bun', cli, docsJson], {
+            stdout: 'pipe',
+            stderr: 'pipe',
+        });
+        // Drain BOTH pipes while awaiting exit. An undrained pipe fills its buffer, the
+        // child blocks writing to it, and we deadlock waiting for an exit that can't come.
+        const [out, err, code] = await Promise.all([
+            new Response(child.stdout).text(),
+            new Response(child.stderr).text(),
+            child.exited,
+        ]);
+        if (err.trim())
+            console.warn(err.trim());
+        if (code !== 0)
+            throw new Error(`check-examples exited ${code}`);
+        return JSON.parse(out);
     }
-    catch {
-        return ['tjs-lang/editors/codemirror'];
+    catch (e) {
+        console.warn(`⚠️  example check: could not run it in a child (${String(e)}) — ` +
+            `falling back to in-process.`);
+        const corpus = JSON.parse(await Bun.file(docsJson).text());
+        return checkExamples(corpus);
     }
 }
 export async function buildSite(config) {
+    // Look at the machine before adding load to it — a build runs on every watch
+    // rebuild, so a dying box (a runaway dev server elsewhere, a VM stall) is caught
+    // here too, not just at dev-server launch. Returns false rather than exiting: this
+    // is a public export, and an adopter's `await buildSite(cfg); await deploy()` must
+    // not be killed from inside a health check they never asked for.
+    if (!(await preflight({
+        label: 'Build',
+        devLimitMb: config.memoryLimitMb,
+        mode: config.preflight,
+    }))) {
+        return false;
+    }
     const PROJECT_ROOT = './';
     const PUBLIC = path.resolve(PROJECT_ROOT, config.outputDir ?? 'docs');
     const DIST = path.resolve(PROJECT_ROOT, 'dist');
@@ -86,8 +189,7 @@ export async function buildSite(config) {
     // display-only `typescript`. Runs on the whole corpus, so it catches breakage
     // on pages the browser test never navigates to.
     if (config.checkExamples !== false) {
-        const corpus = JSON.parse(await Bun.file(DOCS_JSON).text());
-        const problems = await checkExamples(corpus);
+        const problems = await checkExamplesInChild(DOCS_JSON);
         if (problems.length) {
             throw new Error(`doc-site build: ${problems.length} live example(s) failed to build:\n\n` +
                 formatExampleProblems(problems) +
@@ -142,36 +244,47 @@ export async function buildSite(config) {
     // consumer supplies (e.g. via staticDirs or an absolute URL).
     const scriptName = (config.scriptUrl ?? '/iife.js').replace(/^\//, '');
     if (config.bundleEntry) {
-        const result = await Bun.build({
-            entrypoints: [config.bundleEntry],
-            outdir: DIST,
-            sourcemap: 'linked',
-            format: 'iife',
-            minify: true,
-            naming: scriptName,
-            // tjs-lang's TRANSPILER (browser bundles) is dynamically import()'d at
-            // runtime (same-origin `/tjs/` copy, else CDN), so keep it out of the bundle.
-            //
-            // `tjs-lang/editors/codemirror` (the CodeMirror language + autocomplete) is
-            // different: it MUST share the editor's CodeMirror instance, so it has to be
-            // bundled IN — a separately loaded copy carries its own `@codemirror/state`
-            // and silently no-ops. So we do NOT externalize it *when tjs-lang is
-            // installed*. But tjs-lang is an OPTIONAL peer; if it's absent, bundling would
-            // fail to resolve, so we externalize it in that case (the runtime import then
-            // no-ops to plain TS highlighting — graceful degradation).
-            external: [
-                'tjs-lang/browser',
-                'tjs-lang/browser/from-ts',
-                ...tjsEditorExternal(PROJECT_ROOT),
-                ...(config.bundleExternals ?? []),
-            ],
-        });
-        if (!result.success) {
+        // tjs-lang transpiles live examples; it's dynamically import()'d at
+        // runtime (falling back to CDN), so keep it out of the bundle.
+        const externals = [
+            'tjs-lang',
+            'tjs-lang/browser',
+            'tjs-lang/browser/from-ts',
+            ...(config.bundleExternals ?? []),
+        ];
+        // Bundle in a CHILD PROCESS, not via Bun.build().
+        //
+        // Bun.build() never gives back the bundler's native arena: measured at ~9MB
+        // of RSS per call and rising, monotonic, with no plateau (40 sequential
+        // builds of one real entry = +367MB, still climbing ~5MB/build at the end),
+        // while the JS heap stays flat — so it is invisible to Bun.gc() and to any
+        // heap profiler. devServer() calls this once per rebuild in a process that
+        // lives for DAYS, so it compounds: a ~2-day watch session reached 136GB RSS
+        // and took the machine down with it. Filed as oven-sh/bun#34053.
+        //
+        // The CLI does identical work in a child whose memory the OS reclaims on
+        // exit: the same 15 bundles leave the parent +0.5MB instead of +192MB.
+        // Keep these flags in sync with the Bun.build() options they replace.
+        const bundle = spawn([
+            'bun',
+            'build',
+            config.bundleEntry,
+            '--outdir',
+            DIST,
+            '--sourcemap=linked',
+            '--format=iife',
+            '--minify',
+            '--entry-naming',
+            scriptName,
+            ...externals.flatMap((ext) => ['--external', ext]),
+        ], { stdout: 'inherit', stderr: 'inherit' });
+        if ((await bundle.exited) !== 0) {
             console.error('bundle build failed');
-            for (const message of result.logs)
-                console.error(message);
             return false;
         }
+        await $ `cp ${DIST}/${scriptName} ${PUBLIC}`.text();
+        const bundleFile = await Bun.file(`${DIST}/${scriptName}`).arrayBuffer();
+        const bundleJs = Buffer.from(bundleFile).toString('utf8');
         // Warn only when an external actually compiled to a synchronous require()
         // shim, which throws at module-eval ("Dynamic require of … is not
         // supported"). That only happens for a *static* `import x from 'ext'`. A
@@ -180,26 +293,37 @@ export async function buildSite(config) {
         // stay silent. The config alone can't tell the two apart; the emitted
         // bundle can, so we inspect the actual output.
         if (config.bundleExternals && config.bundleExternals.length > 0) {
-            const js = (await Promise.all(result.outputs
-                .filter((o) => o.kind === 'entry-point' || o.path.endsWith('.js'))
-                .map((o) => o.text()))).join('\n');
-            const broken = config.bundleExternals.filter((ext) => new RegExp(`(?:__require|\\brequire)\\(\\s*["'\`]${escapeRegExp(ext)}["'\`]`).test(js));
+            const broken = config.bundleExternals.filter((ext) => new RegExp(`(?:__require|\\brequire)\\(\\s*["'\`]${escapeRegExp(ext)}["'\`]`).test(bundleJs));
             if (broken.length > 0) {
                 console.warn(`⚠️  bundleExternals compiled to a synchronous require() shim that throws at runtime (${broken.join(', ')}). Reference these via a dynamic import() (kept async by the bundler)\n` +
                     `    or an importmap, not a static import.`);
             }
         }
-        await $ `cp ${DIST}/${scriptName} ${PUBLIC}`.text();
-        const bundleFile = await Bun.file(`${DIST}/${scriptName}`).arrayBuffer();
         // import.meta is illegal in a classic <script> — if it survived bundling
         // (a branch the bundler couldn't eliminate) the IIFE will SyntaxError.
-        if (Buffer.from(bundleFile).toString('utf8').includes('import.meta')) {
+        if (bundleJs.includes('import.meta')) {
             console.warn(`⚠️  ${scriptName} contains \`import.meta\`, which is a SyntaxError in a classic <script>.\n` +
                 `    A dependency referenced it in a branch the bundler couldn't drop — mark that dep external\n` +
                 `    (+ importmap) or choose a browser-only entry point.`);
         }
-        const bundleGzip = gzipSync(Buffer.from(bundleFile));
-        console.log(`${scriptName}: ${(bundleFile.byteLength / 1024).toFixed(1)}kb (${(bundleGzip.length / 1024).toFixed(1)}kb gzip)`);
+        // Gzip in a CHILD, not in-process. zlib's gzip is native, so it strands memory the
+        // JS heap never sees; this runs on every rebuild over a ~1MB bundle to print ONE
+        // size line, in a process that lives for days. Deliberately zlib-in-a-child rather
+        // than the `gzip` CLI: the two disagree by ~1.6%, and this number is tracked across
+        // releases, so moving the work must not move the measurement. Best-effort — a
+        // missing size line must never fail a build.
+        let gzipKb = '';
+        try {
+            const bytes = Number((await $ `bun -e ${`const {gzipSync}=require('zlib');const b=await Bun.file(${JSON.stringify(`${DIST}/${scriptName}`)}).arrayBuffer();process.stdout.write(String(gzipSync(Buffer.from(b)).length))`}`
+                .quiet()
+                .text()).trim());
+            if (bytes > 0)
+                gzipKb = ` (${(bytes / 1024).toFixed(1)}kb gzip)`;
+        }
+        catch {
+            // couldn't measure — report the raw size alone
+        }
+        console.log(`${scriptName}: ${(bundleFile.byteLength / 1024).toFixed(1)}kb${gzipKb}`);
     }
     else if (!/^(https?:)?\/\//.test(config.scriptUrl ?? '/iife.js')) {
         // No custom bundleEntry (the normal case for a pure-docs / book site) and a
@@ -319,7 +443,7 @@ export async function buildSite(config) {
     // the corpus and survives the `rm -rf <outputDir>` at the top of the NEXT build
     // (otherwise a dev rebuild would silently drop it). Cheap (~0.4s).
     if (config.epub) {
-        await buildEpub(config, typeof config.epub === 'object' ? config.epub : {});
+        await buildEpubInChild(config, typeof config.epub === 'object' ? config.epub : {});
     }
     console.timeEnd('build');
     return true;

package/dist/doc-system/site/preflight.d.ts

@@ -0,0 +1,152 @@
+export interface ProcInfo {
+    pid: number;
+    rssMb: number;
+    /** elapsed time as `ps` reports it, e.g. `01:23:45` or `2-04:11:07` */
+    etime: string;
+    command: string;
+}
+export interface Assessment {
+    level: 'ok' | 'warn' | 'fail';
+    /** processes worth naming, worst first */
+    offenders: ProcInfo[];
+    /** human-readable reason, empty when ok */
+    reason: string;
+}
+/**
+ * Parse `ps -eo pid=,rss=,etime=,args=` output. Bytes are KB in `ps`; the command
+ * is everything after the third field, so it may contain spaces.
+ */
+export declare function parsePs(output: string): ProcInfo[];
+/** `ps` etime — `[[dd-]hh:]mm:ss` — as hours. */
+export declare function etimeHours(etime: string): number;
+/**
+ * How much RAM is actually available to *dev work* — the denominator every
+ * percentage below is taken against.
+ *
+ * This is the whole trick, and it is worth being explicit about why it is not
+ * `totalmem()`. **Percentages of physical RAM scale with the hardware; headroom does
+ * not.** Buy 96GB more memory and spend all of it on a resident model, and the slack
+ * available to a runaway build is exactly what it was before — but every
+ * percentage-of-RAM threshold has quadrupled, so the guard gets four times laxer on
+ * the machine where it needed to be just as strict. On a 128GB box with a 96GB model
+ * up, a 50%-of-RAM "catastrophic" bar is 64GB: double the headroom that exists, so it
+ * can never fire in time.
+ *
+ * Budget = RAM − (what exempt processes hold) − (`reservedMb`), floored at
+ * `minBudgetPct` of RAM so a machine that is genuinely fully committed still gets
+ * sane thresholds rather than absurd ones. On the 128GB/96GB box that yields a ~32GB
+ * budget — the same absolute thresholds that are right on a 32GB laptop, because the
+ * slack is the same. Unload the model and the budget floats back up on its own.
+ *
+ * `devLimitMb` is deliberately NOT a percentage: it survives a hardware change
+ * precisely because it is absolute. 4GB is too big for a dev server on any machine.
+ */
+export declare function memoryBudgetMb(procs: ProcInfo[], opts: {
+    totalRamMb: number;
+    reservedMb?: number;
+    minBudgetPct?: number;
+}): number;
+/**
+ * Judge the process table against physical RAM.
+ *
+ * THREE per-process triggers. Note what is deliberately NOT here: a rule that SUMS
+ * RSS across processes. It is the obvious idea and it is wrong on macOS — at the
+ * moment of the fatal crash, total RSS across 629 processes was ~225GB against 32GB
+ * of RAM, and the box was still limping. Compression and swap mean summed footprint
+ * routinely exceeds physical RAM on a perfectly healthy machine, so an aggregate-RSS
+ * rule would fire during ordinary work and get switched off within a week. A SINGLE
+ * process at half the machine is genuinely anomalous; the sum is not. (For a real
+ * machine-level signal, see `assessMemoryPressure` below — VM pressure, not RSS
+ * arithmetic.)
+ *
+ * Every percentage is taken against the **dev memory budget** (`memoryBudgetMb`), NOT
+ * against physical RAM — see that function for why. Exempt processes (a resident LLM,
+ * say) can never be offenders; they shrink the budget instead.
+ *
+ * - **Any process over `catastrophicPct` of the budget** (default 50%). Nothing
+ *   healthy is half the available machine. Not limited to dev processes — if
+ *   something else is eating the box, adding a build to it is still the wrong move.
+ * - **A dev process over `hugeDevPct` of the budget** (default 25%), *regardless of
+ *   age*. The age gate below is a real blind spot: the runaway that took the machine
+ *   down went 0→100GB in about twenty minutes, so it spent its entire climb through
+ *   the 4–16GB band both too young to be "stale" and too small to be "catastrophic" —
+ *   i.e. invisible for the whole ascent. No legitimate dev process is a quarter of
+ *   the available machine, young or not.
+ * - **A long-lived dev process over `devLimitMb`** (default: the dev server's own RSS
+ *   ceiling). Absolute, not a percentage — 4GB is too big for a dev server on any
+ *   machine, and an absolute limit survives a hardware upgrade. Age-gated so a build
+ *   legitimately peaking mid-run isn't flagged.
+ *
+ * `selfPid` is excluded throughout — this process's own growth is the memory
+ * watchdog's job, and it is bounded there.
+ */
+export declare function assessProcesses(procs: ProcInfo[], opts: {
+    totalRamMb: number;
+    selfPid?: number;
+    devLimitMb?: number;
+    catastrophicPct?: number;
+    hugeDevPct?: number;
+    minAgeHours?: number;
+    /** RAM to hold back for known-heavy non-dev work, on top of exempt processes */
+    reservedMb?: number;
+}): Assessment;
+export interface VmPressure {
+    freeMb: number;
+    compressorMb: number;
+    swapUsedMb: number;
+}
+/** Parse `vm_stat`. Page size is in its header ("page size of 16384 bytes"). */
+export declare function parseVmStat(output: string): {
+    freeMb: number;
+    compressorMb: number;
+};
+/** Parse `sysctl vm.swapusage` → used MB. */
+export declare function parseSwapUsage(output: string): number;
+/**
+ * Judge VM pressure.
+ *
+ * Requires BOTH a swollen compressor AND almost no free memory. Either alone is
+ * normal: macOS keeps free memory low on purpose (it is caching, not idling), and a
+ * few GB of compressor on a busy machine is routine. It is the combination — the
+ * compressor has eaten a large slice of RAM *and* there is nothing left to hand out
+ * — that means the VM system is about to stall. Swap is reported for context and as
+ * a corroborating trigger, not on its own: a Mac can sit on GBs of swap indefinitely
+ * while perfectly healthy.
+ */
+export declare function assessMemoryPressure(vm: VmPressure, opts: {
+    totalRamMb: number;
+    compressorPct?: number;
+    freeFloorMb?: number;
+    swapFloorMb?: number;
+}): {
+    level: 'ok' | 'fail';
+    reason: string;
+};
+/**
+ * Check the machine before adding load to it.
+ *
+ * Returns **true when it is safe to proceed**. It never calls `process.exit` — this is
+ * library code (`buildSite`/`devServer` are public exports of `tosijs-ui/site`), and an
+ * adopter's `await buildSite(cfg); await publishToS3()` must not be killed from inside
+ * a health check it did not ask for. The caller decides what a `false` means.
+ *
+ * Modes (`mode`, or `preflight` in SiteConfig):
+ *   'fail' — refuse (return false). Auto-downgraded to 'warn' in CI / non-TTY.
+ *   'warn' — print and proceed.
+ *   'off'  — skip entirely. Also `DEV_SKIP_PREFLIGHT=1`.
+ *
+ * Best-effort by construction: `ps` is POSIX-only, and a health check that breaks the
+ * build when IT fails is worse than no health check at all.
+ */
+export type PreflightMode = 'off' | 'warn' | 'fail';
+export declare function preflight(opts?: {
+    devLimitMb?: number;
+    label?: string;
+    mode?: PreflightMode | false;
+    /** injected for tests; defaults to a real `ps` snapshot */
+    procs?: ProcInfo[];
+    /** injected for tests; defaults to this machine's physical RAM */
+    totalRamMb?: number;
+    /** injected for tests; defaults to a real vm_stat/swapusage read (null off darwin) */
+    vm?: VmPressure | null;
+}): Promise<boolean>;