rvlib-shipkit 1.0.1 → 1.1.0

package/src/serve/hub-app/cols-svc.ts

@@ -5,7 +5,7 @@
 // the card.
 import { makeAutoObservable, runInAction } from 'mobx'
 import type { HubSt } from './hub-st.ts'
-import type { Col, ColMeta, Resource, Schedule } from './hub-types.ts'
+import type { Col, ColMeta, HubProblem, Resource, Schedule } from './hub-types.ts'
 
 export class ColsSvc {
     /** exactly what the server last sent; `resources` is this plus the live
@@ -14,6 +14,10 @@ export class ColsSvc {
     colMeta: ColMeta[] = []
     /** the hub's daily health pass timer, as the last poll saw it */
     schedule: Schedule | null = null
+    /** what the server says is wrong with the hub itself */
+    hubProblems: HubProblem[] = []
+    /** the last poll failed: the page cannot reach its own server */
+    fetchErr: string | null = null
     actErr = ''
     remarkDraft = ''
     proofDraft = ''
@@ -37,20 +41,18 @@ export class ColsSvc {
         try {
             const r = await fetch('/api/resources.json')
             if (!r.ok) throw new Error(await r.text())
-            const j = (await r.json()) as { collections: ColMeta[]; resources: Resource[]; schedule?: Schedule | null }
+            const j = (await r.json()) as { collections: ColMeta[]; resources: Resource[]; schedule?: Schedule | null; hub?: HubProblem[] }
             runInAction(() => {
                 this.colMeta = j.collections
                 this.fetched = j.resources
                 this.schedule = j.schedule ?? null
+                this.hubProblems = j.hub ?? []
+                this.fetchErr = null
             })
         } catch (e) {
-            // a dead server must SAY so on the tree, never render an empty
-            // repo that reads like a healthy one
-            const text = `resources fetch broke: ${(e as Error).message}`
-            runInAction(() => {
-                this.colMeta = [{ id: 'hub', title: 'hub', media: 'health' }]
-                this.fetched = [{ id: 'hub:resources-broken', collection: 'hub', type: 'health', path: 'health/hub-fetch', virtual: true, title: text, text, state: 'red', severity: 'red' }]
-            })
+            // a dead server must SAY so, in the header, never leave a tree
+            // that reads like a healthy repo. The last rows stay, dated by it
+            runInAction(() => { this.fetchErr = `the page cannot reach this hub: ${(e as Error).message}` })
         }
     }
 
@@ -62,6 +64,11 @@ export class ColsSvc {
     }
 
     /** the collections, as a grouping of the one array */
+    /** the header's red: the server's own problems, then an unreachable server */
+    get hubStatus(): HubProblem[] {
+        return [...this.hubProblems, ...(this.fetchErr == null ? [] : [{ id: 'fetch', text: this.fetchErr, severity: 'red' as const }])]
+    }
+
     get cols(): Col[] {
         return this.colMeta.map((m) => ({ ...m, instances: this.resources.filter((r) => r.collection === m.id) }))
     }

package/src/serve/hub-app/dur.ts

@@ -31,9 +31,18 @@ const group = (n: number): string => {
 
 export const fmtMs = (ms: number | null | undefined): string => (ms == null ? '' : `${group(Math.round(ms))}ms`)
 
-export function durationTone(ms: number | null | undefined, ceilingMs?: number | null): DurTone | null {
+/** the tone of one duration. A check's own BUDGET, when the caller knows it,
+ *  replaces the table above it: over the budget is red, past half of it
+ *  orange, and a suite well inside a generous budget never reads red because
+ *  a rule would be slow at that length (core/budget.ts) */
+export function durationTone(ms: number | null | undefined, ceilingMs?: number | null, budgetMs?: number | null): DurTone | null {
     if (ms == null) return null
     if (ceilingMs != null && ceilingMs > 0 && ms > ceilingMs * CEILING_SHARE) return 'bad'
+    if (budgetMs != null && budgetMs > 0) {
+        if (ms > budgetMs) return 'bad'
+        if (ms > budgetMs / 2) return 'slow'
+        return ms < FAST_MS ? 'fast' : 'plain'
+    }
     if (ms < FAST_MS) return 'fast'
     if (ms < PLAIN_MS) return 'plain'
     if (ms < SLOW_MS) return 'slow'

package/src/serve/hub-app/header-bar.tsx

@@ -46,6 +46,16 @@ const RunState = observer(({ app }: { app: HubSt }) => {
     )
 })
 
+/** what is wrong with the hub itself, one chip per problem. Nothing when all
+ *  is well: the header states trouble, never health */
+const HubStatus = observer(({ app }: { app: HubSt }) => (
+    <>
+        {app.cols.hubStatus.map((p) => (
+            <span key={p.id} className={`hstatus ${p.severity === 'red' ? 'red' : 'warn'}`} title={p.text}>{p.text}</span>
+        ))}
+    </>
+))
+
 export const HeaderBar = observer(({ app }: { app: HubSt }) => {
     const u = app.checks
     return (
@@ -57,6 +67,7 @@ export const HeaderBar = observer(({ app }: { app: HubSt }) => {
                 ))}
             </nav>
             <span className="sp" />
+            <HubStatus app={app} />
             <RunState app={app} />
             <button className="primary" disabled={u.busy} onClick={() => void u.run()}>run the health pass</button>
             <button disabled={u.busy} onClick={() => void u.run(undefined, { gate: true, label: 'the gate' })}>run the gate</button>

package/src/serve/hub-app/help.tsx

@@ -14,6 +14,7 @@ export const TERMS: Record<string, string> = {
     advisory: 'it can fail without stopping anything: the gate reports it and goes on',
     blocking: 'its failure stops the gate, and stops you',
     ceiling: 'the time it is killed at: a check creeping toward its ceiling is visible before it trips',
+    budget: 'what a healthy run takes: a run that passes over it warns and reads yellow, and never blocks. Set it with budgetMs in the check file, or budgets in repo.config.ts',
     cached: 'it did not run: the answer came from the cache, because nothing it depends on moved since it last ran',
     ran: 'it ran and was timed, so the number is what it costs today',
     slowest: 'the worst run on record, reds included',

package/src/serve/hub-app/hub-st.ts

@@ -134,10 +134,9 @@ export class HubSt {
         // the runs page is a page, not a resource, so no collection lists it
         out.push({ id: 'page:runs', label: 'runs', section: 'page', run: () => this.cols.select('runs', RUNS_ID) })
         for (const col of this.cols.sidebarCols) {
-            const health = col.id === 'health'
-            out.push({ id: `col:${col.id}`, label: col.title, section: 'section', run: () => (health ? this.showStatus() : this.showCol(col.id)) })
+            out.push({ id: `col:${col.id}`, label: col.title, section: 'section', run: () => this.showCol(col.id) })
             for (const i of col.instances)
-                out.push({ id: `item:${i.id}`, label: i.text, section: col.title, run: () => (health ? this.showStatus() : this.cols.select(col.id, i.id)) })
+                out.push({ id: `item:${i.id}`, label: i.text, section: col.title, run: () => this.cols.select(col.id, i.id) })
         }
         return out
     }
@@ -203,7 +202,7 @@ export class HubSt {
         this.cols.select(r.collection, r.id)
     }
 
-    /** collection view — one card per instance (health has StatusCard instead) */
+    /** collection view — one card per instance */
     showCol(id: string): void {
         this.setSel({ t: 'col', id })
     }

package/src/serve/hub-app/hub-types.ts

@@ -105,6 +105,7 @@ export type CheckFacts = {
     ttl?: string
     never?: string
     timeoutMs: number
+    budgetMs: number
     status: CheckStatus | null
     replayed: boolean
     lastMs: number | null
@@ -121,6 +122,10 @@ export type CheckFacts = {
     recent?: RecentRun[]
 }
 
+/** what is wrong with the hub itself, as the resources feed sends it
+ *  (serve/hub-problems.ts): the header shows it, never the tree */
+export type HubProblem = { id: string; text: string; severity: 'red' | 'yellow' }
+
 /** the hub's daily health pass timer, as the resources feed sends it
  *  (serve/health-schedule.ts). null = this hub runs no timer */
 export type Schedule = { everyMs: number; lastPassAt: number | null; nextAt: number; running: boolean }

package/src/serve/hub-app/hub.css

@@ -37,6 +37,9 @@ body{margin:0;background:var(--bg);color:var(--ink);font:14px/1.5 system-ui,sans
 .hnav button:hover{color:var(--ink);border-color:transparent;border-bottom-color:var(--line)}
 .hnav button.on{color:var(--accent);border-bottom-color:var(--accent)}
 /* the run state holds its width whatever it says, so a run never moves the buttons beside it */
+.hstatus{max-width:420px;flex:0 1 auto;font-size:11.5px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
+.hstatus.red{color:var(--red)}
+.hstatus.warn{color:var(--warn)}
 .hstate{display:inline-flex;gap:var(--s1);align-items:center;width:260px;min-width:260px;flex:none;justify-content:flex-end;font-size:11.5px;color:var(--dim);overflow:hidden}
 .hstate.on{color:var(--ink)}
 .hstate .htext{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}

package/src/serve/hub-app/timeline-view.tsx

@@ -163,7 +163,7 @@ const TestFiles = ({ app, rows, recorded }: { app: HubSt; rows: SuiteToFix[]; re
     )
 }
 
-const FLAG_WORD = { over: 'at its ceiling', near: 'past half its ceiling' } as const
+const FLAG_WORD = { over: 'at its ceiling', near: 'past half its ceiling', budget: 'over its budget' } as const
 
 /** two numbers side by side need their names once, above the column */
 const Slowest = ({ app, rows }: { app: HubSt; rows: SlowCheck[] }) => {
@@ -174,9 +174,9 @@ const Slowest = ({ app, rows }: { app: HubSt; rows: SlowCheck[] }) => {
             {rows.map((r) => (
                 <div key={r.id} className="rline rslow" onClick={() => app.timeline.openCheck(r.id)}>
                     <span className="rtag">{r.id}</span>
-                    <span className={`rnum ${durationTone(r.meanMs, r.ceilingMs) ?? ''}`} title="mean of the runs that ran">{fmtMs(r.meanMs)}</span>
-                    <span className={`rnum ${durationTone(r.worstMs, r.ceilingMs) ?? ''}`} title="the worst run that ran">{fmtMs(r.worstMs)}</span>
-                    <span className={r.flag === 'over' ? 'rflag red' : 'rflag warn'} title={r.ceilingMs == null ? undefined : `ceiling ${fmtMs(r.ceilingMs)}`}>{r.flag == null ? '' : FLAG_WORD[r.flag]}</span>
+                    <span className={`rnum ${durationTone(r.meanMs, r.ceilingMs, r.budgetMs) ?? ''}`} title="mean of the runs that ran">{fmtMs(r.meanMs)}</span>
+                    <span className={`rnum ${durationTone(r.worstMs, r.ceilingMs, r.budgetMs) ?? ''}`} title="the worst run that ran">{fmtMs(r.worstMs)}</span>
+                    <span className={r.flag === 'over' ? 'rflag red' : 'rflag warn'} title={[r.budgetMs == null ? null : `budget ${fmtMs(r.budgetMs)}`, r.ceilingMs == null ? null : `ceiling ${fmtMs(r.ceilingMs)}`].filter((x) => x != null).join(', ') || undefined}>{r.flag == null ? '' : FLAG_WORD[r.flag]}</span>
                 </div>
             ))}
         </>

package/src/serve/hub-app/tree-build.ts

@@ -335,7 +335,7 @@ export function rowSlots(row: TreeRow, running: boolean): RowSlots {
         state: verdict[0],
         tone: verdict[1],
         dur: fmtMs(u.lastMs),
-        durTone: durationTone(u.lastMs, u.timeoutMs),
+        durTone: durationTone(u.lastMs, u.timeoutMs, u.budgetMs),
         note: WHEN_NOTE[u.when],
         count: null,
         hint: [
@@ -391,7 +391,6 @@ export const TYPE_MARK: Record<string, TypeMark> = {
     workflow: { glyph: '\u29c9', hue: 'k6', word: 'a comfy workflow module' },
     wiki: { glyph: '\u2b21', hue: 'k3', word: 'a markdown wiki, opened as its link graph' },
     contract: { glyph: '\u00a7', hue: 'k4', word: 'a release contract of a venue' },
-    health: { glyph: '\u229e', hue: 'k5', word: 'a health row that belongs to no other kind' },
     check: { glyph: '\u2299', hue: 'k6', word: 'a gate check: one question `shipkit ci` or `shipkit check` asks, whether it builds, guards a repo shape or probes the world. same page and same switch for all of them' },
     test: { glyph: '\u2263', hue: 'k5', word: 'a test file: the check that runs it, its last runs against its budget, the command that runs it alone' },
     reflection: { glyph: '\u203b', hue: 'k2', word: 'a reflection: friction this repo logged against the wiki, waiting to be folded in' },

package/src/serve/hub-problems.ts

@@ -0,0 +1,35 @@
+// what is wrong with the hub ITSELF, as opposed to the repo it serves: a
+// collection it could not list, a restart it could not do. No row owns
+// these, so they ride the hub's status: the header on the page, and the
+// problems frame the hub task prints (problems/problem-board.ts), so the
+// pane and the page count the same reds.
+
+export type HubProblem = { id: string; text: string; severity: 'red' | 'yellow' }
+
+export class HubProblems {
+    private readonly byId = new Map<string, HubProblem>()
+
+    constructor(private readonly onChange: (all: HubProblem[]) => void = () => {}) {}
+
+    /** a problem under this id, or none (null). Tells onChange only when the
+     *  list moved */
+    set(id: string, p: Omit<HubProblem, 'id'> | null): void {
+        const before = this.byId.get(id)
+        if (p == null) {
+            if (before == null) return
+            this.byId.delete(id)
+        } else {
+            if (before != null && before.text === p.text && before.severity === p.severity) return
+            this.byId.set(id, { id, ...p })
+        }
+        this.onChange(this.list())
+    }
+
+    list(): HubProblem[] {
+        return [...this.byId.values()].sort((a, b) => a.id.localeCompare(b.id))
+    }
+
+    /** what projectResources / projectNeeds report per collection */
+    collector = (collection: string, error: string | null): void =>
+        this.set(`collector:${collection}`, error == null ? null : { text: `the hub could not list ${collection}: ${error}`, severity: 'red' })
+}

package/src/serve/run-serve.ts

@@ -9,7 +9,6 @@
 import { existsSync, readFileSync, statSync } from 'fs'
 import { homedir } from 'os'
 import { basename, dirname, join, resolve, sep } from 'path'
-import { fileURLToPath } from 'url'
 import { loadKit, type ResolvedKit } from '../assets/kit-config.ts'
 import { studioRoutes, type StudioRoutes } from '../assets/server/studio-server.ts'
 import { projectNeeds } from '../collections/collection-types.ts'
@@ -42,9 +41,13 @@ import { BUILD_TIMEOUT_MS, CHECK_TIMEOUT_MS } from '../collections/checks.ts'
 import { HealthSchedule } from './health-schedule.ts'
 import { GateTrigger, watchTracked } from './gate-trigger.ts'
 import { ProblemBoard } from '../problems/problem-board.ts'
+import { HubProblems } from './hub-problems.ts'
+import { StaleRestart } from './stale-restart.ts'
+import { reexecBlocker, reexecSelf } from '../core/reexec.ts'
 import { serveHub, type HubDeps } from './serve-hub.ts'
 import { defaultHubPort } from './hub-port.ts'
 import { LAYOUT } from '../core/layout.ts'
+import { diskLibVersion, LIB_VERSION } from '../core/lib-version.ts'
 export { defaultHubPort }
 
 export type ServeIo = { print: (l: string) => void; printErr: (l: string) => void }
@@ -153,7 +156,7 @@ export async function resolveHubDeps(root: string, boot: RepoConfig, io: ServeIo
     const hasComfy = scanComfyWorkflows(root).length > 0
     const panel = hasComfy ? new ComfyPanel({ root, port: c.serve?.comfyPort ?? defaultComfyPanelPort(c.name) }) : null
     const allCollections = () => [
-        ...buildCollections(root, c, kit, { bootVersion: BOOT_VERSION, libVersion }),
+        ...buildCollections(root, c, kit),
         ...(hasComfy ? [comfyCollection(root)] : []),
         wikisCollection(graphs, root),
     ]
@@ -167,6 +170,11 @@ export async function resolveHubDeps(root: string, boot: RepoConfig, io: ServeIo
     // the latest verdict of every check, printed as one problem frame after
     // each run for the vscode problems pane (problems/problem-line.ts)
     const board = new ProblemBoard()
+    // what is wrong with the hub itself rides the same frame, the moment it
+    // moves, so the pane never reads 0 while the page shows a hub red
+    const hubProblems = new HubProblems((all) => {
+        if (board.setHub(all)) for (const l of board.frame()) io.print(l)
+    })
     const runChecks: HubDeps['runChecks'] = (tap, o) => {
         const run = chain.then(async () => {
             active++
@@ -267,13 +275,18 @@ export async function resolveHubDeps(root: string, boot: RepoConfig, io: ServeIo
         repoHub: (repo) => others.hubOf(repo),
         timeline: async () => {
             const ceilings: Record<string, number> = {}
+            const budgets: Record<string, number> = {}
             const inventory = await gateInventory(root, c)
-            for (const row of inventory) ceilings[row.decl.id] = row.timeoutMs ?? (row.build ? BUILD_TIMEOUT_MS : CHECK_TIMEOUT_MS)
+            for (const row of inventory) {
+                ceilings[row.decl.id] = row.timeoutMs ?? (row.build ? BUILD_TIMEOUT_MS : CHECK_TIMEOUT_MS)
+                budgets[row.decl.id] = row.budgetMs
+            }
             return shapeTimeline({
                 runs: readRuns(record),
                 invocations: readInvocations(record),
                 suites: readSuites(record),
                 ceilings,
+                budgets,
                 inventory: inventory.map((row) => row.decl.id),
                 declaredSlow: declaredSlowFiles(c),
                 cap: parallelCap(),
@@ -287,7 +300,9 @@ export async function resolveHubDeps(root: string, boot: RepoConfig, io: ServeIo
             return check == null || check === '' ? shapeTestsRoot(input) : shapeTestCheck(check, input)
         },
         prose: async () => proseView(CheckCtx.open({ root, config: c }), c),
-        needs: () => projectNeeds(allCollections()),
+        needs: () => projectNeeds(allCollections(), hubProblems.collector),
+        hubProblems,
+        busy: () => active > 0,
         collections: allCollections,
         asksDir: c.asks?.dir ?? LAYOUT.asks,
         comfy: panel == null ? null : { ensure: () => panel.ensure(), restart: () => panel.restart(), box: () => comfyBox(root), stop: () => panel.stop() },
@@ -319,14 +334,6 @@ export function hubGateOpts(o: { only?: string[]; auto: boolean }): Pick<CiOpts,
     return { onlyIds: o.only, fix: whole, status: whole }
 }
 
-// the hub must KNOW when it is outdated — a stale morning
-// process serves broken-looking picks all day: version read at
-// boot vs on disk per attention load; mismatch = red line telling him to
-// restart the task
-const libVersion = (): string => {
-    try { return (JSON.parse(readFileSync(fileURLToPath(new URL('../../package.json', import.meta.url)), 'utf8')) as { version?: string }).version ?? '?' } catch { return '?' }
-}
-const BOOT_VERSION = libVersion()
 
 // the / dashboard's data now comes from the collection providers
 // (src/collections/providers.ts) — never a parallel projection.
@@ -368,12 +375,44 @@ export async function runServe(root: string, io: ServeIo, o?: { port?: number })
         })
         const watcher = watchTracked(root, () => trigger.touched(), { onError: (e) => io.printErr(`WARN the file watch stopped, so the problems pane refreshes only on a hub run: ${e.message}`) })
         if (watcher.error != null) io.printErr(`WARN no file watch (${watcher.error}), so the problems pane refreshes only on a hub run`)
+        // a lib that moved under this process (under bun --watch the version
+        // import already restarted it, so what is left is a plain serve), or
+        // a file table the watch restarts leaked: it re-executes itself in
+        // place (same pid, argv, port) once no run is in flight. Only a
+        // restart that throws is said, in the hub's own status
+        const stale = new StaleRestart({
+            boot: LIB_VERSION,
+            disk: diskLibVersion,
+            busy: () => deps.busy?.() === true,
+            restart: (why) => {
+                // a refusal throws while the hub still serves, so the page can say it
+                const blocked = reexecBlocker()
+                if (blocked != null) throw new Error(blocked)
+                io.print(`ok: ${why}: restarting in place`)
+                stale.stop()
+                trigger.stop()
+                watcher.close()
+                hub.stop()
+                try {
+                    reexecSelf()
+                } catch (e) {
+                    // the server is already down: nothing is left to show a status
+                    io.printErr(`FAIL: this hub stopped to restart and could not: ${(e as Error).message}. Restart the hub task`)
+                    process.exit(1)
+                }
+            },
+            say: (text) => deps.hubProblems?.set('stale', text == null ? null : { text, severity: 'red' }),
+        })
+        stale.start()
+        // a watch restart that leaked sheds it before the first run spawns anything
+        stale.tick()
         trigger.request()
         // a child process (the comfy panel) outlives a parent nobody stopped,
         // and the next launch then adopts one holding the PREVIOUS run's
         // modules. A hard kill still leaks one, which is why adoption exists
         for (const sig of ['SIGINT', 'SIGTERM'] as const)
             process.on(sig, () => {
+                stale.stop()
                 trigger.stop()
                 watcher.close()
                 hub.stop()

package/src/serve/serve-hub.ts

@@ -10,7 +10,7 @@ import { graphRoutes } from 'rvlib-shipkit/src/wiki/serve-graph.ts'
 import type { StudioRoutes } from '../assets/server/studio-server.ts'
 import { comfyBoxUp, type ComfyBox, type ComfyPanelUp } from '../comfy/comfy-panel.ts'
 import { appendAskRemark, appendAssetNote, appendPostFeedback, askDone, askProof, markPosted, pickPostVariant } from '../collections/actions.ts'
-import { projectResources, type Collection, type Instance, type NeedItem } from '../collections/collection-types.ts'
+import { brokenRow, projectResources, type Collection, type Instance, type NeedItem } from '../collections/collection-types.ts'
 import type { CheckEvent } from '../core/run-check.ts'
 import { configDirty, readGateBlock, writeGateBlock, type GateEdit } from '../core/when-block.ts'
 import type { InvokedBy } from '../ledger/run-log.ts'
@@ -22,6 +22,7 @@ import type { LiveView } from '../ledger/live-timeline.ts'
 
 export type { LiveView }
 import type { HealthSchedule } from './health-schedule.ts'
+import type { HubProblems } from './hub-problems.ts'
 import hubIndex from './hub-app/index.html'
 import { projectGutter } from './project-gutter-route.ts'
 import { sameOrigin } from './same-origin.ts'
@@ -78,6 +79,11 @@ export type HubDeps = {
     /** every human-action item, red/yellow, gathered per request from the
      *  collection providers (run-serve builds it) */
     needs: () => Promise<NeedItem[]>
+    /** a run is in flight (a restart now would kill it) */
+    busy?: () => boolean
+    /** what is wrong with the hub itself: the page header shows it, and the
+     *  hub task's problems frame counts it. Absent = nothing is tracked */
+    hubProblems?: HubProblems
     /** BROWSE-ALL: the collections with every instance incl. healthy —
      *  the page shows everything that exists, dimmed never hidden */
     collections: () => Collection[]
@@ -152,8 +158,9 @@ export function serveHub(d: HubDeps): Hub {
                 const cols = d.collections()
                 return Response.json({
                     collections: cols.map((c) => ({ id: c.id, title: c.title, media: c.media })),
-                    resources: await projectResources(cols),
+                    resources: await projectResources(cols, d.hubProblems?.collector),
                     schedule: d.schedule?.state() ?? null,
+                    hub: d.hubProblems?.list() ?? [],
                 })
             }
             // the previous address, answered while pages opened before the
@@ -166,7 +173,7 @@ export function serveHub(d: HubDeps): Hub {
                     try {
                         out.push({ id: col.id, title: col.title, media: col.media, instances: await col.list() })
                     } catch (e) {
-                        out.push({ id: col.id, title: col.title, media: col.media, instances: [{ id: `${col.id}:broken`, type: 'health', path: `health/${col.id}-collector`, virtual: true, text: `collector broke: ${(e as Error).message}`, severity: 'red' }] })
+                        out.push({ id: col.id, title: col.title, media: col.media, instances: [brokenRow(col, e)] })
                     }
                 }
                 return Response.json({ collections: out })

package/src/serve/stale-restart.ts

@@ -0,0 +1,77 @@
+// a hub that went stale restarts itself. Stale is two things. Its lib moved
+// under it: under `bun --watch` the version is a module import
+// (core/lib-version.ts), so a bump restarts the hub before this ever sees a
+// difference, and a hub NOT under a watch sees the package.json on disk move
+// away from the version it booted with. Or its file table leaked
+// (core/fd-table.ts): every `bun --watch` restart keeps the previous image's
+// watch fds, and a full table fails every spawn with EBADF. Either way it
+// re-executes in place (core/reexec.ts) once no run is in flight, which is a
+// clean table too. Only a restart that throws reaches a human, once, through
+// `say`.
+import { assertLeakHealable, FD_HEALED_ENV, fdTableLeak } from '../core/fd-table.ts'
+
+export type StaleTick = 'fresh' | 'busy' | 'restarted' | 'impossible'
+
+export type StaleRestartOpts = {
+    /** the version this process booted with */
+    boot: string
+    /** the version on disk now, `?` when unreadable */
+    disk: () => string
+    /** why the file table leaked, null while it did not. Default: this
+     *  process's own table (core/fd-table.ts) */
+    leak?: () => string | null
+    /** a run is in flight: a restart now would kill it */
+    busy: () => boolean
+    /** never returns when it works. `why` is one line on what went stale */
+    restart: (why: string) => void
+    /** the hub's status line: a text when the restart is impossible, null
+     *  once there is nothing to say */
+    say: (text: string | null) => void
+    everyMs?: number
+}
+
+export class StaleRestart {
+    private timer: ReturnType<typeof setInterval> | null = null
+    private failed: string | null = null
+
+    constructor(private readonly o: StaleRestartOpts) {}
+
+    tick(): StaleTick {
+        const stale = this.why()
+        if (stale == null) {
+            if (this.failed != null) { this.failed = null; this.o.say(null) }
+            return 'fresh'
+        }
+        if (this.failed != null) return 'impossible'
+        if (this.o.busy()) return 'busy'
+        try {
+            if (stale.leak) {
+                assertLeakHealable()
+                process.env[FD_HEALED_ENV] = String(Date.now())
+            }
+            this.o.restart(stale.why)
+            return 'restarted'
+        } catch (e) {
+            this.failed = `${stale.why}, and this hub could not restart itself: ${(e as Error).message}. Restart the hub task`
+            this.o.say(this.failed)
+            return 'impossible'
+        }
+    }
+
+    private why(): { why: string; leak: boolean } | null {
+        const disk = this.o.disk()
+        // a package.json half written by a bump reads as `?`, and is no verdict
+        if (disk !== '?' && disk !== this.o.boot) return { why: `this hub runs shipkit ${this.o.boot} but ${disk} is on disk`, leak: false }
+        const leak = (this.o.leak ?? fdTableLeak)()
+        return leak == null ? null : { why: leak, leak: true }
+    }
+
+    start(): void {
+        this.timer ??= setInterval(() => this.tick(), this.o.everyMs ?? 2_000)
+    }
+
+    stop(): void {
+        if (this.timer != null) clearInterval(this.timer)
+        this.timer = null
+    }
+}

package/src/suite/suite-cli.ts

@@ -6,8 +6,9 @@
 // `Ran N tests` so the gate can still report what ran.
 //
 // usage: suite-cli [--budget-ms N] [--slow file]... [--serial file]... [--declared-in <where>] -- <bun test args>
-import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'fs'
-import { availableParallelism, loadavg, tmpdir } from 'os'
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'fs'
+import { availableParallelism, tmpdir } from 'os'
+import { machineLoad } from '../core/budget.ts'
 import { join } from 'path'
 import { FULL_TESTS_ENV } from '../ledger/gate-checks.ts'
 import { CHECK_ID_ENV, recordSuites, type SuitePhase } from '../ledger/suite-log.ts'
@@ -15,16 +16,19 @@ import { killTree } from '../core/spawn-timeout.ts'
 import { parallelCap } from '../core/parallel.ts'
 import { poolDir, poolOccupancy } from '../core/slots.ts'
 import { keepTimings, seedTimings, timingsHome } from './kept-timings.ts'
-import { aloneRunsNeeded, bunfigTest, readPhaseCpu, SUITE_CPU_ENV, type AloneRun, type Confirmed, confirmOver, crashedWorkers, type MachineLoad, WAIT_FACTOR, DEFAULT_BUDGET_MS, FULL_CEILING_MS, failedSuites, failLines, judgeSuites, normSuitePath, parseSuiteFlags, planPhases, QUICK_CEILING_MS, poolWorkers } from './judge-suites.ts'
+import { aloneRunsNeeded, bunfigTest, readPhaseCpu, SUITE_CPU_ENV, type AloneRun, type Confirmed, confirmOver, crashedWorkers, WAIT_FACTOR, DEFAULT_BUDGET_MS, FULL_CEILING_MS, failedSuites, failLines, judgeSuites, normSuitePath, parseSuiteFlags, planPhases, QUICK_CEILING_MS, poolWorkers } from './judge-suites.ts'
 
 type Phase = { code: number; signal: string | null; stalled: boolean; out: string; timings: Record<string, number>; cpuMs: number | null }
 
-async function runPhase(args: string[], timingsFile: string | null, ceilingMs: number, echo = true, cpuFile: string | null = null): Promise<Phase> {
+async function runPhase(args: string[], timingsFile: string | null, ceilingMs: number, tmp: string, echo = true, cpuFile: string | null = null): Promise<Phase> {
     const timing = timingsFile != null ? [`--timings=${timingsFile}`, '--update-timings'] : []
     // the per suite CPU figure rides in a preload of the runner's own, beside
     // the repo's (a command line preload adds to bunfig's, never replaces it)
     const cpu = cpuFile != null ? ['--preload', CPU_PRELOAD] : []
-    const env = cpuFile != null ? { ...process.env, [SUITE_CPU_ENV]: cpuFile } : process.env
+    // TMPDIR is the run's own dir, deleted with it: whatever a suite forgets to
+    // clean dies here, never in the machine temp dir, whose listing bun reads on
+    // the first import from a fixture there (90k leaked dirs cost ~100ms each time)
+    const env = { ...process.env, TMPDIR: tmp, ...(cpuFile != null ? { [SUITE_CPU_ENV]: cpuFile } : {}) }
     const proc = Bun.spawn(['bun', 'test', ...cpu, ...timing, ...args], { stdout: 'pipe', stderr: 'pipe', stdin: 'ignore', env })
     // the workers are children of `bun test`, and a stuck one is the process
     // to reach, so the whole tree goes
@@ -59,8 +63,6 @@ const CPU_PRELOAD = join(import.meta.dir, 'suite-cpu-preload.ts')
 
 const secs = (ms: number): string => `${(ms / 1000).toFixed(1)}s`
 
-const machineLoad = (): MachineLoad => ({ load1: loadavg()[0] ?? 0, cores: availableParallelism() })
-
 const overWords = (c: Confirmed): string =>
     c.over === 'cpu' ? `${c.file} ${secs(c.ms)} of CPU (${secs(c.wallMs)} wall)`
         : c.over === 'wait' ? `${c.file} ${secs(c.wallMs)} wall on an idle machine, more than ${WAIT_FACTOR}x the budget, with ${secs(c.cpuMs ?? 0)} of CPU: it waits (a timer, a sleep, a poll)`
@@ -103,6 +105,8 @@ export async function suiteCli(argv: string[]): Promise<number> {
     const workers = parallel ? poolWorkers({ cores, size: parallelCap(), held }) : undefined
     const plan = planPhases({ tests, tier, rawArgs, parallel, noIsolate: helpText.includes('--no-isolate'), configIgnores: bunfig.ignores, workers })
     const dir = mkdtempSync(join(tmpdir(), 'shipkit-suite-'))
+    const tmp = join(dir, 'temp')
+    mkdirSync(tmp)
     const notes: string[] = []
     if (!helpText.includes('--parallel')) notes.push(`skip: parallel suites (bun ${Bun.version} has no --parallel)`)
     if (!timed) notes.push(`skip: suite budget (bun ${Bun.version} writes no --timings)`)
@@ -114,7 +118,7 @@ export async function suiteCli(argv: string[]): Promise<number> {
         if (kept != null) seedTimings(kept, join(dir, 'parallel.json'))
         const cpuFile = timed && parallel ? join(dir, 'cpu.tsv') : null
         const p1At = Date.now()
-        const p1 = await runPhase(plan.parallel, timed ? join(dir, 'parallel.json') : null, ceilingMs, true, cpuFile)
+        const p1 = await runPhase(plan.parallel, timed ? join(dir, 'parallel.json') : null, ceilingMs, tmp, true, cpuFile)
         const p1Ms = Date.now() - p1At
         // bun merges what it read into what it writes, so a suite deleted since
         // the last run is judged and kept only if the workers say it ran here
@@ -130,7 +134,7 @@ export async function suiteCli(argv: string[]): Promise<number> {
             }
         }
         const p2At = Date.now()
-        const p2 = plan.serial != null ? await runPhase(plan.serial, timed ? join(dir, 'serial.json') : null, ceilingMs) : null
+        const p2 = plan.serial != null ? await runPhase(plan.serial, timed ? join(dir, 'serial.json') : null, ceilingMs, tmp) : null
         const p2Ms = p2 == null ? null : Date.now() - p2At
         const phases = [p1, ...(p2 != null ? [p2] : [])]
         const ran = phases.reduce((n, p) => n + lastCount(p.out, /Ran (\d+) tests?/g), 0)
@@ -149,7 +153,7 @@ export async function suiteCli(argv: string[]): Promise<number> {
         const { rerun, cleared: clearedInPhase } = aloneRunsNeeded(judged.over, parallelCpu, budget, load)
         const aloneAt = Date.now()
         for (const [i, t] of rerun.entries()) {
-            const run = await runPhase([...rawArgs, t.file], join(dir, `alone-${i}.json`), ceilingMs, false)
+            const run = await runPhase([...rawArgs, t.file], join(dir, `alone-${i}.json`), ceilingMs, tmp, false)
             const wallMs = Object.entries(run.timings).find(([f]) => normSuitePath(f) === t.file)?.[1]
             if (wallMs != null) aloneRuns[t.file] = { wallMs, cpuMs: run.cpuMs }
         }

package/src/ts/source-graph.ts

@@ -146,7 +146,8 @@ export function buildSourceGraph(cfg: RvtConfig = loadConfig()): SourceGraph {
  * knows a missing declared path makes a check uncacheable, and a throw here
  * would turn a stale declaration into a dead gate.
  */
-export function importClosure(cfg: RvtConfig, entries: string[], o: { dynamic?: boolean } = {}): string[] {
+export function importClosure(cfg: RvtConfig, entries: string[], o: { dynamic?: boolean; read?: (rel: string) => string } = {}): string[] {
+   const read = o.read ?? ((rel: string): string => readFileSync(join(cfg.root, rel), 'utf8'))
    const probe = new Map<string, boolean>()
    const has = (p: string): boolean => {
       let r = probe.get(p)
@@ -163,7 +164,7 @@ export function importClosure(cfg: RvtConfig, entries: string[], o: { dynamic?:
       const file = queue.shift()!
       let content: string
       try {
-         content = readFileSync(join(cfg.root, file), 'utf8')
+         content = read(file)
       } catch {
          continue
       }