rvlib-shipkit 2.4.0 → 2.5.0

package/src/core/last-good.ts

@@ -0,0 +1,217 @@
+// the last good copy of the toolkit: what `sk` and the home run while the
+// checkout fails to load. The daemon clones the source after a load that
+// works (home/lib-watch.ts); the boot (cli.ts) runs the copy when the tree
+// fails to load, with one line that names the file.
+//
+// the one constraint a reader breaks first: cli.ts imports this file
+// statically, so it imports node builtins only. A lib import here would let
+// a broken file stop the boot before it can fall back.
+import { constants, copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync, readlinkSync, realpathSync, renameSync, rmSync, symlinkSync, writeFileSync } from 'fs'
+import { homedir } from 'os'
+import { join, relative } from 'path'
+
+/** set on a process that runs the copy: the Held record, as json */
+export const LAST_GOOD_ENV = 'SHIPKIT_LAST_GOOD'
+
+/** why a process runs code older than its tree */
+export type Held = {
+    /** epoch ms the tree was first seen failing */
+    since: number
+    /** the file that fails, relative to the lib (`src/trailer/graph.ts:12`) */
+    file: string
+    error: string
+    /** the lib root whose tree fails */
+    from: string
+    /** epoch ms the copy was taken, null when this process runs its own boot image */
+    copyAt: number | null
+}
+
+/** how long a replaced copy stays, from when it stopped being current: a
+ *  process that runs it may still import a module from it lazily */
+export const OLD_COPY_MS = 10 * 60_000
+
+/** this lib's root (its package.json beside src) */
+export const OWN_LIB = join(import.meta.dir, '../..')
+
+const real = (p: string): string => {
+    try {
+        return realpathSync(p)
+    } catch {
+        return p
+    }
+}
+
+export const lastGoodHome = (env: Record<string, string | undefined> = process.env): string =>
+    env.SHIPKIT_LAST_GOOD_HOME != null && env.SHIPKIT_LAST_GOOD_HOME !== '' ? env.SHIPKIT_LAST_GOOD_HOME : join(homedir(), '.cache/shipkit/last-good')
+
+/** one folder per lib checkout, so a worktree never runs the main checkout's copy */
+export const lastGoodDir = (lib: string, home = lastGoodHome()): string => join(home, Bun.hash(real(lib)).toString(16).padStart(16, '0'))
+
+const STAMP = 'stamp.json'
+/** in a copy that is no longer current: when it stopped being */
+const RETIRED = 'retired'
+
+/** a copy carries a stamp: the lib it was taken from and when */
+export function copyStamp(root: string): { at: number; from: string } | null {
+    try {
+        const j = JSON.parse(readFileSync(join(root, STAMP), 'utf8')) as { at?: number; from?: string }
+        return typeof j.at === 'number' && typeof j.from === 'string' ? { at: j.at, from: j.from } : null
+    } catch {
+        return null
+    }
+}
+
+/** the copy's cli and when it was taken, null when the lib has none */
+export function lastGood(lib: string, home = lastGoodHome()): { cli: string; at: number } | null {
+    const cur = join(lastGoodDir(lib, home), 'current')
+    const s = copyStamp(cur)
+    const cli = join(cur, 'src/cli.ts')
+    return s == null || !existsSync(cli) ? null : { cli, at: s.at }
+}
+
+/** clone the lib's source as the last good copy (APFS clone: about 0.2 s for
+ *  the whole source), its node_modules linked, probe it in its own folder,
+ *  then point `current` at it. Throws when the clone does not load */
+export async function takeLastGood(lib: string, o: { home?: string; now?: number } = {}): Promise<{ dir: string; ms: number }> {
+    const t0 = performance.now()
+    const now = o.now ?? Date.now()
+    const base = lastGoodDir(lib, o.home)
+    const name = `v${now}`
+    const dir = join(base, name)
+    mkdirSync(dir, { recursive: true })
+    const cp = Bun.spawn(['cp', '-c', '-R', join(lib, 'src'), join(dir, 'src')], { stdout: 'ignore', stderr: 'pipe' })
+    if ((await cp.exited) !== 0) {
+        const err = (await new Response(cp.stderr).text()).trim()
+        rmSync(dir, { recursive: true, force: true })
+        throw new Error(`could not clone ${lib}/src: ${err}`)
+    }
+    for (const f of ['package.json', 'tsconfig.json']) if (existsSync(join(lib, f))) copyFileSync(join(lib, f), join(dir, f), constants.COPYFILE_FICLONE)
+    symlinkSync(join(real(lib), 'node_modules'), join(dir, 'node_modules'))
+    writeFileSync(join(dir, STAMP), `${JSON.stringify({ at: now, from: real(lib) })}\n`)
+    // a write that landed during the clone is in it: the copy loads in its own folder, or never becomes current
+    const { probeLoad, bootEntries } = await import('./load-probe.ts')
+    const a = await probeLoad(dir, bootEntries(dir, 'all'))
+    if (!a.ok) {
+        rmSync(dir, { recursive: true, force: true })
+        throw new Error(`the clone fails to load (${a.file}: ${a.error}), so the copy before it stays current`)
+    }
+    // a link renamed over the old one: a reader sees the old copy or the new, never none
+    const link = join(base, `.current-${process.pid}`)
+    rmSync(link, { force: true })
+    symlinkSync(name, link)
+    let was: string | null = null
+    try { was = readlinkSync(join(base, 'current')) } catch { /* the first copy */ }
+    renameSync(link, join(base, 'current'))
+    // a copy ages from when it stopped being current: a process that runs it keeps its modules
+    if (was != null && was !== name && existsSync(join(base, was))) writeFileSync(join(base, was, RETIRED), `${now}\n`)
+    for (const e of readdirSync(base)) {
+        const at = /^v(\d+)$/.exec(e)
+        if (at == null || e === name) continue
+        let since = Number(at[1])
+        try { since = Number(readFileSync(join(base, e, RETIRED), 'utf8')) } catch { /* never current: a clone that failed or was cut */ }
+        if (now - since > OLD_COPY_MS) rmSync(join(base, e), { recursive: true, force: true })
+    }
+    return { dir, ms: performance.now() - t0 }
+}
+
+/** the lib file that imports `name` from `module`, read only on a failure */
+function importerOf(lib: string, name: string, module: string): string | null {
+    const want = real(module).replace(/\.[cm]?[jt]sx?$/, '')
+    const re = new RegExp(`(?:import|export)\\s*(?:type\\s*)?\\{[^}]*\\b${name.replace(/[^\w$]/g, '')}\\b[^}]*\\}\\s*from\\s*['"]([^'"]+)['"]`, 'g')
+    for (const f of new Bun.Glob('src/**/*.{ts,tsx}').scanSync(lib)) {
+        const full = join(lib, f)
+        let text: string
+        try { text = readFileSync(full, 'utf8') } catch { continue }
+        if (!text.includes(name)) continue
+        for (const m of text.matchAll(re)) {
+            if (!m[1]!.startsWith('.')) continue
+            if (real(join(full, '..', m[1]!)).replace(/\.[cm]?[jt]sx?$/, '') === want) return full
+        }
+    }
+    return null
+}
+
+/** the file a load error names, relative to `lib`, and its first line */
+export function loadFailure(e: unknown, lib: string): { file: string; error: string } {
+    const x = (e ?? {}) as { message?: string; position?: { file?: string; line?: number } | null; referrer?: string; stack?: string }
+    const rel = (p: string): string => {
+        const r = relative(real(lib), real(p))
+        return r.startsWith('..') ? p : r
+    }
+    const error = String(x.message ?? e).split('\n')[0]!.slice(0, 200)
+    // bun names the module a missing export was asked of, never the file that asked
+    const miss = /Export named '([^']+)' not found in module '([^']+)'/.exec(error)
+    if (miss != null) {
+        const importer = importerOf(lib, miss[1]!, miss[2]!)
+        return { file: rel(importer ?? miss[2]!), error: `imports '${miss[1]}', which ${rel(miss[2]!)} does not export` }
+    }
+    if (x.position?.file != null) return { file: `${rel(x.position.file)}${x.position.line != null ? `:${x.position.line}` : ''}`, error }
+    if (typeof x.referrer === 'string' && x.referrer !== '') return { file: rel(x.referrer), error }
+    const at = /\(?((?:\/[^\s():]+)+\.[cm]?[jt]sx?):(\d+)/.exec(x.stack ?? '')
+    return { file: at == null ? '?' : `${rel(at[1]!)}:${at[2]}`, error }
+}
+
+const hhmm = (t: number): string => new Date(t).toTimeString().slice(0, 5)
+
+/** when a copy was taken: HH:MM within the last hour of today, else its
+ *  date, time and age, so an old copy is never mistaken for a fresh one */
+export function copyWhen(at: number, now = Date.now()): string {
+    const d = new Date(at)
+    if (d.toDateString() === new Date(now).toDateString() && now - at < 3_600_000) return hhmm(at)
+    const age = now - at
+    const old = age < 48 * 3_600_000 ? `${Math.round(age / 3_600_000)} h old` : `${Math.floor(age / 86_400_000)} d old`
+    const day = `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')}`
+    return `${day} ${hhmm(at)}, ${old}`
+}
+
+/** the one line every surface shows (the hub, the overview, a log) */
+export const heldText = (h: Held): string =>
+    `running last good code since ${hhmm(h.since)} because ${h.file} fails to load: ${h.error}${h.copyAt == null ? '' : ` (the copy from ${copyWhen(h.copyAt)})`}`
+
+let held: Held | null = null
+
+/** this process's own state: a refused restart set it, or the boot that ran
+ *  the copy left it in the env */
+export function heldNow(): Held | null {
+    if (held != null) return held
+    const raw = process.env[LAST_GOOD_ENV]
+    if (raw == null || raw === '' || copyStamp(OWN_LIB) == null) return null
+    try {
+        return JSON.parse(raw) as Held
+    } catch {
+        return null
+    }
+}
+
+export function setHeld(h: Held | null): void {
+    held = h
+}
+
+/** the tree this process should go back to: its own, or the one its copy stands in for */
+export const homeLib = (): string => (copyStamp(OWN_LIB)?.from ?? OWN_LIB)
+
+/** the boot of a tree that loads: an env a copy left is not this process's */
+export function bootLoaded(): void {
+    if (copyStamp(OWN_LIB) == null) delete process.env[LAST_GOOD_ENV]
+}
+
+/** the boot of a tree that failed to load: runs the copy (never returns), or
+ *  returns when there is none so the caller throws the error itself */
+export function runLastGood(e: unknown, argv: string[]): void {
+    const f = loadFailure(e, OWN_LIB)
+    // the copy itself failing is the error to show, never a loop
+    if (copyStamp(OWN_LIB) != null) return
+    const copy = lastGood(OWN_LIB)
+    if (copy == null) {
+        process.stderr.write(`FAIL shipkit: ${f.file} fails to load (${f.error}), and this checkout has no last good copy: the home's daemon keeps one\n`)
+        return
+    }
+    const h: Held = { since: Date.now(), file: f.file, error: f.error, from: real(OWN_LIB), copyAt: copy.at }
+    process.stderr.write(`WARN shipkit: ${f.file} fails to load (${f.error}), so this runs the last good copy from ${copyWhen(copy.at)}\n`)
+    const env = Object.fromEntries(Object.entries({ ...process.env, [LAST_GOOD_ENV]: JSON.stringify(h) }).filter((x): x is [string, string] => x[1] != null))
+    const args = [process.execPath, ...process.execArgv, copy.cli, ...argv]
+    const execve = (process as { execve?: (file: string, args: string[], env: Record<string, string>) => never }).execve
+    if (typeof execve === 'function') execve.call(process, process.execPath, args, env)
+    const child = Bun.spawnSync(args, { cwd: process.cwd(), stdout: 'inherit', stderr: 'inherit', stdin: 'inherit', env })
+    process.exit(child.exitCode ?? 1)
+}

package/src/core/load-probe.ts

@@ -0,0 +1,71 @@
+// does a tree load: a child bun imports its entries and says which file
+// fails. Asked before a machine process restarts onto new code, and before
+// the daemon keeps a copy as the last good one. Run as a child so a module
+// that fails, or one whose top level runs, never touches the asking process.
+import { join } from 'path'
+import { loadFailure } from './last-good.ts'
+import type { LoadVerdict } from '../serve/stale-restart.ts'
+
+export type LoadAnswer = { ok: true; ms: number } | { ok: false; ms: number; file: string; error: string }
+
+export type Role = 'daemon' | 'pm' | 'web' | 'serve'
+
+const RUNNER: Record<Role, string> = { daemon: 'src/home/run-daemon.ts', pm: 'src/home/run-pm.ts', web: 'src/home/run-web.ts', serve: 'src/serve/run-serve.ts' }
+
+/** the modules a restart through the cli loads: the boot's (cli.ts
+ *  `loadBoot`: the registry and what runs before it) and the role's runner;
+ *  `all`: every role's, what a copy every process may run must load */
+export const bootEntries = (lib: string, role?: Role | 'all'): string[] => [
+    join(lib, 'src/cli.ts'),
+    ...(role == null ? [] : (role === 'all' ? Object.values(RUNNER) : [RUNNER[role]]).map((r) => join(lib, r))),
+]
+
+/** import `entries` in a child bun from `lib`'s own probe file */
+export async function probeLoad(lib: string, entries: string[]): Promise<LoadAnswer> {
+    const t0 = performance.now()
+    const p = Bun.spawn([process.execPath, join(lib, 'src/core/load-probe.ts'), ...entries], { cwd: lib, stdout: 'pipe', stderr: 'pipe', env: { ...process.env, SHIPKIT_NO_HOME: '1' } })
+    const [out, err, code] = await Promise.all([new Response(p.stdout).text(), new Response(p.stderr).text(), p.exited])
+    const ms = performance.now() - t0
+    if (code === 0) return { ok: true, ms }
+    try {
+        const j = JSON.parse(out.trim().split('\n').at(-1) ?? '') as { file: string; error: string }
+        return { ok: false, ms, file: j.file, error: j.error }
+    } catch {
+        // the probe file itself did not load
+        const line = err.split('\n').find((l) => /error/i.test(l)) ?? err.trim().split('\n')[0] ?? `exit ${code}`
+        return { ok: false, ms, file: 'src/core/load-probe.ts', error: line.trim().slice(0, 200) }
+    }
+}
+
+/** one probe per state of the tree: asked again only once the key moves */
+export class TreeProbe {
+    private key: string | null = null
+    private verdict: LoadVerdict = 'pending'
+
+    constructor(private readonly run: () => Promise<LoadAnswer>, private readonly keyOf: () => string) {}
+
+    loads(): LoadVerdict {
+        const k = this.keyOf()
+        if (k !== this.key) {
+            this.key = k
+            this.verdict = 'pending'
+            const land = (v: LoadVerdict): void => { if (this.key === k) this.verdict = v }
+            void this.run().then((a) => land(a.ok ? 'ok' : { file: a.file, error: a.error }), (e: Error) => land({ file: '?', error: e.message }))
+        }
+        return this.verdict
+    }
+}
+
+if (import.meta.main) {
+    const lib = join(import.meta.dir, '../..')
+    for (const e of process.argv.slice(2)) {
+        try {
+            const m = (await import(e)) as { loadBoot?: () => Promise<unknown> }
+            if (typeof m.loadBoot === 'function') await m.loadBoot()
+        } catch (x) {
+            console.log(JSON.stringify(loadFailure(x, lib)))
+            process.exit(3)
+        }
+    }
+    process.exit(0)
+}

package/src/core/reexec.ts

@@ -5,7 +5,9 @@
 // code, which keeps the terminal but leaves this process waiting under it.
 import { existsSync } from 'fs'
 
-const argvOf = (): string[] => [process.execPath, ...process.execArgv, ...process.argv.slice(1)]
+/** `entry` in place of the running script: a process on the last good copy
+ *  goes back to the tree's own cli (core/last-good.ts) */
+const argvOf = (entry?: string): string[] => [process.execPath, ...process.execArgv, ...(entry == null ? process.argv.slice(1) : [entry, ...process.argv.slice(2)])]
 
 /** why this invocation cannot run again, null when it can. Asked BEFORE a
  *  caller tears anything down, so a refusal leaves the process serving */
@@ -16,8 +18,8 @@ export function reexecBlocker(): string | null {
 }
 
 /** never returns. `env` defaults to this process's own */
-export function reexecSelf(env: Record<string, string | undefined> = process.env): never {
-    const argv = argvOf()
+export function reexecSelf(env: Record<string, string | undefined> = process.env, entry?: string): never {
+    const argv = argvOf(entry)
     const clean = Object.fromEntries(Object.entries(env).filter((e): e is [string, string] => e[1] != null))
     const execve = (process as { execve?: (file: string, args: string[], env: Record<string, string>) => never }).execve
     if (typeof execve === 'function') execve.call(process, process.execPath, argv, clean)

package/src/fleet/push-fleet.ts

@@ -0,0 +1,80 @@
+// push every house repo that holds commits its remote does not: `sk fleet
+// push` and the activity page's push button run this one path. A plain
+// `git push`, never a force: a branch that also fell behind its remote waits
+// for a pull, a branch with no upstream goes to its one remote with -u, and
+// git itself refuses anything that is not a fast-forward. Local git only, no
+// fetch first: "behind" is as of the last fetch, and the push is what finds
+// out for sure.
+import { parseStatus } from '../home/activity.ts'
+
+export type PushAction = 'push' | 'push-upstream' | 'none' | 'skip'
+export type PushPlan = { name: string; path: string; branch: string | null; remote: string | null; ahead: number; action: PushAction; why: string }
+export type PushResult = PushPlan & { ok: boolean; line: string }
+
+/** at most this many pushes at once */
+const PARALLEL = 4
+const PUSH_MS = 60_000
+
+async function git(cwd: string, args: string[], ms = 15_000): Promise<{ code: number; out: string; err: string }> {
+    // a push that would ask for a password fails instead of hanging the page
+    const p = Bun.spawn(['git', ...args], { cwd, stdout: 'pipe', stderr: 'pipe', env: { ...process.env, GIT_TERMINAL_PROMPT: '0' } })
+    const timer = setTimeout(() => p.kill(), ms)
+    const [out, err, code] = await Promise.all([new Response(p.stdout).text(), new Response(p.stderr).text(), p.exited])
+    clearTimeout(timer)
+    return { code, out, err }
+}
+
+/** what to do with one repo, from `git status --porcelain=v2 --branch`, its
+ *  remotes and its commit count. Pure: the tests feed it text */
+export function planOf(repo: { name: string; path: string }, status: string, remotes: string[], commits: number): PushPlan {
+    const s = parseStatus(status)
+    const base = { name: repo.name, path: repo.path, branch: s.branch, remote: null as string | null, ahead: s.ahead ?? 0 }
+    if (s.branch == null) return { ...base, action: 'skip', why: s.detached != null ? `detached at ${s.detached}: no branch to push` : 'no commit yet' }
+    if (s.upstream == null) {
+        if (commits === 0) return { ...base, action: 'none', why: 'no commit yet' }
+        const remote = remotes.length === 1 ? remotes[0]! : remotes.includes('origin') ? 'origin' : null
+        if (remote == null) return { ...base, action: 'skip', why: remotes.length === 0 ? 'no remote to push to' : `no upstream, and ${remotes.length} remotes: set one with git push -u <remote> ${s.branch}` }
+        return { ...base, remote, ahead: commits, action: 'push-upstream', why: `no upstream: push -u ${remote} ${s.branch}` }
+    }
+    if (s.gone) return { ...base, action: 'skip', why: `its upstream ${s.upstream} is gone` }
+    const ahead = s.ahead ?? 0
+    const behind = s.behind ?? 0
+    if (ahead === 0) return { ...base, action: 'none', why: 'pushed' }
+    if (behind > 0) return { ...base, action: 'skip', why: `${ahead} ahead and ${behind} behind ${s.upstream}: pull first, never a force` }
+    return { ...base, remote: s.upstream.split('/')[0] ?? null, action: 'push', why: `${ahead} unpushed to ${s.upstream}` }
+}
+
+export async function pushPlan(repo: { name: string; path: string }): Promise<PushPlan> {
+    const [status, remotes, count] = await Promise.all([
+        git(repo.path, ['status', '--porcelain=v2', '--branch']),
+        git(repo.path, ['remote']),
+        git(repo.path, ['rev-list', '--count', 'HEAD']),
+    ])
+    if (status.code !== 0) return { name: repo.name, path: repo.path, branch: null, remote: null, ahead: 0, action: 'skip', why: `git status failed: ${status.err.trim().split('\n')[0]}` }
+    return planOf(repo, status.out, remotes.out.split('\n').filter((r) => r !== ''), Number(count.out.trim()) || 0)
+}
+
+async function pushOne(p: PushPlan): Promise<PushResult> {
+    if (p.action !== 'push' && p.action !== 'push-upstream') return { ...p, ok: p.action === 'none', line: `${p.action === 'none' ? 'ok' : 'skip'}: ${p.name}: ${p.why}` }
+    const args = p.action === 'push' ? ['push'] : ['push', '-u', p.remote!, p.branch!]
+    const r = await git(p.path, args, PUSH_MS)
+    if (r.code === 0) return { ...p, ok: true, line: `ok: ${p.name}: pushed ${p.ahead} commit${p.ahead === 1 ? '' : 's'} (${p.branch})` }
+    const why = r.err.split('\n').map((l) => l.trim()).filter((l) => l !== '' && !l.startsWith('hint:')).slice(-1)[0] ?? `git push exited ${r.code}`
+    return { ...p, ok: false, line: `FAIL: ${p.name}: git ${args.join(' ')}: ${why}` }
+}
+
+/** every repo planned, the ones that need it pushed, a few at a time; `dry`
+ *  plans only. Results come back in the repos' order */
+export async function pushFleet(repos: { name: string; path: string }[], o: { dry?: boolean } = {}): Promise<PushResult[]> {
+    const plans = await Promise.all(repos.map(pushPlan))
+    if (o.dry === true) return plans.map((p) => ({ ...p, ok: p.action !== 'skip', line: `${p.action === 'skip' ? 'skip' : p.action === 'none' ? 'ok' : 'would push'}: ${p.name}: ${p.why}` }))
+    const out: PushResult[] = new Array(plans.length)
+    let next = 0
+    await Promise.all(Array.from({ length: PARALLEL }, async () => {
+        while (next < plans.length) {
+            const i = next++
+            out[i] = await pushOne(plans[i]!)
+        }
+    }))
+    return out
+}

package/src/home/home-routes.ts

@@ -7,6 +7,7 @@ import { existsSync } from 'fs'
 import { dirname, join } from 'path'
 import { fileURLToPath } from 'url'
 import { installedVersion, isBehind } from '../fleet/scan-fleet.ts'
+import { heldNow, heldText } from '../core/last-good.ts'
 import { LIB_VERSION } from '../core/lib-version.ts'
 import { projectGutter, type ProjectRow } from '../serve/project-gutter-route.ts'
 import type { WiredRepo } from '../serve/project-gutter.ts'
@@ -20,6 +21,7 @@ import { ActivityCache, repoActivity, windowMinutes } from './activity.ts'
 import { devRepos, InsightsCache, type Scope } from './insights.ts'
 import { GLOBAL_CONTEXT_EVERY_MS, houseContext, writeIfChanged } from './global-context.ts'
 import { HouseRuns } from '../serve/house-runs.ts'
+import { pushFleet } from '../fleet/push-fleet.ts'
 
 /** how far back a repo's git page lists its commits */
 export const GIT_PAGE_MS = 7 * 24 * 3600_000
@@ -38,6 +40,8 @@ export type HomeAnswer = {
     /** repo → its installed shipkit, for every repo behind this home's */
     behind: Record<string, string>
     modernize: Record<string, ModernizeView>
+    /** the daemon runs older code than its tree, and why (core/last-good.ts) */
+    held?: string | null
 }
 
 export type HomeDeps = {
@@ -156,7 +160,8 @@ export function homeRoutes(d: HomeDeps) {
         }
         const modernize: Record<string, ModernizeView> = {}
         for (const [k, m] of modernizing) modernize[k] = { status: m.status, code: m.code, tail: tailLines(m.log, 40) }
-        return { version: LIB_VERSION, workers: d.hubs.views(repos), active: d.state.data.active, behind, modernize }
+        const h = heldNow()
+        return { version: LIB_VERSION, workers: d.hubs.views(repos), active: d.state.data.active, behind, modernize, held: h == null ? null : heldText(h) }
     }
 
     const find = (name: unknown): WiredRepo | null => repos.find((r) => r.name === name) ?? null
@@ -171,9 +176,14 @@ export function homeRoutes(d: HomeDeps) {
         }
         if (path === '/hub.json')
             // the identity a client probes (homeUp), through the web or here
-            return Response.json({ hub: 'shipkit-home', name: d.name, root: d.root, version: LIB_VERSION }, { headers: { 'access-control-allow-origin': '*' } })
+            return Response.json({ hub: 'shipkit-home', name: d.name, root: d.root, version: LIB_VERSION, held: heldNow() == null ? null : heldText(heldNow()!) }, { headers: { 'access-control-allow-origin': '*' } })
         if (!sameOrigin(req)) return new Response('refused: a request that changes something must come from the home page', { status: 403 })
         if (path.startsWith('/api/projects')) return d.gutter.handle(req, path)
+        // the activity page's push button: every house repo with unpushed commits, never a force (fleet/push-fleet.ts)
+        if (path === '/api/home/push-all' && req.method === 'POST') {
+            if (repos.length === 0) await refresh()
+            return Response.json(await pushFleet(repos.map((r) => ({ name: r.name, path: r.path }))))
+        }
         // the web asks which page routes to serve: one per house repo
         if (path === '/api/home/repos.json') {
             if (repos.length === 0) await refresh()

package/src/home/lib-watch.ts

@@ -5,10 +5,14 @@
 // started through cli.ts holds every command (one edit would restart all
 // three). This watch covers the lib source, filtered to the files the role's
 // own entry module reaches, and says a change once the burst of writes settles.
-import { existsSync, readFileSync, watch, type FSWatcher } from 'fs'
+// A restart lands only on code that loads (core/load-probe.ts): while a file
+// of the tree fails, the role keeps its running image and says why once. The
+// daemon also keeps the last good copy of the tree (core/last-good.ts).
+import { existsSync, readFileSync, realpathSync, statSync, watch, type FSWatcher } from 'fs'
 import { dirname, extname, join, relative, resolve } from 'path'
-import { LIB_ROOT } from '../core/lib-root.ts'
-import { diskLibVersion, LIB_VERSION } from '../core/lib-version.ts'
+import { copyStamp, heldNow, homeLib, LAST_GOOD_ENV, lastGood, OWN_LIB, setHeld, takeLastGood } from '../core/last-good.ts'
+import { LIB_VERSION } from '../core/lib-version.ts'
+import { bootEntries, probeLoad, TreeProbe } from '../core/load-probe.ts'
 import { reexecBlocker, reexecSelf } from '../core/reexec.ts'
 import { StaleRestart } from '../serve/stale-restart.ts'
 
@@ -56,6 +60,9 @@ export function libGraph(entry: string, o: { bundles?: boolean } = {}): Set<stri
 export class LibWatch {
     private w: FSWatcher | null = null
     private last: { file: string; at: number } | null = null
+    private any: { file: string; at: number } | null = null
+    /** every code write under src, own or not */
+    seq = 0
     readonly error: string | null = null
     private now: () => number
     private settleMs: number
@@ -64,11 +71,25 @@ export class LibWatch {
     constructor(private root: string, o: { settleMs?: number; now?: () => number; only?: Set<string> } = {}) {
         this.now = o.now ?? Date.now
         this.settleMs = o.settleMs ?? 1_000
+        // an APFS clone of the tree (the last good copy, a scratch checkout)
+        // fires a change on every source file it reads: only a moved mtime,
+        // or a file gone, is a write
+        const mtimes = new Map<string, number>()
+        const mtimeOf = (f: string): number | null => {
+            try { return statSync(f).mtimeMs } catch { return null }
+        }
+        for (const f of new Bun.Glob('**/*').scanSync(join(root, 'src'))) if (CODE.test(f)) mtimes.set(join(root, 'src', f), mtimeOf(join(root, 'src', f)) ?? -1)
         try {
             this.w = watch(join(root, 'src'), { recursive: true }, (_ev, file) => {
                 if (file == null || !CODE.test(String(file))) return
+                const full = join(root, 'src', String(file))
+                const m = mtimeOf(full)
+                if (m != null && m === mtimes.get(full)) return
+                mtimes.set(full, m ?? -1)
+                this.seq++
+                this.any = { file: String(file), at: this.now() }
                 if (o.only != null && !o.only.has(join(root, 'src', String(file)))) return
-                this.last = { file: String(file), at: this.now() }
+                this.last = this.any
             })
         } catch (e) {
             this.error = (e as Error).message
@@ -86,41 +107,92 @@ export class LibWatch {
         return `the shipkit source moved (src/${this.last.file})`
     }
 
+    /** no write anywhere under src for settleMs */
+    settled(): boolean {
+        return this.any == null || this.now() - this.any.at >= this.settleMs
+    }
+
+    /** one line on the last write anywhere under src, once settled, else null */
+    whyAny(): string | null {
+        return this.any == null || !this.settled() ? null : `the shipkit source moved (src/${this.any.file})`
+    }
+
     close(): void {
         this.w?.close()
         this.w = null
     }
 }
 
+/** how often the daemon may probe the tree to keep its last good copy */
+export const KEEP_EVERY_MS = 60_000
+
+const realOf = (p: string): string => {
+    try { return realpathSync(p) } catch { return p }
+}
+
+const versionAt = (root: string): string => {
+    try { return (JSON.parse(readFileSync(join(root, 'package.json'), 'utf8')) as { version?: string }).version ?? '?' } catch { return '?' }
+}
+
 /** a machine process restarts itself in place, same pid, once its own code
- *  moved and nothing it runs is in flight. `before` tears down what must not
- *  outlive the image (its server); what it started detached keeps running */
+ *  moved, nothing it runs is in flight and the new code loads. `before`
+ *  tears down what must not outlive the image (its server); what it started
+ *  detached keeps running. On the last good copy it watches the tree and
+ *  goes back to it once it loads. `keep`: this process keeps the copy */
 export function reloadOnOwnCode(o: {
-    role: string
+    role: 'pm' | 'daemon' | 'web'
     /** the role's runner module: its graph is what the role reloads on */
     entry: string
     bundles?: boolean
     busy?: () => boolean
     before: () => void
+    keep?: boolean
     io: { print: (l: string) => void; printErr: (l: string) => void }
 }): { stop: () => void; files: number } {
-    const graph = libGraph(o.entry, { bundles: o.bundles })
-    const lib = new LibWatch(LIB_ROOT, { only: graph })
+    const onCopy = copyStamp(OWN_LIB) != null
+    const tree = homeLib()
+    const graph = libGraph(onCopy ? join(tree, relative(OWN_LIB, o.entry)) : o.entry, { bundles: o.bundles })
+    const lib = new LibWatch(tree, { only: onCopy ? undefined : graph })
     if (lib.error != null) o.io.printErr(`WARN no watch on the shipkit source (${lib.error}), so an edit of the ${o.role} needs a restart of it`)
+    const probe = new TreeProbe(() => probeLoad(tree, bootEntries(tree, o.role)), () => `${lib.seq}|${versionAt(tree)}`)
+    let since: number | null = null
+    let said = ''
+    const hold = (v: { file: string; error: string } | null): void => {
+        if (v == null) {
+            if (since != null) o.io.print(`ok: the shipkit source loads again`)
+            since = null
+            said = ''
+            if (!onCopy) setHeld(null)
+            return
+        }
+        since ??= heldNow()?.since ?? Date.now()
+        const text = `${v.file} fails to load (${v.error})`
+        if (text !== said) {
+            said = text
+            o.io.printErr(`FAIL: ${text}: sk runs the last good copy, and the ${o.role} keeps its running code until the tree loads again`)
+        }
+        setHeld({ since, file: v.file, error: v.error, from: tree, copyAt: lastGood(tree)?.at ?? null })
+    }
     const stale = new StaleRestart({
         boot: LIB_VERSION,
-        disk: diskLibVersion,
-        moved: () => lib.why(),
+        disk: () => versionAt(tree),
+        // on the copy, any write may heal the tree, and the first look is at once
+        moved: () => (onCopy ? (lib.whyAny() ?? (lib.seq === 0 ? 'this process runs the last good copy' : null)) : lib.why()),
         busy: o.busy ?? (() => false),
+        loads: () => probe.loads(),
+        hold,
         restart: (why) => {
             const blocked = reexecBlocker()
             if (blocked != null) throw new Error(blocked)
-            o.io.print(`ok: ${why}: the ${o.role} restarts in place`)
-            stale.stop()
-            lib.close()
+            o.io.print(`ok: ${why}: the ${o.role} restarts in place${onCopy ? ' on the tree' : ''}`)
+            stop()
             o.before()
             try {
-                reexecSelf()
+                if (!onCopy) reexecSelf()
+                const env = { ...process.env }
+                delete env[LAST_GOOD_ENV]
+                // the same script in the tree: cli.ts for most roles, run-pm.ts for pm
+                reexecSelf(env, join(tree, relative(realOf(OWN_LIB), realOf(process.argv[1] ?? join(OWN_LIB, 'src/cli.ts')))))
             } catch (e) {
                 o.io.printErr(`FAIL: the ${o.role} stopped to restart and could not: ${(e as Error).message}`)
                 process.exit(1)
@@ -128,12 +200,30 @@ export function reloadOnOwnCode(o: {
         },
         say: (text) => { if (text != null) o.io.printErr(`WARN ${text}`) },
     })
+    // the daemon keeps the copy: after a settled burst that loads, at most every KEEP_EVERY_MS
+    // once a minute counts from the last copy's own stamp (a restart keeps it), or from a try that failed
+    let kept = -1
+    let triedAt = 0
+    const keeper = o.keep === true && !onCopy
+        ? setInterval(() => {
+            if (lib.seq === kept || !lib.settled() || Date.now() - Math.max(lastGood(tree)?.at ?? 0, triedAt) < KEEP_EVERY_MS) return
+            const v = probe.loads()
+            if (v === 'pending') return
+            if (v !== 'ok') return hold(v)
+            hold(null)
+            kept = lib.seq
+            triedAt = Date.now()
+            void takeLastGood(tree).catch((e: Error) => o.io.printErr(`WARN the last good copy of the shipkit source could not be kept: ${e.message}`))
+        }, 2_000)
+        : null
+    function stop(): void {
+        stale.stop()
+        lib.close()
+        if (keeper != null) clearInterval(keeper)
+    }
     stale.start()
     return {
-        files: [...graph].filter((f) => !relative(LIB_ROOT, f).startsWith('..')).length,
-        stop: () => {
-            stale.stop()
-            lib.close()
-        },
+        files: [...graph].filter((f) => !relative(tree, f).startsWith('..')).length,
+        stop,
     }
 }