rvlib-shipkit 1.1.1 → 1.3.0

package/src/home/hubs.ts

@@ -0,0 +1,286 @@
+// every house repo's hub, IN the home process: one runtime per repo
+// (serve/hub-runtime.ts), loaded at boot one after the other, served under
+// /r/<repo>/ of the home's one server. A repo whose config or deps fail to
+// load reads crashed with the error, and the others keep serving.
+//
+// the one constraint a reader breaks first: nothing here starts a gate. Every
+// run nobody asked for comes from the machine's one plan (plan/planner.ts),
+// which each loaded hub joins. An edit of a repo config reloads that repo's
+// hub alone, once its gate is done: the daemon runs with no bun watch, so one
+// repo's edit never restarts the others (lib-watch.ts).
+import { appendFileSync, existsSync, mkdirSync, readFileSync } from 'fs'
+import { dirname, join } from 'path'
+import type { WiredRepo } from '../serve/project-gutter.ts'
+import type { HubRuntime, ServeIo } from '../serve/hub-runtime.ts'
+import { configStamp, PING_EVERY_MS } from '../serve/hub-runtime.ts'
+import { capLog, tailLines, type ProcStatus } from './supervise.ts'
+import { hubBase, hubLog } from './home-paths.ts'
+import { NOT_INSTALLED } from './home-words.ts'
+
+/** what the home needs of a loaded hub */
+export type LoadedHub = Pick<HubRuntime, 'handler' | 'start' | 'stop' | 'ping' | 'busy'> & { uses: string[] }
+
+export type HubLoader = (repo: WiredRepo, io: ServeIo, o: { fresh: boolean }) => Promise<LoadedHub>
+
+export type WorkerView = {
+    id: string
+    repo: string
+    status: ProcStatus
+    /** the home's pid while the hub is loaded: it runs in the home */
+    pid: number | null
+    /** epoch ms of the last status change */
+    since: number
+    /** why it is not up, when that is known and `blocked` does not say it */
+    note: string | null
+    /** the home's port, which serves every hub */
+    port: number
+    /** the hub page, on the home's origin */
+    url: string
+    /** why the home cannot load it at all */
+    blocked: string | null
+    /** the last lines of its log, while it is not up */
+    tail: string[]
+    /** the machine resources its config names in serve.uses */
+    uses: string[]
+}
+
+type Entry = {
+    repo: WiredRepo
+    status: ProcStatus
+    since: number
+    note: string | null
+    rt: LoadedHub | null
+    loading: Promise<void> | null
+    /** loads done, so a view tells a reload from the first one */
+    loads: number
+    /** stopped by hand: nothing but a restart loads it again */
+    held: boolean
+    /** its repo.config.ts as the last load read it */
+    stamp: string
+}
+
+/** why the home cannot load this repo's hub, or null when it can */
+export function hubBlocker(path: string): string | null {
+    if (!existsSync(join(path, 'repo.config.ts'))) return 'no repo.config.ts: this repo is not wired to shipkit'
+    return null
+}
+
+/** a load that failed on a fresh clone: its dependencies are not installed */
+function notInstalled(path: string): boolean {
+    if (existsSync(join(path, 'node_modules'))) return false
+    try {
+        const pkg = JSON.parse(readFileSync(join(path, 'package.json'), 'utf8')) as { dependencies?: object; devDependencies?: object }
+        return Object.keys({ ...pkg.dependencies, ...pkg.devDependencies }).length > 0
+    } catch {
+        return false
+    }
+}
+
+export class Hubs {
+    private entries = new Map<string, Entry>()
+    private queue: Promise<void> = Promise.resolve()
+    private now: () => number
+    private lastPing = -Infinity
+
+    /** `prepare` runs before each load and may edit the repo's config (the
+     *  identity fill): the stamp is taken after it, so its edit is no reload */
+    constructor(private o: { dir: string; port: number; load: HubLoader; prepare?: (repo: WiredRepo, io: ServeIo) => Promise<void>; now?: () => number }) {
+        this.now = o.now ?? Date.now
+    }
+
+    private io(e: Entry): ServeIo {
+        const file = hubLog(this.o.dir, e.repo.name)
+        const put = (l: string): void => {
+            try {
+                mkdirSync(dirname(file), { recursive: true })
+                appendFileSync(file, `${l}\n`)
+            } catch { /* a full disk loses a log line, never the hub */ }
+        }
+        return { print: put, printErr: put }
+    }
+
+    private entry(repo: WiredRepo): Entry {
+        let e = this.entries.get(repo.name)
+        if (e == null) {
+            e = { repo, status: 'down', since: this.now(), note: null, rt: null, loading: null, loads: 0, held: false, stamp: '' }
+            this.entries.set(repo.name, e)
+        }
+        e.repo = repo
+        return e
+    }
+
+    private set(e: Entry, status: ProcStatus, note: string | null = null): void {
+        e.status = status
+        e.note = note
+        e.since = this.now()
+    }
+
+    /** queue a load: one at a time for the machine, in the order asked */
+    private schedule(e: Entry, fresh: boolean): Promise<void> {
+        if (e.loading != null) return e.loading
+        this.set(e, 'starting')
+        const run = this.queue.then(async () => {
+            const blocked = hubBlocker(e.repo.path)
+            if (blocked != null) {
+                this.set(e, 'down', blocked)
+                return
+            }
+            const io = this.io(e)
+            try {
+                if (this.o.prepare != null) await this.o.prepare(e.repo, io).catch((err) => io.printErr(`WARN ${e.repo.name}: ${(err as Error).message}`))
+                e.stamp = configStamp(e.repo.path)
+                const rt = await this.o.load(e.repo, io, { fresh: fresh || e.loads > 0 })
+                // a stop by hand came while it loaded, and wins
+                if (e.held) {
+                    rt.stop({ procs: true })
+                    this.set(e, 'down', 'stopped by hand')
+                    return
+                }
+                rt.start()
+                e.rt = rt
+                e.loads++
+                this.set(e, 'up')
+            } catch (err) {
+                if (e.held) return
+                const msg = (err as Error).message
+                io.printErr(`FAIL: ${e.repo.name}'s hub did not load: ${msg}`)
+                this.set(e, 'crashed', msg)
+            }
+        }).finally(() => { e.loading = null })
+        e.loading = run
+        this.queue = run.catch(() => {})
+        return run
+    }
+
+    /** the list of house repos moved: a new one loads, a gone one stops. The
+     *  first call is the boot: every repo queues, in rail order */
+    sync(repos: WiredRepo[]): void {
+        const names = new Set(repos.map((r) => r.name))
+        for (const [name, e] of this.entries) {
+            if (names.has(name)) continue
+            e.rt?.stop({ procs: true })
+            this.entries.delete(name)
+        }
+        for (const r of repos) {
+            const known = this.entries.has(r.name)
+            const e = this.entry(r)
+            if (!known) void this.schedule(e, false)
+        }
+    }
+
+    /** load it when it is down, and say where it stands */
+    ensure(repo: WiredRepo): WorkerView {
+        const e = this.entry(repo)
+        if (e.status === 'down' && e.rt == null && !e.held && hubBlocker(repo.path) == null) void this.schedule(e, false)
+        return this.view(e)
+    }
+
+    /** held down until a restart; a load in flight is dropped when it lands */
+    stop(repo: WiredRepo): WorkerView {
+        const e = this.entry(repo)
+        e.held = true
+        e.rt?.stop({ procs: true })
+        e.rt = null
+        this.set(e, 'down', 'stopped by hand')
+        return this.view(e)
+    }
+
+    /** stop, then load again with the config read fresh */
+    async restart(repo: WiredRepo): Promise<WorkerView> {
+        const e = this.entry(repo)
+        await e.loading
+        e.held = false
+        e.rt?.stop({ procs: true })
+        e.rt = null
+        await this.schedule(e, true)
+        return this.view(e)
+    }
+
+    /** a request under /r/<repo>/: waits for a load in flight, then the
+     *  repo's own handler answers with the path below its base */
+    async dispatch(req: Request, path: string, srv: Parameters<LoadedHub['handler']['handle']>[2]): Promise<Response> {
+        const m = /^\/r\/([^/]+)(\/.*)?$/.exec(path)
+        if (m == null) return new Response('not found', { status: 404 })
+        const name = decodeURIComponent(m[1]!)
+        const rest = m[2]
+        if (rest == null) return Response.redirect(`${hubBase(name)}/${new URL(req.url).search}`, 308)
+        const e = this.entries.get(name)
+        if (e == null) return new Response(`not a house repo: ${name}`, { status: 404 })
+        await e.loading
+        if (e.rt == null) return new Response(`${name}'s hub is ${e.status}${e.note != null ? `: ${e.note}` : ''}`, { status: 409 })
+        return e.rt.handler.handle(req, rest, srv)
+    }
+
+    /** one pass: logs stay capped, an edited config reloads its hub, and the
+     *  ACTIVE repos get their RED ping every PING_EVERY_MS */
+    async tick(active: Set<string>): Promise<void> {
+        for (const e of this.entries.values()) {
+            capLog(hubLog(this.o.dir, e.repo.name))
+            this.reloadIfEdited(e)
+        }
+        const now = this.now()
+        const up = [...this.entries.values()].filter((e) => e.rt != null && active.has(e.repo.name))
+        if (now - this.lastPing >= PING_EVERY_MS) {
+            this.lastPing = now
+            for (const e of up) await e.rt?.ping()
+        }
+    }
+
+    /** its config moved since it loaded: reload it with the config read
+     *  fresh, its processes left running for the new load to adopt. A gate in
+     *  flight defers it to a later tick */
+    private reloadIfEdited(e: Entry): void {
+        if (e.loading != null || e.held) return
+        const now = configStamp(e.repo.path)
+        if (now === e.stamp || now === '') return
+        // a hub that crashed on its config gets its next try from the edit
+        if (e.rt == null) {
+            if (e.status !== 'crashed') return
+            this.io(e).print(`ok: ${e.repo.name}'s repo.config.ts changed: loading its hub again`)
+            void this.schedule(e, true)
+            return
+        }
+        if (e.rt.busy()) return
+        this.io(e).print(`ok: ${e.repo.name}'s repo.config.ts changed: reloading its hub`)
+        e.rt.stop()
+        e.rt = null
+        void this.schedule(e, true)
+    }
+
+    /** a run is in flight in some hub */
+    busy(): boolean {
+        for (const e of this.entries.values()) if (e.rt?.busy() === true) return true
+        return false
+    }
+
+    /** the repos it knows, loaded or not */
+    names(): string[] {
+        return [...this.entries.keys()]
+    }
+
+    private view(e: Entry): WorkerView {
+        const blocked = hubBlocker(e.repo.path) ?? (e.status === 'crashed' && notInstalled(e.repo.path) ? NOT_INSTALLED : null)
+        return {
+            id: e.repo.name,
+            repo: e.repo.name,
+            status: e.status,
+            pid: e.rt != null ? process.pid : null,
+            since: e.since,
+            note: blocked != null && e.note === blocked ? null : e.note,
+            port: this.o.port,
+            url: `${hubBase(e.repo.name)}/`,
+            blocked,
+            tail: e.status === 'up' ? [] : tailLines(hubLog(this.o.dir, e.repo.name), 30),
+            uses: e.rt?.uses ?? [],
+        }
+    }
+
+    views(repos: WiredRepo[]): WorkerView[] {
+        return repos.map((r) => this.view(this.entry(r)))
+    }
+
+    /** every hub stops, its processes left running for the next home to adopt */
+    stopAll(): void {
+        for (const e of this.entries.values()) e.rt?.stop()
+    }
+}

package/src/home/lib-watch.ts

@@ -0,0 +1,139 @@
+// each machine process watches its OWN code and restarts in place on it: pm,
+// the daemon and the web each run with no bun watch. A bun watch follows the
+// whole module graph, which for the daemon holds every house repo config and
+// check file (one repo's edit would restart every repo), and for any process
+// started through cli.ts holds every command (one edit would restart all
+// three). This watch covers the lib source, filtered to the files the role's
+// own entry module reaches, and says a change once the burst of writes settles.
+import { existsSync, readFileSync, watch, type FSWatcher } from 'fs'
+import { dirname, extname, join, relative, resolve } from 'path'
+import { LIB_ROOT } from '../core/lib-root.ts'
+import { diskLibVersion, LIB_VERSION } from '../core/lib-version.ts'
+import { reexecBlocker, reexecSelf } from '../core/reexec.ts'
+import { StaleRestart } from '../serve/stale-restart.ts'
+
+const CODE = /\.(ts|tsx|js|jsx|json|css|html)$/
+const LOADER: Record<string, 'ts' | 'tsx' | 'js' | 'jsx'> = { '.ts': 'ts', '.tsx': 'tsx', '.js': 'js', '.jsx': 'jsx' }
+const TRY = ['', '.ts', '.tsx', '.js', '/index.ts', '/index.tsx']
+
+function resolveRel(from: string, spec: string): string | null {
+    const base = resolve(dirname(from), spec)
+    for (const s of TRY) if (existsSync(base + s) && extname(base + s) !== '') return base + s
+    return null
+}
+
+/** every lib file `entry` reaches through its relative imports, type only
+ *  imports left out (an edit there changes no running code). With `bundles`,
+ *  an imported html page is followed into the scripts and styles it names:
+ *  the web serves those, the daemon only holds the page object */
+export function libGraph(entry: string, o: { bundles?: boolean } = {}): Set<string> {
+    const out = new Set<string>()
+    const todo = [resolve(entry)]
+    while (todo.length > 0) {
+        const f = todo.pop()!
+        if (out.has(f)) continue
+        out.add(f)
+        const ext = extname(f)
+        let text: string
+        try { text = readFileSync(f, 'utf8') } catch { continue }
+        const specs: string[] = []
+        if (LOADER[ext] != null) {
+            try {
+                for (const i of new Bun.Transpiler({ loader: LOADER[ext] }).scanImports(text)) specs.push(i.path)
+            } catch { /* a file that does not parse is still watched, its imports are not */ }
+        } else if (ext === '.html' && o.bundles === true) {
+            for (const m of text.matchAll(/(?:src|href)="(\.[^"]+)"/g)) specs.push(m[1]!)
+        }
+        for (const s of specs) {
+            if (!s.startsWith('.')) continue
+            const r = resolveRel(f, s)
+            if (r != null) todo.push(r)
+        }
+    }
+    return out
+}
+
+export class LibWatch {
+    private w: FSWatcher | null = null
+    private last: { file: string; at: number } | null = null
+    readonly error: string | null = null
+    private now: () => number
+    private settleMs: number
+
+    /** `only`: absolute paths; an edit of any other file is not a change */
+    constructor(private root: string, o: { settleMs?: number; now?: () => number; only?: Set<string> } = {}) {
+        this.now = o.now ?? Date.now
+        this.settleMs = o.settleMs ?? 1_000
+        try {
+            this.w = watch(join(root, 'src'), { recursive: true }, (_ev, file) => {
+                if (file == null || !CODE.test(String(file))) return
+                if (o.only != null && !o.only.has(join(root, 'src', String(file)))) return
+                this.last = { file: String(file), at: this.now() }
+            })
+        } catch (e) {
+            this.error = (e as Error).message
+        }
+    }
+
+    /** a change was seen, settled or not */
+    pending(): boolean {
+        return this.last != null
+    }
+
+    /** one line on what moved, once no write came for settleMs, else null */
+    why(): string | null {
+        if (this.last == null || this.now() - this.last.at < this.settleMs) return null
+        return `the shipkit source moved (src/${this.last.file})`
+    }
+
+    close(): void {
+        this.w?.close()
+        this.w = null
+    }
+}
+
+/** a machine process restarts itself in place, same pid, once its own code
+ *  moved and nothing it runs is in flight. `before` tears down what must not
+ *  outlive the image (its server); what it started detached keeps running */
+export function reloadOnOwnCode(o: {
+    role: string
+    /** the role's runner module: its graph is what the role reloads on */
+    entry: string
+    bundles?: boolean
+    busy?: () => boolean
+    before: () => void
+    io: { print: (l: string) => void; printErr: (l: string) => void }
+}): { stop: () => void; files: number } {
+    const graph = libGraph(o.entry, { bundles: o.bundles })
+    const lib = new LibWatch(LIB_ROOT, { only: graph })
+    if (lib.error != null) o.io.printErr(`WARN no watch on the shipkit source (${lib.error}), so an edit of the ${o.role} needs a restart of it`)
+    const stale = new StaleRestart({
+        boot: LIB_VERSION,
+        disk: diskLibVersion,
+        moved: () => lib.why(),
+        busy: o.busy ?? (() => false),
+        restart: (why) => {
+            const blocked = reexecBlocker()
+            if (blocked != null) throw new Error(blocked)
+            o.io.print(`ok: ${why}: the ${o.role} restarts in place`)
+            stale.stop()
+            lib.close()
+            o.before()
+            try {
+                reexecSelf()
+            } catch (e) {
+                o.io.printErr(`FAIL: the ${o.role} stopped to restart and could not: ${(e as Error).message}`)
+                process.exit(1)
+            }
+        },
+        say: (text) => { if (text != null) o.io.printErr(`WARN ${text}`) },
+    })
+    stale.start()
+    return {
+        files: [...graph].filter((f) => !relative(LIB_ROOT, f).startsWith('..')).length,
+        stop: () => {
+            stale.stop()
+            lib.close()
+        },
+    }
+}

package/src/home/machine-stores.ts

@@ -0,0 +1,115 @@
+// the machine's own stores and their caps (core/retention.ts owns the model):
+// every log under the home dir, and the gui window's Chrome profile. And the
+// daemon's timer that sweeps every store, the repos' and these.
+import { existsSync, lstatSync, readdirSync, rmSync, statSync } from 'fs'
+import { join } from 'path'
+import { repoStores, sweepStores, walkUse, type Store } from '../core/retention.ts'
+import { capLog, LOG_MAX_BYTES } from './supervise.ts'
+
+/** a log nothing wrote for this long belongs to a process that is gone (a
+ *  repo removed, a resource dropped) */
+export const LOG_KEEP_MS = 30 * 24 * 60 * 60 * 1000
+
+/** the profile's size cap: one page's worth of cookies and storage */
+export const GUI_PROFILE_MAX = 128 * 1024 * 1024
+
+/** what Chrome rebuilds on its own, so trimming them loses nothing the window needs */
+export const GUI_CACHE_DIRS = [
+    'Default/Cache',
+    'Default/Code Cache',
+    'Default/GPUCache',
+    'GraphiteDawnCache',
+    'GrShaderCache',
+    'ShaderCache',
+    'component_crx_cache',
+    'optimization_guide_model_store',
+    'OnDeviceHeadSuggestModel',
+    'WasmTtsEngine',
+    'Safe Browsing',
+]
+
+const isLog = (n: string): boolean => n.endsWith('.log')
+const notProfile = (n: string): boolean => n === 'gui-profile'
+
+/** every `*.log` under the home dir, at any depth, the profile aside */
+function logFiles(dir: string): string[] {
+    const out: string[] = []
+    const walk = (d: string): void => {
+        let list
+        try { list = readdirSync(d, { withFileTypes: true }) } catch { return }
+        for (const e of list) {
+            if (e.isDirectory()) { if (!notProfile(e.name)) walk(join(d, e.name)); continue }
+            if (e.isFile() && isLog(e.name)) out.push(join(d, e.name))
+        }
+    }
+    walk(dir)
+    return out
+}
+
+/** Chrome holds the profile while a window runs: its lock is a symlink */
+const profileInUse = (profile: string): boolean => {
+    try { return lstatSync(join(profile, 'SingletonLock')).isSymbolicLink() } catch { return false }
+}
+
+export function machineStores(dir: string): Store[] {
+    const profile = join(dir, 'gui-profile')
+    return [
+        {
+            id: 'home logs',
+            where: `${dir}/**/*.log`,
+            cap: { maxFileBytes: LOG_MAX_BYTES, maxAgeMs: LOG_KEEP_MS },
+            measure: () => walkUse(dir, { match: isLog, skip: notProfile }),
+            sweep: (now) => {
+                for (const f of logFiles(dir)) {
+                    try {
+                        if (statSync(f).mtimeMs < now - LOG_KEEP_MS) rmSync(f, { force: true })
+                        else capLog(f)
+                    } catch { /* raced: gone */ }
+                }
+            },
+        },
+        {
+            id: 'gui profile',
+            where: profile,
+            cap: { maxBytes: GUI_PROFILE_MAX },
+            measure: () => walkUse(profile),
+            // only while no window holds it: Chrome would rebuild a cache
+            // dir under a running window, and races the delete
+            sweep: () => {
+                if (!existsSync(profile) || profileInUse(profile)) return
+                for (const d of GUI_CACHE_DIRS) rmSync(join(profile, d), { recursive: true, force: true })
+            },
+        },
+    ]
+}
+
+/** how often the daemon sweeps, after the one on boot */
+export const SWEEP_EVERY_MS = 60 * 60 * 1000
+/** the boot sweep waits this long, so loading the repos goes first */
+export const BOOT_SWEEP_DELAY_MS = 10_000
+
+export type Sweeper = { stop: () => void; once: () => Promise<void> }
+
+/** the daemon's one retention sweep: every repo it loads, then the machine,
+ *  on boot and every hour. A store whose sweep throws is one printed line */
+export function startSweeps(o: { roots: () => string[]; dir: string; printErr: (l: string) => void; everyMs?: number; bootDelayMs?: number; now?: () => number }): Sweeper {
+    let running = false
+    const once = async (): Promise<void> => {
+        if (running) return
+        running = true
+        try {
+            const now = (o.now ?? Date.now)()
+            for (const root of o.roots()) {
+                for (const r of await sweepStores(repoStores(root), now)) if (r.error != null) o.printErr(`WARN retention sweep of ${r.id} in ${root} failed: ${r.error}`)
+            }
+            for (const r of await sweepStores(machineStores(o.dir), now)) if (r.error != null) o.printErr(`WARN retention sweep of ${r.id} failed: ${r.error}`)
+        } finally {
+            running = false
+        }
+    }
+    const boot = setTimeout(() => void once(), o.bootDelayMs ?? BOOT_SWEEP_DELAY_MS)
+    const every = setInterval(() => void once(), o.everyMs ?? SWEEP_EVERY_MS)
+    boot.unref()
+    every.unref()
+    return { stop: () => { clearTimeout(boot); clearInterval(every) }, once }
+}

package/src/home/open-app.ts

@@ -0,0 +1,16 @@
+// opens one shipkit app window on a url, in its own short process, so pm
+// never loads what building the app needs:
+//
+//   bun open-app.ts <url>
+import { LIB_VERSION } from '../core/lib-version.ts'
+import { openWindow } from './home-start.ts'
+
+if (import.meta.main) {
+    const url = process.argv[2]
+    if (url == null) {
+        console.error('usage: bun open-app.ts <url>')
+        process.exit(2)
+    }
+    const cmd = openWindow(url, { version: LIB_VERSION, warn: (l) => console.error(l) })
+    console.log(`shipkit: opened ${url} as an app window: ${cmd.join(' ')}`)
+}

package/src/home/pm-config.ts

@@ -0,0 +1,37 @@
+// what pm needs from the house repos (whether this one hosts the house wiki,
+// and the global processes every house repo declares in its `.shipkit.ts`
+// files), read in a short child process so pm itself never imports a repo
+// file. That graph would make pm large and reload it on every edit there.
+//
+//   bun pm-config.ts <root>   prints { index, processes, errors } as json on stdout
+import { homedir } from 'os'
+import type { GlobalProcessDecl } from './global-units.ts'
+
+export type PmConfig = {
+    /** the house wiki index the repo includes, null when it includes none */
+    index: string | null
+    processes: GlobalProcessDecl[]
+    /** the unit files that would not load */
+    errors: string[]
+}
+
+if (import.meta.main) {
+    const root = process.argv[2]
+    if (root == null) {
+        console.error('usage: bun pm-config.ts <root>')
+        process.exit(2)
+    }
+    const { houseRepos } = await import('../serve/project-gutter.ts')
+    const house = houseRepos(root, homedir())
+    const out: PmConfig = { index: house.index, processes: [], errors: [] }
+    if (house.index != null) {
+        // a unit file imports the lib by name: this copy, like the daemon's
+        const { useOneLib } = await import('../core/one-lib.ts')
+        const { scanGlobalUnits } = await import('./global-units.ts')
+        useOneLib()
+        const u = await scanGlobalUnits(house.repos.map((r) => ({ name: r.name, path: r.path })))
+        out.processes = u.processes
+        out.errors = u.errors
+    }
+    process.stdout.write(JSON.stringify(out))
+}