rvlib-shipkit 1.1.0 → 1.2.0

package/src/home/lib-watch.ts

@@ -0,0 +1,139 @@
+// each machine process watches its OWN code and restarts in place on it: pm,
+// the daemon and the web each run with no bun watch. A bun watch follows the
+// whole module graph, which for the daemon holds every house repo config and
+// check file (one repo's edit would restart every repo), and for any process
+// started through cli.ts holds every command (one edit would restart all
+// three). This watch covers the lib source, filtered to the files the role's
+// own entry module reaches, and says a change once the burst of writes settles.
+import { existsSync, readFileSync, watch, type FSWatcher } from 'fs'
+import { dirname, extname, join, relative, resolve } from 'path'
+import { LIB_ROOT } from '../core/lib-root.ts'
+import { diskLibVersion, LIB_VERSION } from '../core/lib-version.ts'
+import { reexecBlocker, reexecSelf } from '../core/reexec.ts'
+import { StaleRestart } from '../serve/stale-restart.ts'
+
+const CODE = /\.(ts|tsx|js|jsx|json|css|html)$/
+const LOADER: Record<string, 'ts' | 'tsx' | 'js' | 'jsx'> = { '.ts': 'ts', '.tsx': 'tsx', '.js': 'js', '.jsx': 'jsx' }
+const TRY = ['', '.ts', '.tsx', '.js', '/index.ts', '/index.tsx']
+
+function resolveRel(from: string, spec: string): string | null {
+    const base = resolve(dirname(from), spec)
+    for (const s of TRY) if (existsSync(base + s) && extname(base + s) !== '') return base + s
+    return null
+}
+
+/** every lib file `entry` reaches through its relative imports, type only
+ *  imports left out (an edit there changes no running code). With `bundles`,
+ *  an imported html page is followed into the scripts and styles it names:
+ *  the web serves those, the daemon only holds the page object */
+export function libGraph(entry: string, o: { bundles?: boolean } = {}): Set<string> {
+    const out = new Set<string>()
+    const todo = [resolve(entry)]
+    while (todo.length > 0) {
+        const f = todo.pop()!
+        if (out.has(f)) continue
+        out.add(f)
+        const ext = extname(f)
+        let text: string
+        try { text = readFileSync(f, 'utf8') } catch { continue }
+        const specs: string[] = []
+        if (LOADER[ext] != null) {
+            try {
+                for (const i of new Bun.Transpiler({ loader: LOADER[ext] }).scanImports(text)) specs.push(i.path)
+            } catch { /* a file that does not parse is still watched, its imports are not */ }
+        } else if (ext === '.html' && o.bundles === true) {
+            for (const m of text.matchAll(/(?:src|href)="(\.[^"]+)"/g)) specs.push(m[1]!)
+        }
+        for (const s of specs) {
+            if (!s.startsWith('.')) continue
+            const r = resolveRel(f, s)
+            if (r != null) todo.push(r)
+        }
+    }
+    return out
+}
+
+export class LibWatch {
+    private w: FSWatcher | null = null
+    private last: { file: string; at: number } | null = null
+    readonly error: string | null = null
+    private now: () => number
+    private settleMs: number
+
+    /** `only`: absolute paths; an edit of any other file is not a change */
+    constructor(private root: string, o: { settleMs?: number; now?: () => number; only?: Set<string> } = {}) {
+        this.now = o.now ?? Date.now
+        this.settleMs = o.settleMs ?? 1_000
+        try {
+            this.w = watch(join(root, 'src'), { recursive: true }, (_ev, file) => {
+                if (file == null || !CODE.test(String(file))) return
+                if (o.only != null && !o.only.has(join(root, 'src', String(file)))) return
+                this.last = { file: String(file), at: this.now() }
+            })
+        } catch (e) {
+            this.error = (e as Error).message
+        }
+    }
+
+    /** a change was seen, settled or not */
+    pending(): boolean {
+        return this.last != null
+    }
+
+    /** one line on what moved, once no write came for settleMs, else null */
+    why(): string | null {
+        if (this.last == null || this.now() - this.last.at < this.settleMs) return null
+        return `the shipkit source moved (src/${this.last.file})`
+    }
+
+    close(): void {
+        this.w?.close()
+        this.w = null
+    }
+}
+
+/** a machine process restarts itself in place, same pid, once its own code
+ *  moved and nothing it runs is in flight. `before` tears down what must not
+ *  outlive the image (its server); what it started detached keeps running */
+export function reloadOnOwnCode(o: {
+    role: string
+    /** the role's runner module: its graph is what the role reloads on */
+    entry: string
+    bundles?: boolean
+    busy?: () => boolean
+    before: () => void
+    io: { print: (l: string) => void; printErr: (l: string) => void }
+}): { stop: () => void; files: number } {
+    const graph = libGraph(o.entry, { bundles: o.bundles })
+    const lib = new LibWatch(LIB_ROOT, { only: graph })
+    if (lib.error != null) o.io.printErr(`WARN no watch on the shipkit source (${lib.error}), so an edit of the ${o.role} needs a restart of it`)
+    const stale = new StaleRestart({
+        boot: LIB_VERSION,
+        disk: diskLibVersion,
+        moved: () => lib.why(),
+        busy: o.busy ?? (() => false),
+        restart: (why) => {
+            const blocked = reexecBlocker()
+            if (blocked != null) throw new Error(blocked)
+            o.io.print(`ok: ${why}: the ${o.role} restarts in place`)
+            stale.stop()
+            lib.close()
+            o.before()
+            try {
+                reexecSelf()
+            } catch (e) {
+                o.io.printErr(`FAIL: the ${o.role} stopped to restart and could not: ${(e as Error).message}`)
+                process.exit(1)
+            }
+        },
+        say: (text) => { if (text != null) o.io.printErr(`WARN ${text}`) },
+    })
+    stale.start()
+    return {
+        files: [...graph].filter((f) => !relative(LIB_ROOT, f).startsWith('..')).length,
+        stop: () => {
+            stale.stop()
+            lib.close()
+        },
+    }
+}

package/src/home/machine-stores.ts

@@ -0,0 +1,115 @@
+// the machine's own stores and their caps (core/retention.ts owns the model):
+// every log under the home dir, and the gui window's Chrome profile. And the
+// daemon's timer that sweeps every store, the repos' and these.
+import { existsSync, lstatSync, readdirSync, rmSync, statSync } from 'fs'
+import { join } from 'path'
+import { repoStores, sweepStores, walkUse, type Store } from '../core/retention.ts'
+import { capLog, LOG_MAX_BYTES } from './supervise.ts'
+
+/** a log nothing wrote for this long belongs to a process that is gone (a
+ *  repo removed, a resource dropped) */
+export const LOG_KEEP_MS = 30 * 24 * 60 * 60 * 1000
+
+/** the profile's size cap: one page's worth of cookies and storage */
+export const GUI_PROFILE_MAX = 128 * 1024 * 1024
+
+/** what Chrome rebuilds on its own, so trimming them loses nothing the window needs */
+export const GUI_CACHE_DIRS = [
+    'Default/Cache',
+    'Default/Code Cache',
+    'Default/GPUCache',
+    'GraphiteDawnCache',
+    'GrShaderCache',
+    'ShaderCache',
+    'component_crx_cache',
+    'optimization_guide_model_store',
+    'OnDeviceHeadSuggestModel',
+    'WasmTtsEngine',
+    'Safe Browsing',
+]
+
+const isLog = (n: string): boolean => n.endsWith('.log')
+const notProfile = (n: string): boolean => n === 'gui-profile'
+
+/** every `*.log` under the home dir, at any depth, the profile aside */
+function logFiles(dir: string): string[] {
+    const out: string[] = []
+    const walk = (d: string): void => {
+        let list
+        try { list = readdirSync(d, { withFileTypes: true }) } catch { return }
+        for (const e of list) {
+            if (e.isDirectory()) { if (!notProfile(e.name)) walk(join(d, e.name)); continue }
+            if (e.isFile() && isLog(e.name)) out.push(join(d, e.name))
+        }
+    }
+    walk(dir)
+    return out
+}
+
+/** Chrome holds the profile while a window runs: its lock is a symlink */
+const profileInUse = (profile: string): boolean => {
+    try { return lstatSync(join(profile, 'SingletonLock')).isSymbolicLink() } catch { return false }
+}
+
+export function machineStores(dir: string): Store[] {
+    const profile = join(dir, 'gui-profile')
+    return [
+        {
+            id: 'home logs',
+            where: `${dir}/**/*.log`,
+            cap: { maxFileBytes: LOG_MAX_BYTES, maxAgeMs: LOG_KEEP_MS },
+            measure: () => walkUse(dir, { match: isLog, skip: notProfile }),
+            sweep: (now) => {
+                for (const f of logFiles(dir)) {
+                    try {
+                        if (statSync(f).mtimeMs < now - LOG_KEEP_MS) rmSync(f, { force: true })
+                        else capLog(f)
+                    } catch { /* raced: gone */ }
+                }
+            },
+        },
+        {
+            id: 'gui profile',
+            where: profile,
+            cap: { maxBytes: GUI_PROFILE_MAX },
+            measure: () => walkUse(profile),
+            // only while no window holds it: Chrome would rebuild a cache
+            // dir under a running window, and races the delete
+            sweep: () => {
+                if (!existsSync(profile) || profileInUse(profile)) return
+                for (const d of GUI_CACHE_DIRS) rmSync(join(profile, d), { recursive: true, force: true })
+            },
+        },
+    ]
+}
+
+/** how often the daemon sweeps, after the one on boot */
+export const SWEEP_EVERY_MS = 60 * 60 * 1000
+/** the boot sweep waits this long, so loading the repos goes first */
+export const BOOT_SWEEP_DELAY_MS = 10_000
+
+export type Sweeper = { stop: () => void; once: () => Promise<void> }
+
+/** the daemon's one retention sweep: every repo it loads, then the machine,
+ *  on boot and every hour. A store whose sweep throws is one printed line */
+export function startSweeps(o: { roots: () => string[]; dir: string; printErr: (l: string) => void; everyMs?: number; bootDelayMs?: number; now?: () => number }): Sweeper {
+    let running = false
+    const once = async (): Promise<void> => {
+        if (running) return
+        running = true
+        try {
+            const now = (o.now ?? Date.now)()
+            for (const root of o.roots()) {
+                for (const r of await sweepStores(repoStores(root), now)) if (r.error != null) o.printErr(`WARN retention sweep of ${r.id} in ${root} failed: ${r.error}`)
+            }
+            for (const r of await sweepStores(machineStores(o.dir), now)) if (r.error != null) o.printErr(`WARN retention sweep of ${r.id} failed: ${r.error}`)
+        } finally {
+            running = false
+        }
+    }
+    const boot = setTimeout(() => void once(), o.bootDelayMs ?? BOOT_SWEEP_DELAY_MS)
+    const every = setInterval(() => void once(), o.everyMs ?? SWEEP_EVERY_MS)
+    boot.unref()
+    every.unref()
+    return { stop: () => { clearTimeout(boot); clearInterval(every) }, once }
+}

package/src/home/open-app.ts

@@ -0,0 +1,16 @@
+// opens one shipkit app window on a url, in its own short process, so pm
+// never loads what building the app needs:
+//
+//   bun open-app.ts <url>
+import { LIB_VERSION } from '../core/lib-version.ts'
+import { openWindow } from './home-start.ts'
+
+if (import.meta.main) {
+    const url = process.argv[2]
+    if (url == null) {
+        console.error('usage: bun open-app.ts <url>')
+        process.exit(2)
+    }
+    const cmd = openWindow(url, { version: LIB_VERSION, warn: (l) => console.error(l) })
+    console.log(`shipkit: opened ${url} as an app window: ${cmd.join(' ')}`)
+}

package/src/home/pm-config.ts

@@ -0,0 +1,37 @@
+// what pm needs from the house repos (whether this one hosts the house wiki,
+// and the global processes every house repo declares in its `.shipkit.ts`
+// files), read in a short child process so pm itself never imports a repo
+// file. That graph would make pm large and reload it on every edit there.
+//
+//   bun pm-config.ts <root>   prints { index, processes, errors } as json on stdout
+import { homedir } from 'os'
+import type { GlobalProcessDecl } from './global-units.ts'
+
+export type PmConfig = {
+    /** the house wiki index the repo includes, null when it includes none */
+    index: string | null
+    processes: GlobalProcessDecl[]
+    /** the unit files that would not load */
+    errors: string[]
+}
+
+if (import.meta.main) {
+    const root = process.argv[2]
+    if (root == null) {
+        console.error('usage: bun pm-config.ts <root>')
+        process.exit(2)
+    }
+    const { houseRepos } = await import('../serve/project-gutter.ts')
+    const house = houseRepos(root, homedir())
+    const out: PmConfig = { index: house.index, processes: [], errors: [] }
+    if (house.index != null) {
+        // a unit file imports the lib by name: this copy, like the daemon's
+        const { useOneLib } = await import('../core/one-lib.ts')
+        const { scanGlobalUnits } = await import('./global-units.ts')
+        useOneLib()
+        const u = await scanGlobalUnits(house.repos.map((r) => ({ name: r.name, path: r.path })))
+        out.processes = u.processes
+        out.errors = u.errors
+    }
+    process.stdout.write(JSON.stringify(out))
+}

package/src/home/pm.ts

@@ -0,0 +1,141 @@
+// pm, the machine's process manager: it keeps the daemon and the web running
+// (supervise.ts: detached, pid records, crash restarts with backoff, adoption)
+// and owns the machine's global processes (resources.ts). Its api answers on
+// its own port, and the web proxies /api/pm/* to it, so the page switches a
+// process and restarts a child through the one origin it knows.
+//
+// the one constraint a reader breaks first: pm never takes a child down with
+// it. A reload of pm, or pm killed, leaves both running, and the next pm
+// adopts them from their pid records. Only a stop asked of pm stops them.
+import { LIB_VERSION } from '../core/lib-version.ts'
+import { sameOrigin } from '../serve/same-origin.ts'
+import { childLog, childPid } from './home-paths.ts'
+import type { Resources, ResourceView } from './resources.ts'
+import { capLog, Proc, realDeps, tailLines, type ProcView, type SuperviseDeps } from './supervise.ts'
+
+export type Role = 'daemon' | 'web'
+export const ROLES: Role[] = ['daemon', 'web']
+
+/** how often pm looks at its children and the global processes */
+export const PM_TICK_MS = 2_000
+
+export type ChildView = ProcView & { tail: string[] }
+
+export type PmAnswer = {
+    pm: 'shipkit-pm'
+    pid: number
+    version: string
+    children: ChildView[]
+    resources: ResourceView[]
+    /** the `.shipkit.ts` files that would not load, as pm last read them */
+    unitErrors: string[]
+}
+
+export type PmDeps = {
+    dir: string
+    /** the house repo: where the children run */
+    root: string
+    /** the command that runs one role, e.g. [bun, cli.ts, 'daemon'] */
+    cmd: (role: Role) => string[]
+    /** true once the role serves */
+    probe: (role: Role) => Promise<boolean>
+    resources: Resources
+    deps?: SuperviseDeps
+    /** runs a stop or restart of the web once its answer is out: the web
+     *  proxies that very request (default: 200 ms later) */
+    later?: (f: () => void) => void
+}
+
+const WEB_ANSWER_MS = 200
+
+export class Pm {
+    readonly children: Map<Role, Proc>
+
+    constructor(private d: PmDeps) {
+        this.children = new Map(ROLES.map((role) => [role, new Proc({
+            id: role,
+            cmd: d.cmd(role),
+            cwd: d.root,
+            log: childLog(d.dir, role),
+            pidKey: childPid(d.dir, role),
+            probe: () => d.probe(role),
+            // a crash loop while its code is being edited heals itself
+            neverGiveUp: true,
+        }, d.deps ?? realDeps)]))
+    }
+
+    get resources(): Resources {
+        return this.d.resources
+    }
+
+    /** every child up: adopted from its pid record, else started, unless
+     *  something else already answers its port */
+    async boot(): Promise<void> {
+        await Promise.all([...this.children.values()].map((p) => p.ensure()))
+    }
+
+    async tick(): Promise<void> {
+        await Promise.all([...[...this.children.values()].map((p) => p.tick()), this.d.resources.tick()])
+        for (const role of ROLES) capLog(childLog(this.d.dir, role))
+    }
+
+    /** stop both children and wait until they are gone, SIGKILL included */
+    async stopAll(o: { waitMs?: number; stepMs?: number } = {}): Promise<void> {
+        for (const p of this.children.values()) p.stop()
+        const until = Date.now() + (o.waitMs ?? 12_000)
+        while (Date.now() < until && [...this.children.values()].some((p) => p.status === 'stopping')) {
+            await Bun.sleep(o.stepMs ?? 200)
+            await Promise.all([...this.children.values()].map((p) => p.tick()))
+        }
+    }
+
+    answer(): PmAnswer {
+        const children = [...this.children.values()].map((p) => ({ ...p.view(), tail: p.status === 'up' ? [] : tailLines(p.spec.log, 20) }))
+        return { pm: 'shipkit-pm', pid: process.pid, version: LIB_VERSION, children, resources: this.d.resources.views(), unitErrors: this.d.resources.errors }
+    }
+
+    async handle(req: Request): Promise<Response> {
+        const path = new URL(req.url).pathname
+        if (!sameOrigin(req)) return new Response('refused: a request that changes something must come from the home page', { status: 403 })
+        if (path === '/api/pm/status.json') return Response.json(this.answer())
+        if (req.method !== 'POST') return new Response('not found', { status: 404 })
+        const b = (await req.json().catch(() => ({}))) as Record<string, unknown>
+        if (path === '/api/pm/child') {
+            const p = this.children.get(b.id as Role)
+            if (p == null) return new Response(`no child '${String(b.id)}': pm runs ${ROLES.join(' and ')}`, { status: 404 })
+            if (b.act === 'start') p.start()
+            else if (b.act === 'stop' || b.act === 'restart') {
+                const act = b.act === 'stop' ? () => p.stop() : () => p.restart()
+                if (b.id === 'web') (this.d.later ?? ((f) => { setTimeout(f, WEB_ANSWER_MS) }))(act)
+                else act()
+            } else return new Response('act must be start, stop or restart', { status: 400 })
+            return Response.json(p.view())
+        }
+        if (path === '/api/pm/resource') {
+            const r = this.d.resources.get(String(b.id))
+            if (r == null) return new Response(`no global process '${String(b.id)}': none of the house repos' .shipkit.ts files declares it`, { status: 404 })
+            const act = b.act ?? (b.on === true ? 'start' : b.on === false ? 'stop' : null)
+            // a bring-up or an action can take minutes: the page follows it on its poll
+            const report = (p: Promise<void>): void => { p.catch((e: Error) => { r.note = e.message }) }
+            if (act === 'start') report(r.set('up'))
+            else if (act === 'stop') report(r.set('down'))
+            else if (act === 'restart') report(r.restart())
+            else if (act === 'open' || act === 'app') {
+                if (r.decl.url == null) return new Response(`${r.decl.id} declares no url to open`, { status: 400 })
+                if (act === 'app' && !r.decl.app) return new Response(`${r.decl.id} does not declare app: true, so it has no app window`, { status: 400 })
+                report(r.open(act === 'app' ? 'app' : 'browser'))
+            } else if (act === 'takeover') {
+                const v = r.takeOverView
+                if (v == null) return new Response(`${r.decl.id} does not run outside the supervisor: nothing to take over`, { status: 409 })
+                if (!v.ok) return new Response(`${r.decl.id}: ${v.why}`, { status: 400 })
+                report(r.takeOver())
+            } else if (act === 'action') {
+                if (!r.decl.actions.some((a) => a.id === b.action)) return new Response(`${r.decl.id} declares no action '${String(b.action)}'`, { status: 404 })
+                if (r.running != null) return new Response(`${r.decl.id} is already running its action '${r.running}'`, { status: 409 })
+                report(r.action(String(b.action)))
+            } else return new Response('act must be start, stop, restart, open, app, takeover or action', { status: 400 })
+            return Response.json(r.view())
+        }
+        return new Response('not found', { status: 404 })
+    }
+}