rvlib-shipkit 1.1.0 → 1.2.0

package/src/core/retention.ts

@@ -0,0 +1,213 @@
+// NOTHING GROWS UNBOUND: every store shipkit writes declares a cap here (an
+// age, a count, a size), and one sweep enforces all of them. The daemon runs
+// it on boot and hourly for every repo it loads and for the machine; rule
+// SK023 fails when a store is past its cap, so a sweep that stopped running
+// is loud. Each store's sweep lives beside the code that writes it, so a gate
+// on a machine with no daemon still keeps its own stores in bounds.
+//
+// The one constraint a reader breaks first: a sweep never removes what a live
+// process may still use. Every sweep below checks a pid or an age first.
+import { existsSync, readdirSync, statSync } from 'fs'
+import { join } from 'path'
+import { INFLIGHT_KEEP_MS, sweepSlots } from './attach.ts'
+import { RUN_DIR_KEEP_MS, RUN_DIR_MAX, runEntries, sweepRunDirs } from './check-runner.ts'
+import { gitRoot } from './git-root.ts'
+import { JOBS, JOBS_KEEP_MS, JOBS_MAX, sweepJobs } from './jobs.ts'
+import { CI_LOG_DIR } from './layout.ts'
+import { compactDb, dbPath, shipkitRoot } from '../ledger/db.ts'
+import { SCAN_INDEX_KEEP_MS, sweepScanIndexes } from '../ledger/git-scan.ts'
+import { capLog, LOG_MAX_BYTES } from '../home/supervise.ts'
+import { processDir } from '../serve/repo-processes.ts'
+import { KEEP_MS as RECORD_KEEP_MS, MAX_ROWS, recordUse, sweepRuns } from '../ledger/run-log.ts'
+import { answersDb, answersUse, importLegacyStore, KEEP_MS as ANSWERS_KEEP_MS, legacyStoreDir, MAX_ANSWERS, sweepAnswers } from '../ledger/store.ts'
+
+export type Cap = {
+    /** the oldest entry may be this old */
+    maxAgeMs?: number
+    /** at most this many entries */
+    maxCount?: number
+    /** every file at most this big */
+    maxFileBytes?: number
+    /** all of it at most this big */
+    maxBytes?: number
+}
+
+export type Use = { count: number; oldestAt: number | null; bytes: number; largestFile: number }
+
+export type Store = {
+    id: string
+    /** where it lives, as a person would look for it */
+    where: string
+    cap: Cap
+    /** what it holds now, null when it cannot be read */
+    measure(): Use | null
+    /** bring it back under its cap */
+    sweep(now: number): void | Promise<void>
+}
+
+/** how far past its cap a store may drift between two sweeps before SK023
+ *  calls it over: the daemon sweeps hourly, and gates write in between */
+export const SLACK = { factor: 2, ageMs: 2 * 60 * 60 * 1000 }
+
+/** why a store is past its cap (with the drift a sweep interval allows), or
+ *  null while it is inside */
+export function overCap(use: Use, cap: Cap, now: number, slack = SLACK): string | null {
+    if (cap.maxCount != null && use.count > Math.max(cap.maxCount * slack.factor, cap.maxCount)) return `${use.count} entries, cap ${cap.maxCount}`
+    if (cap.maxAgeMs != null && use.oldestAt != null && now - use.oldestAt > cap.maxAgeMs + slack.ageMs) return `oldest ${fmtAge(now - use.oldestAt)} old, cap ${fmtAge(cap.maxAgeMs)}`
+    if (cap.maxFileBytes != null && use.largestFile > cap.maxFileBytes * slack.factor) return `a file of ${fmtBytes(use.largestFile)}, cap ${fmtBytes(cap.maxFileBytes)}`
+    if (cap.maxBytes != null && use.bytes > cap.maxBytes * slack.factor) return `${fmtBytes(use.bytes)}, cap ${fmtBytes(cap.maxBytes)}`
+    return null
+}
+
+export const fmtBytes = (n: number): string => (n >= 1 << 20 ? `${(n / (1 << 20)).toFixed(1)} MB` : n >= 1024 ? `${Math.round(n / 1024)} KB` : `${n} B`)
+export const fmtAge = (ms: number): string => (ms >= 86_400_000 ? `${Math.round(ms / 86_400_000)}d` : ms >= 3_600_000 ? `${Math.round(ms / 3_600_000)}h` : `${Math.round(ms / 60_000)}m`)
+
+/** every file under a dir (not following links), skipping dirs `skip` names */
+export function walkUse(dir: string, o: { match?: (name: string) => boolean; skip?: (name: string) => boolean } = {}): Use | null {
+    if (!existsSync(dir)) return { count: 0, oldestAt: null, bytes: 0, largestFile: 0 }
+    const use: Use = { count: 0, oldestAt: null, bytes: 0, largestFile: 0 }
+    const walk = (d: string): void => {
+        let list
+        try { list = readdirSync(d, { withFileTypes: true }) } catch { return }
+        for (const e of list) {
+            const p = join(d, e.name)
+            if (e.isDirectory()) { if (o.skip?.(e.name) !== true) walk(p); continue }
+            if (!e.isFile() || (o.match != null && !o.match(e.name))) continue
+            try {
+                const s = statSync(p)
+                use.count++
+                use.bytes += s.size
+                use.largestFile = Math.max(use.largestFile, s.size)
+                use.oldestAt = use.oldestAt == null ? s.mtimeMs : Math.min(use.oldestAt, s.mtimeMs)
+            } catch { /* raced: gone */ }
+        }
+    }
+    walk(dir)
+    return use
+}
+
+const counted = (count: number, oldestAt: number | null): Use => ({ count, oldestAt, bytes: 0, largestFile: 0 })
+
+/** the stores one repo holds */
+export function repoStores(root: string): Store[] {
+    const sk = shipkitRoot(root)
+    const logs = join(root, CI_LOG_DIR)
+    const jobs = join(root, JOBS)
+    const gitDir = gitRoot(root)?.gitDir
+    const stores: Store[] = [
+        {
+            id: 'answers',
+            where: `${answersDb(root)} (answers)`,
+            cap: { maxAgeMs: ANSWERS_KEEP_MS, maxCount: MAX_ANSWERS },
+            measure: () => {
+                if (!existsSync(answersDb(root))) return counted(0, null)
+                const u = answersUse(answersDb(root))
+                return u == null ? null : counted(u.count, u.oldestUsed)
+            },
+            sweep: (now) => { sweepAnswers(answersDb(root), now) },
+        },
+        {
+            // the json store this table replaced: read once, then gone
+            id: 'json answers',
+            where: legacyStoreDir(root),
+            cap: { maxCount: 0 },
+            measure: () => walkUse(legacyStoreDir(root)),
+            sweep: async (now) => { await importLegacyStore(root, now) },
+        },
+        {
+            id: 'run record',
+            where: `${dbPath(root)} (runs, invocations, suites)`,
+            cap: { maxAgeMs: RECORD_KEEP_MS, maxCount: MAX_ROWS },
+            measure: () => {
+                if (!existsSync(dbPath(root))) return counted(0, null)
+                const u = recordUse(dbPath(root))
+                return u == null ? null : counted(u.count, u.oldest)
+            },
+            // the last db store: the answers above and these rows are one
+            // file, which the deletes leave full of free pages
+            sweep: (now) => {
+                sweepRuns(dbPath(root), now)
+                compactDb(dbPath(root))
+            },
+        },
+        {
+            id: 'inflight',
+            where: join(sk, 'inflight'),
+            cap: { maxAgeMs: INFLIGHT_KEEP_MS },
+            measure: () => walkUse(join(sk, 'inflight')),
+            sweep: (now) => sweepSlots(join(sk, 'inflight'), now),
+        },
+        {
+            id: 'run logs',
+            where: logs,
+            cap: { maxAgeMs: RUN_DIR_KEEP_MS, maxCount: RUN_DIR_MAX },
+            measure: () => {
+                const all = runEntries(logs)
+                return counted(all.length, all.length === 0 ? null : Math.min(...all.map((e) => e.started)))
+            },
+            sweep: (now) => sweepRunDirs(logs, now),
+        },
+        {
+            id: 'jobs',
+            where: jobs,
+            cap: { maxAgeMs: JOBS_KEEP_MS, maxCount: JOBS_MAX },
+            measure: () => {
+                let names: string[] = []
+                try { names = readdirSync(jobs) } catch { return counted(0, null) }
+                let oldest: number | null = null
+                for (const n of names) try { const m = statSync(join(jobs, n)).mtimeMs; oldest = oldest == null ? m : Math.min(oldest, m) } catch { /* raced */ }
+                return counted(names.length, oldest)
+            },
+            sweep: (now) => sweepJobs(root, now),
+        },
+        {
+            // the dev servers the hub runs for the repo (serve.processes)
+            id: 'process logs',
+            where: `${processDir(root)}/*.log`,
+            cap: { maxFileBytes: LOG_MAX_BYTES },
+            measure: () => walkUse(processDir(root), { match: (n) => n.endsWith('.log') }),
+            sweep: () => {
+                let names: string[] = []
+                try { names = readdirSync(processDir(root)) } catch { return }
+                for (const n of names) if (n.endsWith('.log')) capLog(join(processDir(root), n))
+            },
+        },
+    ]
+    if (gitDir != null) {
+        const dir = join(gitDir, 'shipkit')
+        stores.push({
+            id: 'scan indexes',
+            where: `${dir}/scan-<pid>.idx`,
+            // one per process scanning at once, and a machine runs a handful
+            cap: { maxAgeMs: SCAN_INDEX_KEEP_MS, maxCount: 16 },
+            measure: () => walkUse(dir, { match: (n) => /^scan-\d+\.idx(\.tmp)?$/.test(n), skip: () => true }),
+            sweep: (now) => sweepScanIndexes(dir, now),
+        })
+    }
+    return stores
+}
+
+export type Measured = { store: Store; use: Use | null; over: string | null }
+
+export function measureStores(stores: Store[], now = Date.now()): Measured[] {
+    return stores.map((store) => {
+        const use = store.measure()
+        return { store, use, over: use == null ? null : overCap(use, store.cap, now) }
+    })
+}
+
+/** sweep every store, one after the other, yielding between them so a
+ *  daemon keeps serving. A sweep that throws is reported, never fatal */
+export async function sweepStores(stores: Store[], now = Date.now()): Promise<{ id: string; error: string | null }[]> {
+    const out: { id: string; error: string | null }[] = []
+    for (const s of stores) {
+        try {
+            await s.sweep(now)
+            out.push({ id: s.id, error: null })
+        } catch (e) {
+            out.push({ id: s.id, error: (e as Error).message })
+        }
+        await Bun.sleep(0)
+    }
+    return out
+}

package/src/core/run-check.ts

@@ -12,7 +12,7 @@ import { join } from 'path'
 import { composeChecks, envTier, gateInventory, type GateCheck, type TestTier } from '../ledger/gate-checks.ts'
 import { Ledger } from '../ledger/ledger.ts'
 import { RunLog, type InvokedBy } from '../ledger/run-log.ts'
-import { storeRoot } from '../ledger/store.ts'
+import { answersDb } from '../ledger/store.ts'
 import { gateBinary } from '../ledger/toolchain-identity.ts'
 import { runChecks, selectChecks, type Audience, type CheckEvent, type CheckResult, type RunnableCheck } from './check-runner.ts'
 import { loadRepoConfig, type RepoConfig } from '../repo-config.ts'
@@ -55,6 +55,9 @@ export type CheckOpts = {
     onlyBlocking?: boolean
     /** which test tier the caller is asking for */
     tier?: TestTier
+    /** colour the wire lines. Default: this process's stdout is a terminal. A
+     *  hub running a terminal's gate passes the terminal's answer */
+    color?: boolean
     /** which question: the gate's set (`shipkit ci`) or the health pass's */
     audience?: Audience
     /** the caller narrowed the run to these ids (`--only`) */
@@ -187,11 +190,12 @@ export async function runCheckPass(root: string, o: CheckOpts = {}): Promise<Che
         root,
         config: c,
         logDir: c.logDir ?? join(root, CI_LOG_DIR),
-        cache: { storeDir: storeRoot(root), ns: c.name, now: Date.now },
+        cache: { db: answersDb(root), ns: c.name, now: Date.now },
         ledger,
         log,
         childEnv: { [SHARE_ENV]: sharePath, ...(o.fresh === true ? { SHIPKIT_FRESH: '1' } : {}) },
         bail: o.bail,
+        color: o.color,
         print: sink,
         onCheck: (e) => { collected.push(...e.lines); o.onCheck?.(e) },
     })

package/src/core/run-ci.ts

@@ -53,6 +53,8 @@ export type CiOpts = {
      *  own panel, and running them twice per refresh proves the same thing
      *  twice. */
     buildOnly?: boolean
+    /** colour the wire lines (run-check.ts CheckOpts.color) */
+    color?: boolean
     /** line sink — commands pass their context stream; defaults to console */
     print?: (line: string) => void
     /** per-CHECK completion tap, handed straight to the runner. The hub passes
@@ -187,6 +189,7 @@ export async function runCi(root: string, o: CiOpts = {}): Promise<number> {
         buildOnly: o.buildOnly,
         bail: o.bail,
         fresh: o.fresh,
+        color: o.color,
         tier,
     }
     try {

package/src/core/slots.ts

@@ -30,6 +30,11 @@ export const poolDir = (): string => {
     return env != null && env !== '' ? env : join(homedir(), '.cache', 'shipkit', 'slots')
 }
 
+/** the TURN the background gates take, one at a time on the whole machine:
+ *  every batch the plan runs, for every repo. A gate someone asked for
+ *  (a terminal, a button) never takes it, so it never waits behind one */
+export const autoGatePool = (): SlotPool => ({ dir: `${poolDir()}-auto-gate`, size: 1 })
+
 /** this process's pool, or null when it runs inside a check that holds one */
 export function machinePool(): SlotPool | null {
     return process.env[SLOT_HELD_ENV] === '1' ? null : { dir: poolDir(), size: parallelCap() }

package/src/core/statement-end.ts

@@ -0,0 +1,17 @@
+// where a statement of a config file ends, read as text. Pure, so the page
+// can import it too.
+
+/** a statement that starts on `i`, ending where its brackets balance and the
+ *  line is not obviously continued. Deliberately simple: anything it cannot
+ *  read lands as a hand pass rather than as a half rewritten config. */
+export function statementEnd(lines: string[], i: number): number {
+    let depth = 0
+    for (let j = i; j < lines.length; j++) {
+        for (const ch of lines[j]!) {
+            if (ch === '(' || ch === '[' || ch === '{') depth++
+            else if (ch === ')' || ch === ']' || ch === '}') depth--
+        }
+        if (depth <= 0) return j
+    }
+    return lines.length - 1
+}

package/src/home/attach.ts

@@ -0,0 +1,95 @@
+// `shipkit serve` while a home runs: the hub itself runs in the home, and
+// this process only streams its lines. So the VS Code folder-open task keeps its
+// problem matcher and its clickable url, with no change to any repo's
+// tasks.json, and closing VS Code no longer stops the hub.
+//
+// the log is what the home prints for that hub, so the frames the matcher
+// reads arrive as the hub prints them. On attach, the last complete frame is replayed first,
+// so the problems pane fills at once instead of at the next run.
+import { closeSync, existsSync, openSync, readSync, statSync } from 'fs'
+import { FRAME_BEGIN, FRAME_END_HEAD } from '../problems/problem-line.ts'
+import { daemonUrl, homeDir, homeUrl, hubBase, hubLog } from './home-paths.ts'
+
+/** the last complete problem frame in a log's text, begin to end line, or [] */
+export function lastFrame(text: string): string[] {
+    const lines = text.split('\n')
+    for (let end = lines.length - 1; end >= 0; end--) {
+        if (!lines[end]!.startsWith(FRAME_END_HEAD)) continue
+        for (let begin = end - 1; begin >= 0; begin--) {
+            if (lines[begin] === FRAME_BEGIN) return lines.slice(begin, end + 1)
+            if (lines[begin]!.startsWith(FRAME_END_HEAD)) break
+        }
+    }
+    return []
+}
+
+/** how often the log is looked at for new bytes */
+export const FOLLOW_MS = 300
+/** the home is asked again this often, so a stream says when it went away */
+export const ENSURE_MS = 60_000
+
+type EnsureAnswer = { status?: string; url?: string; blocked?: string | null; note?: string | null }
+
+async function ensure(name: string): Promise<EnsureAnswer | string> {
+    try {
+        const r = await fetch(`${daemonUrl()}/api/home/hub`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ name, act: 'ensure' }), signal: AbortSignal.timeout(3_000) })
+        if (!r.ok) return `${r.status} ${await r.text()}`
+        return (await r.json()) as EnsureAnswer
+    } catch (e) {
+        return (e as Error).message
+    }
+}
+
+/** hand the hub to the home and stream its log until killed. 'serve' when
+ *  the home will not run it (not one of its repos): the caller serves itself */
+export async function attachToHome(name: string, io: { print: (l: string) => void; printErr: (l: string) => void }): Promise<number | 'serve'> {
+    const first = await ensure(name)
+    if (typeof first === 'string' || first.blocked != null) {
+        io.print(`  the home did not take ${name}'s hub (${typeof first === 'string' ? first : first.blocked}), so this process serves it`)
+        return 'serve'
+    }
+    io.print(`${name} hub → ${homeUrl().replace('127.0.0.1', 'localhost')}${first.url ?? `${hubBase(name)}/`} (run by the home)`)
+    const log = hubLog(homeDir(), name)
+    let pos = 0
+    if (existsSync(log)) {
+        const text = await Bun.file(log).text()
+        for (const l of lastFrame(text)) io.print(l)
+        pos = Buffer.byteLength(text)
+    }
+    let partial = ''
+    const follow = (): void => {
+        let size: number
+        try {
+            size = statSync(log).size
+        } catch {
+            return
+        }
+        // the home cut the log (capLog) or restarted the hub on a new one
+        if (size < pos) pos = 0
+        if (size === pos) return
+        const fd = openSync(log, 'r')
+        try {
+            const buf = Buffer.alloc(size - pos)
+            readSync(fd, buf, 0, buf.length, pos)
+            pos = size
+            const text = partial + buf.toString('utf8')
+            const lines = text.split('\n')
+            partial = lines.pop() ?? ''
+            for (const l of lines) io.print(l)
+        } finally {
+            closeSync(fd)
+        }
+    }
+    setInterval(follow, FOLLOW_MS)
+    let warned = false
+    setInterval(() => {
+        void ensure(name).then((r) => {
+            if (typeof r === 'string') {
+                if (!warned) io.printErr(`WARN the home stopped answering (${r}), so this repo's hub is down with it until launchd brings the home back`)
+                warned = true
+            } else warned = false
+        })
+    }, ENSURE_MS)
+    await new Promise(() => {})
+    return 0
+}

package/src/home/autostart.ts

@@ -0,0 +1,108 @@
+// any shipkit command on a machine with a house starts pm first when it does
+// not answer: one localhost probe, then a launchd kickstart, and the agent is
+// installed from the house repo the first time. The command itself never waits
+// for pm: a client that needs the daemon (ci, check, serve) waits on its own.
+//
+// the one constraint a reader breaks first: a `shipkit pm uninstall` is a
+// machine saying no. It leaves an off marker, and nothing here installs pm
+// again until `shipkit pm install` lifts it.
+import { existsSync, mkdirSync, rmSync, writeFileSync } from 'fs'
+import { join } from 'path'
+import { fileURLToPath } from 'url'
+import { SLOT_HELD_ENV } from '../core/slots.ts'
+import { homeDir, pmUp, WORKER_ENV } from './home-paths.ts'
+import { installHome, LABEL } from './home-launchd.ts'
+import { homeInstalled, houseCli, houseRootOf } from './home-start.ts'
+
+type Env = Record<string, string | undefined>
+const set = (v: string | undefined): boolean => v != null && v !== ''
+
+/** the marker `pm uninstall` leaves: pm stays down until `pm install` */
+export const offMarker = (dir = homeDir()): string => join(dir, 'autostart-off')
+
+export function markAutostart(on: boolean, dir = homeDir()): void {
+    if (on) rmSync(offMarker(dir), { force: true })
+    else {
+        mkdirSync(dir, { recursive: true })
+        writeFileSync(offMarker(dir), 'shipkit pm uninstall stopped pm here. `shipkit pm install` starts it again\n')
+    }
+}
+
+/** the commands that ARE the home, or talk to launchd themselves */
+const OWN = new Set(['pm', 'home', 'daemon', 'web', 'gui'])
+
+/** why this command does not start pm, or null when it should */
+export function autostartSkip(argv: string[], env: Env, o: { platform?: string; off?: () => boolean } = {}): string | null {
+    if (OWN.has(argv[0] ?? '')) return 'a home command'
+    if (set(env.CI)) return 'CI'
+    if (set(env.SHIPKIT_NO_HOME)) return 'SHIPKIT_NO_HOME'
+    if (set(env[WORKER_ENV])) return 'inside the home'
+    if (set(env[SLOT_HELD_ENV])) return 'inside a check'
+    if ((o.platform ?? process.platform) !== 'darwin') return 'no launchd'
+    if ((o.off ?? (() => existsSync(offMarker())))()) return 'pm uninstalled'
+    return null
+}
+
+export type AutostartDeps = {
+    up: () => Promise<boolean>
+    installed: () => boolean
+    /** the repo that hosts the house wiki, or null on a machine with none */
+    house: () => string | null
+    install: (root: string) => Promise<number>
+    kickstart: () => Promise<number>
+    printErr: (l: string) => void
+}
+
+export type AutostartResult = 'up' | 'kicked' | 'installed' | 'no house' | 'failed'
+
+/** pm answers, or is on its way. Never throws: a command runs even when pm
+ *  cannot start, and says so once */
+export async function ensurePm(d: AutostartDeps): Promise<AutostartResult> {
+    if (await d.up()) return 'up'
+    try {
+        if (!d.installed()) {
+            const root = d.house()
+            if (root == null) return 'no house'
+            if ((await d.install(root)) !== 0) {
+                d.printErr(`WARN pm is down and its LaunchAgent would not install from ${root}: \`shipkit pm install\` there says why`)
+                return 'failed'
+            }
+            return 'installed'
+        }
+        const code = await d.kickstart()
+        if (code !== 0) {
+            d.printErr(`WARN pm is down and launchctl kickstart ${LABEL} failed (exit ${code}): \`shipkit pm install\` from the house repo reinstalls it`)
+            return 'failed'
+        }
+        return 'kicked'
+    } catch (e) {
+        d.printErr(`WARN pm is down and could not be started: ${(e as Error).message}`)
+        return 'failed'
+    }
+}
+
+const OWN_CLI = fileURLToPath(new URL('../cli.ts', import.meta.url))
+const LIB_DIR = fileURLToPath(new URL('../..', import.meta.url))
+
+/** the real machine's wiring: cwd first, else the house this lib lives in */
+export function machineAutostart(cwd: string): AutostartDeps {
+    const quiet = { print: (l: string) => { if (l.startsWith('FAIL')) process.stderr.write(`${l}\n`) } }
+    return {
+        up: () => pmUp(undefined, 300),
+        installed: homeInstalled,
+        house: () => houseRootOf(cwd) ?? houseRootOf(LIB_DIR),
+        install: (root) => installHome({ root, cli: houseCli(root, OWN_CLI), ...quiet }),
+        kickstart: async () => {
+            const p = Bun.spawn(['launchctl', 'kickstart', `gui/${process.getuid?.() ?? 501}/${LABEL}`], { stdout: 'ignore', stderr: 'ignore' })
+            return p.exited
+        },
+        printErr: (l) => process.stderr.write(`${l}\n`),
+    }
+}
+
+/** the cli's first step: start pm when this command should */
+export async function autostart(argv: string[], env: Env = process.env, cwd = process.cwd()): Promise<AutostartResult | string> {
+    const skip = autostartSkip(argv, env)
+    if (skip != null) return skip
+    return ensurePm(machineAutostart(cwd))
+}

package/src/home/global-units.ts

@@ -0,0 +1,86 @@
+// the `.shipkit.ts` units of every house repo that belong to the machine, not
+// to one repo: its global processes (pm's) and its widgets (the daemon's).
+// Read by importing each file, so the reader must have registered the one
+// copy of the lib first (core/one-lib.ts): a unit file imports it by name.
+import { relative, resolve } from 'path'
+import { importFresh, repoUnitFiles } from '../core/discover-checks.ts'
+import { unitKind, type GlobalProcessUnit, type WidgetUnit } from '../shipkit-files.ts'
+
+/** one button of a global process, as pm runs it */
+export type ActionDecl = { id: string; label: string; cmd: string[] | null }
+
+/** a global process as pm holds it: plain data, paths absolute */
+export type GlobalProcessDecl = {
+    id: string
+    label: string
+    description: string | null
+    where: string | null
+    cmd: string[]
+    down: string[] | null
+    /** absolute: where cmd, down and every action run */
+    cwd: string
+    url: string | null
+    probe: string | null
+    app: boolean
+    actions: ActionDecl[]
+    /** the repo that declares it, and the file, repo-relative */
+    repo: string
+    root: string
+    file: string
+}
+
+export type RepoRef = { name: string; path: string }
+
+export type LoadedWidget = { repo: string; file: string; def: WidgetUnit }
+
+export type GlobalUnits = {
+    processes: GlobalProcessDecl[]
+    widgets: LoadedWidget[]
+    /** a file that would not load, or a process id declared twice */
+    errors: string[]
+}
+
+export function toDecl(u: GlobalProcessUnit, repo: RepoRef, file: string): GlobalProcessDecl {
+    return {
+        id: u.id,
+        label: u.label ?? u.id,
+        description: u.description ?? null,
+        where: u.where ?? null,
+        cmd: u.cmd,
+        down: u.down ?? null,
+        cwd: resolve(repo.path, u.cwd ?? '.'),
+        url: u.url ?? null,
+        probe: u.probe ?? u.url ?? null,
+        app: u.app === true,
+        actions: Object.entries(u.actions ?? {}).map(([id, a]) => ({ id, label: a.label, cmd: a.cmd ?? null })),
+        repo: repo.name,
+        root: repo.path,
+        file: relative(repo.path, file).replace(/\\/g, '/'),
+    }
+}
+
+/** every global process and widget the repos declare. A process id is the
+ *  machine's: a second declaration of one is an error, the first one wins */
+export async function scanGlobalUnits(repos: RepoRef[], o: { files?: (root: string) => string[] } = {}): Promise<GlobalUnits> {
+    const out: GlobalUnits = { processes: [], widgets: [], errors: [] }
+    for (const repo of repos) {
+        for (const f of (o.files ?? repoUnitFiles)(repo.path)) {
+            const rel = relative(repo.path, f).replace(/\\/g, '/')
+            let d: unknown
+            try {
+                d = await importFresh(f)
+            } catch (e) {
+                out.errors.push(`${repo.name}/${rel}: ${(e as Error).message}`)
+                continue
+            }
+            const kind = unitKind(d)
+            if (kind === 'globalProcess') {
+                const u = d as GlobalProcessUnit
+                const first = out.processes.find((p) => p.id === u.id)
+                if (first != null) out.errors.push(`${repo.name}/${rel}: global process '${u.id}' is already declared by ${first.repo}/${first.file}`)
+                else out.processes.push(toDecl(u, repo, f))
+            } else if (kind === 'widget') out.widgets.push({ repo: repo.name, file: rel, def: d as WidgetUnit })
+        }
+    }
+    return out
+}