rvlib-shipkit 1.1.1 → 1.3.0

package/src/home/activity.ts

@@ -0,0 +1,249 @@
+// the activity page's data: for every house repo, its branch, how far it is
+// from its upstream, what it holds uncommitted and what it committed in a
+// window. Plain local git, never a fetch, so "ahead" is against the upstream
+// as this disk last saw it.
+//
+// the one constraint a reader breaks first: this runs only when the page asks
+// (no tick). A pass reads ACTIVITY_CONCURRENCY repos at once, and an answer is
+// shared for ACTIVITY_TTL_MS per window, never served past it, so a page left
+// open costs about one pass a minute however many tabs show it, and a closed
+// one costs nothing.
+import { gitBin } from '../core/git-bin.ts'
+import { readLastStatusReadonly } from '../ledger/run-log.ts'
+import { checkStateOf, recordOf } from '../serve/project-gutter.ts'
+import type { Line } from '../core/wire.ts'
+import { MAX_COMMITS, MAX_FILES, type ActivityAnswer, type ActivityCommit, type ActivityRepo, type ActivityRow, type FileKind, type StatusCheck } from './activity-shape.ts'
+
+export * from './activity-shape.ts'
+
+/** how long one window's answer is shared between readers: half the page's
+ *  60s cadence, so tabs a few seconds apart share a pass and none is shown
+ *  an answer older than this */
+export const ACTIVITY_TTL_MS = 30_000
+
+export type ParsedStatus = Pick<ActivityRow, 'branch' | 'detached' | 'upstream' | 'gone' | 'ahead' | 'behind' | 'counts' | 'files'>
+
+/** `git status --porcelain=v2 --branch`, read */
+export function parseStatus(text: string): ParsedStatus {
+    const out: ParsedStatus = { branch: null, detached: null, upstream: null, gone: false, ahead: null, behind: null, counts: { staged: 0, modified: 0, untracked: 0 }, files: [] }
+    let oid = ''
+    const add = (kind: FileKind, path: string): void => {
+        if (out.files.length < MAX_FILES) out.files.push({ kind, path })
+    }
+    for (const line of text.split('\n')) {
+        if (line.startsWith('# branch.oid ')) oid = line.slice(13)
+        else if (line.startsWith('# branch.head ')) {
+            const h = line.slice(14)
+            if (h !== '(detached)') out.branch = h
+        } else if (line.startsWith('# branch.upstream ')) out.upstream = line.slice(18)
+        else if (line.startsWith('# branch.ab ')) {
+            const m = /^\+(\d+) -(\d+)$/.exec(line.slice(12))
+            if (m != null) {
+                out.ahead = Number(m[1])
+                out.behind = Number(m[2])
+            }
+        } else if (line.startsWith('1 ') || line.startsWith('2 ')) {
+            // `1 XY sub mH mI mW hH hI path`, `2 … Xscore path<TAB>orig`
+            const f = line.split(' ')
+            const xy = f[1] ?? '..'
+            const path = f.slice(line.startsWith('1 ') ? 8 : 9).join(' ').split('\t')[0]!
+            if (xy[0] !== '.') out.counts.staged++
+            if (xy[1] !== '.') out.counts.modified++
+            add(xy[1] !== '.' ? 'modified' : 'staged', path)
+        } else if (line.startsWith('u ')) {
+            out.counts.modified++
+            add('modified', line.split(' ').slice(10).join(' '))
+        } else if (line.startsWith('? ')) {
+            out.counts.untracked++
+            add('untracked', line.slice(2))
+        }
+    }
+    if (out.branch == null && oid !== '' && oid !== '(initial)') out.detached = oid.slice(0, 7)
+    if (out.upstream != null && out.ahead == null) out.gone = true
+    return out
+}
+
+const SEP = '\x1f'
+const END = '\x1e'
+const FORMAT = `--format=%h${SEP}%ct${SEP}%an${SEP}%S${SEP}%s${SEP}%H${END}`
+
+/** `git log` in FORMAT, read; `unpushed` holds the full shas no remote has */
+export function parseLog(text: string, onHead: boolean, unpushed: (full: string) => boolean = () => false): ActivityCommit[] {
+    const out: ActivityCommit[] = []
+    for (const rec of text.split(END)) {
+        const f = rec.replace(/^\n/, '').split(SEP)
+        if (f.length < 6) continue
+        const src = f[3]!.replace(/^refs\/heads\//, '')
+        out.push({ sha: f[0]!, at: Number(f[1]) * 1000, author: f[2]!, branch: onHead || src === '' ? null : src, title: f[4]!, pushed: !unpushed(f[5]!) })
+    }
+    return out
+}
+
+async function git(cwd: string, args: string[]): Promise<{ code: number; out: string; err: string }> {
+    // no optional locks: a look never takes the index lock an agent's commit needs
+    const p = Bun.spawn([gitBin(), ...args], { cwd, stdout: 'pipe', stderr: 'pipe', env: { ...process.env, GIT_OPTIONAL_LOCKS: '0' } })
+    const [out, err, code] = await Promise.all([new Response(p.stdout).text(), new Response(p.stderr).text(), p.exited])
+    return { code, out, err: err.trim() }
+}
+
+/** which check ids a repo declares today: the daemon's plan; null = not
+ *  known, every recorded check counts */
+export type Declared = (name: string) => ReadonlySet<string> | null
+
+/** each declared check's title by id; null = not known, the id stands in */
+export type Titles = (name: string) => ReadonlyMap<string, string> | null
+
+/** the line a check printed about its problem: its first line of the same
+ *  weight (a fail for a failure, a warning for a warning), else its first
+ *  line that is not a pass, without the id its line leads with */
+export function findingOf(lines: string, status: 'fail' | 'warn', id: string): string {
+    let parsed: Line[]
+    try {
+        parsed = JSON.parse(lines) as Line[]
+    } catch {
+        return ''
+    }
+    if (!Array.isArray(parsed)) return ''
+    const kinds = status === 'fail' ? ['fail', 'fix-fail'] : ['warn', 'fix-warn']
+    const hit = parsed.find((l) => kinds.includes(l.kind)) ?? parsed.find((l) => l.kind !== 'ok' && l.kind !== 'skip')
+    if (hit == null) return ''
+    const t = hit.text.trim()
+    // `[ID] text`, and a runner line's `ID — text`: the list names the id already
+    for (const lead of [`[${id}]`, `${id} —`]) if (t.startsWith(lead)) return t.slice(lead.length).trim()
+    return t
+}
+
+/** a repo's checks as its run record last saw them, counted over what it
+ *  declares, and the failing and warning ones with what each said */
+export function statusOf(repo: ActivityRepo, declared: Declared | null = null, titles: Titles | null = null): Pick<ActivityRow, 'status' | 'failing'> {
+    const record = recordOf(repo.path)
+    if (record == null) return { status: null, failing: [] }
+    const rows = readLastStatusReadonly(record, { lines: true })
+    const status = checkStateOf(rows, declared?.(repo.name) ?? null)
+    if (status == null) return { status, failing: [] }
+    const byId = new Map(rows.map((r) => [r.id, r]))
+    const named = titles?.(repo.name) ?? null
+    const one = (id: string, st: 'fail' | 'warn'): StatusCheck => {
+        const r = byId.get(id)
+        return { id, title: named?.get(id) ?? id, status: st, at: status.since[id] ?? null, line: r == null ? '' : findingOf(r.lines, st, id) }
+    }
+    return { status, failing: [...status.reds.map((id) => one(id, 'fail')), ...status.warns.map((id) => one(id, 'warn'))] }
+}
+
+/** one repo, from `since` (epoch ms) to now */
+export async function repoActivity(repo: ActivityRepo, since: number, declared: Declared | null = null, titles: Titles | null = null): Promise<ActivityRow> {
+    const empty = parseStatus('')
+    const row: ActivityRow = { name: repo.name, ...empty, commits: [], last: null, ...statusOf(repo, declared, titles), error: null }
+    try {
+        const iso = new Date(since).toISOString()
+        const range = [`--since=${iso}`, `--max-count=${MAX_COMMITS}`, '--no-show-signature', FORMAT]
+        // every ask that needs no other answer goes at once: git calls are
+        // ~5ms each but queue behind a busy daemon, so waiting for them one
+        // by one multiplies the wait. The upstream diff and the last commit
+        // are cheap enough to ask even when the answer goes unused
+        const [st, up, head, refs, one] = await Promise.all([
+            git(repo.path, ['status', '--porcelain=v2', '--branch']),
+            git(repo.path, ['rev-list', `--max-count=${MAX_COMMITS}`, '@{upstream}..HEAD']),
+            // a repo with no commit yet has no HEAD to walk: nothing committed
+            git(repo.path, ['log', ...range, 'HEAD']),
+            // the other local branches (an agent's worktree) whose tip moved in
+            // the window, minus what HEAD has: a repo with hundreds of branches
+            // walks only the few that moved
+            git(repo.path, ['for-each-ref', '--format=%(refname:short)%09%(committerdate:unix)%09%(upstream:short)%09%(upstream:track)', 'refs/heads']),
+            git(repo.path, ['log', '-1', '--no-show-signature', FORMAT, 'HEAD']),
+        ])
+        if (st.code !== 0) return { ...row, error: st.err || `git status exited ${st.code}` }
+        Object.assign(row, parseStatus(st.out))
+        // HEAD's unpushed commits: all of them without an upstream, else the
+        // `ahead` ones
+        const headAway = row.upstream == null || row.gone
+        const away = !headAway && (row.ahead ?? 0) > 0 && up.code === 0 ? new Set(up.out.split('\n').filter((l) => l !== '')) : new Set<string>()
+        const own = head.code === 0 ? parseLog(head.out, true, (h) => headAway || away.has(h)) : []
+        const moved = (refs.code === 0 ? refs.out.split('\n') : [])
+            .map((l) => l.split('\t'))
+            .filter((f) => f.length >= 2 && f[0] !== row.branch && Number(f[1]) * 1000 >= since)
+            // an upstream that is gone holds nothing
+            .map((f) => (f[3] === '[gone]' ? [f[0]!, f[1]!, ''] : f))
+        let other: ActivityCommit[] = []
+        if (moved.length > 0) {
+            const side = await git(repo.path, ['log', ...range, '--source', ...moved.map((f) => `refs/heads/${f[0]}`), '--not', 'HEAD', '--'])
+            // a side branch's commit is pushed once its own upstream holds it;
+            // on a failed ask every one reads unpushed, never pushed
+            const tracked = moved.filter((f) => (f[2] ?? '') !== '')
+            let sideAway = new Set<string>()
+            if (tracked.length > 0) {
+                const r = await git(repo.path, ['rev-list', `--since=${iso}`, ...tracked.map((f) => `refs/heads/${f[0]}`), '--not', ...tracked.map((f) => f[2]!), '--'])
+                if (r.code === 0) sideAway = new Set(r.out.split('\n').filter((l) => l !== ''))
+                else sideAway = new Set(['*'])
+            }
+            const untracked = new Set(moved.filter((f) => (f[2] ?? '') === '').map((f) => f[0]!))
+            other = side.code === 0 ? parseLog(side.out, false, (h) => sideAway.has('*') || sideAway.has(h)).map((c) => (c.branch != null && untracked.has(c.branch) ? { ...c, pushed: false } : c)) : []
+        }
+        row.commits = [...own, ...other].sort((a, b) => b.at - a.at).slice(0, MAX_COMMITS)
+        // an empty window still says what came last on HEAD
+        if (row.commits.length === 0) {
+            row.last = one.code === 0 ? parseLog(one.out, true, (h) => headAway || away.has(h))[0] ?? null : null
+        }
+        return row
+    } catch (e) {
+        return { ...row, error: (e as Error).message }
+    }
+}
+
+/** how many repos a pass reads at once: five git calls each, so a pass
+ *  never has more than 30 git processes alive */
+export const ACTIVITY_CONCURRENCY = 6
+
+/** every repo, ACTIVITY_CONCURRENCY at a time, rows in the order given.
+ *  THE computation: the daemon's route and `shipkit cross-repo-activity`
+ *  both call it, and only render it differently */
+export async function activityOf(repos: ActivityRepo[], minutes: number, now = Date.now(), declared: Declared | null = null, titles: Titles | null = null): Promise<ActivityAnswer> {
+    const t0 = performance.now()
+    const rows: ActivityRow[] = new Array(repos.length)
+    let next = 0
+    const worker = async (): Promise<void> => {
+        while (next < repos.length) {
+            const i = next++
+            rows[i] = await repoActivity(repos[i]!, now - minutes * 60_000, declared, titles)
+        }
+    }
+    await Promise.all(Array.from({ length: Math.min(ACTIVITY_CONCURRENCY, repos.length) }, worker))
+    return { now, minutes, rows, ms: Math.round(performance.now() - t0) }
+}
+
+/** one answer per window, shared for ttlMs. Past it a read waits for a new
+ *  pass: an old answer is never served as current. Callers during a pass
+ *  share it. */
+export class ActivityCache {
+    private kept = new Map<number, ActivityAnswer>()
+    private running = new Map<number, Promise<ActivityAnswer>>()
+    private ttlMs: number
+    private now: () => number
+    private declared: Declared | null
+    private titles: Titles | null
+
+    constructor(o: { ttlMs?: number; now?: () => number; declared?: Declared; titles?: Titles } = {}) {
+        this.ttlMs = o.ttlMs ?? ACTIVITY_TTL_MS
+        this.now = o.now ?? Date.now
+        this.declared = o.declared ?? null
+        this.titles = o.titles ?? null
+    }
+
+    private pass(repos: ActivityRepo[], minutes: number): Promise<ActivityAnswer> {
+        const going = this.running.get(minutes)
+        if (going != null) return going
+        const p = activityOf(repos, minutes, this.now(), this.declared, this.titles)
+            .then((a) => { this.kept.set(minutes, a); return a })
+            .finally(() => this.running.delete(minutes))
+        this.running.set(minutes, p)
+        return p
+    }
+
+    /** the kept answer while it is younger than the ttl, else a new pass.
+     *  `fresh` always waits for a pass: the page's refresh button */
+    get(repos: ActivityRepo[], minutes: number, o: { fresh?: boolean } = {}): Promise<ActivityAnswer> {
+        const k = this.kept.get(minutes)
+        if (k == null || o.fresh === true || this.now() - k.now >= this.ttlMs) return this.pass(repos, minutes)
+        return Promise.resolve(k)
+    }
+}

package/src/home/attach.ts

@@ -0,0 +1,95 @@
+// `shipkit serve` while a home runs: the hub itself runs in the home, and
+// this process only streams its lines. So the VS Code folder-open task keeps its
+// problem matcher and its clickable url, with no change to any repo's
+// tasks.json, and closing VS Code no longer stops the hub.
+//
+// the log is what the home prints for that hub, so the frames the matcher
+// reads arrive as the hub prints them. On attach, the last complete frame is replayed first,
+// so the problems pane fills at once instead of at the next run.
+import { closeSync, existsSync, openSync, readSync, statSync } from 'fs'
+import { FRAME_BEGIN, FRAME_END_HEAD } from '../problems/problem-line.ts'
+import { daemonUrl, homeDir, homeUrl, hubBase, hubLog } from './home-paths.ts'
+
+/** the last complete problem frame in a log's text, begin to end line, or [] */
+export function lastFrame(text: string): string[] {
+    const lines = text.split('\n')
+    for (let end = lines.length - 1; end >= 0; end--) {
+        if (!lines[end]!.startsWith(FRAME_END_HEAD)) continue
+        for (let begin = end - 1; begin >= 0; begin--) {
+            if (lines[begin] === FRAME_BEGIN) return lines.slice(begin, end + 1)
+            if (lines[begin]!.startsWith(FRAME_END_HEAD)) break
+        }
+    }
+    return []
+}
+
+/** how often the log is looked at for new bytes */
+export const FOLLOW_MS = 300
+/** the home is asked again this often, so a stream says when it went away */
+export const ENSURE_MS = 60_000
+
+type EnsureAnswer = { status?: string; url?: string; blocked?: string | null; note?: string | null }
+
+async function ensure(name: string): Promise<EnsureAnswer | string> {
+    try {
+        const r = await fetch(`${daemonUrl()}/api/home/hub`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ name, act: 'ensure' }), signal: AbortSignal.timeout(3_000) })
+        if (!r.ok) return `${r.status} ${await r.text()}`
+        return (await r.json()) as EnsureAnswer
+    } catch (e) {
+        return (e as Error).message
+    }
+}
+
+/** hand the hub to the home and stream its log until killed. 'serve' when
+ *  the home will not run it (not one of its repos): the caller serves itself */
+export async function attachToHome(name: string, io: { print: (l: string) => void; printErr: (l: string) => void }): Promise<number | 'serve'> {
+    const first = await ensure(name)
+    if (typeof first === 'string' || first.blocked != null) {
+        io.print(`  the home did not take ${name}'s hub (${typeof first === 'string' ? first : first.blocked}), so this process serves it`)
+        return 'serve'
+    }
+    io.print(`${name} hub → ${homeUrl().replace('127.0.0.1', 'localhost')}${first.url ?? `${hubBase(name)}/`} (run by the home)`)
+    const log = hubLog(homeDir(), name)
+    let pos = 0
+    if (existsSync(log)) {
+        const text = await Bun.file(log).text()
+        for (const l of lastFrame(text)) io.print(l)
+        pos = Buffer.byteLength(text)
+    }
+    let partial = ''
+    const follow = (): void => {
+        let size: number
+        try {
+            size = statSync(log).size
+        } catch {
+            return
+        }
+        // the home cut the log (capLog) or restarted the hub on a new one
+        if (size < pos) pos = 0
+        if (size === pos) return
+        const fd = openSync(log, 'r')
+        try {
+            const buf = Buffer.alloc(size - pos)
+            readSync(fd, buf, 0, buf.length, pos)
+            pos = size
+            const text = partial + buf.toString('utf8')
+            const lines = text.split('\n')
+            partial = lines.pop() ?? ''
+            for (const l of lines) io.print(l)
+        } finally {
+            closeSync(fd)
+        }
+    }
+    setInterval(follow, FOLLOW_MS)
+    let warned = false
+    setInterval(() => {
+        void ensure(name).then((r) => {
+            if (typeof r === 'string') {
+                if (!warned) io.printErr(`WARN the home stopped answering (${r}), so this repo's hub is down with it until launchd brings the home back`)
+                warned = true
+            } else warned = false
+        })
+    }, ENSURE_MS)
+    await new Promise(() => {})
+    return 0
+}

package/src/home/autostart.ts

@@ -0,0 +1,108 @@
+// any shipkit command on a machine with a house starts pm first when it does
+// not answer: one localhost probe, then a launchd kickstart, and the agent is
+// installed from the house repo the first time. The command itself never waits
+// for pm: a client that needs the daemon (ci, check, serve) waits on its own.
+//
+// the one constraint a reader breaks first: a `shipkit pm uninstall` is a
+// machine saying no. It leaves an off marker, and nothing here installs pm
+// again until `shipkit pm install` lifts it.
+import { existsSync, mkdirSync, rmSync, writeFileSync } from 'fs'
+import { join } from 'path'
+import { fileURLToPath } from 'url'
+import { SLOT_HELD_ENV } from '../core/slots.ts'
+import { homeDir, pmUp, WORKER_ENV } from './home-paths.ts'
+import { installHome, LABEL } from './home-launchd.ts'
+import { homeInstalled, houseCli, houseRootOf } from './home-start.ts'
+
+type Env = Record<string, string | undefined>
+const set = (v: string | undefined): boolean => v != null && v !== ''
+
+/** the marker `pm uninstall` leaves: pm stays down until `pm install` */
+export const offMarker = (dir = homeDir()): string => join(dir, 'autostart-off')
+
+export function markAutostart(on: boolean, dir = homeDir()): void {
+    if (on) rmSync(offMarker(dir), { force: true })
+    else {
+        mkdirSync(dir, { recursive: true })
+        writeFileSync(offMarker(dir), 'shipkit pm uninstall stopped pm here. `shipkit pm install` starts it again\n')
+    }
+}
+
+/** the commands that ARE the home, or talk to launchd themselves */
+const OWN = new Set(['pm', 'home', 'daemon', 'web', 'gui'])
+
+/** why this command does not start pm, or null when it should */
+export function autostartSkip(argv: string[], env: Env, o: { platform?: string; off?: () => boolean } = {}): string | null {
+    if (OWN.has(argv[0] ?? '')) return 'a home command'
+    if (set(env.CI)) return 'CI'
+    if (set(env.SHIPKIT_NO_HOME)) return 'SHIPKIT_NO_HOME'
+    if (set(env[WORKER_ENV])) return 'inside the home'
+    if (set(env[SLOT_HELD_ENV])) return 'inside a check'
+    if ((o.platform ?? process.platform) !== 'darwin') return 'no launchd'
+    if ((o.off ?? (() => existsSync(offMarker())))()) return 'pm uninstalled'
+    return null
+}
+
+export type AutostartDeps = {
+    up: () => Promise<boolean>
+    installed: () => boolean
+    /** the repo that hosts the house wiki, or null on a machine with none */
+    house: () => string | null
+    install: (root: string) => Promise<number>
+    kickstart: () => Promise<number>
+    printErr: (l: string) => void
+}
+
+export type AutostartResult = 'up' | 'kicked' | 'installed' | 'no house' | 'failed'
+
+/** pm answers, or is on its way. Never throws: a command runs even when pm
+ *  cannot start, and says so once */
+export async function ensurePm(d: AutostartDeps): Promise<AutostartResult> {
+    if (await d.up()) return 'up'
+    try {
+        if (!d.installed()) {
+            const root = d.house()
+            if (root == null) return 'no house'
+            if ((await d.install(root)) !== 0) {
+                d.printErr(`WARN pm is down and its LaunchAgent would not install from ${root}: \`shipkit pm install\` there says why`)
+                return 'failed'
+            }
+            return 'installed'
+        }
+        const code = await d.kickstart()
+        if (code !== 0) {
+            d.printErr(`WARN pm is down and launchctl kickstart ${LABEL} failed (exit ${code}): \`shipkit pm install\` from the house repo reinstalls it`)
+            return 'failed'
+        }
+        return 'kicked'
+    } catch (e) {
+        d.printErr(`WARN pm is down and could not be started: ${(e as Error).message}`)
+        return 'failed'
+    }
+}
+
+const OWN_CLI = fileURLToPath(new URL('../cli.ts', import.meta.url))
+const LIB_DIR = fileURLToPath(new URL('../..', import.meta.url))
+
+/** the real machine's wiring: cwd first, else the house this lib lives in */
+export function machineAutostart(cwd: string): AutostartDeps {
+    const quiet = { print: (l: string) => { if (l.startsWith('FAIL')) process.stderr.write(`${l}\n`) } }
+    return {
+        up: () => pmUp(undefined, 300),
+        installed: homeInstalled,
+        house: () => houseRootOf(cwd) ?? houseRootOf(LIB_DIR),
+        install: (root) => installHome({ root, cli: houseCli(root, OWN_CLI), ...quiet }),
+        kickstart: async () => {
+            const p = Bun.spawn(['launchctl', 'kickstart', `gui/${process.getuid?.() ?? 501}/${LABEL}`], { stdout: 'ignore', stderr: 'ignore' })
+            return p.exited
+        },
+        printErr: (l) => process.stderr.write(`${l}\n`),
+    }
+}
+
+/** the cli's first step: start pm when this command should */
+export async function autostart(argv: string[], env: Env = process.env, cwd = process.cwd()): Promise<AutostartResult | string> {
+    const skip = autostartSkip(argv, env)
+    if (skip != null) return skip
+    return ensurePm(machineAutostart(cwd))
+}

package/src/home/global-units.ts

@@ -0,0 +1,87 @@
+// the `.shipkit.ts` units of every house repo that belong to the machine, not
+// to one repo: its global processes (pm's) and its widgets (the daemon's).
+// Read by importing each file, so the reader must have registered the one
+// copy of the lib first (core/one-lib.ts): a unit file imports it by name.
+import { relative, resolve } from 'path'
+import { importFresh, repoUnitFiles } from '../core/discover-checks.ts'
+import { unitKind, type GlobalProcessUnit, type WidgetUnit } from '../shipkit-files.ts'
+
+/** one button of a global process, as pm runs it */
+export type ActionDecl = { id: string; label: string; cmd: string[] | null }
+
+/** a global process as pm holds it: plain data, paths absolute */
+export type GlobalProcessDecl = {
+    id: string
+    label: string
+    description: string | null
+    where: string | null
+    cmd: string[]
+    down: string[] | null
+    /** absolute: where cmd, down and every action run */
+    cwd: string
+    url: string | null
+    probe: string | null
+    /** as declared: an emoji, or an image path relative to root */
+    icon: string | null
+    actions: ActionDecl[]
+    /** the repo that declares it, and the file, repo-relative */
+    repo: string
+    root: string
+    file: string
+}
+
+export type RepoRef = { name: string; path: string }
+
+export type LoadedWidget = { repo: string; file: string; def: WidgetUnit }
+
+export type GlobalUnits = {
+    processes: GlobalProcessDecl[]
+    widgets: LoadedWidget[]
+    /** a file that would not load, or a process id declared twice */
+    errors: string[]
+}
+
+export function toDecl(u: GlobalProcessUnit, repo: RepoRef, file: string): GlobalProcessDecl {
+    return {
+        id: u.id,
+        label: u.label ?? u.id,
+        description: u.description ?? null,
+        where: u.where ?? null,
+        cmd: u.cmd,
+        down: u.down ?? null,
+        cwd: resolve(repo.path, u.cwd ?? '.'),
+        url: u.url ?? null,
+        probe: u.probe ?? u.url ?? null,
+        icon: u.icon ?? null,
+        actions: Object.entries(u.actions ?? {}).map(([id, a]) => ({ id, label: a.label, cmd: a.cmd ?? null })),
+        repo: repo.name,
+        root: repo.path,
+        file: relative(repo.path, file).replace(/\\/g, '/'),
+    }
+}
+
+/** every global process and widget the repos declare. A process id is the
+ *  machine's: a second declaration of one is an error, the first one wins */
+export async function scanGlobalUnits(repos: RepoRef[], o: { files?: (root: string) => string[] } = {}): Promise<GlobalUnits> {
+    const out: GlobalUnits = { processes: [], widgets: [], errors: [] }
+    for (const repo of repos) {
+        for (const f of (o.files ?? repoUnitFiles)(repo.path)) {
+            const rel = relative(repo.path, f).replace(/\\/g, '/')
+            let d: unknown
+            try {
+                d = await importFresh(f)
+            } catch (e) {
+                out.errors.push(`${repo.name}/${rel}: ${(e as Error).message}`)
+                continue
+            }
+            const kind = unitKind(d)
+            if (kind === 'globalProcess') {
+                const u = d as GlobalProcessUnit
+                const first = out.processes.find((p) => p.id === u.id)
+                if (first != null) out.errors.push(`${repo.name}/${rel}: global process '${u.id}' is already declared by ${first.repo}/${first.file}`)
+                else out.processes.push(toDecl(u, repo, f))
+            } else if (kind === 'widget') out.widgets.push({ repo: repo.name, file: rel, def: d as WidgetUnit })
+        }
+    }
+    return out
+}