rvlib-shipkit 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +67 -0
- package/package.json +3 -1
- package/src/check.ts +24 -0
- package/src/checks/SK022-machine-runs-the-pin.check.ts +43 -0
- package/src/checks/SK023-stores-under-cap.check.ts +49 -0
- package/src/checks/SK024-identity-colors.check.ts +49 -0
- package/src/cli/cmd_Check.ts +26 -1
- package/src/cli/cmd_Ci.ts +18 -5
- package/src/cli/cmd_Home.ts +170 -0
- package/src/cli.ts +22 -2
- package/src/comfy/comfy-panel.ts +1 -16
- package/src/core/attach.ts +14 -9
- package/src/core/check-file-names.ts +8 -0
- package/src/core/check-runner.ts +40 -18
- package/src/core/discover-checks.ts +36 -18
- package/src/core/identity-color.ts +203 -0
- package/src/core/identity.ts +91 -0
- package/src/core/job-follow.ts +102 -0
- package/src/core/job-run.ts +27 -0
- package/src/core/jobs.ts +87 -27
- package/src/core/legacy-shapes.ts +1 -14
- package/src/core/machine-db.ts +83 -0
- package/src/core/one-lib.ts +85 -0
- package/src/core/port-holders.ts +43 -0
- package/src/core/retention.ts +213 -0
- package/src/core/run-check.ts +6 -2
- package/src/core/run-ci.ts +3 -0
- package/src/core/slots.ts +5 -0
- package/src/core/statement-end.ts +17 -0
- package/src/home/attach.ts +95 -0
- package/src/home/autostart.ts +108 -0
- package/src/home/global-units.ts +86 -0
- package/src/home/gui-app/ShipkitGui.swift +159 -0
- package/src/home/gui-app.ts +99 -0
- package/src/home/handoff.ts +51 -0
- package/src/home/home-launchd.ts +104 -0
- package/src/home/home-paths.ts +88 -0
- package/src/home/home-routes.ts +208 -0
- package/src/home/home-start.ts +105 -0
- package/src/home/home-state.ts +43 -0
- package/src/home/home-words.ts +12 -0
- package/src/home/hubs.ts +286 -0
- package/src/home/lib-watch.ts +139 -0
- package/src/home/machine-stores.ts +115 -0
- package/src/home/open-app.ts +16 -0
- package/src/home/pm-config.ts +37 -0
- package/src/home/pm.ts +141 -0
- package/src/home/resources.ts +338 -0
- package/src/home/run-daemon.ts +137 -0
- package/src/home/run-pm.ts +149 -0
- package/src/home/run-unit-action.ts +25 -0
- package/src/home/run-web.ts +37 -0
- package/src/home/serve-daemon.ts +31 -0
- package/src/home/serve-web.ts +172 -0
- package/src/home/supervise.ts +346 -0
- package/src/home/via-hub.ts +142 -0
- package/src/home/widgets.ts +102 -0
- package/src/ledger/check-stats.ts +19 -45
- package/src/ledger/db.ts +69 -17
- package/src/ledger/gate-checks.ts +5 -0
- package/src/ledger/git-scan.ts +31 -7
- package/src/ledger/ledger.ts +40 -14
- package/src/ledger/run-log.ts +58 -23
- package/src/ledger/store.ts +228 -103
- package/src/ledger/world-cache.ts +15 -13
- package/src/plan/cadence.ts +237 -0
- package/src/plan/planner.ts +303 -0
- package/src/plan/repo-source.ts +100 -0
- package/src/problems/problem-line.ts +1 -1
- package/src/repo-config.ts +51 -2
- package/src/serve/hub-app/check-view.tsx +9 -9
- package/src/serve/hub-app/check-words.ts +1 -1
- package/src/serve/hub-app/checks-svc.ts +12 -8
- package/src/serve/hub-app/col-card.tsx +4 -4
- package/src/serve/hub-app/cols-svc.ts +92 -13
- package/src/serve/hub-app/comfy-svc.ts +12 -6
- package/src/serve/hub-app/comfy-view.tsx +3 -3
- package/src/serve/hub-app/floating.tsx +49 -0
- package/src/serve/hub-app/geometry-view.tsx +4 -4
- package/src/serve/hub-app/hash-route.ts +1 -1
- package/src/serve/hub-app/header-bar.tsx +44 -12
- package/src/serve/hub-app/health-svc.ts +3 -3
- package/src/serve/hub-app/help.tsx +10 -7
- package/src/serve/hub-app/home-st.ts +241 -0
- package/src/serve/hub-app/home-strip.css +70 -0
- package/src/serve/hub-app/home-strip.tsx +245 -0
- package/src/serve/hub-app/home.html +17 -0
- package/src/serve/hub-app/hub-app.tsx +201 -93
- package/src/serve/hub-app/hub-types.ts +0 -3
- package/src/serve/hub-app/hub.css +43 -23
- package/src/serve/hub-app/icons.tsx +14 -0
- package/src/serve/hub-app/index.html +3 -1
- package/src/serve/hub-app/keys.ts +25 -0
- package/src/serve/hub-app/layout-st.ts +55 -0
- package/src/serve/hub-app/live-svc.ts +12 -9
- package/src/serve/hub-app/machine-st.ts +270 -0
- package/src/serve/hub-app/media-svc.ts +63 -44
- package/src/serve/hub-app/media-view.tsx +20 -20
- package/src/serve/hub-app/next-run.ts +24 -30
- package/src/serve/hub-app/palette-st.ts +88 -0
- package/src/serve/hub-app/planned-st.ts +171 -0
- package/src/serve/hub-app/planned-view.tsx +109 -0
- package/src/serve/hub-app/planned.css +39 -0
- package/src/serve/hub-app/processes-svc.ts +63 -0
- package/src/serve/hub-app/project-list.css +97 -0
- package/src/serve/hub-app/project-list.tsx +167 -0
- package/src/serve/hub-app/projects-svc.ts +139 -17
- package/src/serve/hub-app/prose-hits-st.ts +3 -3
- package/src/serve/hub-app/prose-hits-view.tsx +6 -6
- package/src/serve/hub-app/{hub-st.ts → repo-st.ts} +155 -84
- package/src/serve/hub-app/runs-tabs.ts +1 -1
- package/src/serve/hub-app/screen-cache.ts +74 -0
- package/src/serve/hub-app/test-view.tsx +4 -4
- package/src/serve/hub-app/tests-svc.ts +3 -3
- package/src/serve/hub-app/tests-view.tsx +10 -10
- package/src/serve/hub-app/timeline-chart.tsx +24 -12
- package/src/serve/hub-app/timeline-svc.ts +5 -4
- package/src/serve/hub-app/timeline-view.tsx +13 -13
- package/src/serve/hub-app/tree-row.tsx +2 -2
- package/src/serve/hub-app/tree-svc.ts +2 -2
- package/src/serve/hub-app/wikis-view.tsx +10 -6
- package/src/serve/hub-runtime.ts +542 -0
- package/src/serve/project-gutter-route.ts +21 -0
- package/src/serve/project-gutter.ts +3 -2
- package/src/serve/repo-processes.ts +89 -0
- package/src/serve/run-serve.ts +110 -402
- package/src/serve/serve-hub.ts +142 -44
- package/src/serve/stale-restart.ts +5 -0
- package/src/serve/{gate-trigger.ts → tracked-watch.ts} +4 -64
- package/src/serve/warm-cell.ts +95 -0
- package/src/shipkit-files.ts +114 -0
- package/src/shipkit.ts +2 -1
- package/src/suite/kept-timings.ts +1 -1
- package/src/wiki/graph-positions.ts +1 -1
- package/src/serve/health-schedule.ts +0 -102
- package/src/serve/hub-app/project-gutter.css +0 -65
- package/src/serve/hub-app/project-gutter.tsx +0 -128
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
// `shipkit web`: the page on the home port, in front of the daemon and pm. It
|
|
2
|
+
// loads no repo and no config, so it starts in a moment and a daemon restart
|
|
3
|
+
// never moves it: the page stays up and says the daemon is down. An edit of
|
|
4
|
+
// its own code, the page included, restarts it in place (lib-watch.ts).
|
|
5
|
+
import homePage from '../serve/hub-app/home.html'
|
|
6
|
+
import { fileURLToPath } from 'url'
|
|
7
|
+
import { daemonPort, daemonUrl, homePort, homeUp, pmUrl } from './home-paths.ts'
|
|
8
|
+
import { reloadOnOwnCode } from './lib-watch.ts'
|
|
9
|
+
import { serveWeb } from './serve-web.ts'
|
|
10
|
+
import type { HomeIo } from './run-daemon.ts'
|
|
11
|
+
|
|
12
|
+
export async function runWeb(io: HomeIo, o: { port?: number } = {}): Promise<number> {
|
|
13
|
+
const port = o.port ?? homePort()
|
|
14
|
+
try {
|
|
15
|
+
const web = serveWeb({ port, daemon: daemonUrl(daemonPort()), pm: pmUrl(), page: homePage })
|
|
16
|
+
const code = reloadOnOwnCode({ role: 'web', entry: fileURLToPath(import.meta.url), bundles: true, before: () => web.stop(), io })
|
|
17
|
+
io.print(`home → http://localhost:${web.server.port} (the daemon: ${daemonUrl()}, pm: ${pmUrl()}), reloads on ${code.files} lib files`)
|
|
18
|
+
for (const sig of ['SIGINT', 'SIGTERM'] as const)
|
|
19
|
+
process.on(sig, () => {
|
|
20
|
+
code.stop()
|
|
21
|
+
web.stop()
|
|
22
|
+
process.exit(0)
|
|
23
|
+
})
|
|
24
|
+
await new Promise(() => {})
|
|
25
|
+
return 0
|
|
26
|
+
} catch (e) {
|
|
27
|
+
if (e instanceof Error && /in use|EADDRINUSE/.test(e.message)) {
|
|
28
|
+
if (await homeUp(port)) {
|
|
29
|
+
io.print(`home → http://localhost:${port} (already running)`)
|
|
30
|
+
return 0
|
|
31
|
+
}
|
|
32
|
+
io.printErr(`FAIL: port ${port} is taken by another process. Set SHIPKIT_HOME_PORT to a free one`)
|
|
33
|
+
return 1
|
|
34
|
+
}
|
|
35
|
+
throw e
|
|
36
|
+
}
|
|
37
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
// the daemon's one server, on its internal port: the home's api and every
|
|
2
|
+
// house repo's hub under /r/<repo>/ (home-routes.ts), and no page. The web
|
|
3
|
+
// (serve-web.ts) serves the page and proxies here, so a browser only ever
|
|
4
|
+
// sees the web origin. A terminal client (via-hub.ts, attach.ts) talks here
|
|
5
|
+
// directly, so a gate never depends on the web.
|
|
6
|
+
import { homeRoutes, TICK_MS, type HomeDeps } from './home-routes.ts'
|
|
7
|
+
|
|
8
|
+
export function serveDaemon(d: HomeDeps & { daemonPort: number }) {
|
|
9
|
+
const routes = homeRoutes(d)
|
|
10
|
+
const fetch = async (req: Request, srv: Bun.Server<unknown>): Promise<Response> =>
|
|
11
|
+
(await routes.handle(req, srv)) ?? new Response('not found', { status: 404 })
|
|
12
|
+
const server = Bun.serve({ port: d.daemonPort, hostname: '127.0.0.1', fetch })
|
|
13
|
+
let stopped = false
|
|
14
|
+
let running = false
|
|
15
|
+
const timer = setInterval(() => {
|
|
16
|
+
if (running || stopped) return
|
|
17
|
+
running = true
|
|
18
|
+
void routes.tick().catch((e) => console.error(`WARN the daemon tick broke: ${(e as Error).message}`)).finally(() => { running = false })
|
|
19
|
+
}, TICK_MS)
|
|
20
|
+
void routes.tick().catch(() => {})
|
|
21
|
+
return {
|
|
22
|
+
server,
|
|
23
|
+
routes,
|
|
24
|
+
stop: () => {
|
|
25
|
+
stopped = true
|
|
26
|
+
clearInterval(timer)
|
|
27
|
+
d.hubs.stopAll()
|
|
28
|
+
server.stop(true)
|
|
29
|
+
},
|
|
30
|
+
}
|
|
31
|
+
}
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
// the web: ONE Bun.serve on the home port. It serves the one page bundle (the
|
|
2
|
+
// overview at `/`, the plan at `/planned`, a repo's hub page at `/r/<repo>/`,
|
|
3
|
+
// all views of one app) and proxies every api,
|
|
4
|
+
// graph and media request to the daemon, streams included, and /api/pm/* to
|
|
5
|
+
// pm. A browser only ever sees this origin.
|
|
6
|
+
//
|
|
7
|
+
// the one constraint a reader breaks first: the same origin guard runs HERE,
|
|
8
|
+
// against this origin, and the proxied request reaches the daemon with no
|
|
9
|
+
// origin header. The daemon's own guard then refuses any page that calls its
|
|
10
|
+
// port directly, and lets this door through.
|
|
11
|
+
import { sameOrigin } from '../serve/same-origin.ts'
|
|
12
|
+
import { hubBase } from './home-paths.ts'
|
|
13
|
+
|
|
14
|
+
/** the page routes: ONE app for the overview, the plan, and the hub page of
|
|
15
|
+
* each known repo, which moves between them in place (home-st.ts) */
|
|
16
|
+
export function pageRoutes<T>(names: string[], page: T): Record<string, T> {
|
|
17
|
+
const out: Record<string, T> = { '/': page, '/planned': page }
|
|
18
|
+
for (const n of names) out[`${hubBase(n)}/`] = page
|
|
19
|
+
return out
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** a request the web hands to the daemon, as it goes there */
|
|
23
|
+
export function proxied(path: string): boolean {
|
|
24
|
+
if (path === '/hub.json' || path.startsWith('/api/')) return true
|
|
25
|
+
return /^\/r\/[^/]+\/./.test(path)
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** the headers a browser sent, minus what names the browser's origin or this hop */
|
|
29
|
+
const DROP = new Set(['host', 'origin', 'referer', 'sec-fetch-site', 'sec-fetch-mode', 'sec-fetch-dest', 'connection', 'keep-alive', 'content-length'])
|
|
30
|
+
|
|
31
|
+
/** what the daemon being down reads like, as a page and as an api answer */
|
|
32
|
+
export const daemonDownText = (daemon: string, why: string): string =>
|
|
33
|
+
`the shipkit daemon does not answer on ${daemon} (${why}). \`shipkit pm\` runs it; this page comes back by itself once it answers`
|
|
34
|
+
|
|
35
|
+
export const pmDownText = (pm: string, why: string): string =>
|
|
36
|
+
`shipkit pm does not answer on ${pm} (${why}), so the resources and the daemon and web states are unknown. launchd runs it (\`shipkit pm install\`), or run \`shipkit pm\` from the house repo`
|
|
37
|
+
|
|
38
|
+
/** the requests pm answers, the resources and the children */
|
|
39
|
+
export const toPm = (path: string): boolean => path.startsWith('/api/pm/')
|
|
40
|
+
|
|
41
|
+
const downPage = (text: string): string => `<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"><title>shipkit: daemon down</title>
|
|
42
|
+
<meta http-equiv="refresh" content="2"><style>body{font:14px/1.5 ui-sans-serif,system-ui;margin:0;padding:32px;background:#141619;color:#e6e6e6}
|
|
43
|
+
@media (prefers-color-scheme: light){body{background:#fafafa;color:#1a1a1a}}b{color:#e5484d}</style></head>
|
|
44
|
+
<body><p><b>daemon down</b></p><p>${text.replace(/&/g, '&').replace(/</g, '<')}</p></body></html>`
|
|
45
|
+
|
|
46
|
+
/** how long the web waits for the daemon's answer headers: the home api
|
|
47
|
+
* answers from memory, a repo request can wait on that repo's load */
|
|
48
|
+
export const MACHINE_ANSWER_MS = 5_000
|
|
49
|
+
export const REPO_ANSWER_MS = 30_000
|
|
50
|
+
/** how often the web asks the daemon's repo list once it has one, and while it has none */
|
|
51
|
+
export const LIST_EVERY_MS = 30_000
|
|
52
|
+
export const LIST_RETRY_MS = 1_000
|
|
53
|
+
|
|
54
|
+
/** a run or a check run streams for minutes, and a silent one is never cut */
|
|
55
|
+
export const streams = (path: string): boolean => /\/api\/(run|check\/run)$/.test(path)
|
|
56
|
+
|
|
57
|
+
export type WebDeps<T> = {
|
|
58
|
+
port: number
|
|
59
|
+
/** the daemon's url, e.g. http://127.0.0.1:4798 */
|
|
60
|
+
daemon: string
|
|
61
|
+
/** pm's url, e.g. http://127.0.0.1:4797 */
|
|
62
|
+
pm: string
|
|
63
|
+
/** the one page bundle, served at every page route */
|
|
64
|
+
page: T
|
|
65
|
+
machineMs?: number
|
|
66
|
+
repoMs?: number
|
|
67
|
+
/** one pace for the repo list, in place of LIST_RETRY_MS and LIST_EVERY_MS */
|
|
68
|
+
listEveryMs?: number
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export function serveWeb<T extends Bun.HTMLBundle | Response>(d: WebDeps<T>) {
|
|
72
|
+
let known = ''
|
|
73
|
+
let wasDown = false
|
|
74
|
+
let stopped = false
|
|
75
|
+
let listTimer: ReturnType<typeof setTimeout> | null = null
|
|
76
|
+
/** the repo names the daemon knows, or the reason it did not answer */
|
|
77
|
+
async function names(): Promise<string[] | Error> {
|
|
78
|
+
try {
|
|
79
|
+
const r = await fetch(`${d.daemon}/api/home/repos.json`, { signal: AbortSignal.timeout(3_000) })
|
|
80
|
+
if (!r.ok) return new Error(`${r.status} ${await r.text()}`)
|
|
81
|
+
return ((await r.json()) as { repos: string[] }).repos
|
|
82
|
+
} catch (e) {
|
|
83
|
+
return e as Error
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
async function proxy(req: Request, url: URL, srv: Bun.Server<unknown>): Promise<Response> {
|
|
88
|
+
if (!sameOrigin(req)) return new Response('refused: a request that changes something must come from the home page', { status: 403 })
|
|
89
|
+
const headers = new Headers()
|
|
90
|
+
req.headers.forEach((v, k) => { if (!DROP.has(k)) headers.set(k, v) })
|
|
91
|
+
// a run or a live stream can stay silent for minutes
|
|
92
|
+
srv.timeout(req, 0)
|
|
93
|
+
const body = req.method === 'GET' || req.method === 'HEAD' ? undefined : await req.arrayBuffer()
|
|
94
|
+
const path = url.pathname
|
|
95
|
+
const pm = toPm(path)
|
|
96
|
+
const target = pm ? d.pm : d.daemon
|
|
97
|
+
const wait = streams(path) ? null : path.startsWith('/r/') ? (d.repoMs ?? REPO_ANSWER_MS) : (d.machineMs ?? MACHINE_ANSWER_MS)
|
|
98
|
+
// the wait covers the answer headers only: a body already flowing is never cut
|
|
99
|
+
const ctl = new AbortController()
|
|
100
|
+
const timer = wait == null ? null : setTimeout(() => ctl.abort(new Error(`no answer in ${wait / 1000}s: it runs but hangs`)), wait)
|
|
101
|
+
try {
|
|
102
|
+
const r = await fetch(`${target}${path}${url.search}`, { method: req.method, headers, body, redirect: 'manual', decompress: false, signal: ctl.signal } as RequestInit)
|
|
103
|
+
if (timer != null) clearTimeout(timer)
|
|
104
|
+
if (wasDown && !pm) {
|
|
105
|
+
wasDown = false
|
|
106
|
+
follow()
|
|
107
|
+
}
|
|
108
|
+
const out = new Headers(r.headers)
|
|
109
|
+
out.delete('content-length')
|
|
110
|
+
out.delete('transfer-encoding')
|
|
111
|
+
out.delete('connection')
|
|
112
|
+
return new Response(r.body, { status: r.status, statusText: r.statusText, headers: out })
|
|
113
|
+
} catch (e) {
|
|
114
|
+
if (timer != null) clearTimeout(timer)
|
|
115
|
+
const why = ctl.signal.aborted ? (ctl.signal.reason as Error).message : (e as Error).message
|
|
116
|
+
if (pm) return new Response(pmDownText(d.pm, why), { status: 502, headers: { 'content-type': 'text/plain; charset=utf-8', 'x-shipkit-pm': 'down' } })
|
|
117
|
+
wasDown = true
|
|
118
|
+
return new Response(daemonDownText(d.daemon, why), { status: 502, headers: { 'content-type': 'text/plain; charset=utf-8', 'x-shipkit-daemon': 'down' } })
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
const fetchFn = async (req: Request, srv: Bun.Server<unknown>): Promise<Response> => {
|
|
123
|
+
const url = new URL(req.url)
|
|
124
|
+
const path = url.pathname
|
|
125
|
+
if (proxied(path)) return proxy(req, url, srv)
|
|
126
|
+
const bare = /^\/r\/([^/]+)$/.exec(path)
|
|
127
|
+
if (bare != null) return Response.redirect(`${path}/${url.search}`, 308)
|
|
128
|
+
// a repo page the routes do not have yet: ask the daemon, set them, ask again
|
|
129
|
+
if (/^\/r\/[^/]+\/$/.test(path)) {
|
|
130
|
+
const n = await names()
|
|
131
|
+
if (n instanceof Error) return new Response(downPage(daemonDownText(d.daemon, n.message)), { status: 503, headers: { 'content-type': 'text/html; charset=utf-8' } })
|
|
132
|
+
const name = decodeURIComponent(path.slice(3, -1))
|
|
133
|
+
// known: its route is set now (here, or by a list that landed meanwhile)
|
|
134
|
+
if (n.includes(name)) {
|
|
135
|
+
sync(n)
|
|
136
|
+
return Response.redirect(`${path}${url.search}`, 307)
|
|
137
|
+
}
|
|
138
|
+
return new Response(`not a house repo: ${name}. The home lists its repos at /`, { status: 404 })
|
|
139
|
+
}
|
|
140
|
+
return new Response('not found: the home page is at /', { status: 404 })
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
const server = Bun.serve({ port: d.port, hostname: '127.0.0.1', routes: pageRoutes([], d.page), fetch: fetchFn })
|
|
144
|
+
/** the page routes follow the daemon's repo list; true when they changed */
|
|
145
|
+
function sync(list: string[]): boolean {
|
|
146
|
+
const sorted = [...list].sort()
|
|
147
|
+
if (sorted.join('\0') === known) return false
|
|
148
|
+
known = sorted.join('\0')
|
|
149
|
+
server.reload({ routes: pageRoutes(sorted, d.page), fetch: fetchFn })
|
|
150
|
+
return true
|
|
151
|
+
}
|
|
152
|
+
/** the routes follow the daemon's list: asked again soon while the
|
|
153
|
+
* daemon does not answer, slowly once it does, and at once on its return */
|
|
154
|
+
function follow(): void {
|
|
155
|
+
if (listTimer != null) clearTimeout(listTimer)
|
|
156
|
+
listTimer = null
|
|
157
|
+
void names().then((n) => {
|
|
158
|
+
if (stopped) return
|
|
159
|
+
if (!(n instanceof Error)) sync(n)
|
|
160
|
+
listTimer = setTimeout(follow, d.listEveryMs ?? (n instanceof Error ? LIST_RETRY_MS : LIST_EVERY_MS))
|
|
161
|
+
})
|
|
162
|
+
}
|
|
163
|
+
follow()
|
|
164
|
+
return {
|
|
165
|
+
server,
|
|
166
|
+
stop: () => {
|
|
167
|
+
stopped = true
|
|
168
|
+
if (listTimer != null) clearTimeout(listTimer)
|
|
169
|
+
server.stop(true)
|
|
170
|
+
},
|
|
171
|
+
}
|
|
172
|
+
}
|
|
@@ -0,0 +1,346 @@
|
|
|
1
|
+
// THE one way shipkit keeps a long running process alive: a global
|
|
2
|
+
// resource, a repo's own dev server. Each process runs
|
|
3
|
+
// DETACHED in its own process group, so the process that started it can die
|
|
4
|
+
// or restart without taking it down, and its pid record in the machine db
|
|
5
|
+
// (core/machine-db.ts) lets the next image adopt it instead of starting a
|
|
6
|
+
// second copy.
|
|
7
|
+
//
|
|
8
|
+
// Truth is the pid and the probe, never this object's memory: tick() asks the
|
|
9
|
+
// kernel whether the pid lives and the probe whether it answers.
|
|
10
|
+
import { closeSync, existsSync, mkdirSync, openSync, readSync, rmSync, statSync, writeFileSync } from 'fs'
|
|
11
|
+
import { dirname } from 'path'
|
|
12
|
+
import { importMachineFile, machineDelete, machineGet, machineSet } from '../core/machine-db.ts'
|
|
13
|
+
|
|
14
|
+
export type ProcStatus = 'down' | 'starting' | 'up' | 'crashed' | 'stopping'
|
|
15
|
+
|
|
16
|
+
export type ProcSpec = {
|
|
17
|
+
id: string
|
|
18
|
+
cmd: string[]
|
|
19
|
+
cwd: string
|
|
20
|
+
env?: Record<string, string>
|
|
21
|
+
/** stdout and stderr land here, appended */
|
|
22
|
+
log: string
|
|
23
|
+
/** names its pid record, what the next image adopts: the machine db row
|
|
24
|
+
* `pid:<pidKey>`. A path, because an older copy wrote a pid FILE there,
|
|
25
|
+
* which is read once into the row and deleted */
|
|
26
|
+
pidKey: string
|
|
27
|
+
/** true once the process serves. No probe = up as soon as the pid lives */
|
|
28
|
+
probe?: () => Promise<boolean>
|
|
29
|
+
/** never left down after CRASH_LIMIT crashes: it keeps retrying at the
|
|
30
|
+
* longest backoff, so a code fix brings it back with no hand */
|
|
31
|
+
neverGiveUp?: boolean
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export type ProcView = {
|
|
35
|
+
id: string
|
|
36
|
+
status: ProcStatus
|
|
37
|
+
pid: number | null
|
|
38
|
+
/** epoch ms of the last status change */
|
|
39
|
+
since: number
|
|
40
|
+
/** crash restarts in the current window */
|
|
41
|
+
restarts: number
|
|
42
|
+
/** why it is not up, when that is known */
|
|
43
|
+
note: string | null
|
|
44
|
+
/** up, and started by something else: not ours to stop */
|
|
45
|
+
external: boolean
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export type SuperviseDeps = {
|
|
49
|
+
spawn: (spec: ProcSpec) => number
|
|
50
|
+
alive: (pid: number) => boolean
|
|
51
|
+
/** signal the whole group the process leads */
|
|
52
|
+
kill: (pid: number, sig: 'SIGTERM' | 'SIGKILL') => void
|
|
53
|
+
now: () => number
|
|
54
|
+
/** epoch ms the process started, null when unknown. A pid that started
|
|
55
|
+
* after its pid record was written was reused by another process */
|
|
56
|
+
startedAt?: (pid: number) => number | null
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** a process starts a moment before its pid record is written */
|
|
60
|
+
const RECORD_SLACK_MS = 2_000
|
|
61
|
+
|
|
62
|
+
/** a crash restarts after BACKOFF_MS[n], the last entry repeating */
|
|
63
|
+
export const BACKOFF_MS = [1_000, 2_000, 5_000, 15_000, 60_000]
|
|
64
|
+
/** this many crashes inside CRASH_WINDOW_MS and it stays down until asked */
|
|
65
|
+
export const CRASH_LIMIT = 5
|
|
66
|
+
export const CRASH_WINDOW_MS = 5 * 60_000
|
|
67
|
+
/** a stop that SIGTERM did not end becomes a SIGKILL after this */
|
|
68
|
+
export const KILL_AFTER_MS = 5_000
|
|
69
|
+
|
|
70
|
+
/** the process is started by a shell that exits at once: it belongs to init
|
|
71
|
+
* from its first moment, so a supervisor that restarts in place (execve)
|
|
72
|
+
* never holds it as a zombie that reads alive forever. `set -m` gives it its
|
|
73
|
+
* own process group, which a stop signals whole */
|
|
74
|
+
const LAUNCH = 'set -m; "$@" >>"$0" 2>&1 </dev/null & echo $!'
|
|
75
|
+
|
|
76
|
+
export const realDeps: SuperviseDeps = {
|
|
77
|
+
spawn: (spec) => {
|
|
78
|
+
mkdirSync(dirname(spec.log), { recursive: true })
|
|
79
|
+
closeSync(openSync(spec.log, 'a'))
|
|
80
|
+
const r = Bun.spawnSync(['/bin/sh', '-c', LAUNCH, spec.log, ...spec.cmd], { cwd: spec.cwd, env: { ...process.env, ...spec.env }, stdout: 'pipe', stderr: 'pipe' })
|
|
81
|
+
const pid = Number(r.stdout.toString().trim())
|
|
82
|
+
if (r.exitCode !== 0 || !Number.isInteger(pid) || pid <= 0) throw new Error(`the launch shell exited ${r.exitCode}: ${r.stderr.toString().trim()}`)
|
|
83
|
+
return pid
|
|
84
|
+
},
|
|
85
|
+
alive: (pid) => {
|
|
86
|
+
try {
|
|
87
|
+
process.kill(pid, 0)
|
|
88
|
+
return true
|
|
89
|
+
} catch (e) {
|
|
90
|
+
// EPERM: it lives, under another user
|
|
91
|
+
return (e as NodeJS.ErrnoException).code === 'EPERM'
|
|
92
|
+
}
|
|
93
|
+
},
|
|
94
|
+
kill: (pid, sig) => {
|
|
95
|
+
try {
|
|
96
|
+
process.kill(-pid, sig)
|
|
97
|
+
} catch {
|
|
98
|
+
try { process.kill(pid, sig) } catch { /* already gone */ }
|
|
99
|
+
}
|
|
100
|
+
},
|
|
101
|
+
now: Date.now,
|
|
102
|
+
startedAt: (pid) => {
|
|
103
|
+
const r = Bun.spawnSync(['ps', '-o', 'lstart=', '-p', String(pid)], { stdout: 'pipe', stderr: 'ignore' })
|
|
104
|
+
const t = Date.parse(r.stdout.toString().trim())
|
|
105
|
+
// lstart has whole seconds
|
|
106
|
+
return Number.isNaN(t) ? null : t + 999
|
|
107
|
+
},
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
export type PidRecord = { pid: number; cmd: string[]; at: number }
|
|
111
|
+
|
|
112
|
+
const pidRow = (key: string): string => `pid:${key}`
|
|
113
|
+
|
|
114
|
+
export function readPid(key: string): PidRecord | null {
|
|
115
|
+
importMachineFile<PidRecord>(pidRow(key), key)
|
|
116
|
+
const r = machineGet<PidRecord>(pidRow(key))
|
|
117
|
+
return r != null && Number.isInteger(r.pid) && r.pid > 0 ? r : null
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
function writePid(key: string, r: PidRecord): void {
|
|
121
|
+
machineSet(pidRow(key), r)
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
const dropPid = (key: string): void => {
|
|
125
|
+
machineDelete(pidRow(key))
|
|
126
|
+
try { rmSync(key, { force: true }) } catch { /* an older copy's file, gone either way */ }
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
export class Proc {
|
|
130
|
+
status: ProcStatus = 'down'
|
|
131
|
+
pid: number | null = null
|
|
132
|
+
since: number
|
|
133
|
+
note: string | null = null
|
|
134
|
+
/** what the owner asked for: a crash restarts only a process wanted up */
|
|
135
|
+
want: 'up' | 'down' = 'down'
|
|
136
|
+
private crashes: number[] = []
|
|
137
|
+
private retryAt: number | null = null
|
|
138
|
+
private killAt: number | null = null
|
|
139
|
+
private probing = false
|
|
140
|
+
/** a restart: start again once the stop landed */
|
|
141
|
+
private again = false
|
|
142
|
+
/** it answers its probe with no pid of ours behind it */
|
|
143
|
+
external = false
|
|
144
|
+
/** the pid came from the pid record an earlier image wrote, not from a start */
|
|
145
|
+
adopted = false
|
|
146
|
+
|
|
147
|
+
constructor(readonly spec: ProcSpec, private d: SuperviseDeps = realDeps) {
|
|
148
|
+
this.since = d.now()
|
|
149
|
+
const rec = readPid(spec.pidKey)
|
|
150
|
+
// an earlier image started it and it still runs: adopt, never double
|
|
151
|
+
if (rec != null && d.alive(rec.pid) && !((d.startedAt?.(rec.pid) ?? 0) > rec.at + RECORD_SLACK_MS)) {
|
|
152
|
+
this.pid = rec.pid
|
|
153
|
+
this.adopted = true
|
|
154
|
+
this.want = 'up'
|
|
155
|
+
this.set('starting')
|
|
156
|
+
} else if (rec != null) dropPid(spec.pidKey)
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
private set(s: ProcStatus, note: string | null = null): void {
|
|
160
|
+
if (s !== this.status) this.since = this.d.now()
|
|
161
|
+
this.status = s
|
|
162
|
+
this.note = note
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/** start it unless it already runs; clears a crash give up */
|
|
166
|
+
start(): void {
|
|
167
|
+
this.want = 'up'
|
|
168
|
+
this.crashes = []
|
|
169
|
+
this.retryAt = null
|
|
170
|
+
if (this.pid != null && this.d.alive(this.pid)) return
|
|
171
|
+
// a second copy would lose the port to the first and read as a crash
|
|
172
|
+
if (this.external) return
|
|
173
|
+
this.launch()
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/** start it, but look first: something else may already serve it (a hub
|
|
177
|
+
* a VS Code task started, a dev server run by hand), and a second copy
|
|
178
|
+
* would lose the port and linger. Every owner starts through this */
|
|
179
|
+
async ensure(): Promise<void> {
|
|
180
|
+
if (this.pid == null && !this.external && this.spec.probe != null && this.want === 'down') await this.tick()
|
|
181
|
+
this.start()
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
private launch(): void {
|
|
185
|
+
try {
|
|
186
|
+
const pid = this.d.spawn(this.spec)
|
|
187
|
+
this.pid = pid
|
|
188
|
+
this.adopted = false
|
|
189
|
+
writePid(this.spec.pidKey, { pid, cmd: this.spec.cmd, at: this.d.now() })
|
|
190
|
+
this.set('starting')
|
|
191
|
+
} catch (e) {
|
|
192
|
+
this.pid = null
|
|
193
|
+
this.set('crashed', `could not start ${this.spec.cmd.join(' ')}: ${(e as Error).message}`)
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/** stop it, then start it again once it is gone */
|
|
198
|
+
restart(): void {
|
|
199
|
+
if (this.pid == null || !this.d.alive(this.pid)) {
|
|
200
|
+
this.start()
|
|
201
|
+
return
|
|
202
|
+
}
|
|
203
|
+
this.stop()
|
|
204
|
+
this.again = true
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/** something else serves it (external): stop that owner, then start our
|
|
208
|
+
* own copy. `targets` are what to signal (a process group, else the pid),
|
|
209
|
+
* `owners` the pids that must be gone before the start. SIGTERM, then
|
|
210
|
+
* SIGKILL once KILL_AFTER_MS passed with an owner still alive */
|
|
211
|
+
async takeOver(owners: number[], targets: number[], sleep: (ms: number) => Promise<void>): Promise<void> {
|
|
212
|
+
const left = (): boolean => owners.some((pid) => this.d.alive(pid))
|
|
213
|
+
for (const t of targets) this.d.kill(t, 'SIGTERM')
|
|
214
|
+
this.set('stopping', 'taking it over: stopping the copy started elsewhere')
|
|
215
|
+
let until = this.d.now() + KILL_AFTER_MS
|
|
216
|
+
while (left() && this.d.now() < until) await sleep(100)
|
|
217
|
+
if (left()) {
|
|
218
|
+
for (const t of targets) this.d.kill(t, 'SIGKILL')
|
|
219
|
+
until = this.d.now() + KILL_AFTER_MS
|
|
220
|
+
while (left() && this.d.now() < until) await sleep(100)
|
|
221
|
+
}
|
|
222
|
+
if (left()) {
|
|
223
|
+
this.set('up', `take over failed: pid ${owners.filter((pid) => this.d.alive(pid)).join(', ')} outlived SIGKILL`)
|
|
224
|
+
return
|
|
225
|
+
}
|
|
226
|
+
this.external = false
|
|
227
|
+
this.set('down')
|
|
228
|
+
this.start()
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
stop(): void {
|
|
232
|
+
this.want = 'down'
|
|
233
|
+
this.again = false
|
|
234
|
+
this.retryAt = null
|
|
235
|
+
if (this.pid == null || !this.d.alive(this.pid)) {
|
|
236
|
+
this.gone()
|
|
237
|
+
// started elsewhere: it keeps running, and says so
|
|
238
|
+
if (this.external) this.set('up', 'started outside the supervisor, so it is not stopped from here')
|
|
239
|
+
else this.set('down')
|
|
240
|
+
return
|
|
241
|
+
}
|
|
242
|
+
this.d.kill(this.pid, 'SIGTERM')
|
|
243
|
+
this.killAt = this.d.now() + KILL_AFTER_MS
|
|
244
|
+
this.set('stopping')
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
/** read through a call, so a check after an await is not narrowed away */
|
|
248
|
+
private stopping(): boolean {
|
|
249
|
+
return this.status === 'stopping'
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
private gone(): void {
|
|
253
|
+
this.pid = null
|
|
254
|
+
this.killAt = null
|
|
255
|
+
dropPid(this.spec.pidKey)
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/** one look at the world: the pid, the probe, a due restart or kill */
|
|
259
|
+
async tick(): Promise<void> {
|
|
260
|
+
const now = this.d.now()
|
|
261
|
+
if (this.pid != null && !this.d.alive(this.pid)) {
|
|
262
|
+
this.gone()
|
|
263
|
+
if (this.again) {
|
|
264
|
+
this.again = false
|
|
265
|
+
this.want = 'up'
|
|
266
|
+
this.launch()
|
|
267
|
+
} else if (this.want === 'down') this.set('down')
|
|
268
|
+
else {
|
|
269
|
+
this.crashes = [...this.crashes.filter((t) => now - t < CRASH_WINDOW_MS), now]
|
|
270
|
+
if (this.crashes.length >= CRASH_LIMIT && this.spec.neverGiveUp !== true) {
|
|
271
|
+
this.want = 'down'
|
|
272
|
+
this.set('crashed', `exited ${this.crashes.length} times in ${Math.round(CRASH_WINDOW_MS / 60_000)} min, left down: see the log`)
|
|
273
|
+
} else {
|
|
274
|
+
this.retryAt = now + BACKOFF_MS[Math.min(this.crashes.length - 1, BACKOFF_MS.length - 1)]!
|
|
275
|
+
this.set('crashed', `exited, restart in ${Math.round((this.retryAt - now) / 1000)}s`)
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
if (this.status === 'stopping' && this.pid != null && this.killAt != null && now >= this.killAt) {
|
|
280
|
+
this.d.kill(this.pid, 'SIGKILL')
|
|
281
|
+
this.killAt = now + KILL_AFTER_MS
|
|
282
|
+
}
|
|
283
|
+
if (this.want === 'up' && this.pid == null && this.retryAt != null && now >= this.retryAt) {
|
|
284
|
+
this.retryAt = null
|
|
285
|
+
this.launch()
|
|
286
|
+
}
|
|
287
|
+
if (this.probing || this.status === 'stopping') return
|
|
288
|
+
this.probing = true
|
|
289
|
+
try {
|
|
290
|
+
if (this.pid != null && (this.status === 'starting' || this.status === 'up')) {
|
|
291
|
+
const ok = this.spec.probe == null ? true : await this.spec.probe().catch(() => false)
|
|
292
|
+
this.external = false
|
|
293
|
+
// a stop may have landed while the probe was out
|
|
294
|
+
if (this.pid != null && !this.stopping()) this.set(ok ? 'up' : 'starting', ok ? null : this.status === 'up' ? 'stopped answering its probe' : null)
|
|
295
|
+
} else if (this.pid == null && this.retryAt == null && this.spec.probe != null && (this.status === 'down' || this.status === 'up')) {
|
|
296
|
+
// no pid of ours: something else may serve it, a hub a VS Code
|
|
297
|
+
// task started or a tunnel opened by hand
|
|
298
|
+
const ok = await this.spec.probe().catch(() => false)
|
|
299
|
+
if (this.pid != null) return
|
|
300
|
+
this.external = ok
|
|
301
|
+
this.set(ok ? 'up' : 'down', ok ? 'started outside the supervisor' : null)
|
|
302
|
+
}
|
|
303
|
+
} finally {
|
|
304
|
+
this.probing = false
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
view(): ProcView {
|
|
309
|
+
return { id: this.spec.id, status: this.status, pid: this.pid, since: this.since, restarts: this.crashes.length, note: this.note, external: this.external }
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
/** the last `n` lines of a log, read from its tail only */
|
|
314
|
+
export function tailLines(file: string, n: number, maxBytes = 64 * 1024): string[] {
|
|
315
|
+
if (!existsSync(file)) return []
|
|
316
|
+
try {
|
|
317
|
+
const size = statSync(file).size
|
|
318
|
+
const fd = openSync(file, 'r')
|
|
319
|
+
try {
|
|
320
|
+
const len = Math.min(size, maxBytes)
|
|
321
|
+
const buf = Buffer.alloc(len)
|
|
322
|
+
readSync(fd, buf, 0, len, size - len)
|
|
323
|
+
return buf.toString('utf8').split('\n').filter((l) => l !== '').slice(-n)
|
|
324
|
+
} finally {
|
|
325
|
+
closeSync(fd)
|
|
326
|
+
}
|
|
327
|
+
} catch {
|
|
328
|
+
return []
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
/** the most a process log grows before capLog cuts it */
|
|
333
|
+
export const LOG_MAX_BYTES = 8 * 1024 * 1024
|
|
334
|
+
|
|
335
|
+
/** a log past `max` bytes is cut to its last `keep` bytes, so a process that
|
|
336
|
+
* runs for weeks never fills the disk. Appends in flight land after the cut */
|
|
337
|
+
export function capLog(file: string, max = LOG_MAX_BYTES, keep = 1024 * 1024): boolean {
|
|
338
|
+
try {
|
|
339
|
+
if (statSync(file).size <= max) return false
|
|
340
|
+
const tail = tailLines(file, Number.MAX_SAFE_INTEGER, keep)
|
|
341
|
+
writeFileSync(file, `${tail.join('\n')}\n`)
|
|
342
|
+
return true
|
|
343
|
+
} catch {
|
|
344
|
+
return false
|
|
345
|
+
}
|
|
346
|
+
}
|