spinuptui 1.2.0 → 1.3.0-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -1
- package/README.md +13 -1
- package/package.json +1 -1
- package/src/config.ts +45 -0
- package/src/index.tsx +37 -38
- package/src/lib/appUpdate.test.ts +26 -0
- package/src/lib/appUpdate.ts +33 -7
- package/src/lib/beacon.test.ts +140 -0
- package/src/lib/beacon.ts +349 -0
- package/src/lib/beaconSetup.test.ts +65 -0
- package/src/lib/beaconSetup.ts +143 -0
- package/src/lib/cliBeacon.ts +290 -0
- package/src/lib/cliScheduler.ts +534 -0
- package/src/lib/flock.ts +51 -0
- package/src/lib/prompt.ts +47 -0
- package/src/lib/schedule.ts +156 -0
- package/src/lib/scheduler.test.ts +353 -0
- package/src/lib/schedulerGit.ts +119 -0
- package/src/lib/schedulerInstall.ts +325 -0
- package/src/lib/schedulerJobs.ts +288 -0
- package/src/lib/schedulerRun.ts +246 -0
- package/src/lib/schedulerTick.ts +307 -0
- package/src/modules/beacon.ts +52 -0
- package/src/modules/contract.ts +82 -0
- package/src/modules/index.ts +29 -0
- package/src/modules/scheduler.ts +146 -0
- package/src/ui/App.tsx +15 -1
- package/src/ui/Explain.tsx +15 -2
- package/src/ui/Header.tsx +2 -0
- package/src/ui/Help.tsx +1 -1
- package/src/ui/main.tsx +42 -0
- package/src/ui/store.tsx +13 -1
- package/src/ui/views/BeaconSetup.tsx +268 -0
- package/src/ui/views/Modules.tsx +232 -0
- package/src/ui/views/SchedulerInstall.tsx +172 -0
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
// Running one Scheduler job: the kinds, the agent adapters, the report and state
|
|
2
|
+
// it leaves behind, and the Beacon message about it.
|
|
3
|
+
//
|
|
4
|
+
// The report/state layout and the agent rules (read-only by default, a fixed
|
|
5
|
+
// always-deny list, a SUMMARY: paragraph as the digest, a lapsed login retried
|
|
6
|
+
// rather than counted as a run) carry over from the git-backed job runner; see
|
|
7
|
+
// docs/2026-10-20_scheduler-spec.md.
|
|
8
|
+
|
|
9
|
+
import { existsSync, mkdirSync, writeFileSync } from "node:fs"
|
|
10
|
+
import { hostname, homedir } from "node:os"
|
|
11
|
+
import { join } from "node:path"
|
|
12
|
+
import { loadConfig } from "../config.ts"
|
|
13
|
+
import { beaconFromConfig, beaconSend } from "./beacon.ts"
|
|
14
|
+
import { spawn } from "./spawn.ts"
|
|
15
|
+
import { fmt } from "./schedule.ts"
|
|
16
|
+
import { agentOf, DEFAULT_TIMEOUT_MIN, expandHome, loadState, runsDir, saveState, splitArgs, type Job, type Source } from "./schedulerJobs.ts"
|
|
17
|
+
|
|
18
|
+
export const DEFAULT_ALLOW = "Read,Grep,Glob"
|
|
19
|
+
// Never allowed in a claude job, whatever its allow: line says.
|
|
20
|
+
export const ALWAYS_DENY = "Bash(git push *),Bash(git reset *),Bash(git clean *),Bash(rm -rf *),Bash(sudo *)"
|
|
21
|
+
const AUTH_FAIL_RE =
|
|
22
|
+
/Failed to authenticate|OAuth (session|token)|could not be refreshed|Invalid API key|API key.*(expired|invalid)|Please run .?\/login|not (logged in|authenticated)/i
|
|
23
|
+
const REPORT_CAP = 200_000
|
|
24
|
+
// Tools a launchd PATH doesn't include by default.
|
|
25
|
+
const EXTRA_PATH = [join(homedir(), ".local/bin"), join(homedir(), ".bun/bin"), "/opt/homebrew/bin", "/usr/local/bin"]
|
|
26
|
+
|
|
27
|
+
export type RunStatus = "ok" | "failed" | "timeout" | "error" | "auth"
|
|
28
|
+
|
|
29
|
+
function preamble(id: string, kind: string, createdBy: string, runs: string): string {
|
|
30
|
+
return `You are running unattended on ${hostname().split(".")[0]} as scheduled job \`${id}\` (kind: ${kind}), queued by: ${createdBy}.
|
|
31
|
+
Nobody is watching this run and nobody can answer questions. Do the task below, then stop.
|
|
32
|
+
Rules: report, don't act. Do not modify files outside ${runs}, do not push, deploy, post to clients or email anyone, unless the task explicitly says to.
|
|
33
|
+
Your final message is the report. Start it with a line \`SUMMARY:\` followed by one short paragraph (under 600 characters, plain text) that stands on its own in a chat message. Then give the details in Markdown.
|
|
34
|
+
|
|
35
|
+
--- TASK ---
|
|
36
|
+
`
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
// How to start this same SpinupTUI: the Bun running now plus its entry point, or
|
|
40
|
+
// just the executable when it's a compiled binary. Module jobs and `spinuptui`
|
|
41
|
+
// jobs use it, so they always run the dispatcher's own version.
|
|
42
|
+
export function selfCommand(): string[] {
|
|
43
|
+
const compiled = Bun.main.startsWith("/$bunfs") || !/\bbun(\.exe)?$/.test(process.execPath)
|
|
44
|
+
return compiled ? [process.execPath] : [process.execPath, Bun.main]
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export function jobEnv(): Record<string, string> {
|
|
48
|
+
const env: Record<string, string> = {}
|
|
49
|
+
for (const [k, v] of Object.entries(process.env)) if (v !== undefined) env[k] = v
|
|
50
|
+
const parts = (env.PATH || "/usr/bin:/bin:/usr/sbin:/sbin").split(":")
|
|
51
|
+
env.PATH = [...EXTRA_PATH.filter((p) => !parts.includes(p)), ...parts].join(":")
|
|
52
|
+
// A job is not a dispatcher: a nested `scheduler tick` must refuse.
|
|
53
|
+
delete env.SPINUPTUI_SCHEDULER_DISPATCHER
|
|
54
|
+
return env
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export interface ProcResult {
|
|
58
|
+
code: number | null
|
|
59
|
+
output: string
|
|
60
|
+
timedOut: boolean
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// Runs a command with stdout and stderr interleaved into one stream, in its own
|
|
64
|
+
// process group so a timeout can stop everything it started.
|
|
65
|
+
export async function runProcess(cmd: string[], opts: { cwd: string; env: Record<string, string>; timeoutMs: number }): Promise<ProcResult> {
|
|
66
|
+
const proc = spawn(["/bin/sh", "-c", 'exec "$@" 2>&1', "sh", ...cmd], {
|
|
67
|
+
cwd: opts.cwd,
|
|
68
|
+
env: opts.env,
|
|
69
|
+
stdin: "ignore",
|
|
70
|
+
stdout: "pipe",
|
|
71
|
+
stderr: "ignore",
|
|
72
|
+
detached: true,
|
|
73
|
+
} as Parameters<typeof spawn>[1])
|
|
74
|
+
let timedOut = false
|
|
75
|
+
const kill = (sig: NodeJS.Signals) => {
|
|
76
|
+
try {
|
|
77
|
+
process.kill(-proc.pid, sig)
|
|
78
|
+
} catch {
|
|
79
|
+
// Already gone.
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
const timer = setTimeout(() => {
|
|
83
|
+
timedOut = true
|
|
84
|
+
kill("SIGTERM")
|
|
85
|
+
setTimeout(() => kill("SIGKILL"), 20_000).unref?.()
|
|
86
|
+
}, opts.timeoutMs)
|
|
87
|
+
const output = await new Response(proc.stdout as ReadableStream).text()
|
|
88
|
+
const code = await proc.exited
|
|
89
|
+
clearTimeout(timer)
|
|
90
|
+
return { code: timedOut ? null : code, output, timedOut }
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
export function summaryOf(job: Job, output: string): string {
|
|
94
|
+
const text = output.trim()
|
|
95
|
+
if (agentOf(job.header)) {
|
|
96
|
+
const m = text.match(/SUMMARY:\s*([\s\S]+?)(\n\s*\n|$)/)
|
|
97
|
+
return (m ? m[1]! : text).trim().slice(0, 900)
|
|
98
|
+
}
|
|
99
|
+
if (job.header.kind === "spinuptui") {
|
|
100
|
+
// A failed JSON result says what went wrong better than its tail does.
|
|
101
|
+
const last = text.split("\n").reverse().find((l) => l.trim().startsWith("{"))
|
|
102
|
+
if (last) {
|
|
103
|
+
try {
|
|
104
|
+
const r = JSON.parse(last) as { ok?: boolean; message?: string }
|
|
105
|
+
if (r.ok === false && r.message) return r.message.slice(0, 900)
|
|
106
|
+
} catch {
|
|
107
|
+
// Not JSON; fall through to the tail.
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
return text.split("\n").slice(-15).join("\n").slice(-900)
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function buildCommand(job: Job, workdir: string, env: Record<string, string>, runs: string): string[] | string {
|
|
115
|
+
const h = job.header
|
|
116
|
+
const kind = h.kind!
|
|
117
|
+
if (kind === "script") return ["/bin/bash", "-c", job.body]
|
|
118
|
+
if (kind === "spinuptui") return [...selfCommand(), ...splitArgs(job.body)]
|
|
119
|
+
const agent = agentOf(h)!
|
|
120
|
+
const prompt = preamble(h.id!, agent === "codex" ? "agent/codex" : "agent/claude", h.created_by!, runs) + job.body
|
|
121
|
+
const bin = Bun.which(agent, { PATH: env.PATH })
|
|
122
|
+
if (!bin) return `${agent} not found on PATH`
|
|
123
|
+
if (agent === "codex") return [bin, "exec", "-s", h.sandbox || "read-only", "-C", workdir, "--skip-git-repo-check", prompt]
|
|
124
|
+
const cmd = [bin, "-p", prompt, "--output-format", "text", "--permission-mode", "default", "--allowedTools", h.allow || DEFAULT_ALLOW, "--disallowedTools", ALWAYS_DENY]
|
|
125
|
+
for (const d of (h.add_dirs ?? "").split(",")) if (d.trim()) cmd.push("--add-dir", expandHome(d.trim()))
|
|
126
|
+
if (h.model) cmd.push("--model", h.model)
|
|
127
|
+
return cmd
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
export interface RunOutcome {
|
|
131
|
+
status: RunStatus
|
|
132
|
+
report?: string // source-relative
|
|
133
|
+
notified?: "sent" | "skipped" | string // string = why it failed
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
const pad2 = (n: number) => String(n).padStart(2, "0")
|
|
137
|
+
const stamp = (d: Date) => fmt(d).replace(" ", "-").replace(":", "")
|
|
138
|
+
const duration = (s: number) => `${Math.floor(s / 60)}m ${pad2(s % 60)}s`
|
|
139
|
+
|
|
140
|
+
export async function runJob(source: Source, job: Job, occurrence: Date, now = new Date()): Promise<RunOutcome> {
|
|
141
|
+
const h = job.header
|
|
142
|
+
const id = h.id!
|
|
143
|
+
const env = jobEnv()
|
|
144
|
+
const workdir = expandHome(h.workdir || source.path)
|
|
145
|
+
const timeoutMs = Number(h.timeout || DEFAULT_TIMEOUT_MIN) * 60_000
|
|
146
|
+
const runs = join(runsDir(source), id)
|
|
147
|
+
mkdirSync(runs, { recursive: true })
|
|
148
|
+
const reportRel = `runs/${id}/${stamp(now)}.md`
|
|
149
|
+
|
|
150
|
+
const started = new Date()
|
|
151
|
+
let status: RunStatus
|
|
152
|
+
let code: number | null = null
|
|
153
|
+
let output: string
|
|
154
|
+
const cmd = existsSync(workdir) ? buildCommand(job, workdir, env, runs) : `workdir does not exist on this machine: ${workdir}`
|
|
155
|
+
if (typeof cmd === "string") {
|
|
156
|
+
status = "error"
|
|
157
|
+
output = cmd
|
|
158
|
+
} else {
|
|
159
|
+
try {
|
|
160
|
+
const r = await runProcess(cmd, { cwd: workdir, env, timeoutMs })
|
|
161
|
+
code = r.code
|
|
162
|
+
output = r.output
|
|
163
|
+
status = r.timedOut ? "timeout" : r.code === 0 ? "ok" : "failed"
|
|
164
|
+
if (r.timedOut) output += `\n\n[scheduler] killed after ${timeoutMs / 60_000} minutes`
|
|
165
|
+
} catch (err) {
|
|
166
|
+
status = "error"
|
|
167
|
+
output = `could not start: ${(err as Error).message}`
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
if (agentOf(h) && status !== "ok" && AUTH_FAIL_RE.test(output)) status = "auth"
|
|
171
|
+
return finish(source, job, occurrence, started, new Date(), status, code, output, reportRel)
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
async function finish(
|
|
175
|
+
source: Source,
|
|
176
|
+
job: Job,
|
|
177
|
+
occurrence: Date,
|
|
178
|
+
started: Date,
|
|
179
|
+
finished: Date,
|
|
180
|
+
status: RunStatus,
|
|
181
|
+
code: number | null,
|
|
182
|
+
output: string,
|
|
183
|
+
reportRel: string,
|
|
184
|
+
): Promise<RunOutcome> {
|
|
185
|
+
const h = job.header
|
|
186
|
+
const id = h.id!
|
|
187
|
+
const seconds = Math.round((finished.getTime() - started.getTime()) / 1000)
|
|
188
|
+
const state = loadState(source, id)
|
|
189
|
+
|
|
190
|
+
if (status === "auth") {
|
|
191
|
+
// The agent isn't logged in, so nothing ran: no report. Retry in 30 minutes,
|
|
192
|
+
// and page only once per scheduled slot.
|
|
193
|
+
const agent = agentOf(h)!
|
|
194
|
+
state.last_status = status
|
|
195
|
+
state.retry_after = fmt(new Date(Date.now() + 30 * 60_000))
|
|
196
|
+
const first = state.auth_alerted_for !== fmt(occurrence)
|
|
197
|
+
state.auth_alerted_for = fmt(occurrence)
|
|
198
|
+
saveState(source, id, state)
|
|
199
|
+
if (!first) return { status, notified: "skipped" }
|
|
200
|
+
const how = agent === "claude" ? "claude, then /login" : "codex login"
|
|
201
|
+
const notified = await notify(
|
|
202
|
+
`⚠️ Job ${id} did not run: the ${agent} login on ${hostname().split(".")[0]} has lapsed. Log in there (run \`${how}\`); it retries every 30 min until ${h.expires}.`,
|
|
203
|
+
h.to,
|
|
204
|
+
)
|
|
205
|
+
return { status, notified }
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
if (output.length > REPORT_CAP) output = output.slice(0, REPORT_CAP) + `\n\n[scheduler] output truncated at ${REPORT_CAP} characters`
|
|
209
|
+
const kindLabel = agentOf(h) ? `agent (${agentOf(h)})` : h.kind!
|
|
210
|
+
writeFileSync(
|
|
211
|
+
join(source.path, reportRel),
|
|
212
|
+
`# ${id} — ${status}\n\n` +
|
|
213
|
+
`- Scheduled: ${fmt(occurrence)}\n- Started: ${fmt(started)}:${pad2(started.getSeconds())}\n- Duration: ${duration(seconds)}\n` +
|
|
214
|
+
`- Exit code: ${code ?? "None"}\n- Kind: ${kindLabel}\n- Job file: jobs/${id}.md\n- Queued by: ${h.created_by}\n\n---\n\n` +
|
|
215
|
+
output.trimEnd() +
|
|
216
|
+
"\n",
|
|
217
|
+
)
|
|
218
|
+
state.last_status = status
|
|
219
|
+
state.last_report = reportRel
|
|
220
|
+
delete state.retry_after
|
|
221
|
+
delete state.auth_alerted_for
|
|
222
|
+
state.runs = (state.runs ?? 0) + 1
|
|
223
|
+
state.last_run = fmt(started)
|
|
224
|
+
state.last_scheduled = fmt(occurrence)
|
|
225
|
+
saveState(source, id, state)
|
|
226
|
+
|
|
227
|
+
let notified: RunOutcome["notified"] = "skipped"
|
|
228
|
+
if ((h.notify || "beacon") !== "none") {
|
|
229
|
+
const icon = { ok: "✅", failed: "❌", timeout: "⏱️", error: "❌" }[status as Exclude<RunStatus, "auth">] ?? "•"
|
|
230
|
+
notified = await notify(
|
|
231
|
+
`${icon} Job ${id} (${status}, ${duration(seconds)})\n\n${summaryOf(job, output) || "(no output)"}\n\nReport: ${source.name}/${reportRel}\nQueued by: ${h.created_by}`,
|
|
232
|
+
h.to,
|
|
233
|
+
)
|
|
234
|
+
}
|
|
235
|
+
return { status, report: reportRel, notified }
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
// In-process Beacon send. Never throws; returns "sent" or why it wasn't.
|
|
239
|
+
export async function notify(text: string, to?: string): Promise<string> {
|
|
240
|
+
try {
|
|
241
|
+
const r = await beaconSend(beaconFromConfig(loadConfig()), text, { to: to || null, silent: false })
|
|
242
|
+
return r.ok ? "sent" : `${r.reason}: ${r.message}`
|
|
243
|
+
} catch (err) {
|
|
244
|
+
return `error: ${(err as Error).message}`
|
|
245
|
+
}
|
|
246
|
+
}
|
|
@@ -0,0 +1,307 @@
|
|
|
1
|
+
// One Scheduler tick: what the dispatcher runs every 5 minutes.
|
|
2
|
+
//
|
|
3
|
+
// 1. Take the tick lock (a tick still running from last time wins).
|
|
4
|
+
// 2. Run due module jobs (ScheduledJobDecl), paging only when one starts or
|
|
5
|
+
// stops failing.
|
|
6
|
+
// 3. For each source: take its lock, pull it if it's a git repository, run
|
|
7
|
+
// each due follow-up in turn, commit and push runs/.
|
|
8
|
+
// 4. Record the tick in last-tick.json (every dispatcher tick; status() reads
|
|
9
|
+
// it) and in tick.log (any tick where something ran or failed).
|
|
10
|
+
//
|
|
11
|
+
// See docs/2026-10-20_scheduler-spec.md.
|
|
12
|
+
|
|
13
|
+
import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, statSync, writeFileSync } from "node:fs"
|
|
14
|
+
import { join } from "node:path"
|
|
15
|
+
import { loadConfig } from "../config.ts"
|
|
16
|
+
import { MODULES } from "../modules/index.ts"
|
|
17
|
+
import type { ScheduledJobDecl } from "../modules/contract.ts"
|
|
18
|
+
import { tryLock } from "./flock.ts"
|
|
19
|
+
import { declSchedule, dueOccurrence, fmt, parseEvery, parseWhen } from "./schedule.ts"
|
|
20
|
+
import {
|
|
21
|
+
ID_RE,
|
|
22
|
+
lastTickPath,
|
|
23
|
+
lateGraceMs,
|
|
24
|
+
listSources,
|
|
25
|
+
loadJobs,
|
|
26
|
+
loadState,
|
|
27
|
+
saveState,
|
|
28
|
+
schedulerStateDir,
|
|
29
|
+
sourceLockPath,
|
|
30
|
+
tickLogPath,
|
|
31
|
+
type JobHeader,
|
|
32
|
+
type Source,
|
|
33
|
+
type TickRecord,
|
|
34
|
+
type TickRun,
|
|
35
|
+
} from "./schedulerJobs.ts"
|
|
36
|
+
import { jobEnv, notify, runJob, runProcess, selfCommand } from "./schedulerRun.ts"
|
|
37
|
+
import { commitAndPush, decideClaim, gitPull, readClaim, stationName } from "./schedulerGit.ts"
|
|
38
|
+
|
|
39
|
+
const LOG_MAX_BYTES = 1_000_000
|
|
40
|
+
const MODULE_JOB_TIMEOUT_MS = 10 * 60_000
|
|
41
|
+
|
|
42
|
+
export type TickResult = ({ ok: true } & TickRecord) | { ok: true; skipped: "locked"; message: string }
|
|
43
|
+
|
|
44
|
+
// ---- module jobs -----------------------------------------------------------------
|
|
45
|
+
|
|
46
|
+
export interface ModuleJobState {
|
|
47
|
+
last_run?: string
|
|
48
|
+
last_scheduled?: string
|
|
49
|
+
last_status?: "ok" | "failed"
|
|
50
|
+
failing_since?: string
|
|
51
|
+
last_output?: string
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export interface ModuleJobDeps {
|
|
55
|
+
stateDir: string
|
|
56
|
+
now: Date
|
|
57
|
+
run(decl: ScheduledJobDecl): Promise<{ ok: boolean; output: string }>
|
|
58
|
+
page(text: string): Promise<string>
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function moduleJobStatePath(dir: string, id: string) {
|
|
62
|
+
return join(dir, "module-jobs", `${id}.json`)
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// Runs whichever declarations are due. Pages when a job starts failing and when
|
|
66
|
+
// it recovers — never on every failed run, so a broken heartbeat can't train
|
|
67
|
+
// anyone to ignore the pager.
|
|
68
|
+
export async function runModuleJobs(decls: ScheduledJobDecl[], deps: ModuleJobDeps): Promise<{ ran: TickRun[]; errors: string[] }> {
|
|
69
|
+
const ran: TickRun[] = []
|
|
70
|
+
const errors: string[] = []
|
|
71
|
+
for (const decl of decls) {
|
|
72
|
+
const path = moduleJobStatePath(deps.stateDir, decl.id)
|
|
73
|
+
let state: ModuleJobState = {}
|
|
74
|
+
try {
|
|
75
|
+
state = JSON.parse(readFileSync(path, "utf8")) as ModuleJobState
|
|
76
|
+
} catch {
|
|
77
|
+
// First run.
|
|
78
|
+
}
|
|
79
|
+
let occ: Date | null
|
|
80
|
+
try {
|
|
81
|
+
if (decl.command[0] === "scheduler") throw new Error("a module job can't run `scheduler`")
|
|
82
|
+
occ = dueOccurrence({ schedule: declSchedule(decl) }, state, deps.now)
|
|
83
|
+
} catch (err) {
|
|
84
|
+
errors.push(`module job ${decl.id}: ${(err as Error).message}`)
|
|
85
|
+
continue
|
|
86
|
+
}
|
|
87
|
+
if (!occ) continue
|
|
88
|
+
// The tick's time, not the wall clock after earlier jobs: intervals stay
|
|
89
|
+
// aligned to ticks instead of drifting by each run's duration.
|
|
90
|
+
const started = deps.now
|
|
91
|
+
const r = await deps.run(decl)
|
|
92
|
+
const status = r.ok ? "ok" : "failed"
|
|
93
|
+
const tail = r.output.trim().split("\n").slice(-10).join("\n").slice(-600)
|
|
94
|
+
let notified: string | undefined
|
|
95
|
+
if (status === "failed" && state.last_status !== "failed") {
|
|
96
|
+
notified = await deps.page(`⚠️ ${decl.id} failed (spinuptui ${decl.command.join(" ")})\n\n${tail || "(no output)"}`)
|
|
97
|
+
state.failing_since = fmt(started)
|
|
98
|
+
} else if (status === "ok" && state.last_status === "failed") {
|
|
99
|
+
notified = await deps.page(`✅ ${decl.id} is working again (failing since ${state.failing_since ?? "?"})`)
|
|
100
|
+
delete state.failing_since
|
|
101
|
+
}
|
|
102
|
+
state.last_status = status
|
|
103
|
+
state.last_run = fmt(started)
|
|
104
|
+
state.last_scheduled = fmt(occ)
|
|
105
|
+
state.last_output = tail
|
|
106
|
+
mkdirSync(join(deps.stateDir, "module-jobs"), { recursive: true })
|
|
107
|
+
writeFileSync(path, JSON.stringify(state, null, 2) + "\n")
|
|
108
|
+
ran.push({ source: "module", id: decl.id, status, ...(notified ? { notified } : {}) })
|
|
109
|
+
}
|
|
110
|
+
return { ran, errors }
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
async function configuredModuleJobs(): Promise<ScheduledJobDecl[]> {
|
|
114
|
+
const out: ScheduledJobDecl[] = []
|
|
115
|
+
for (const m of MODULES) {
|
|
116
|
+
if (m.jobs.length === 0) continue
|
|
117
|
+
const s = await m.status().catch(() => null)
|
|
118
|
+
if (s && s.state !== "not_configured") out.push(...m.jobs)
|
|
119
|
+
}
|
|
120
|
+
return out
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// ---- sources ---------------------------------------------------------------------
|
|
124
|
+
|
|
125
|
+
// `late:`: is this slot too old to run? Only fixed-time slots can be late; an
|
|
126
|
+
// interval job's slot is always "now".
|
|
127
|
+
export function isTooLate(header: JobHeader, occ: Date, now: Date): boolean {
|
|
128
|
+
const grace = lateGraceMs(header.late)
|
|
129
|
+
if (grace === null || grace === undefined) return false
|
|
130
|
+
if (header.every && parseEvery(header.every).type === "interval") return false
|
|
131
|
+
return now.getTime() - occ.getTime() > grace
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
const hm = (d: Date) => fmt(d).slice(11)
|
|
135
|
+
|
|
136
|
+
async function tickSource(
|
|
137
|
+
source: Source,
|
|
138
|
+
now: Date,
|
|
139
|
+
only: string | null,
|
|
140
|
+
force: boolean,
|
|
141
|
+
via: "dispatcher" | "manual",
|
|
142
|
+
): Promise<{ ran: TickRun[]; errors: string[]; notRunHere?: string }> {
|
|
143
|
+
const ran: TickRun[] = []
|
|
144
|
+
const errors: string[] = []
|
|
145
|
+
if (!existsSync(source.path)) return { ran, errors: [`source ${source.name}: ${source.path} does not exist`] }
|
|
146
|
+
const lock = tryLock(sourceLockPath(source))
|
|
147
|
+
if (!lock) return { ran, errors: [`source ${source.name}: busy (another dispatcher holds its lock); skipped this tick`] }
|
|
148
|
+
try {
|
|
149
|
+
if (source.git) {
|
|
150
|
+
const e = await gitPull(source.path)
|
|
151
|
+
if (e) errors.push(`source ${source.name}: ${e}`)
|
|
152
|
+
// One station per git source. Only the dispatcher claims; a manual `run`
|
|
153
|
+
// on a machine that doesn't hold the claim is deliberate, so it proceeds.
|
|
154
|
+
if (via === "dispatcher") {
|
|
155
|
+
const d = await decideClaim(source)
|
|
156
|
+
if (!d.run) {
|
|
157
|
+
if (d.holder === null && d.why.startsWith("couldn't")) errors.push(`source ${d.why}`)
|
|
158
|
+
return { ran, errors, notRunHere: d.why }
|
|
159
|
+
}
|
|
160
|
+
} else {
|
|
161
|
+
const holder = readClaim(source)
|
|
162
|
+
if (holder && holder !== stationName()) process.stderr.write(`Warning: ${source.name} is run by ${holder}; running it here anyway, because you asked.\n`)
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
for (const job of loadJobs(source)) {
|
|
166
|
+
const id = job.header.id
|
|
167
|
+
if (only && id !== only) continue
|
|
168
|
+
if (job.problems.length) {
|
|
169
|
+
if (!id || !ID_RE.test(id)) continue
|
|
170
|
+
const sig = job.problems.join("; ")
|
|
171
|
+
const state = loadState(source, id)
|
|
172
|
+
if (state.invalid !== sig) {
|
|
173
|
+
state.invalid = sig
|
|
174
|
+
saveState(source, id, state)
|
|
175
|
+
const notified = await notify(`⚠️ Job file ${source.name}/jobs/${job.name} is invalid and was skipped: ${sig}`)
|
|
176
|
+
ran.push({ source: source.name, id, status: "invalid", notified })
|
|
177
|
+
}
|
|
178
|
+
continue
|
|
179
|
+
}
|
|
180
|
+
const state = loadState(source, id!)
|
|
181
|
+
if (state.invalid !== undefined) {
|
|
182
|
+
delete state.invalid
|
|
183
|
+
saveState(source, id!, state)
|
|
184
|
+
}
|
|
185
|
+
let occ: Date | null
|
|
186
|
+
try {
|
|
187
|
+
occ = force ? forcedOccurrence(job.header.when, now) : dueOccurrence(job.header, state, now)
|
|
188
|
+
} catch (err) {
|
|
189
|
+
errors.push(`job ${source.name}/${id}: ${(err as Error).message}`)
|
|
190
|
+
continue
|
|
191
|
+
}
|
|
192
|
+
if (!occ) continue
|
|
193
|
+
if (!force && isTooLate(job.header, occ, now)) {
|
|
194
|
+
state.last_scheduled = fmt(occ)
|
|
195
|
+
state.last_status = "skipped"
|
|
196
|
+
delete state.retry_after
|
|
197
|
+
saveState(source, id!, state)
|
|
198
|
+
const notified =
|
|
199
|
+
(job.header.notify || "beacon") === "none"
|
|
200
|
+
? "skipped"
|
|
201
|
+
: await notify(
|
|
202
|
+
`⏭ Skipped job ${id}: it was due at ${fmt(occ)}, and this station only got to it at ${hm(now)}. Its late: ${job.header.late} policy says not to run it late.\n\nQueued by: ${job.header.created_by}`,
|
|
203
|
+
job.header.to,
|
|
204
|
+
)
|
|
205
|
+
ran.push({ source: source.name, id: id!, status: "skipped", notified })
|
|
206
|
+
continue
|
|
207
|
+
}
|
|
208
|
+
process.stderr.write(`→ running ${source.name}/${id} (scheduled ${fmt(occ)})\n`)
|
|
209
|
+
const outcome = await runJob(source, job, occ, now)
|
|
210
|
+
ran.push({ source: source.name, id: id!, ...outcome })
|
|
211
|
+
}
|
|
212
|
+
if (source.git) {
|
|
213
|
+
const mine = ran.filter((r) => r.source === source.name)
|
|
214
|
+
const e = await commitAndPush(source.path, ["runs"], "runs: " + (mine.length ? mine.map((r) => `${r.id} (${r.status})`).join(", ") : "state"))
|
|
215
|
+
if (e) errors.push(`source ${source.name}: ${e}`)
|
|
216
|
+
}
|
|
217
|
+
} finally {
|
|
218
|
+
lock.release()
|
|
219
|
+
}
|
|
220
|
+
return { ran, errors }
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
// `scheduler run`: a one-shot whose time has passed is recorded as its own slot,
|
|
224
|
+
// so the next tick doesn't run it again; anything else runs as "now".
|
|
225
|
+
function forcedOccurrence(when: string | undefined, now: Date): Date {
|
|
226
|
+
if (when) {
|
|
227
|
+
const w = parseWhen(when)
|
|
228
|
+
if (w <= now) return w
|
|
229
|
+
}
|
|
230
|
+
return new Date(new Date(now).setSeconds(0, 0))
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
// ---- tick ------------------------------------------------------------------------
|
|
234
|
+
|
|
235
|
+
export interface TickOptions {
|
|
236
|
+
via: "dispatcher" | "manual"
|
|
237
|
+
// `scheduler run <id>`: one job, now, whether due or not.
|
|
238
|
+
only?: { source: string; id: string }
|
|
239
|
+
now?: Date
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
export async function tick(opts: TickOptions): Promise<TickResult> {
|
|
243
|
+
const dir = schedulerStateDir()
|
|
244
|
+
mkdirSync(dir, { recursive: true })
|
|
245
|
+
const lock = tryLock(join(dir, "tick.lock"))
|
|
246
|
+
if (!lock) return { ok: true, skipped: "locked", message: "A previous tick is still running." }
|
|
247
|
+
const startedAt = Date.now()
|
|
248
|
+
const now = opts.now ?? new Date()
|
|
249
|
+
const ran: TickRun[] = []
|
|
250
|
+
const errors: string[] = []
|
|
251
|
+
const notRunHere: string[] = []
|
|
252
|
+
try {
|
|
253
|
+
if (!opts.only) {
|
|
254
|
+
const decls = await configuredModuleJobs()
|
|
255
|
+
if (decls.length) {
|
|
256
|
+
const r = await runModuleJobs(decls, {
|
|
257
|
+
stateDir: dir,
|
|
258
|
+
now,
|
|
259
|
+
run: async (decl) => {
|
|
260
|
+
const p = await runProcess([...selfCommand(), ...decl.command], { cwd: dir, env: jobEnv(), timeoutMs: MODULE_JOB_TIMEOUT_MS })
|
|
261
|
+
return { ok: p.code === 0, output: p.output }
|
|
262
|
+
},
|
|
263
|
+
page: (text) => notify(text),
|
|
264
|
+
})
|
|
265
|
+
ran.push(...r.ran)
|
|
266
|
+
errors.push(...r.errors)
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
for (const source of listSources(loadConfig())) {
|
|
270
|
+
if (opts.only && source.name !== opts.only.source) continue
|
|
271
|
+
const r = await tickSource(source, now, opts.only?.id ?? null, !!opts.only, opts.via)
|
|
272
|
+
ran.push(...r.ran)
|
|
273
|
+
errors.push(...r.errors)
|
|
274
|
+
if (r.notRunHere) notRunHere.push(r.notRunHere)
|
|
275
|
+
}
|
|
276
|
+
for (const r of ran) if (r.notified && r.notified !== "sent" && r.notified !== "skipped") errors.push(`${r.source}/${r.id}: not paged (${r.notified})`)
|
|
277
|
+
const record: TickRecord = {
|
|
278
|
+
at: new Date().toISOString(),
|
|
279
|
+
via: opts.via,
|
|
280
|
+
durationMs: Date.now() - startedAt,
|
|
281
|
+
ran,
|
|
282
|
+
errors,
|
|
283
|
+
...(notRunHere.length ? { notRunHere } : {}),
|
|
284
|
+
}
|
|
285
|
+
// Only the dispatcher's ticks count as "the station is alive": a manual
|
|
286
|
+
// `run` or `tick --here` must not mask a dispatcher that has stopped.
|
|
287
|
+
if (opts.via === "dispatcher") {
|
|
288
|
+
writeFileSync(`${lastTickPath()}.tmp`, JSON.stringify(record, null, 2) + "\n")
|
|
289
|
+
renameSync(`${lastTickPath()}.tmp`, lastTickPath())
|
|
290
|
+
}
|
|
291
|
+
if (ran.length || errors.length) appendTickLog(record)
|
|
292
|
+
return { ok: true, ...record }
|
|
293
|
+
} finally {
|
|
294
|
+
lock.release()
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
function appendTickLog(record: TickRecord): void {
|
|
299
|
+
try {
|
|
300
|
+
const path = tickLogPath()
|
|
301
|
+
const size = existsSync(path) ? statSync(path).size : 0
|
|
302
|
+
if (size > LOG_MAX_BYTES) renameSync(path, `${path}.1`)
|
|
303
|
+
appendFileSync(path, JSON.stringify(record) + "\n")
|
|
304
|
+
} catch {
|
|
305
|
+
// The log is history; losing a line must not fail the tick.
|
|
306
|
+
}
|
|
307
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
// Beacon's module declaration: the first implementation of the contract.
|
|
2
|
+
|
|
3
|
+
import { loadConfig } from "../config.ts"
|
|
4
|
+
import { beaconFromConfig, lastSendLogEntry, maskChatId, maskToken } from "../lib/beacon.ts"
|
|
5
|
+
import { moduleStateDir, moduleWorkDir, type ModuleCheck, type ModuleDefinition, type ModuleStatus } from "./contract.ts"
|
|
6
|
+
|
|
7
|
+
export const beaconModule: ModuleDefinition = {
|
|
8
|
+
id: "beacon",
|
|
9
|
+
name: "Beacon",
|
|
10
|
+
summary: "Tells you when something needs you: one pager for the fleet and every module.",
|
|
11
|
+
dependsOn: [],
|
|
12
|
+
configKey: "beacon",
|
|
13
|
+
stateDir: () => moduleStateDir("beacon"),
|
|
14
|
+
workDir: () => moduleWorkDir("beacon"),
|
|
15
|
+
jobs: [],
|
|
16
|
+
async status(): Promise<ModuleStatus> {
|
|
17
|
+
const beacon = beaconFromConfig(loadConfig())
|
|
18
|
+
if (beacon.channels.length === 0) {
|
|
19
|
+
return {
|
|
20
|
+
id: "beacon",
|
|
21
|
+
state: "not_configured",
|
|
22
|
+
summary: "No channels configured",
|
|
23
|
+
checks: [],
|
|
24
|
+
remedy: "spinuptui beacon add telegram <name> --env-file <path>",
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
const checks: ModuleCheck[] = beacon.channels.map((c) => ({
|
|
28
|
+
name: `channel ${c.name}`,
|
|
29
|
+
ok: true,
|
|
30
|
+
detail: `telegram bot ${maskToken(c.channel.botToken)} → chat ${maskChatId(c.channel.chatId)}${c.env ? " (from environment)" : ""}${c.name === beacon.defaultChannel ? ", default" : ""}`,
|
|
31
|
+
}))
|
|
32
|
+
checks.push({
|
|
33
|
+
name: "default channel",
|
|
34
|
+
ok: beacon.defaultChannel !== null,
|
|
35
|
+
detail: beacon.defaultChannel ?? "none — senders must pass --to",
|
|
36
|
+
})
|
|
37
|
+
const last = lastSendLogEntry()
|
|
38
|
+
checks.push({
|
|
39
|
+
name: "last send",
|
|
40
|
+
ok: last?.ok ?? true,
|
|
41
|
+
detail: last ? `${last.ok ? "delivered" : `failed (${last.error})`} via ${last.channel} at ${last.at}` : "nothing sent yet",
|
|
42
|
+
})
|
|
43
|
+
const failing = checks.find((c) => !c.ok)
|
|
44
|
+
return {
|
|
45
|
+
id: "beacon",
|
|
46
|
+
state: failing ? "degraded" : "ok",
|
|
47
|
+
summary: failing ? `${failing.name}: ${failing.detail}` : `${beacon.channels.length} channel${beacon.channels.length === 1 ? "" : "s"}, default ${beacon.defaultChannel}`,
|
|
48
|
+
checks,
|
|
49
|
+
...(failing ? { remedy: "spinuptui beacon test" } : {}),
|
|
50
|
+
}
|
|
51
|
+
},
|
|
52
|
+
}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
// The module contract (docs/2026-10-16_beacon-and-module-contract-spec.md).
|
|
2
|
+
//
|
|
3
|
+
// A module is a feature that keeps working after the TUI is closed. Each one
|
|
4
|
+
// describes itself with a ModuleDefinition so `spinuptui modules`, the future
|
|
5
|
+
// Modules screen and `station setup` are views over the same data rather than
|
|
6
|
+
// knowing every module by hand.
|
|
7
|
+
//
|
|
8
|
+
// Rules the shape encodes:
|
|
9
|
+
// - Config is a machine-level top-level key in config.json named after the
|
|
10
|
+
// module, written only by deliberate setup commands.
|
|
11
|
+
// - Anything a module writes while running goes in its state directory, never
|
|
12
|
+
// config.json (whose writes are neither atomic nor locked).
|
|
13
|
+
// - The user's work (jobs, reports, write-ups — anything a person reads, keeps
|
|
14
|
+
// or shares) goes in its work directory, ~/SpinupTUI/<id>/, not next to the
|
|
15
|
+
// tokens in the hidden config directory.
|
|
16
|
+
// - Recurring work is declared in `jobs` for Scheduler to run; a module never
|
|
17
|
+
// installs an OS scheduler unit of its own.
|
|
18
|
+
// - status() is computed on demand, is cheap, and never sends anything.
|
|
19
|
+
|
|
20
|
+
import { homedir } from "node:os"
|
|
21
|
+
import { join } from "node:path"
|
|
22
|
+
import { configDir, loadConfig } from "../config.ts"
|
|
23
|
+
|
|
24
|
+
export type ModuleId = "beacon" | "scheduler" | "sentinel" | "samurai"
|
|
25
|
+
|
|
26
|
+
export interface ScheduledJobDecl {
|
|
27
|
+
id: string // e.g. "sentinel.tick"
|
|
28
|
+
every?: string // e.g. "5m"
|
|
29
|
+
at?: string // e.g. "08:00"
|
|
30
|
+
// A spinuptui subcommand's arguments, never a path into a checkout.
|
|
31
|
+
command: string[]
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export interface ModuleCheck {
|
|
35
|
+
name: string
|
|
36
|
+
ok: boolean
|
|
37
|
+
detail: string
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export interface ModuleStatus {
|
|
41
|
+
id: ModuleId
|
|
42
|
+
state: "not_configured" | "ok" | "degraded"
|
|
43
|
+
summary: string
|
|
44
|
+
checks: ModuleCheck[]
|
|
45
|
+
remedy?: string
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export interface ModuleDefinition {
|
|
49
|
+
id: ModuleId
|
|
50
|
+
name: string
|
|
51
|
+
// Outcome copy, shared with the landing page's module card.
|
|
52
|
+
summary: string
|
|
53
|
+
dependsOn: ModuleId[]
|
|
54
|
+
configKey: string
|
|
55
|
+
// Machine state: locks, logs, high-water marks.
|
|
56
|
+
stateDir(): string
|
|
57
|
+
// The user's work. Created on first use, never at install.
|
|
58
|
+
workDir(): string
|
|
59
|
+
jobs: ScheduledJobDecl[]
|
|
60
|
+
status(): Promise<ModuleStatus>
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export function moduleStateDir(id: ModuleId): string {
|
|
64
|
+
return join(configDir(), "modules", id)
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const expandHome = (p: string) => (p === "~" ? homedir() : p.startsWith("~/") ? join(homedir(), p.slice(2)) : p)
|
|
68
|
+
|
|
69
|
+
// ~/SpinupTUI by default: visible and easy to back up or turn into a git
|
|
70
|
+
// repository. Deliberately not ~/Documents, which iCloud may sync (a second
|
|
71
|
+
// syncer writing inside a .git corrupts it).
|
|
72
|
+
export function spinuptuiHome(): string {
|
|
73
|
+
const p = process.env.SPINUPTUI_HOME?.trim() || loadConfig().homeDir || join(homedir(), "SpinupTUI")
|
|
74
|
+
return expandHome(p)
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export function moduleWorkDir(id: ModuleId): string {
|
|
78
|
+
return join(spinuptuiHome(), id)
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// A module that hasn't shipped yet: just enough to draw its card.
|
|
82
|
+
export type UpcomingModule = Pick<ModuleDefinition, "id" | "name" | "summary" | "dependsOn">
|