spinuptui 1.3.0-alpha.1 → 1.3.0-alpha.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -1
- package/README.md +13 -3
- package/package.json +1 -1
- package/src/config.ts +25 -0
- package/src/index.tsx +26 -38
- package/src/lib/cliHome.ts +107 -0
- package/src/lib/cliScheduler.ts +630 -0
- package/src/lib/flock.ts +51 -0
- package/src/lib/heartbeat.test.ts +86 -0
- package/src/lib/heartbeat.ts +245 -0
- package/src/lib/home.test.ts +102 -0
- package/src/lib/home.ts +442 -0
- package/src/lib/schedule.ts +156 -0
- package/src/lib/scheduler.test.ts +353 -0
- package/src/lib/schedulerGit.ts +119 -0
- package/src/lib/schedulerInstall.ts +325 -0
- package/src/lib/schedulerJobs.ts +290 -0
- package/src/lib/schedulerRun.ts +247 -0
- package/src/lib/schedulerTick.ts +322 -0
- package/src/lib/spinupweb.test.ts +22 -0
- package/src/lib/spinupweb.ts +11 -0
- package/src/lib/stackCache.test.ts +43 -0
- package/src/lib/stackCache.ts +35 -1
- package/src/modules/beacon.ts +2 -1
- package/src/modules/contract.ts +22 -1
- package/src/modules/index.ts +2 -7
- package/src/modules/scheduler.ts +159 -0
- package/src/ui/App.tsx +19 -1
- package/src/ui/Details.tsx +7 -1
- package/src/ui/Onboarding.tsx +71 -3
- package/src/ui/main.tsx +42 -0
- package/src/ui/store.tsx +87 -3
- package/src/ui/views/AccountSlugPrompt.tsx +102 -0
- package/src/ui/views/Browser.tsx +5 -5
- package/src/ui/views/HeartbeatSetup.tsx +284 -0
- package/src/ui/views/HomeSync.tsx +299 -0
- package/src/ui/views/Modules.tsx +69 -5
- package/src/ui/views/SchedulerInstall.tsx +172 -0
- package/src/ui/views/Search.tsx +5 -6
- package/src/ui/views/ServerActions.tsx +6 -4
- package/src/ui/views/Stacks.tsx +22 -9
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
// Running one Scheduler job: the kinds, the agent adapters, the report and state
|
|
2
|
+
// it leaves behind, and the Beacon message about it.
|
|
3
|
+
//
|
|
4
|
+
// The report/state layout and the agent rules (read-only by default, a fixed
|
|
5
|
+
// always-deny list, a SUMMARY: paragraph as the digest, a lapsed login retried
|
|
6
|
+
// rather than counted as a run) carry over from the git-backed job runner; see
|
|
7
|
+
// docs/2026-10-20_scheduler-spec.md.
|
|
8
|
+
|
|
9
|
+
import { existsSync, writeFileSync } from "node:fs"
|
|
10
|
+
import { hostname, homedir } from "node:os"
|
|
11
|
+
import { join } from "node:path"
|
|
12
|
+
import { loadConfig } from "../config.ts"
|
|
13
|
+
import { beaconFromConfig, beaconSend } from "./beacon.ts"
|
|
14
|
+
import { spawn } from "./spawn.ts"
|
|
15
|
+
import { prepareWorkDir } from "./home.ts"
|
|
16
|
+
import { fmt } from "./schedule.ts"
|
|
17
|
+
import { agentOf, DEFAULT_TIMEOUT_MIN, expandHome, loadState, runsDir, saveState, splitArgs, type Job, type Source } from "./schedulerJobs.ts"
|
|
18
|
+
|
|
19
|
+
export const DEFAULT_ALLOW = "Read,Grep,Glob"
|
|
20
|
+
// Never allowed in a claude job, whatever its allow: line says.
|
|
21
|
+
export const ALWAYS_DENY = "Bash(git push *),Bash(git reset *),Bash(git clean *),Bash(rm -rf *),Bash(sudo *)"
|
|
22
|
+
const AUTH_FAIL_RE =
|
|
23
|
+
/Failed to authenticate|OAuth (session|token)|could not be refreshed|Invalid API key|API key.*(expired|invalid)|Please run .?\/login|not (logged in|authenticated)/i
|
|
24
|
+
const REPORT_CAP = 200_000
|
|
25
|
+
// Tools a launchd PATH doesn't include by default.
|
|
26
|
+
const EXTRA_PATH = [join(homedir(), ".local/bin"), join(homedir(), ".bun/bin"), "/opt/homebrew/bin", "/usr/local/bin"]
|
|
27
|
+
|
|
28
|
+
export type RunStatus = "ok" | "failed" | "timeout" | "error" | "auth"
|
|
29
|
+
|
|
30
|
+
function preamble(id: string, kind: string, createdBy: string, runs: string): string {
|
|
31
|
+
return `You are running unattended on ${hostname().split(".")[0]} as scheduled job \`${id}\` (kind: ${kind}), queued by: ${createdBy}.
|
|
32
|
+
Nobody is watching this run and nobody can answer questions. Do the task below, then stop.
|
|
33
|
+
Rules: report, don't act. Do not modify files outside ${runs}, do not push, deploy, post to clients or email anyone, unless the task explicitly says to.
|
|
34
|
+
Your final message is the report. Start it with a line \`SUMMARY:\` followed by one short paragraph (under 600 characters, plain text) that stands on its own in a chat message. Then give the details in Markdown.
|
|
35
|
+
|
|
36
|
+
--- TASK ---
|
|
37
|
+
`
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
// How to start this same SpinupTUI: the Bun running now plus its entry point, or
|
|
41
|
+
// just the executable when it's a compiled binary. Module jobs and `spinuptui`
|
|
42
|
+
// jobs use it, so they always run the dispatcher's own version.
|
|
43
|
+
export function selfCommand(): string[] {
|
|
44
|
+
const compiled = Bun.main.startsWith("/$bunfs") || !/\bbun(\.exe)?$/.test(process.execPath)
|
|
45
|
+
return compiled ? [process.execPath] : [process.execPath, Bun.main]
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export function jobEnv(): Record<string, string> {
|
|
49
|
+
const env: Record<string, string> = {}
|
|
50
|
+
for (const [k, v] of Object.entries(process.env)) if (v !== undefined) env[k] = v
|
|
51
|
+
const parts = (env.PATH || "/usr/bin:/bin:/usr/sbin:/sbin").split(":")
|
|
52
|
+
env.PATH = [...EXTRA_PATH.filter((p) => !parts.includes(p)), ...parts].join(":")
|
|
53
|
+
// A job is not a dispatcher: a nested `scheduler tick` must refuse.
|
|
54
|
+
delete env.SPINUPTUI_SCHEDULER_DISPATCHER
|
|
55
|
+
return env
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export interface ProcResult {
|
|
59
|
+
code: number | null
|
|
60
|
+
output: string
|
|
61
|
+
timedOut: boolean
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// Runs a command with stdout and stderr interleaved into one stream, in its own
|
|
65
|
+
// process group so a timeout can stop everything it started.
|
|
66
|
+
export async function runProcess(cmd: string[], opts: { cwd: string; env: Record<string, string>; timeoutMs: number }): Promise<ProcResult> {
|
|
67
|
+
const proc = spawn(["/bin/sh", "-c", 'exec "$@" 2>&1', "sh", ...cmd], {
|
|
68
|
+
cwd: opts.cwd,
|
|
69
|
+
env: opts.env,
|
|
70
|
+
stdin: "ignore",
|
|
71
|
+
stdout: "pipe",
|
|
72
|
+
stderr: "ignore",
|
|
73
|
+
detached: true,
|
|
74
|
+
} as Parameters<typeof spawn>[1])
|
|
75
|
+
let timedOut = false
|
|
76
|
+
const kill = (sig: NodeJS.Signals) => {
|
|
77
|
+
try {
|
|
78
|
+
process.kill(-proc.pid, sig)
|
|
79
|
+
} catch {
|
|
80
|
+
// Already gone.
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
const timer = setTimeout(() => {
|
|
84
|
+
timedOut = true
|
|
85
|
+
kill("SIGTERM")
|
|
86
|
+
setTimeout(() => kill("SIGKILL"), 20_000).unref?.()
|
|
87
|
+
}, opts.timeoutMs)
|
|
88
|
+
const output = await new Response(proc.stdout as ReadableStream).text()
|
|
89
|
+
const code = await proc.exited
|
|
90
|
+
clearTimeout(timer)
|
|
91
|
+
return { code: timedOut ? null : code, output, timedOut }
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
export function summaryOf(job: Job, output: string): string {
|
|
95
|
+
const text = output.trim()
|
|
96
|
+
if (agentOf(job.header)) {
|
|
97
|
+
const m = text.match(/SUMMARY:\s*([\s\S]+?)(\n\s*\n|$)/)
|
|
98
|
+
return (m ? m[1]! : text).trim().slice(0, 900)
|
|
99
|
+
}
|
|
100
|
+
if (job.header.kind === "spinuptui") {
|
|
101
|
+
// A failed JSON result says what went wrong better than its tail does.
|
|
102
|
+
const last = text.split("\n").reverse().find((l) => l.trim().startsWith("{"))
|
|
103
|
+
if (last) {
|
|
104
|
+
try {
|
|
105
|
+
const r = JSON.parse(last) as { ok?: boolean; message?: string }
|
|
106
|
+
if (r.ok === false && r.message) return r.message.slice(0, 900)
|
|
107
|
+
} catch {
|
|
108
|
+
// Not JSON; fall through to the tail.
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
return text.split("\n").slice(-15).join("\n").slice(-900)
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function buildCommand(job: Job, workdir: string, env: Record<string, string>, runs: string): string[] | string {
|
|
116
|
+
const h = job.header
|
|
117
|
+
const kind = h.kind!
|
|
118
|
+
if (kind === "script") return ["/bin/bash", "-c", job.body]
|
|
119
|
+
if (kind === "spinuptui") return [...selfCommand(), ...splitArgs(job.body)]
|
|
120
|
+
const agent = agentOf(h)!
|
|
121
|
+
const prompt = preamble(h.id!, agent === "codex" ? "agent/codex" : "agent/claude", h.created_by!, runs) + job.body
|
|
122
|
+
const bin = Bun.which(agent, { PATH: env.PATH })
|
|
123
|
+
if (!bin) return `${agent} not found on PATH`
|
|
124
|
+
if (agent === "codex") return [bin, "exec", "-s", h.sandbox || "read-only", "-C", workdir, "--skip-git-repo-check", prompt]
|
|
125
|
+
const cmd = [bin, "-p", prompt, "--output-format", "text", "--permission-mode", "default", "--allowedTools", h.allow || DEFAULT_ALLOW, "--disallowedTools", ALWAYS_DENY]
|
|
126
|
+
for (const d of (h.add_dirs ?? "").split(",")) if (d.trim()) cmd.push("--add-dir", expandHome(d.trim()))
|
|
127
|
+
if (h.model) cmd.push("--model", h.model)
|
|
128
|
+
return cmd
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
export interface RunOutcome {
|
|
132
|
+
status: RunStatus
|
|
133
|
+
report?: string // source-relative
|
|
134
|
+
notified?: "sent" | "skipped" | string // string = why it failed
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
const pad2 = (n: number) => String(n).padStart(2, "0")
|
|
138
|
+
const stamp = (d: Date) => fmt(d).replace(" ", "-").replace(":", "")
|
|
139
|
+
const duration = (s: number) => `${Math.floor(s / 60)}m ${pad2(s % 60)}s`
|
|
140
|
+
|
|
141
|
+
export async function runJob(source: Source, job: Job, occurrence: Date, now = new Date()): Promise<RunOutcome> {
|
|
142
|
+
const h = job.header
|
|
143
|
+
const id = h.id!
|
|
144
|
+
const env = jobEnv()
|
|
145
|
+
const workdir = expandHome(h.workdir || source.path)
|
|
146
|
+
const timeoutMs = Number(h.timeout || DEFAULT_TIMEOUT_MIN) * 60_000
|
|
147
|
+
const runs = join(runsDir(source), id)
|
|
148
|
+
prepareWorkDir(runs)
|
|
149
|
+
const reportRel = `runs/${id}/${stamp(now)}.md`
|
|
150
|
+
|
|
151
|
+
const started = new Date()
|
|
152
|
+
let status: RunStatus
|
|
153
|
+
let code: number | null = null
|
|
154
|
+
let output: string
|
|
155
|
+
const cmd = existsSync(workdir) ? buildCommand(job, workdir, env, runs) : `workdir does not exist on this machine: ${workdir}`
|
|
156
|
+
if (typeof cmd === "string") {
|
|
157
|
+
status = "error"
|
|
158
|
+
output = cmd
|
|
159
|
+
} else {
|
|
160
|
+
try {
|
|
161
|
+
const r = await runProcess(cmd, { cwd: workdir, env, timeoutMs })
|
|
162
|
+
code = r.code
|
|
163
|
+
output = r.output
|
|
164
|
+
status = r.timedOut ? "timeout" : r.code === 0 ? "ok" : "failed"
|
|
165
|
+
if (r.timedOut) output += `\n\n[scheduler] killed after ${timeoutMs / 60_000} minutes`
|
|
166
|
+
} catch (err) {
|
|
167
|
+
status = "error"
|
|
168
|
+
output = `could not start: ${(err as Error).message}`
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
if (agentOf(h) && status !== "ok" && AUTH_FAIL_RE.test(output)) status = "auth"
|
|
172
|
+
return finish(source, job, occurrence, started, new Date(), status, code, output, reportRel)
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
async function finish(
|
|
176
|
+
source: Source,
|
|
177
|
+
job: Job,
|
|
178
|
+
occurrence: Date,
|
|
179
|
+
started: Date,
|
|
180
|
+
finished: Date,
|
|
181
|
+
status: RunStatus,
|
|
182
|
+
code: number | null,
|
|
183
|
+
output: string,
|
|
184
|
+
reportRel: string,
|
|
185
|
+
): Promise<RunOutcome> {
|
|
186
|
+
const h = job.header
|
|
187
|
+
const id = h.id!
|
|
188
|
+
const seconds = Math.round((finished.getTime() - started.getTime()) / 1000)
|
|
189
|
+
const state = loadState(source, id)
|
|
190
|
+
|
|
191
|
+
if (status === "auth") {
|
|
192
|
+
// The agent isn't logged in, so nothing ran: no report. Retry in 30 minutes,
|
|
193
|
+
// and page only once per scheduled slot.
|
|
194
|
+
const agent = agentOf(h)!
|
|
195
|
+
state.last_status = status
|
|
196
|
+
state.retry_after = fmt(new Date(Date.now() + 30 * 60_000))
|
|
197
|
+
const first = state.auth_alerted_for !== fmt(occurrence)
|
|
198
|
+
state.auth_alerted_for = fmt(occurrence)
|
|
199
|
+
saveState(source, id, state)
|
|
200
|
+
if (!first) return { status, notified: "skipped" }
|
|
201
|
+
const how = agent === "claude" ? "claude, then /login" : "codex login"
|
|
202
|
+
const notified = await notify(
|
|
203
|
+
`⚠️ Job ${id} did not run: the ${agent} login on ${hostname().split(".")[0]} has lapsed. Log in there (run \`${how}\`); it retries every 30 min until ${h.expires}.`,
|
|
204
|
+
h.to,
|
|
205
|
+
)
|
|
206
|
+
return { status, notified }
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
if (output.length > REPORT_CAP) output = output.slice(0, REPORT_CAP) + `\n\n[scheduler] output truncated at ${REPORT_CAP} characters`
|
|
210
|
+
const kindLabel = agentOf(h) ? `agent (${agentOf(h)})` : h.kind!
|
|
211
|
+
writeFileSync(
|
|
212
|
+
join(source.path, reportRel),
|
|
213
|
+
`# ${id} — ${status}\n\n` +
|
|
214
|
+
`- Scheduled: ${fmt(occurrence)}\n- Started: ${fmt(started)}:${pad2(started.getSeconds())}\n- Duration: ${duration(seconds)}\n` +
|
|
215
|
+
`- Exit code: ${code ?? "None"}\n- Kind: ${kindLabel}\n- Job file: jobs/${id}.md\n- Queued by: ${h.created_by}\n\n---\n\n` +
|
|
216
|
+
output.trimEnd() +
|
|
217
|
+
"\n",
|
|
218
|
+
)
|
|
219
|
+
state.last_status = status
|
|
220
|
+
state.last_report = reportRel
|
|
221
|
+
delete state.retry_after
|
|
222
|
+
delete state.auth_alerted_for
|
|
223
|
+
state.runs = (state.runs ?? 0) + 1
|
|
224
|
+
state.last_run = fmt(started)
|
|
225
|
+
state.last_scheduled = fmt(occurrence)
|
|
226
|
+
saveState(source, id, state)
|
|
227
|
+
|
|
228
|
+
let notified: RunOutcome["notified"] = "skipped"
|
|
229
|
+
if ((h.notify || "beacon") !== "none") {
|
|
230
|
+
const icon = { ok: "✅", failed: "❌", timeout: "⏱️", error: "❌" }[status as Exclude<RunStatus, "auth">] ?? "•"
|
|
231
|
+
notified = await notify(
|
|
232
|
+
`${icon} Job ${id} (${status}, ${duration(seconds)})\n\n${summaryOf(job, output) || "(no output)"}\n\nReport: ${source.name}/${reportRel}\nQueued by: ${h.created_by}`,
|
|
233
|
+
h.to,
|
|
234
|
+
)
|
|
235
|
+
}
|
|
236
|
+
return { status, report: reportRel, notified }
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
// In-process Beacon send. Never throws; returns "sent" or why it wasn't.
|
|
240
|
+
export async function notify(text: string, to?: string): Promise<string> {
|
|
241
|
+
try {
|
|
242
|
+
const r = await beaconSend(beaconFromConfig(loadConfig()), text, { to: to || null, silent: false })
|
|
243
|
+
return r.ok ? "sent" : `${r.reason}: ${r.message}`
|
|
244
|
+
} catch (err) {
|
|
245
|
+
return `error: ${(err as Error).message}`
|
|
246
|
+
}
|
|
247
|
+
}
|
|
@@ -0,0 +1,322 @@
|
|
|
1
|
+
// One Scheduler tick: what the dispatcher runs every 5 minutes.
|
|
2
|
+
//
|
|
3
|
+
// 1. Take the tick lock (a tick still running from last time wins).
|
|
4
|
+
// 2. Run due module jobs (ScheduledJobDecl), paging only when one starts or
|
|
5
|
+
// stops failing.
|
|
6
|
+
// 3. For each source: take its lock, pull it if it's a git repository, run
|
|
7
|
+
// each due follow-up in turn, commit and push runs/.
|
|
8
|
+
// 4. Record the tick in last-tick.json (every dispatcher tick; status() reads
|
|
9
|
+
// it) and in tick.log (any tick where something ran or failed).
|
|
10
|
+
//
|
|
11
|
+
// See docs/2026-10-20_scheduler-spec.md.
|
|
12
|
+
|
|
13
|
+
import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, statSync, writeFileSync } from "node:fs"
|
|
14
|
+
import { join } from "node:path"
|
|
15
|
+
import { loadConfig } from "../config.ts"
|
|
16
|
+
import { MODULES } from "../modules/index.ts"
|
|
17
|
+
import type { ScheduledJobDecl } from "../modules/contract.ts"
|
|
18
|
+
import { tryLock } from "./flock.ts"
|
|
19
|
+
import { declSchedule, dueOccurrence, fmt, parseEvery, parseWhen } from "./schedule.ts"
|
|
20
|
+
import {
|
|
21
|
+
ID_RE,
|
|
22
|
+
lastTickPath,
|
|
23
|
+
lateGraceMs,
|
|
24
|
+
listSources,
|
|
25
|
+
loadJobs,
|
|
26
|
+
LOCAL_SOURCE,
|
|
27
|
+
loadState,
|
|
28
|
+
saveState,
|
|
29
|
+
schedulerStateDir,
|
|
30
|
+
sourceLockPath,
|
|
31
|
+
tickLogPath,
|
|
32
|
+
type JobHeader,
|
|
33
|
+
type Source,
|
|
34
|
+
type TickRecord,
|
|
35
|
+
type TickRun,
|
|
36
|
+
} from "./schedulerJobs.ts"
|
|
37
|
+
import { jobEnv, notify, runJob, runProcess, selfCommand } from "./schedulerRun.ts"
|
|
38
|
+
import { commitAndPush, decideClaim, gitPull, readClaim, stationName } from "./schedulerGit.ts"
|
|
39
|
+
import { pagerProblem, sendHeartbeat, tickHeartbeat } from "./heartbeat.ts"
|
|
40
|
+
|
|
41
|
+
const LOG_MAX_BYTES = 1_000_000
|
|
42
|
+
const MODULE_JOB_TIMEOUT_MS = 10 * 60_000
|
|
43
|
+
|
|
44
|
+
export type TickResult = ({ ok: true } & TickRecord) | { ok: true; skipped: "locked"; message: string }
|
|
45
|
+
|
|
46
|
+
// ---- module jobs -----------------------------------------------------------------
|
|
47
|
+
|
|
48
|
+
export interface ModuleJobState {
|
|
49
|
+
last_run?: string
|
|
50
|
+
last_scheduled?: string
|
|
51
|
+
last_status?: "ok" | "failed"
|
|
52
|
+
failing_since?: string
|
|
53
|
+
last_output?: string
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export interface ModuleJobDeps {
|
|
57
|
+
stateDir: string
|
|
58
|
+
now: Date
|
|
59
|
+
run(decl: ScheduledJobDecl): Promise<{ ok: boolean; output: string }>
|
|
60
|
+
page(text: string): Promise<string>
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function moduleJobStatePath(dir: string, id: string) {
|
|
64
|
+
return join(dir, "module-jobs", `${id}.json`)
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
// Runs whichever declarations are due. Pages when a job starts failing and when
|
|
68
|
+
// it recovers — never on every failed run, so a broken heartbeat can't train
|
|
69
|
+
// anyone to ignore the pager.
|
|
70
|
+
export async function runModuleJobs(decls: ScheduledJobDecl[], deps: ModuleJobDeps): Promise<{ ran: TickRun[]; errors: string[] }> {
|
|
71
|
+
const ran: TickRun[] = []
|
|
72
|
+
const errors: string[] = []
|
|
73
|
+
for (const decl of decls) {
|
|
74
|
+
const path = moduleJobStatePath(deps.stateDir, decl.id)
|
|
75
|
+
let state: ModuleJobState = {}
|
|
76
|
+
try {
|
|
77
|
+
state = JSON.parse(readFileSync(path, "utf8")) as ModuleJobState
|
|
78
|
+
} catch {
|
|
79
|
+
// First run.
|
|
80
|
+
}
|
|
81
|
+
let occ: Date | null
|
|
82
|
+
try {
|
|
83
|
+
if (decl.command[0] === "scheduler") throw new Error("a module job can't run `scheduler`")
|
|
84
|
+
occ = dueOccurrence({ schedule: declSchedule(decl) }, state, deps.now)
|
|
85
|
+
} catch (err) {
|
|
86
|
+
errors.push(`module job ${decl.id}: ${(err as Error).message}`)
|
|
87
|
+
continue
|
|
88
|
+
}
|
|
89
|
+
if (!occ) continue
|
|
90
|
+
// The tick's time, not the wall clock after earlier jobs: intervals stay
|
|
91
|
+
// aligned to ticks instead of drifting by each run's duration.
|
|
92
|
+
const started = deps.now
|
|
93
|
+
const r = await deps.run(decl)
|
|
94
|
+
const status = r.ok ? "ok" : "failed"
|
|
95
|
+
const tail = r.output.trim().split("\n").slice(-10).join("\n").slice(-600)
|
|
96
|
+
let notified: string | undefined
|
|
97
|
+
if (status === "failed" && state.last_status !== "failed") {
|
|
98
|
+
notified = await deps.page(`⚠️ ${decl.id} failed (spinuptui ${decl.command.join(" ")})\n\n${tail || "(no output)"}`)
|
|
99
|
+
state.failing_since = fmt(started)
|
|
100
|
+
} else if (status === "ok" && state.last_status === "failed") {
|
|
101
|
+
notified = await deps.page(`✅ ${decl.id} is working again (failing since ${state.failing_since ?? "?"})`)
|
|
102
|
+
delete state.failing_since
|
|
103
|
+
}
|
|
104
|
+
state.last_status = status
|
|
105
|
+
state.last_run = fmt(started)
|
|
106
|
+
state.last_scheduled = fmt(occ)
|
|
107
|
+
state.last_output = tail
|
|
108
|
+
mkdirSync(join(deps.stateDir, "module-jobs"), { recursive: true })
|
|
109
|
+
writeFileSync(path, JSON.stringify(state, null, 2) + "\n")
|
|
110
|
+
ran.push({ source: "module", id: decl.id, status, ...(notified ? { notified } : {}) })
|
|
111
|
+
}
|
|
112
|
+
return { ran, errors }
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
async function configuredModuleJobs(): Promise<ScheduledJobDecl[]> {
|
|
116
|
+
const out: ScheduledJobDecl[] = []
|
|
117
|
+
for (const m of MODULES) {
|
|
118
|
+
if (m.jobs.length === 0) continue
|
|
119
|
+
const s = await m.status().catch(() => null)
|
|
120
|
+
if (s && s.state !== "not_configured") out.push(...m.jobs)
|
|
121
|
+
}
|
|
122
|
+
return out
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
// ---- sources ---------------------------------------------------------------------
|
|
126
|
+
|
|
127
|
+
// `late:`: is this slot too old to run? Only fixed-time slots can be late; an
|
|
128
|
+
// interval job's slot is always "now".
|
|
129
|
+
export function isTooLate(header: JobHeader, occ: Date, now: Date): boolean {
|
|
130
|
+
const grace = lateGraceMs(header.late)
|
|
131
|
+
if (grace === null || grace === undefined) return false
|
|
132
|
+
if (header.every && parseEvery(header.every).type === "interval") return false
|
|
133
|
+
return now.getTime() - occ.getTime() > grace
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
const hm = (d: Date) => fmt(d).slice(11)
|
|
137
|
+
|
|
138
|
+
async function tickSource(
|
|
139
|
+
source: Source,
|
|
140
|
+
now: Date,
|
|
141
|
+
only: string | null,
|
|
142
|
+
force: boolean,
|
|
143
|
+
via: "dispatcher" | "manual",
|
|
144
|
+
): Promise<{ ran: TickRun[]; errors: string[]; notRunHere?: string }> {
|
|
145
|
+
const ran: TickRun[] = []
|
|
146
|
+
const errors: string[] = []
|
|
147
|
+
if (!existsSync(source.path)) {
|
|
148
|
+
// ~/SpinupTUI/scheduler appears on first use, so a missing local folder
|
|
149
|
+
// just means nothing has been queued there yet.
|
|
150
|
+
if (source.name === LOCAL_SOURCE) return { ran, errors }
|
|
151
|
+
return { ran, errors: [`source ${source.name}: ${source.path} does not exist`] }
|
|
152
|
+
}
|
|
153
|
+
const lock = tryLock(sourceLockPath(source))
|
|
154
|
+
if (!lock) return { ran, errors: [`source ${source.name}: busy (another dispatcher holds its lock); skipped this tick`] }
|
|
155
|
+
try {
|
|
156
|
+
if (source.git) {
|
|
157
|
+
const e = await gitPull(source.path)
|
|
158
|
+
if (e) errors.push(`source ${source.name}: ${e}`)
|
|
159
|
+
// One station per git source. Only the dispatcher claims; a manual `run`
|
|
160
|
+
// on a machine that doesn't hold the claim is deliberate, so it proceeds.
|
|
161
|
+
if (via === "dispatcher") {
|
|
162
|
+
const d = await decideClaim(source)
|
|
163
|
+
if (!d.run) {
|
|
164
|
+
if (d.holder === null && d.why.startsWith("couldn't")) errors.push(`source ${d.why}`)
|
|
165
|
+
return { ran, errors, notRunHere: d.why }
|
|
166
|
+
}
|
|
167
|
+
} else {
|
|
168
|
+
const holder = readClaim(source)
|
|
169
|
+
if (holder && holder !== stationName()) process.stderr.write(`Warning: ${source.name} is run by ${holder}; running it here anyway, because you asked.\n`)
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
for (const job of loadJobs(source)) {
|
|
173
|
+
const id = job.header.id
|
|
174
|
+
if (only && id !== only) continue
|
|
175
|
+
if (job.problems.length) {
|
|
176
|
+
if (!id || !ID_RE.test(id)) continue
|
|
177
|
+
const sig = job.problems.join("; ")
|
|
178
|
+
const state = loadState(source, id)
|
|
179
|
+
if (state.invalid !== sig) {
|
|
180
|
+
state.invalid = sig
|
|
181
|
+
saveState(source, id, state)
|
|
182
|
+
const notified = await notify(`⚠️ Job file ${source.name}/jobs/${job.name} is invalid and was skipped: ${sig}`)
|
|
183
|
+
ran.push({ source: source.name, id, status: "invalid", notified })
|
|
184
|
+
}
|
|
185
|
+
continue
|
|
186
|
+
}
|
|
187
|
+
const state = loadState(source, id!)
|
|
188
|
+
if (state.invalid !== undefined) {
|
|
189
|
+
delete state.invalid
|
|
190
|
+
saveState(source, id!, state)
|
|
191
|
+
}
|
|
192
|
+
let occ: Date | null
|
|
193
|
+
try {
|
|
194
|
+
occ = force ? forcedOccurrence(job.header.when, now) : dueOccurrence(job.header, state, now)
|
|
195
|
+
} catch (err) {
|
|
196
|
+
errors.push(`job ${source.name}/${id}: ${(err as Error).message}`)
|
|
197
|
+
continue
|
|
198
|
+
}
|
|
199
|
+
if (!occ) continue
|
|
200
|
+
if (!force && isTooLate(job.header, occ, now)) {
|
|
201
|
+
state.last_scheduled = fmt(occ)
|
|
202
|
+
state.last_status = "skipped"
|
|
203
|
+
delete state.retry_after
|
|
204
|
+
saveState(source, id!, state)
|
|
205
|
+
const notified =
|
|
206
|
+
(job.header.notify || "beacon") === "none"
|
|
207
|
+
? "skipped"
|
|
208
|
+
: await notify(
|
|
209
|
+
`⏭ Skipped job ${id}: it was due at ${fmt(occ)}, and this station only got to it at ${hm(now)}. Its late: ${job.header.late} policy says not to run it late.\n\nQueued by: ${job.header.created_by}`,
|
|
210
|
+
job.header.to,
|
|
211
|
+
)
|
|
212
|
+
ran.push({ source: source.name, id: id!, status: "skipped", notified })
|
|
213
|
+
continue
|
|
214
|
+
}
|
|
215
|
+
process.stderr.write(`→ running ${source.name}/${id} (scheduled ${fmt(occ)})\n`)
|
|
216
|
+
const outcome = await runJob(source, job, occ, now)
|
|
217
|
+
ran.push({ source: source.name, id: id!, ...outcome })
|
|
218
|
+
}
|
|
219
|
+
if (source.git) {
|
|
220
|
+
const mine = ran.filter((r) => r.source === source.name)
|
|
221
|
+
const e = await commitAndPush(source.path, ["runs"], "runs: " + (mine.length ? mine.map((r) => `${r.id} (${r.status})`).join(", ") : "state"))
|
|
222
|
+
if (e) errors.push(`source ${source.name}: ${e}`)
|
|
223
|
+
}
|
|
224
|
+
} finally {
|
|
225
|
+
lock.release()
|
|
226
|
+
}
|
|
227
|
+
return { ran, errors }
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
// `scheduler run`: a one-shot whose time has passed is recorded as its own slot,
|
|
231
|
+
// so the next tick doesn't run it again; anything else runs as "now".
|
|
232
|
+
function forcedOccurrence(when: string | undefined, now: Date): Date {
|
|
233
|
+
if (when) {
|
|
234
|
+
const w = parseWhen(when)
|
|
235
|
+
if (w <= now) return w
|
|
236
|
+
}
|
|
237
|
+
return new Date(new Date(now).setSeconds(0, 0))
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
// ---- tick ------------------------------------------------------------------------
|
|
241
|
+
|
|
242
|
+
export interface TickOptions {
|
|
243
|
+
via: "dispatcher" | "manual"
|
|
244
|
+
// `scheduler run <id>`: one job, now, whether due or not.
|
|
245
|
+
only?: { source: string; id: string }
|
|
246
|
+
now?: Date
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
export async function tick(opts: TickOptions): Promise<TickResult> {
|
|
250
|
+
const dir = schedulerStateDir()
|
|
251
|
+
mkdirSync(dir, { recursive: true })
|
|
252
|
+
const lock = tryLock(join(dir, "tick.lock"))
|
|
253
|
+
if (!lock) return { ok: true, skipped: "locked", message: "A previous tick is still running." }
|
|
254
|
+
const startedAt = Date.now()
|
|
255
|
+
const now = opts.now ?? new Date()
|
|
256
|
+
const ran: TickRun[] = []
|
|
257
|
+
const errors: string[] = []
|
|
258
|
+
const notRunHere: string[] = []
|
|
259
|
+
try {
|
|
260
|
+
if (!opts.only) {
|
|
261
|
+
const decls = await configuredModuleJobs()
|
|
262
|
+
if (decls.length) {
|
|
263
|
+
const r = await runModuleJobs(decls, {
|
|
264
|
+
stateDir: dir,
|
|
265
|
+
now,
|
|
266
|
+
run: async (decl) => {
|
|
267
|
+
const p = await runProcess([...selfCommand(), ...decl.command], { cwd: dir, env: jobEnv(), timeoutMs: MODULE_JOB_TIMEOUT_MS })
|
|
268
|
+
return { ok: p.code === 0, output: p.output }
|
|
269
|
+
},
|
|
270
|
+
page: (text) => notify(text),
|
|
271
|
+
})
|
|
272
|
+
ran.push(...r.ran)
|
|
273
|
+
errors.push(...r.errors)
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
for (const source of listSources(loadConfig())) {
|
|
277
|
+
if (opts.only && source.name !== opts.only.source) continue
|
|
278
|
+
const r = await tickSource(source, now, opts.only?.id ?? null, !!opts.only, opts.via)
|
|
279
|
+
ran.push(...r.ran)
|
|
280
|
+
errors.push(...r.errors)
|
|
281
|
+
if (r.notRunHere) notRunHere.push(r.notRunHere)
|
|
282
|
+
}
|
|
283
|
+
for (const r of ran) if (r.notified && r.notified !== "sent" && r.notified !== "skipped") errors.push(`${r.source}/${r.id}: not paged (${r.notified})`)
|
|
284
|
+
const record: TickRecord = {
|
|
285
|
+
at: new Date().toISOString(),
|
|
286
|
+
via: opts.via,
|
|
287
|
+
durationMs: Date.now() - startedAt,
|
|
288
|
+
ran,
|
|
289
|
+
errors,
|
|
290
|
+
...(notRunHere.length ? { notRunHere } : {}),
|
|
291
|
+
}
|
|
292
|
+
// Only the dispatcher's ticks count as "the station is alive": a manual
|
|
293
|
+
// `run` or `tick --here` must not mask a dispatcher that has stopped. The
|
|
294
|
+
// same goes for the heartbeat, which tells Kuma the clock is running.
|
|
295
|
+
if (opts.via === "dispatcher") {
|
|
296
|
+
const hb = loadConfig().scheduler?.heartbeat
|
|
297
|
+
if (hb) {
|
|
298
|
+
const t = tickHeartbeat(ran.length, errors, pagerProblem())
|
|
299
|
+
const r = await sendHeartbeat(hb.url, t.status, t.msg, record.durationMs)
|
|
300
|
+
record.heartbeat = { ok: r.ok, status: r.status, ...(r.error ? { error: r.error } : {}) }
|
|
301
|
+
if (!r.ok) record.errors.push(`heartbeat: Kuma didn't take the ping (${r.error})`)
|
|
302
|
+
}
|
|
303
|
+
writeFileSync(`${lastTickPath()}.tmp`, JSON.stringify(record, null, 2) + "\n")
|
|
304
|
+
renameSync(`${lastTickPath()}.tmp`, lastTickPath())
|
|
305
|
+
}
|
|
306
|
+
if (ran.length || errors.length) appendTickLog(record)
|
|
307
|
+
return { ok: true, ...record }
|
|
308
|
+
} finally {
|
|
309
|
+
lock.release()
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
function appendTickLog(record: TickRecord): void {
|
|
314
|
+
try {
|
|
315
|
+
const path = tickLogPath()
|
|
316
|
+
const size = existsSync(path) ? statSync(path).size : 0
|
|
317
|
+
if (size > LOG_MAX_BYTES) renameSync(path, `${path}.1`)
|
|
318
|
+
appendFileSync(path, JSON.stringify(record) + "\n")
|
|
319
|
+
} catch {
|
|
320
|
+
// The log is history; losing a line must not fail the tick.
|
|
321
|
+
}
|
|
322
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test"
|
|
2
|
+
import { parseAccountSlug, siteWebUrl } from "./spinupweb.ts"
|
|
3
|
+
|
|
4
|
+
describe("parseAccountSlug", () => {
|
|
5
|
+
test("a bare slug", () => {
|
|
6
|
+
expect(parseAccountSlug(" acme-agency ")).toBe("acme-agency")
|
|
7
|
+
})
|
|
8
|
+
test("a whole dashboard URL", () => {
|
|
9
|
+
expect(parseAccountSlug("https://spinupwp.app/acme-agency/sites/123#git")).toBe("acme-agency")
|
|
10
|
+
expect(parseAccountSlug("spinupwp.app/acme-agency")).toBe("acme-agency")
|
|
11
|
+
expect(parseAccountSlug("https://SpinupWP.app/acme-agency/?tab=servers")).toBe("acme-agency")
|
|
12
|
+
})
|
|
13
|
+
test("anything that isn't a slug", () => {
|
|
14
|
+
expect(parseAccountSlug("")).toBeNull()
|
|
15
|
+
expect(parseAccountSlug("https://spinupwp.app/")).toBeNull()
|
|
16
|
+
expect(parseAccountSlug("acme agency")).toBeNull()
|
|
17
|
+
expect(parseAccountSlug("../etc")).toBeNull()
|
|
18
|
+
})
|
|
19
|
+
test("feeds the deep link", () => {
|
|
20
|
+
expect(siteWebUrl(42, parseAccountSlug("https://spinupwp.app/acme-agency/sites"))).toBe("https://spinupwp.app/acme-agency/sites/42")
|
|
21
|
+
})
|
|
22
|
+
})
|
package/src/lib/spinupweb.ts
CHANGED
|
@@ -12,6 +12,17 @@
|
|
|
12
12
|
|
|
13
13
|
const WEB_BASE = "https://spinupwp.app"
|
|
14
14
|
|
|
15
|
+
// Read an account slug from what a user types: the slug itself, or any dashboard
|
|
16
|
+
// URL pasted whole ("https://spinupwp.app/acme-agency/sites/123" → "acme-agency").
|
|
17
|
+
// Null when it doesn't look like a slug.
|
|
18
|
+
export function parseAccountSlug(input: string): string | null {
|
|
19
|
+
let s = input.trim()
|
|
20
|
+
const at = s.toLowerCase().indexOf("spinupwp.app/")
|
|
21
|
+
if (at >= 0) s = s.slice(at + "spinupwp.app/".length)
|
|
22
|
+
s = s.replace(/^\/+/, "").split(/[/?#]/)[0] ?? ""
|
|
23
|
+
return /^[A-Za-z0-9][A-Za-z0-9_-]*$/.test(s) ? s : null
|
|
24
|
+
}
|
|
25
|
+
|
|
15
26
|
export function serverWebUrl(id: number, accountSlug: string | null): string {
|
|
16
27
|
return accountSlug ? `${WEB_BASE}/${accountSlug}/servers/${id}` : WEB_BASE
|
|
17
28
|
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test"
|
|
2
|
+
import { isRecheckDue, isVersionOld, VERSION_MAX_AGE_MS, type CachedProbe } from "./stackCache.ts"
|
|
3
|
+
|
|
4
|
+
const NOW = 1_800_000_000_000
|
|
5
|
+
|
|
6
|
+
function entry(version: string | null, ageMs: number): CachedProbe {
|
|
7
|
+
return {
|
|
8
|
+
result: { kind: version ? "wordpress" : "whmcs", app: "x", version, label: "x" },
|
|
9
|
+
detectedAt: NOW - ageMs,
|
|
10
|
+
signature: "true:",
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
describe("isVersionOld", () => {
|
|
15
|
+
test("a fresh version reading isn't old", () => {
|
|
16
|
+
expect(isVersionOld(entry("7.1.3", 60_000), NOW)).toBe(false)
|
|
17
|
+
})
|
|
18
|
+
test("a version reading past the max age is old", () => {
|
|
19
|
+
expect(isVersionOld(entry("7.0", VERSION_MAX_AGE_MS + 1), NOW)).toBe(true)
|
|
20
|
+
})
|
|
21
|
+
test("an identity without a version never goes old", () => {
|
|
22
|
+
expect(isVersionOld(entry(null, 365 * VERSION_MAX_AGE_MS), NOW)).toBe(false)
|
|
23
|
+
})
|
|
24
|
+
test("no entry isn't old", () => {
|
|
25
|
+
expect(isVersionOld(undefined, NOW)).toBe(false)
|
|
26
|
+
})
|
|
27
|
+
})
|
|
28
|
+
|
|
29
|
+
describe("isRecheckDue", () => {
|
|
30
|
+
const old = entry("7.0", 3 * VERSION_MAX_AGE_MS)
|
|
31
|
+
test("an old version with no attempt yet is due", () => {
|
|
32
|
+
expect(isRecheckDue(old, NOW)).toBe(true)
|
|
33
|
+
})
|
|
34
|
+
test("an attempt inside the window (even a failed one) holds it off", () => {
|
|
35
|
+
expect(isRecheckDue({ ...old, recheckedAt: NOW - 60_000 }, NOW)).toBe(false)
|
|
36
|
+
})
|
|
37
|
+
test("an attempt from a previous window doesn't", () => {
|
|
38
|
+
expect(isRecheckDue({ ...old, recheckedAt: NOW - VERSION_MAX_AGE_MS - 1 }, NOW)).toBe(true)
|
|
39
|
+
})
|
|
40
|
+
test("a fresh reading is never due", () => {
|
|
41
|
+
expect(isRecheckDue(entry("7.1.3", 60_000), NOW)).toBe(false)
|
|
42
|
+
})
|
|
43
|
+
})
|