spinuptui 0.20.1 → 0.21.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +80 -1
- package/README.md +29 -11
- package/package.json +1 -1
- package/src/config.ts +3 -0
- package/src/index.tsx +44 -1
- package/src/lib/cliIncidents.ts +126 -0
- package/src/lib/cliSsh.ts +161 -0
- package/src/lib/probe.ts +1 -1
- package/src/lib/ssh.ts +46 -0
- package/src/lib/uptimeKuma.ts +121 -5
- package/src/lib/vanitySite.ts +74 -0
- package/src/ui/Header.tsx +10 -2
- package/src/ui/store.tsx +123 -6
- package/src/ui/views/KumaSite.tsx +508 -127
package/src/lib/uptimeKuma.ts
CHANGED
|
@@ -41,6 +41,7 @@ export interface KumaBeat {
|
|
|
41
41
|
time: string
|
|
42
42
|
msg?: string
|
|
43
43
|
ping?: number | null
|
|
44
|
+
important?: boolean // true on a status-transition row (Kuma's own "important events" flag)
|
|
44
45
|
}
|
|
45
46
|
|
|
46
47
|
// A notification provider as configured in Kuma (Settings → Notifications) —
|
|
@@ -66,6 +67,23 @@ interface Ack {
|
|
|
66
67
|
const ACK_TIMEOUT_MS = 15_000
|
|
67
68
|
const CONNECT_TIMEOUT_MS = 10_000
|
|
68
69
|
|
|
70
|
+
// engine.io wraps most transport failures in a TransportError whose `.message`
|
|
71
|
+
// is a fixed literal ("websocket error") with the real reason tucked away in
|
|
72
|
+
// `.description`; a few failure modes (a WebSocket constructor throwing
|
|
73
|
+
// synchronously — seen from Bun's net/tls shim on cert issues) skip that
|
|
74
|
+
// wrapper and hand us a bare Error whose `.message` can be empty. Walk every
|
|
75
|
+
// place a reason might live rather than trusting `.message` alone, so a
|
|
76
|
+
// connect failure never renders as a bare trailing colon.
|
|
77
|
+
function describeConnectError(e: unknown): string {
|
|
78
|
+
const err = e as { message?: string; description?: unknown; code?: string } | undefined
|
|
79
|
+
if (err?.message) return err.message
|
|
80
|
+
const desc = err?.description
|
|
81
|
+
if (desc instanceof Error && desc.message) return desc.message
|
|
82
|
+
if (typeof desc === "string" && desc) return desc
|
|
83
|
+
if (err?.code) return err.code
|
|
84
|
+
return String(e)
|
|
85
|
+
}
|
|
86
|
+
|
|
69
87
|
// 32 chars, [A-Za-z0-9] — same shape the Kuma UI generates for push monitors.
|
|
70
88
|
// (Kuma stores whatever the client sends; the token is chosen client-side.)
|
|
71
89
|
export function genPushToken(): string {
|
|
@@ -154,7 +172,10 @@ export class KumaClient {
|
|
|
154
172
|
|
|
155
173
|
static async connect(url: string): Promise<KumaClient> {
|
|
156
174
|
const base = url.replace(/\/+$/, "")
|
|
157
|
-
|
|
175
|
+
// Polling as a fallback (not websocket-only): a reverse proxy/CDN in front of
|
|
176
|
+
// Kuma that doesn't forward the WebSocket Upgrade headers otherwise hard-fails
|
|
177
|
+
// the connection outright instead of degrading gracefully.
|
|
178
|
+
const socket = io(base, { transports: ["websocket", "polling"], reconnection: false, timeout: CONNECT_TIMEOUT_MS })
|
|
158
179
|
const client = new KumaClient(socket)
|
|
159
180
|
await new Promise<void>((resolve, reject) => {
|
|
160
181
|
const t = setTimeout(() => {
|
|
@@ -168,7 +189,7 @@ export class KumaClient {
|
|
|
168
189
|
socket.once("connect_error", (e: Error) => {
|
|
169
190
|
clearTimeout(t)
|
|
170
191
|
socket.disconnect()
|
|
171
|
-
reject(new KumaError(`Couldn't reach Uptime Kuma at ${base}: ${e
|
|
192
|
+
reject(new KumaError(`Couldn't reach Uptime Kuma at ${base}: ${describeConnectError(e)}`))
|
|
172
193
|
})
|
|
173
194
|
})
|
|
174
195
|
return client
|
|
@@ -403,6 +424,27 @@ export function redisPushMonitorPayload(name: string, pushToken: string): Record
|
|
|
403
424
|
}
|
|
404
425
|
}
|
|
405
426
|
|
|
427
|
+
// The server-wide PHP-fatal sentinel: one per server (like the Redis sentinel),
|
|
428
|
+
// fed by a root-level cron that greps every site's error.log/debug.log for new
|
|
429
|
+
// "PHP Fatal error" lines — catches PHP dying behind a still-warm page cache,
|
|
430
|
+
// which the plain up/down and fingerprint monitors sleep straight through.
|
|
431
|
+
// Unlike every other monitor kind, a down beat's `msg` carries a comma-separated
|
|
432
|
+
// list of the actual site domain(s) that fataled (this monitor's own domain is
|
|
433
|
+
// just the server's vanity anchor, not the site that's actually broken).
|
|
434
|
+
export function fatalPushMonitorPayload(name: string, pushToken: string): Record<string, unknown> {
|
|
435
|
+
return {
|
|
436
|
+
type: "push",
|
|
437
|
+
name,
|
|
438
|
+
pushToken,
|
|
439
|
+
description: "Server-wide PHP fatal sentinel: a root cron greps every site's error/debug log each minute for new PHP fatals and reports up/down. `msg` on a down beat lists the affected domain(s).",
|
|
440
|
+
interval: 60,
|
|
441
|
+
retryInterval: 60,
|
|
442
|
+
maxretries: 2,
|
|
443
|
+
upsideDown: false,
|
|
444
|
+
accepted_statuscodes: ["200-299"],
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
|
|
406
448
|
// The front-page fingerprint monitor (site monitoring Phase 1): a `keyword`
|
|
407
449
|
// monitor asserting the front page still carries the template-identity string
|
|
408
450
|
// calibration derived (lib/siteFingerprint.ts). Catches "the page cache is
|
|
@@ -453,6 +495,53 @@ export async function registerFingerprintMonitor(
|
|
|
453
495
|
return kuma.addMonitor(fingerprintMonitorPayload(name, url, opts.keyword, opts.interval))
|
|
454
496
|
}
|
|
455
497
|
|
|
498
|
+
// The cache-bypass monitor (site monitoring blind-spot fix, part 2): a plain
|
|
499
|
+
// http monitor on the front page, but with a Cookie header that forces
|
|
500
|
+
// SpinupWP's nginx to bypass the page cache — the same bypass siteDoctor.ts
|
|
501
|
+
// already uses and has verified live. Unlike the health/fingerprint monitors,
|
|
502
|
+
// this actually exercises PHP on every check, so it's deliberately opt-in
|
|
503
|
+
// (not automatic) and meant for a much looser interval than either of those.
|
|
504
|
+
export function bypassMonitorPayload(name: string, url: string, interval: number): Record<string, unknown> {
|
|
505
|
+
return {
|
|
506
|
+
type: "http",
|
|
507
|
+
name,
|
|
508
|
+
url,
|
|
509
|
+
method: "GET",
|
|
510
|
+
interval,
|
|
511
|
+
retryInterval: 60,
|
|
512
|
+
resendInterval: 0,
|
|
513
|
+
maxretries: 2,
|
|
514
|
+
timeout: 48,
|
|
515
|
+
accepted_statuscodes: ["200-299"],
|
|
516
|
+
expiryNotification: false, // the plain health monitor already owns cert expiry
|
|
517
|
+
ignoreTls: false,
|
|
518
|
+
upsideDown: false,
|
|
519
|
+
maxredirects: 10,
|
|
520
|
+
headers: JSON.stringify({ Cookie: "wordpress_no_cache=1" }),
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
// Adopt-or-EDIT-or-create, same philosophy as registerFingerprintMonitor:
|
|
525
|
+
// recalibrating the interval updates the existing row in place so history and
|
|
526
|
+
// notification wiring survive.
|
|
527
|
+
export async function registerBypassMonitor(
|
|
528
|
+
kuma: KumaClient,
|
|
529
|
+
domain: string,
|
|
530
|
+
opts: { proto: "http" | "https"; interval: number; knownId?: number | null },
|
|
531
|
+
): Promise<number> {
|
|
532
|
+
const name = `${domain} cache-bypass`
|
|
533
|
+
const url = `${opts.proto}://${domain}/`
|
|
534
|
+
const monitors = await kuma.getMonitors()
|
|
535
|
+
const byId = new Map(monitors.map((m) => [m.id, m]))
|
|
536
|
+
const existingId = (opts.knownId != null && byId.has(opts.knownId) ? opts.knownId : undefined) ?? monitors.find((m) => m.name === name)?.id
|
|
537
|
+
if (existingId != null) {
|
|
538
|
+
const full = await kuma.getMonitor(existingId)
|
|
539
|
+
await kuma.editMonitor({ ...full, type: "http", url, interval: opts.interval, headers: JSON.stringify({ Cookie: "wordpress_no_cache=1" }) })
|
|
540
|
+
return existingId
|
|
541
|
+
}
|
|
542
|
+
return kuma.addMonitor(bypassMonitorPayload(name, url, opts.interval))
|
|
543
|
+
}
|
|
544
|
+
|
|
456
545
|
// Adopt-or-create the monitors for one domain — THE single implementation shared
|
|
457
546
|
// by the vanity wizard's monitor step and the `m` overlay's add/repair, so the
|
|
458
547
|
// two paths cannot drift. Rules:
|
|
@@ -464,8 +553,24 @@ export async function registerFingerprintMonitor(
|
|
|
464
553
|
export async function registerMonitors(
|
|
465
554
|
kuma: KumaClient,
|
|
466
555
|
domain: string,
|
|
467
|
-
opts: {
|
|
468
|
-
|
|
556
|
+
opts: {
|
|
557
|
+
proto: "http" | "https"
|
|
558
|
+
healthzPath: string
|
|
559
|
+
wantPush: boolean
|
|
560
|
+
wantRedis?: boolean
|
|
561
|
+
wantFatal?: boolean
|
|
562
|
+
known?: KumaMonitorRef | null
|
|
563
|
+
pushToken?: string | null
|
|
564
|
+
},
|
|
565
|
+
): Promise<{
|
|
566
|
+
healthId: number
|
|
567
|
+
pushId?: number
|
|
568
|
+
pushToken?: string
|
|
569
|
+
redisId?: number
|
|
570
|
+
redisToken?: string
|
|
571
|
+
fatalId?: number
|
|
572
|
+
fatalToken?: string
|
|
573
|
+
}> {
|
|
469
574
|
const monitors = await kuma.getMonitors()
|
|
470
575
|
const byId = new Map(monitors.map((m) => [m.id, m]))
|
|
471
576
|
const byName = new Map(monitors.map((m) => [m.name, m]))
|
|
@@ -482,7 +587,12 @@ export async function registerMonitors(
|
|
|
482
587
|
const existingRedis = opts.wantRedis && live(known.redisId) == null ? byName.get(`${domain} redis`) : undefined
|
|
483
588
|
if (existingRedis?.pushToken) redisToken = existingRedis.pushToken
|
|
484
589
|
|
|
485
|
-
|
|
590
|
+
// Same independent-token story as Redis — its own push URL, its own cron line.
|
|
591
|
+
let fatalToken = known.fatalToken ?? genPushToken()
|
|
592
|
+
const existingFatal = opts.wantFatal && live(known.fatalId) == null ? byName.get(`${domain} php-fatal`) : undefined
|
|
593
|
+
if (existingFatal?.pushToken) fatalToken = existingFatal.pushToken
|
|
594
|
+
|
|
595
|
+
const [healthId, pushId, redisId, fatalId] = await Promise.all([
|
|
486
596
|
(async () => live(known.healthId) ?? byName.get(domain)?.id ?? (await kuma.addMonitor(healthMonitorPayload(domain, `${opts.proto}://${domain}${opts.healthzPath}`))))(),
|
|
487
597
|
(async () => {
|
|
488
598
|
if (!opts.wantPush) return undefined
|
|
@@ -492,6 +602,10 @@ export async function registerMonitors(
|
|
|
492
602
|
if (!opts.wantRedis) return undefined
|
|
493
603
|
return live(known.redisId) ?? existingRedis?.id ?? (await kuma.addMonitor(redisPushMonitorPayload(`${domain} redis`, redisToken)))
|
|
494
604
|
})(),
|
|
605
|
+
(async () => {
|
|
606
|
+
if (!opts.wantFatal) return undefined
|
|
607
|
+
return live(known.fatalId) ?? existingFatal?.id ?? (await kuma.addMonitor(fatalPushMonitorPayload(`${domain} php-fatal`, fatalToken)))
|
|
608
|
+
})(),
|
|
495
609
|
])
|
|
496
610
|
return {
|
|
497
611
|
healthId,
|
|
@@ -499,6 +613,8 @@ export async function registerMonitors(
|
|
|
499
613
|
pushToken: opts.wantPush ? pushToken : undefined,
|
|
500
614
|
redisId,
|
|
501
615
|
redisToken: opts.wantRedis ? redisToken : undefined,
|
|
616
|
+
fatalId,
|
|
617
|
+
fatalToken: opts.wantFatal ? fatalToken : undefined,
|
|
502
618
|
}
|
|
503
619
|
}
|
|
504
620
|
|
package/src/lib/vanitySite.ts
CHANGED
|
@@ -6,8 +6,10 @@
|
|
|
6
6
|
// docs/vanity-site/index.php is the design reference; keep the two in sync.
|
|
7
7
|
|
|
8
8
|
import { resolve4 } from "node:dns/promises"
|
|
9
|
+
import type { Server } from "../api/types.ts"
|
|
9
10
|
import { SSH_OPTS, sshPort, runProcess, meaningfulError, remoteDocRoot } from "./dbBackup.ts"
|
|
10
11
|
import { pushUrl, LOAD_PUSH_SCALE } from "./uptimeKuma.ts"
|
|
12
|
+
import { runSudoServerOp } from "./ssh.ts"
|
|
11
13
|
|
|
12
14
|
// The vanity convention: the site living at the server's own hostname. Every
|
|
13
15
|
// feature that special-cases vanity sites (full monitoring, page re-seed) keys
|
|
@@ -295,3 +297,75 @@ export async function seedVanityPushCron(t: PushCronTarget): Promise<{ ok: boole
|
|
|
295
297
|
if (r.code !== 0) return { ok: false, error: meaningfulError(r.stderr, "Couldn't install the heartbeat cron over SSH.") }
|
|
296
298
|
return { ok: true }
|
|
297
299
|
}
|
|
300
|
+
|
|
301
|
+
const FATAL_CRON_MARKER = "spinup-kuma-fatal"
|
|
302
|
+
const FATAL_SCRIPT_PATH = "/root/spinup-fatal-check.sh"
|
|
303
|
+
const FATAL_INSTALLED_MARKER = "===FATALCRON_INSTALLED"
|
|
304
|
+
|
|
305
|
+
// Server-wide PHP-fatal sentinel (site monitoring blind-spot fix): SpinupWP's
|
|
306
|
+
// page cache keeps serving `/` even while PHP is fataling on anything the
|
|
307
|
+
// cache doesn't already have — the plain up/down and fingerprint monitors
|
|
308
|
+
// never touch broken PHP as long as the cache stays warm (verified against a
|
|
309
|
+
// real 2026-07-07 incident that sat undetected for 54 of its 68 minutes).
|
|
310
|
+
//
|
|
311
|
+
// Runs as ROOT (via sudo) because it globs every site on the box: a site's
|
|
312
|
+
// own user can't even list `/sites` (confirmed live), let alone read a
|
|
313
|
+
// sibling site's logs once logrotate recreates them `640 root adm`. One
|
|
314
|
+
// monitor covers the whole server rather than one per site, to avoid Kuma
|
|
315
|
+
// "monitor bloat" — a down beat's `msg` carries the actual affected
|
|
316
|
+
// domain(s), since THIS monitor's own name is just the server's vanity
|
|
317
|
+
// anchor, not the site that's actually broken.
|
|
318
|
+
//
|
|
319
|
+
// A PHP fatal can land in either logs/error.log (nginx) or logs/debug.log
|
|
320
|
+
// (WordPress's own log, populated when WP_DEBUG_LOG is on) depending on that
|
|
321
|
+
// site's config — verified live: the real incident above is in debug.log,
|
|
322
|
+
// not error.log. Both are checked for every site.
|
|
323
|
+
const FATAL_CHECK_SCRIPT = `#!/bin/bash
|
|
324
|
+
STATE_DIR=/root/.spinup-fatal-state
|
|
325
|
+
mkdir -p "$STATE_DIR"
|
|
326
|
+
FOUND=""
|
|
327
|
+
for site_dir in /sites/*/; do
|
|
328
|
+
domain=$(basename "$site_dir")
|
|
329
|
+
for logname in error.log debug.log; do
|
|
330
|
+
logfile="\${site_dir}logs/\${logname}"
|
|
331
|
+
[ -f "$logfile" ] || continue
|
|
332
|
+
state_file="$STATE_DIR/\${domain}__\${logname}.offset"
|
|
333
|
+
prev=$(cat "$state_file" 2>/dev/null || echo 0)
|
|
334
|
+
cur=$(stat -c %s "$logfile" 2>/dev/null || echo 0)
|
|
335
|
+
[ "$cur" -lt "$prev" ] && prev=0
|
|
336
|
+
if [ "$cur" -gt "$prev" ] && tail -c +"$((prev + 1))" "$logfile" | grep -q "PHP Fatal error"; then
|
|
337
|
+
FOUND="\${FOUND:+$FOUND,}$domain"
|
|
338
|
+
fi
|
|
339
|
+
echo "$cur" > "$state_file"
|
|
340
|
+
done
|
|
341
|
+
done
|
|
342
|
+
if [ -n "$FOUND" ]; then
|
|
343
|
+
curl -fsS --max-time 20 "__PUSH_URL__?status=down&msg=$FOUND" >/dev/null 2>&1
|
|
344
|
+
else
|
|
345
|
+
curl -fsS --max-time 20 "__PUSH_URL__?status=up&msg=fatal-check-ok" >/dev/null 2>&1
|
|
346
|
+
fi
|
|
347
|
+
`
|
|
348
|
+
|
|
349
|
+
// Adopt-or-refresh: writes the check script (push URL baked in) and installs
|
|
350
|
+
// the one-line-per-minute crontab entry that invokes it, in ROOT's crontab.
|
|
351
|
+
// Idempotent, same strip-marked-line-then-append idiom as seedVanityPushCron.
|
|
352
|
+
export async function seedFatalMonitorCron(
|
|
353
|
+
server: Server,
|
|
354
|
+
opts: { sudoUser: string; sudoPassword: string; kumaUrl: string; pushToken: string },
|
|
355
|
+
): Promise<{ ok: boolean; error?: string }> {
|
|
356
|
+
const push = pushUrl(opts.kumaUrl, opts.pushToken)
|
|
357
|
+
const script = FATAL_CHECK_SCRIPT.replace(/__PUSH_URL__/g, push)
|
|
358
|
+
const scriptB64 = Buffer.from(script, "utf8").toString("base64")
|
|
359
|
+
return runSudoServerOp(
|
|
360
|
+
server,
|
|
361
|
+
{ sudoUser: opts.sudoUser, sudoPassword: opts.sudoPassword },
|
|
362
|
+
() =>
|
|
363
|
+
[
|
|
364
|
+
`printf '%s' '${scriptB64}' | base64 -d > ${FATAL_SCRIPT_PATH}`,
|
|
365
|
+
`chmod +x ${FATAL_SCRIPT_PATH}`,
|
|
366
|
+
`( crontab -l 2>/dev/null | grep -v -e '${FATAL_CRON_MARKER}' ; echo '* * * * * ${FATAL_SCRIPT_PATH} >/dev/null 2>&1 # ${FATAL_CRON_MARKER}' ) | crontab -`,
|
|
367
|
+
`echo ${FATAL_INSTALLED_MARKER}`,
|
|
368
|
+
].join("\n"),
|
|
369
|
+
FATAL_INSTALLED_MARKER,
|
|
370
|
+
)
|
|
371
|
+
}
|
package/src/ui/Header.tsx
CHANGED
|
@@ -28,7 +28,13 @@ const SUBTITLES: Record<Route, string> = {
|
|
|
28
28
|
|
|
29
29
|
export function Header() {
|
|
30
30
|
const { route, servers, sites, loading, lastUpdated, updateInfo, newServerJob, vanityJob, cloneJob, cloneServer, kumaStatus } = useStore()
|
|
31
|
-
|
|
31
|
+
// Name the domain(s) when there are few enough to fit the header row; fall
|
|
32
|
+
// back to a bare count once there are too many to list safely. Deliberately
|
|
33
|
+
// still just the plain up/down signal (kumaStatus.up) — not expanding what
|
|
34
|
+
// counts as "down" here, only how it's displayed.
|
|
35
|
+
const downDomains = [...kumaStatus.entries()].filter(([, s]) => s.up === false).map(([domain]) => domain)
|
|
36
|
+
const kumaDown = downDomains.length
|
|
37
|
+
const kumaDownLabel = kumaDown === 0 ? "" : kumaDown <= 2 ? `${downDomains.map((d) => truncate(d, 24)).join(", ")} down` : `${kumaDown} monitors down`
|
|
32
38
|
const updateReady = updateInfo?.updateAvailable ?? false
|
|
33
39
|
const building = isNewServerInFlight(newServerJob)
|
|
34
40
|
// The sshkey step is the one that waits on the USER (add your key, confirm in
|
|
@@ -103,7 +109,9 @@ export function Header() {
|
|
|
103
109
|
</box>
|
|
104
110
|
)}
|
|
105
111
|
{/* Uptime Kuma: only bad news earns header space — silence when all up. */}
|
|
106
|
-
{kumaDown > 0 &&
|
|
112
|
+
{kumaDown > 0 && (
|
|
113
|
+
<text content={` ▼ ${kumaDownLabel} — 4 to find ${kumaDown === 1 ? "it" : "them"} `} fg={theme.bad} style={{ flexShrink: 0 }} wrapMode="none" />
|
|
114
|
+
)}
|
|
107
115
|
{loading && <Spinner interval={100} />}
|
|
108
116
|
<text content={` ${servers.length} servers · ${sites.length} sites `} fg={theme.textDim} style={{ flexShrink: 0 }} />
|
|
109
117
|
<text content={lastUpdated ? `updated ${timeAgo(lastUpdated.toISOString())}` : "loading…"} fg={theme.textFaint} style={{ flexShrink: 0 }} />
|
package/src/ui/store.tsx
CHANGED
|
@@ -40,8 +40,8 @@ import {
|
|
|
40
40
|
} from "../lib/providers.ts"
|
|
41
41
|
import { ProvidersCache, type VerifiedConn } from "../lib/providersCache.ts"
|
|
42
42
|
import { recordProviderFor, type DnsRecord, type RecordResult } from "../lib/dnsRecords.ts"
|
|
43
|
-
import { deriveSiteUser, seedVanityIndex, seedVanityPushCron, probeServerRedis, aRecordResolves, isVanityPair } from "../lib/vanitySite.ts"
|
|
44
|
-
import { withKuma, KumaClient, registerMonitors, registerFingerprintMonitor, setMonitorNotifications, genPushToken, rotatePushToken, retargetHealthKeyMonitors, LOAD_PUSH_SCALE } from "../lib/uptimeKuma.ts"
|
|
43
|
+
import { deriveSiteUser, seedVanityIndex, seedVanityPushCron, seedFatalMonitorCron, probeServerRedis, aRecordResolves, isVanityPair } from "../lib/vanitySite.ts"
|
|
44
|
+
import { withKuma, KumaClient, registerMonitors, registerFingerprintMonitor, registerBypassMonitor, setMonitorNotifications, genPushToken, rotatePushToken, retargetHealthKeyMonitors, LOAD_PUSH_SCALE } from "../lib/uptimeKuma.ts"
|
|
45
45
|
import { deriveFingerprint } from "../lib/siteFingerprint.ts"
|
|
46
46
|
import type { StoredProviders } from "../config.ts"
|
|
47
47
|
import { fetchRebootInfo, grantSiteSshKey, revokeSiteSshKey, verifySudo, ensureSpinupKey, listPersonalKeys, keyBody, type RebootInfo } from "../lib/ssh.ts"
|
|
@@ -219,6 +219,14 @@ export interface KumaDomainStatus {
|
|
|
219
219
|
// The server's Redis sentinel (vanity domains only): false = the cron reports
|
|
220
220
|
// Redis unreachable while the server itself is beating fine.
|
|
221
221
|
redisUp: boolean | null
|
|
222
|
+
// The server-wide PHP-fatal sentinel (vanity domains only): false = a new
|
|
223
|
+
// fatal was detected on some site on this server (see the down beat's own
|
|
224
|
+
// message for which one — this field is just "is anything on fire").
|
|
225
|
+
fatalUp: boolean | null
|
|
226
|
+
// The cache-bypass monitor (regular sites, opt-in): false = PHP couldn't
|
|
227
|
+
// render a fresh request even with the page cache skipped — distinct from
|
|
228
|
+
// `up`, which a warm cache can keep green even when this is failing.
|
|
229
|
+
bypassUp: boolean | null
|
|
222
230
|
}
|
|
223
231
|
|
|
224
232
|
// Clone a server to a new server (backlog item 5). Five server-level steps wrapping
|
|
@@ -538,6 +546,7 @@ interface StoreValue extends DataState {
|
|
|
538
546
|
clearVanity: () => void
|
|
539
547
|
vanityHealthKeyFor: (domain: string) => string | null // key baked into the seeded page's ?format=json mode
|
|
540
548
|
kumaConfigured: boolean // an Uptime Kuma connection exists (config or env)
|
|
549
|
+
kumaUrl: string | null // base URL for deep-linking to a monitor's own Kuma page
|
|
541
550
|
kumaSite: Site | null // site whose Kuma overlay (m) is open
|
|
542
551
|
setKumaSite: (s: Site | null) => void
|
|
543
552
|
kumaOps: Map<number, KumaOp> // in-flight/settled monitor setups, by site id
|
|
@@ -547,6 +556,8 @@ interface StoreValue extends DataState {
|
|
|
547
556
|
startVanityReseed: (site: Site) => void // refresh an existing vanity page (no Kuma needed)
|
|
548
557
|
startKumaRotate: (site: Site) => void // rotate the push token + health key (vanity; old URLs die immediately)
|
|
549
558
|
startFingerprintSetup: (site: Site, intervalSec: number) => void // calibrate + register the front-page fingerprint monitor
|
|
559
|
+
startBypassMonitorSetup: (site: Site, intervalSec: number) => void // register (or recalibrate) the opt-in cache-bypass monitor
|
|
560
|
+
removeSiteMonitor: (site: Site, kind: "fingerprint" | "bypass") => void // delete an opt-in per-site monitor (front page / cache bypass only)
|
|
550
561
|
fetchKumaAlerts: (site: Site) => Promise<{ ok: true; providers: KumaAlertProvider[]; monitorIds: number[] } | { ok: false; error: string }> // live read of provider/attachment state
|
|
551
562
|
toggleKumaAlert: (site: Site, providerId: number, on: boolean, monitorIds: number[]) => Promise<{ ok: true } | { ok: false; error: string }>
|
|
552
563
|
clearKumaOp: (siteId: number) => void // forget a settled op so the overlay opens fresh
|
|
@@ -2947,7 +2958,7 @@ export function StoreProvider({ children }: { children: ReactNode }) {
|
|
|
2947
2958
|
// very maintenance windows we create around reboots).
|
|
2948
2959
|
const beatUp = (s: number) => s !== 0
|
|
2949
2960
|
for (const [domain, ref] of Object.entries(cfgRef.current.kumaMonitors)) {
|
|
2950
|
-
const status: KumaDomainStatus = { up: null, uptime24: null, responseBeats: [], loadBeats: [], lastLoad: null, fingerprintUp: null, redisUp: null }
|
|
2961
|
+
const status: KumaDomainStatus = { up: null, uptime24: null, responseBeats: [], loadBeats: [], lastLoad: null, fingerprintUp: null, redisUp: null, fatalUp: null, bypassUp: null }
|
|
2951
2962
|
if (ref.healthId && byId.has(ref.healthId)) {
|
|
2952
2963
|
const beats = kuma.beatsFor(ref.healthId)
|
|
2953
2964
|
const last = beats[beats.length - 1]
|
|
@@ -2974,6 +2985,16 @@ export function StoreProvider({ children }: { children: ReactNode }) {
|
|
|
2974
2985
|
const last = beats[beats.length - 1]
|
|
2975
2986
|
status.redisUp = last ? beatUp(last.status) : null
|
|
2976
2987
|
}
|
|
2988
|
+
if (ref.fatalId && byId.has(ref.fatalId)) {
|
|
2989
|
+
const beats = kuma.beatsFor(ref.fatalId)
|
|
2990
|
+
const last = beats[beats.length - 1]
|
|
2991
|
+
status.fatalUp = last ? beatUp(last.status) : null
|
|
2992
|
+
}
|
|
2993
|
+
if (ref.bypassId && byId.has(ref.bypassId)) {
|
|
2994
|
+
const beats = kuma.beatsFor(ref.bypassId)
|
|
2995
|
+
const last = beats[beats.length - 1]
|
|
2996
|
+
status.bypassUp = last ? beatUp(last.status) : null
|
|
2997
|
+
}
|
|
2977
2998
|
map.set(domain, status)
|
|
2978
2999
|
}
|
|
2979
3000
|
return map
|
|
@@ -3106,13 +3127,22 @@ export function StoreProvider({ children }: { children: ReactNode }) {
|
|
|
3106
3127
|
wantRedis = probe.ok
|
|
3107
3128
|
if (!probe.ok) redisNote = probe.error ?? "Redis sentinel skipped."
|
|
3108
3129
|
}
|
|
3130
|
+
// The PHP-fatal sentinel is also server-wide (it globs every site's
|
|
3131
|
+
// logs), but unlike Redis it needs ROOT — a regular site user can't
|
|
3132
|
+
// even list /sites (confirmed live). Only offer it when sudo is
|
|
3133
|
+
// already connected for this server; otherwise skip quietly rather
|
|
3134
|
+
// than blocking the rest of setup, and say what unlocks it.
|
|
3135
|
+
const sudoReady = isVanity && !!server && isSudoConnected(server.id)
|
|
3136
|
+
const wantFatal = sudoReady
|
|
3137
|
+
let fatalNote: string | null = null
|
|
3138
|
+
if (isVanity && server && !sudoReady) fatalNote = "PHP-fatal sentinel skipped — connect sudo (S) first, then re-run to add it."
|
|
3109
3139
|
setOp({ status: "running", detail: "Registering monitors in Uptime Kuma…" })
|
|
3110
3140
|
const known = cfgRef.current.kumaMonitors[domain] ?? {}
|
|
3111
3141
|
// Monitor over the protocol the site actually serves — an https monitor
|
|
3112
3142
|
// on an http-only site would be permanently down.
|
|
3113
3143
|
const proto = site.https?.enabled ? "https" : "http"
|
|
3114
3144
|
const { result, jwt } = await withKuma(conn, (kuma) =>
|
|
3115
|
-
registerMonitors(kuma, domain, { proto, healthzPath: isVanity ? "/?healthz" : "/", wantPush: isVanity, wantRedis, known }),
|
|
3145
|
+
registerMonitors(kuma, domain, { proto, healthzPath: isVanity ? "/?healthz" : "/", wantPush: isVanity, wantRedis, wantFatal, known }),
|
|
3116
3146
|
)
|
|
3117
3147
|
persistKumaAuth(conn, jwt)
|
|
3118
3148
|
persistKumaMonitors(domain, {
|
|
@@ -3122,16 +3152,28 @@ export function StoreProvider({ children }: { children: ReactNode }) {
|
|
|
3122
3152
|
pushToken: result.pushToken ?? known.pushToken,
|
|
3123
3153
|
redisId: result.redisId ?? known.redisId,
|
|
3124
3154
|
redisToken: result.redisToken ?? known.redisToken,
|
|
3155
|
+
fatalId: result.fatalId ?? known.fatalId,
|
|
3156
|
+
fatalToken: result.fatalToken ?? known.fatalToken,
|
|
3125
3157
|
})
|
|
3126
3158
|
if (isVanity && result.pushId && result.pushToken && server) {
|
|
3127
3159
|
setOp({ status: "running", detail: "Installing the heartbeat cron…" })
|
|
3128
3160
|
const cron = await seedVanityPushCron({ ...ssh, kumaUrl: conn.url, pushToken: result.pushToken, redisToken: result.redisId != null ? (result.redisToken ?? known.redisToken ?? null) : null })
|
|
3129
3161
|
if (!cron.ok) throw new Error(cron.error || "Couldn't install the heartbeat cron.")
|
|
3130
3162
|
}
|
|
3163
|
+
if (sudoReady && server && result.fatalId && result.fatalToken) {
|
|
3164
|
+
setOp({ status: "running", detail: "Installing the PHP-fatal check (sudo)…" })
|
|
3165
|
+
const sudoUser = sudoUsers.get(server.id) ?? ""
|
|
3166
|
+
const sudoPassword = sudoPwRef.current.get(server.id) ?? ""
|
|
3167
|
+
const fatalCron = await seedFatalMonitorCron(server, { sudoUser, sudoPassword, kumaUrl: conn.url, pushToken: result.fatalToken })
|
|
3168
|
+
if (!fatalCron.ok) {
|
|
3169
|
+
if (/password was rejected|password is required|can't run sudo|couldn't connect/i.test(fatalCron.error ?? "")) disconnectSudo(server.id)
|
|
3170
|
+
fatalNote = `PHP-fatal sentinel skipped — ${fatalCron.error ?? "couldn't install the cron."}`
|
|
3171
|
+
}
|
|
3172
|
+
}
|
|
3131
3173
|
setOp({
|
|
3132
3174
|
status: "done",
|
|
3133
3175
|
detail: isVanity
|
|
3134
|
-
? `Monitors live: healthz checks + load graph${result.redisId != null ? " + Redis sentinel" : ""} (cron beating every minute).${redisNote ? ` ${redisNote}` : ""}`
|
|
3176
|
+
? `Monitors live: healthz checks + load graph${result.redisId != null ? " + Redis sentinel" : ""}${result.fatalId != null && !fatalNote ? " + PHP-fatal sentinel" : ""} (cron beating every minute).${redisNote ? ` ${redisNote}` : ""}${fatalNote ? ` ${fatalNote}` : ""}`
|
|
3135
3177
|
: "Monitor live: homepage checks + certificate-expiry alerts.",
|
|
3136
3178
|
})
|
|
3137
3179
|
} catch (err) {
|
|
@@ -3140,7 +3182,7 @@ export function StoreProvider({ children }: { children: ReactNode }) {
|
|
|
3140
3182
|
}
|
|
3141
3183
|
void run()
|
|
3142
3184
|
},
|
|
3143
|
-
[servers, reseedVanityPage, persistKumaAuth, persistKumaMonitors],
|
|
3185
|
+
[servers, reseedVanityPage, persistKumaAuth, persistKumaMonitors, isSudoConnected, sudoUsers, disconnectSudo],
|
|
3144
3186
|
)
|
|
3145
3187
|
|
|
3146
3188
|
// Calibrate + register the front-page fingerprint monitor (site monitoring
|
|
@@ -3186,6 +3228,76 @@ export function StoreProvider({ children }: { children: ReactNode }) {
|
|
|
3186
3228
|
[persistKumaAuth, persistKumaMonitors],
|
|
3187
3229
|
)
|
|
3188
3230
|
|
|
3231
|
+
// Register (or recalibrate) the per-site cache-bypass monitor (site
|
|
3232
|
+
// monitoring blind-spot fix, part 2): opt-in, deliberately a much looser
|
|
3233
|
+
// interval than health/fingerprint since it forces a real PHP render on
|
|
3234
|
+
// every check, unlike those two which are served straight from cache.
|
|
3235
|
+
const startBypassMonitorSetup = useCallback(
|
|
3236
|
+
(site: Site, intervalSec: number) => {
|
|
3237
|
+
const conn = cfgRef.current.uptimeKuma
|
|
3238
|
+
if (!conn || isDevMode()) return
|
|
3239
|
+
const setOp = (op: KumaOp) => setKumaOps((prev) => new Map(prev).set(site.id, op))
|
|
3240
|
+
const run = async () => {
|
|
3241
|
+
try {
|
|
3242
|
+
const domain = site.domain
|
|
3243
|
+
const proto = site.https?.enabled ? "https" : "http"
|
|
3244
|
+
const known = cfgRef.current.kumaMonitors[domain] ?? {}
|
|
3245
|
+
setOp({ status: "running", detail: "Registering the cache-bypass monitor…" })
|
|
3246
|
+
const { result: bypassId, jwt } = await withKuma(conn, (kuma) =>
|
|
3247
|
+
registerBypassMonitor(kuma, domain, { proto, interval: intervalSec, knownId: known.bypassId }),
|
|
3248
|
+
)
|
|
3249
|
+
persistKumaAuth(conn, jwt)
|
|
3250
|
+
persistKumaMonitors(domain, { ...known, bypassId })
|
|
3251
|
+
const win = intervalSec % 3600 === 0 ? `${intervalSec / 3600}h` : `${Math.round(intervalSec / 60)}m`
|
|
3252
|
+
setOp({ status: "done", detail: `Cache-bypass check live — forces a fresh render every ${win}, bypassing the page cache.` })
|
|
3253
|
+
} catch (err) {
|
|
3254
|
+
setOp({ status: "error", detail: "", error: (err as Error).message })
|
|
3255
|
+
}
|
|
3256
|
+
}
|
|
3257
|
+
void run()
|
|
3258
|
+
},
|
|
3259
|
+
[persistKumaAuth, persistKumaMonitors],
|
|
3260
|
+
)
|
|
3261
|
+
|
|
3262
|
+
// Remove an opt-in per-site monitor (front-page fingerprint or cache-bypass)
|
|
3263
|
+
// — deliberately NOT offered for health/push/redis/fatal, which are all
|
|
3264
|
+
// re-derived automatically on the next `a`/`R` press (no persisted opt-out
|
|
3265
|
+
// exists for them yet), so "removing" one there would just silently
|
|
3266
|
+
// reappear. These two are genuinely opt-in: pressing `a` after removal
|
|
3267
|
+
// just reopens the picker, nothing re-creates them on its own.
|
|
3268
|
+
const removeSiteMonitor = useCallback(
|
|
3269
|
+
(site: Site, kind: "fingerprint" | "bypass") => {
|
|
3270
|
+
const conn = cfgRef.current.uptimeKuma
|
|
3271
|
+
if (!conn || isDevMode()) return
|
|
3272
|
+
const setOp = (op: KumaOp) => setKumaOps((prev) => new Map(prev).set(site.id, op))
|
|
3273
|
+
const run = async () => {
|
|
3274
|
+
try {
|
|
3275
|
+
const domain = site.domain
|
|
3276
|
+
const known = cfgRef.current.kumaMonitors[domain] ?? {}
|
|
3277
|
+
const id = kind === "fingerprint" ? known.fingerprintId : known.bypassId
|
|
3278
|
+
const label = kind === "fingerprint" ? "Front-page" : "Cache-bypass"
|
|
3279
|
+
if (id == null) return setOp({ status: "done", detail: "Already removed." })
|
|
3280
|
+
setOp({ status: "running", detail: `Removing the ${label.toLowerCase()} monitor…` })
|
|
3281
|
+
const { jwt } = await withKuma(conn, (kuma) => kuma.deleteMonitor(id))
|
|
3282
|
+
persistKumaAuth(conn, jwt)
|
|
3283
|
+
const next = { ...known }
|
|
3284
|
+
if (kind === "fingerprint") {
|
|
3285
|
+
delete next.fingerprintId
|
|
3286
|
+
delete next.fingerprint
|
|
3287
|
+
} else {
|
|
3288
|
+
delete next.bypassId
|
|
3289
|
+
}
|
|
3290
|
+
persistKumaMonitors(domain, next)
|
|
3291
|
+
setOp({ status: "done", detail: `${label} monitor removed.` })
|
|
3292
|
+
} catch (err) {
|
|
3293
|
+
setOp({ status: "error", detail: "", error: (err as Error).message })
|
|
3294
|
+
}
|
|
3295
|
+
}
|
|
3296
|
+
void run()
|
|
3297
|
+
},
|
|
3298
|
+
[persistKumaAuth, persistKumaMonitors],
|
|
3299
|
+
)
|
|
3300
|
+
|
|
3189
3301
|
// Load the alert wiring for one site: every notification provider configured
|
|
3190
3302
|
// in Kuma, and whether each is attached to this site's Spinup monitors. The
|
|
3191
3303
|
// view calls this on demand (`n` in the overlay) — it's a live read, not
|
|
@@ -4083,6 +4195,9 @@ export function StoreProvider({ children }: { children: ReactNode }) {
|
|
|
4083
4195
|
clearVanity,
|
|
4084
4196
|
vanityHealthKeyFor,
|
|
4085
4197
|
kumaConfigured: !isDevMode() && cfgRef.current.uptimeKuma != null,
|
|
4198
|
+
// Base URL for deep-linking straight to a monitor's own Kuma page
|
|
4199
|
+
// (`${kumaUrl}/dashboard/${id}`) — null in Dev Mode / when unconfigured.
|
|
4200
|
+
kumaUrl: !isDevMode() ? (cfgRef.current.uptimeKuma?.url ?? null) : null,
|
|
4086
4201
|
kumaSite,
|
|
4087
4202
|
setKumaSite,
|
|
4088
4203
|
kumaOps,
|
|
@@ -4092,6 +4207,8 @@ export function StoreProvider({ children }: { children: ReactNode }) {
|
|
|
4092
4207
|
startVanityReseed,
|
|
4093
4208
|
startKumaRotate,
|
|
4094
4209
|
startFingerprintSetup,
|
|
4210
|
+
startBypassMonitorSetup,
|
|
4211
|
+
removeSiteMonitor,
|
|
4095
4212
|
fetchKumaAlerts,
|
|
4096
4213
|
toggleKumaAlert,
|
|
4097
4214
|
clearKumaOp,
|