shraga 0.1.31 → 0.1.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/defaults/skills/mcp-server.md +1 -0
- package/defaults/skills/scheduler.md +105 -1
- package/defaults/skills/self-aware.md +12 -1
- package/package.json +1 -1
- package/src/server/boot.ts +51 -4
- package/src/server/downtime.ts +451 -0
- package/src/server/mcp-server.ts +19 -2
- package/src/server/scheduler/engine.ts +256 -42
- package/src/server/scheduler/runner.ts +6 -6
- package/src/server/scheduler/storage.ts +74 -0
- package/src/server/scheduler/types.ts +65 -0
- package/src/server/slack/bot.ts +5 -0
|
@@ -0,0 +1,451 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Downtime recovery — "what happened while I was down?"
|
|
3
|
+
*
|
|
4
|
+
* This box is deliberately NOT always-on. When it comes back, the honest question is not "what
|
|
5
|
+
* should I replay?" but "what did I miss, and what do you want me to do about it?". Phase 1
|
|
6
|
+
* (scheduler run lock + `onMissed`) already stopped the blind replay that finished an 08:00 job at
|
|
7
|
+
* 22:00; this module supplies the other half — the ledger and the on-demand report.
|
|
8
|
+
*
|
|
9
|
+
* Deliberately inert: nothing here starts a run, answers a Slack message, or mutates a schedule.
|
|
10
|
+
* The only writers are the heartbeat, the gap ledger (boot gap + late-tick/suspend gap), and the
|
|
11
|
+
* Slack last-seen cursor.
|
|
12
|
+
* Acting on a finding is always an explicit follow-up (`POST /api/schedules/:id/run`).
|
|
13
|
+
*/
|
|
14
|
+
import { readFileSync, writeFileSync, existsSync, mkdirSync, renameSync } from 'node:fs';
|
|
15
|
+
import { dataPath } from './paths.ts';
|
|
16
|
+
import { loadSchedules } from './scheduler/storage.ts';
|
|
17
|
+
import type { MissedRun } from './scheduler/types.ts';
|
|
18
|
+
|
|
19
|
+
const HEARTBEAT_FILE = dataPath('state/heartbeat.json');
|
|
20
|
+
const DOWNTIME_FILE = dataPath('state/downtime.json');
|
|
21
|
+
const SLACK_CURSORS_FILE = dataPath('state/slack-cursors.json');
|
|
22
|
+
|
|
23
|
+
/** How often liveness is stamped to disk. */
|
|
24
|
+
export const HEARTBEAT_INTERVAL_MS = Number(process.env.HEARTBEAT_INTERVAL_MS ?? 60_000);
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Gap above which an absence counts as downtime — 3 heartbeat intervals (3m by default).
|
|
28
|
+
*
|
|
29
|
+
* The worst case for a CLEAN restart is: the last heartbeat landed a tick before shutdown (up to
|
|
30
|
+
* 1 interval stale) + the process restart itself. One interval is therefore already "normal", and
|
|
31
|
+
* two leaves no margin for a slow boot, a loaded box, or clock/write skew — either would log
|
|
32
|
+
* phantom downtime on every deploy, which is worse than useless (it would drown the real outage).
|
|
33
|
+
* Three intervals is comfortably above every clean-restart case and still far below anything a
|
|
34
|
+
* human would call an outage: a real power cut is minutes-to-hours, not 3 minutes.
|
|
35
|
+
*/
|
|
36
|
+
export const DOWNTIME_THRESHOLD_MS = Number(process.env.DOWNTIME_THRESHOLD_MS ?? 3 * HEARTBEAT_INTERVAL_MS);
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Bounded history: the last 20 outages. This is a "what did I miss" aid, not an uptime archive —
|
|
40
|
+
* the report only ever reads the recent tail, and the joins that make an entry actionable
|
|
41
|
+
* (`missedRun`, Slack history) age out long before 20 outages do. 20 keeps the file trivially
|
|
42
|
+
* small while still covering many months on a box that is off occasionally.
|
|
43
|
+
*/
|
|
44
|
+
export const DOWNTIME_HISTORY_MAX = 20;
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* How long a `suspend` gap keeps colouring the report's `note`.
|
|
48
|
+
*
|
|
49
|
+
* A suspend is never retroactively scanned — no restart happens, so no catch-up ever covers it, and
|
|
50
|
+
* a later `boot` entry does not make it honest. The warning therefore cannot be tied to "is the
|
|
51
|
+
* newest entry a suspend?"; it has to age out on its own. 24h is the horizon of the manual check
|
|
52
|
+
* the note asks for ("which schedule windows fell in the gap?"): nearly every schedule here is
|
|
53
|
+
* daily or tighter, so after a day the same window has come round again and run normally, and the
|
|
54
|
+
* gap is no longer the thing to look at.
|
|
55
|
+
*/
|
|
56
|
+
export const SUSPEND_NOTE_MAX_AGE_MS = Number(process.env.SUSPEND_NOTE_MAX_AGE_MS ?? 24 * 60 * 60 * 1000);
|
|
57
|
+
|
|
58
|
+
export interface DowntimeEntry {
|
|
59
|
+
/** Last proven-alive moment (the final heartbeat before the gap). */
|
|
60
|
+
from: number;
|
|
61
|
+
/** When the process came back. */
|
|
62
|
+
to: number;
|
|
63
|
+
ms: number;
|
|
64
|
+
/**
|
|
65
|
+
* How the gap was noticed. `boot` = the process died (crash, power cut, deploy) and the gap was
|
|
66
|
+
* measured at startup. `suspend` = the process never died — the host slept (lid closed, standby)
|
|
67
|
+
* and a heartbeat tick came back late by more than the threshold. Both are real outages; the
|
|
68
|
+
* distinction matters to the reader, because a `suspend` gap means the process was FROZEN, so
|
|
69
|
+
* nothing at all ran at boot afterwards (no scheduler catch-up scan — see `missedSchedules`).
|
|
70
|
+
* Absent on entries written before this field existed.
|
|
71
|
+
*/
|
|
72
|
+
cause?: 'boot' | 'suspend';
|
|
73
|
+
/**
|
|
74
|
+
* channelId → last Slack ts we had seen when this outage was recorded. Present on `boot` entries
|
|
75
|
+
* ONLY, where it is snapshotted before any live traffic can move the cursors (`recordBootGap()`
|
|
76
|
+
* runs before the Slack ingress mounts — see boot.ts). The live cursor is a tail pointer: one
|
|
77
|
+
* normal message after recovery pushes it past the entire backlog, so a snapshot taken after
|
|
78
|
+
* traffic resumed would hide the outage rather than bound it.
|
|
79
|
+
*
|
|
80
|
+
* Absent on `suspend` entries by design: that gap is recorded from a heartbeat tick, up to
|
|
81
|
+
* HEARTBEAT_INTERVAL_MS after the wake — exactly when the reconnected socket delivers the
|
|
82
|
+
* backlog — so the cursor is no longer provably pre-gap. With no snapshot the backfill floors at
|
|
83
|
+
* the outage start (`from`), which can re-report a few already-seen messages but can never skip
|
|
84
|
+
* unseen ones.
|
|
85
|
+
*/
|
|
86
|
+
slackCursors?: Record<string, string>;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
interface DowntimeFile { entries: DowntimeEntry[] }
|
|
90
|
+
|
|
91
|
+
// ── storage (same conventions as scheduler/storage.ts: tmp file + rename) ──────────────────────
|
|
92
|
+
|
|
93
|
+
function readJson<T>(file: string, fallback: T): T {
|
|
94
|
+
if (!existsSync(file)) return fallback;
|
|
95
|
+
try {
|
|
96
|
+
const parsed = JSON.parse(readFileSync(file, 'utf-8'));
|
|
97
|
+
return parsed && typeof parsed === 'object' ? (parsed as T) : fallback;
|
|
98
|
+
} catch (err) {
|
|
99
|
+
console.error(`[downtime] failed to parse ${file}, starting fresh:`, err);
|
|
100
|
+
return fallback;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
function writeJsonAtomic(file: string, value: unknown): void {
|
|
105
|
+
mkdirSync(dataPath('state'), { recursive: true });
|
|
106
|
+
const tmp = `${file}.tmp`;
|
|
107
|
+
writeFileSync(tmp, JSON.stringify(value, null, 2));
|
|
108
|
+
renameSync(tmp, file);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
// ── heartbeat ─────────────────────────────────────────────────────────────────────────────────
|
|
112
|
+
|
|
113
|
+
export function readHeartbeat(): number | null {
|
|
114
|
+
const hb = readJson<{ at?: number }>(HEARTBEAT_FILE, {});
|
|
115
|
+
return Number.isFinite(hb.at) ? (hb.at as number) : null;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
export function writeHeartbeat(at = Date.now()): void {
|
|
119
|
+
writeJsonAtomic(HEARTBEAT_FILE, { at, pid: process.pid });
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* Stamp liveness every interval, and NOTICE when a tick comes back late.
|
|
124
|
+
*
|
|
125
|
+
* A boot-time check only catches an outage that killed the process. The common failure on this box
|
|
126
|
+
* is the opposite: a MacBook that sleeps with the lid closed. The process is frozen, not killed —
|
|
127
|
+
* it never restarts, so `recordBootGap()` never runs, and a 2.5h outage was completely invisible.
|
|
128
|
+
* A late tick is the one signal that survives a suspend, because the timer resumes on wake and the
|
|
129
|
+
* wall clock has moved on. Unref'd — a heartbeat must never hold the process open.
|
|
130
|
+
*/
|
|
131
|
+
export function startHeartbeat(intervalMs = HEARTBEAT_INTERVAL_MS): () => void {
|
|
132
|
+
writeHeartbeat();
|
|
133
|
+
const timer = setInterval(() => {
|
|
134
|
+
try { recordTickGap(); } catch (err) { console.error('[downtime] heartbeat write failed:', err); }
|
|
135
|
+
}, intervalMs);
|
|
136
|
+
timer.unref?.();
|
|
137
|
+
return () => clearInterval(timer);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
// ── downtime ledger ───────────────────────────────────────────────────────────────────────────
|
|
141
|
+
|
|
142
|
+
export function listDowntime(): DowntimeEntry[] {
|
|
143
|
+
return readJson<DowntimeFile>(DOWNTIME_FILE, { entries: [] }).entries ?? [];
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* The one gap-recording path, shared by the boot check and the late-tick check.
|
|
148
|
+
*
|
|
149
|
+
* Both ask the same question — "the last proven-alive moment is `last`, it is now `now`, is that a
|
|
150
|
+
* hole?" — so they share the same threshold, the same bounds, and the same Slack-cursor snapshot.
|
|
151
|
+
* Writing the heartbeat forward is the caller's job and happens either way (this process is alive
|
|
152
|
+
* NOW regardless of the verdict), which is also what makes an outage record exactly ONCE: the next
|
|
153
|
+
* comparison starts from `now`, not from the stale pre-gap stamp.
|
|
154
|
+
*
|
|
155
|
+
* Never fires anything. It appends a row to a JSON file and logs — that is the whole contract.
|
|
156
|
+
*/
|
|
157
|
+
function recordGap(last: number, now: number, cause: 'boot' | 'suspend'): DowntimeEntry | null {
|
|
158
|
+
const ms = now - last;
|
|
159
|
+
// `<=` also covers a backwards clock jump (negative ms): degrade to "no outage", never invent one.
|
|
160
|
+
if (ms <= DOWNTIME_THRESHOLD_MS) return null;
|
|
161
|
+
|
|
162
|
+
// Freeze the Slack cursors into the entry — but only on the boot path, where they are still
|
|
163
|
+
// provably pre-gap (recordBootGap runs before the Slack ingress mounts). On the suspend path the
|
|
164
|
+
// socket has been back for up to a heartbeat interval and may already have pushed the cursor past
|
|
165
|
+
// the whole backlog; recording that would be worse than recording nothing, so the entry carries
|
|
166
|
+
// no snapshot and the backfill floors at the gap start instead. See DowntimeEntry.slackCursors.
|
|
167
|
+
const snapshot: Record<string, string> = {};
|
|
168
|
+
if (cause === 'boot') for (const [channel, cursor] of Object.entries(listSlackCursors())) snapshot[channel] = cursor.ts;
|
|
169
|
+
|
|
170
|
+
const entry: DowntimeEntry = { from: last, to: now, ms, cause, ...(Object.keys(snapshot).length ? { slackCursors: snapshot } : {}) };
|
|
171
|
+
const entries = [...listDowntime(), entry].slice(-DOWNTIME_HISTORY_MAX);
|
|
172
|
+
writeJsonAtomic(DOWNTIME_FILE, { entries } satisfies DowntimeFile);
|
|
173
|
+
console.log(`[downtime] ${cause} gap of ${Math.round(ms / 60_000)}m recorded — down from ${new Date(last).toISOString()} to ${new Date(now).toISOString()}`);
|
|
174
|
+
return entry;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* Compare the last heartbeat against boot time and, if the gap is real, record it. Returns the
|
|
179
|
+
* entry it recorded, or null — a clean restart (and a first-ever boot, which has no heartbeat to
|
|
180
|
+
* measure from) records NOTHING, so the ledger only ever contains genuine outages.
|
|
181
|
+
*/
|
|
182
|
+
export function recordBootGap(bootTime = Date.now()): DowntimeEntry | null {
|
|
183
|
+
const last = readHeartbeat();
|
|
184
|
+
// Write the new heartbeat regardless: whatever we conclude, this process is alive now.
|
|
185
|
+
writeHeartbeat(bootTime);
|
|
186
|
+
if (last === null) return null;
|
|
187
|
+
return recordGap(last, bootTime, 'boot');
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/**
|
|
191
|
+
* One heartbeat tick: stamp liveness, and record an outage if the tick is late past the threshold.
|
|
192
|
+
*
|
|
193
|
+
* The reference point is the PERSISTED heartbeat — the same value `recordBootGap()` measures from,
|
|
194
|
+
* so boot and suspend can never disagree about where the last proven-alive moment was, and can
|
|
195
|
+
* never double-record one outage: whichever check sees the gap first advances the stamp, and the
|
|
196
|
+
* other then measures from the new one.
|
|
197
|
+
*
|
|
198
|
+
* A long suspend yields ONE entry, not one per missed interval, because a frozen process fires no
|
|
199
|
+
* timers while it sleeps — `setInterval` does not accumulate a backlog of missed ticks — and even
|
|
200
|
+
* if it did, this writes the heartbeat forward before the next tick can compare.
|
|
201
|
+
*/
|
|
202
|
+
export function recordTickGap(now = Date.now()): DowntimeEntry | null {
|
|
203
|
+
const last = readHeartbeat();
|
|
204
|
+
writeHeartbeat(now);
|
|
205
|
+
if (last === null) return null;
|
|
206
|
+
return recordGap(last, now, 'suspend');
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
// ── Slack last-seen cursors ───────────────────────────────────────────────────────────────────
|
|
210
|
+
|
|
211
|
+
/** channelId → { ts: last message ts we observed, at: when we observed it }. */
|
|
212
|
+
export interface SlackCursor { ts: string; at: number }
|
|
213
|
+
type SlackCursors = Record<string, SlackCursor>;
|
|
214
|
+
|
|
215
|
+
export function listSlackCursors(): SlackCursors {
|
|
216
|
+
return readJson<SlackCursors>(SLACK_CURSORS_FILE, {});
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/**
|
|
220
|
+
* Record that we saw `ts` in `channel`. Called for every inbound Slack message the ingress hands
|
|
221
|
+
* us (whether or not the agent chose to answer it) — it is a LIVE TAIL pointer: it says where the
|
|
222
|
+
* stream is now, not what was handled, and after a recovery it runs ahead of the outage backlog
|
|
223
|
+
* within one message (which is why an outage snapshots it, see `DowntimeEntry.slackCursors`).
|
|
224
|
+
* Monotonic: an out-of-order event can't rewind the cursor and cause a re-fetch of already-seen
|
|
225
|
+
* history. Slack ts values carry 16 significant digits, so they are compared as STRINGS — `Number`
|
|
226
|
+
* rounds them to a double and makes same-second messages compare equal.
|
|
227
|
+
*/
|
|
228
|
+
export function noteSlackSeen(channel: string, ts: string): void {
|
|
229
|
+
if (!channel || !ts) return;
|
|
230
|
+
try {
|
|
231
|
+
const cursors = listSlackCursors();
|
|
232
|
+
const prev = cursors[channel];
|
|
233
|
+
if (prev && prev.ts >= ts) return;
|
|
234
|
+
cursors[channel] = { ts, at: Date.now() };
|
|
235
|
+
writeJsonAtomic(SLACK_CURSORS_FILE, cursors);
|
|
236
|
+
} catch (err) {
|
|
237
|
+
console.error('[downtime] failed to record Slack cursor:', err);
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
export interface BackfilledMessage {
|
|
242
|
+
channel: string;
|
|
243
|
+
ts: string;
|
|
244
|
+
user?: string;
|
|
245
|
+
botId?: string;
|
|
246
|
+
text: string;
|
|
247
|
+
clientMsgId?: string;
|
|
248
|
+
/** True when the text mentions the agent (bot or user id) — the subset most likely to need action. */
|
|
249
|
+
mentionsAgent: boolean;
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
export interface SlackBackfill {
|
|
253
|
+
/** `truncated` = the page cap was hit with more still waiting, so the OLDEST part of the outage is missing. */
|
|
254
|
+
channels: { channel: string; from: string; fetched: number; truncated?: boolean; error?: string }[];
|
|
255
|
+
messages: BackfilledMessage[];
|
|
256
|
+
skipped?: string;
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
/** Injected so tests (and any future caller) can drive the join without a live Slack workspace. */
|
|
260
|
+
export type SlackHistoryFn = (method: string, body: Record<string, unknown>) => Promise<any>;
|
|
261
|
+
|
|
262
|
+
export interface BackfillOptions {
|
|
263
|
+
/** Defaults to the shared mcp-slack-use client (`slackPost`) — the app's ONE Slack seam. */
|
|
264
|
+
history?: SlackHistoryFn;
|
|
265
|
+
/** Agent ids to flag mentions against. */
|
|
266
|
+
agentIds?: string[];
|
|
267
|
+
/** Safety valve on a very long outage: pages of 200 per channel. */
|
|
268
|
+
maxPages?: number;
|
|
269
|
+
/**
|
|
270
|
+
* channelId → last-seen ts AS OF the outage (`DowntimeEntry.slackCursors`). The live cursors are
|
|
271
|
+
* deliberately NOT used as a floor: they advance with the stream and would skip the backlog.
|
|
272
|
+
*/
|
|
273
|
+
cursors?: Record<string, string>;
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/**
|
|
277
|
+
* Fetch what arrived during `range` from each channel we know about, on demand.
|
|
278
|
+
*
|
|
279
|
+
* Why history and not event replay: Slack's Events API retries a failed delivery for only ~30
|
|
280
|
+
* minutes and then drops the event PERMANENTLY — after a multi-hour outage there is nothing left
|
|
281
|
+
* to redeliver. `conversations.history` keeps the same messages readable for days (retention), so
|
|
282
|
+
* it, not the event stream, is the durable recovery source. This is also why it is on-demand only
|
|
283
|
+
* and never runs at boot: it is a paid, rate-limited read of someone else's system, and its result
|
|
284
|
+
* is a REPORT, not a work queue.
|
|
285
|
+
*/
|
|
286
|
+
export async function fetchSlackBackfill(range: { from: number; to?: number }, opts: BackfillOptions = {}): Promise<SlackBackfill> {
|
|
287
|
+
// Which channels to ask about: the ones seen at the time of the outage, the ones seen since
|
|
288
|
+
// (a channel that only became active during/after the outage still has missed history), and the
|
|
289
|
+
// agent's own channel — which must be readable on a first deploy, before any traffic at all.
|
|
290
|
+
const snapshot = opts.cursors ?? {};
|
|
291
|
+
const channels = [...new Set([
|
|
292
|
+
...Object.keys(snapshot),
|
|
293
|
+
...Object.keys(listSlackCursors()),
|
|
294
|
+
...(process.env.SLACK_AGENT_CHANNEL ? [process.env.SLACK_AGENT_CHANNEL] : []),
|
|
295
|
+
])];
|
|
296
|
+
if (!channels.length) return { channels: [], messages: [], skipped: 'no channels seen yet (no cursor recorded, no SLACK_AGENT_CHANNEL)' };
|
|
297
|
+
|
|
298
|
+
let history = opts.history;
|
|
299
|
+
let agentIds = (opts.agentIds ?? []).filter(Boolean);
|
|
300
|
+
if (!history) {
|
|
301
|
+
try {
|
|
302
|
+
// The app's ONE Slack seam (slack/api.ts re-exports the mcp-slack-use client). No new HTTP
|
|
303
|
+
// client, no second token-resolution path.
|
|
304
|
+
const api = await import('./slack/api.ts');
|
|
305
|
+
history = (method, body) => api.slackPost(method, body);
|
|
306
|
+
if (!agentIds.length) {
|
|
307
|
+
const ids = await Promise.all([api.getBotUserId().catch(() => null), api.getAgentUserId().catch(() => null)]);
|
|
308
|
+
agentIds = ids.filter((id): id is string => !!id);
|
|
309
|
+
}
|
|
310
|
+
} catch (err) {
|
|
311
|
+
return { channels: [], messages: [], skipped: `Slack client unavailable: ${(err as Error).message}` };
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
const maxPages = opts.maxPages ?? 5;
|
|
316
|
+
const out: SlackBackfill = { channels: [], messages: [] };
|
|
317
|
+
const seen = new Set<string>();
|
|
318
|
+
|
|
319
|
+
for (const channel of channels) {
|
|
320
|
+
// Start at the outage start, raised only by a SNAPSHOT cursor that is later still (a channel
|
|
321
|
+
// whose last-seen message post-dates `range.from` — we already saw those). Never raised by the
|
|
322
|
+
// live cursor: that one has moved on with the stream and would hide the whole backlog.
|
|
323
|
+
const oldest = Math.max(Number(snapshot[channel]) || 0, range.from / 1000);
|
|
324
|
+
let fetched = 0;
|
|
325
|
+
let pageCursor: string | undefined;
|
|
326
|
+
let error: string | undefined;
|
|
327
|
+
let truncated = false;
|
|
328
|
+
try {
|
|
329
|
+
for (let page = 0; page < maxPages; page++) {
|
|
330
|
+
const body: Record<string, unknown> = { channel, oldest: String(oldest), limit: 200 };
|
|
331
|
+
if (range.to) body.latest = String(range.to / 1000);
|
|
332
|
+
if (pageCursor) body.cursor = pageCursor;
|
|
333
|
+
const res = await history('conversations.history', body);
|
|
334
|
+
if (!res?.ok) { error = String(res?.error ?? 'unknown Slack error'); break; }
|
|
335
|
+
for (const m of (res.messages ?? []) as any[]) {
|
|
336
|
+
// Dedupe on client_msg_id (Slack's own idempotency key), falling back to channel+ts for
|
|
337
|
+
// messages that carry none (bot posts, joins). Paging overlap and a re-run of this
|
|
338
|
+
// report must not double-report the same message.
|
|
339
|
+
const key = m.client_msg_id ?? `${channel}:${m.ts}`;
|
|
340
|
+
if (seen.has(key)) continue;
|
|
341
|
+
seen.add(key);
|
|
342
|
+
const text = String(m.text ?? '');
|
|
343
|
+
out.messages.push({
|
|
344
|
+
channel,
|
|
345
|
+
ts: String(m.ts),
|
|
346
|
+
user: m.user,
|
|
347
|
+
botId: m.bot_id,
|
|
348
|
+
text,
|
|
349
|
+
clientMsgId: m.client_msg_id,
|
|
350
|
+
mentionsAgent: agentIds.some(id => text.includes(id)),
|
|
351
|
+
});
|
|
352
|
+
fetched++;
|
|
353
|
+
}
|
|
354
|
+
pageCursor = res.response_metadata?.next_cursor || undefined;
|
|
355
|
+
if (!res.has_more || !pageCursor) break;
|
|
356
|
+
// Still more waiting when the cap is reached: say so. conversations.history returns
|
|
357
|
+
// NEWEST-first, so what we dropped is the START of the outage — the oldest and most likely
|
|
358
|
+
// to have been missed. A silently short report would read as "that's everything".
|
|
359
|
+
if (page === maxPages - 1) truncated = true;
|
|
360
|
+
}
|
|
361
|
+
} catch (err) {
|
|
362
|
+
error = (err as Error).message;
|
|
363
|
+
}
|
|
364
|
+
out.channels.push({ channel, from: String(oldest), fetched, ...(truncated ? { truncated } : {}), ...(error ? { error } : {}) });
|
|
365
|
+
}
|
|
366
|
+
out.messages.sort((a, b) => Number(a.ts) - Number(b.ts));
|
|
367
|
+
return out;
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
// ── the on-demand report ──────────────────────────────────────────────────────────────────────
|
|
371
|
+
|
|
372
|
+
export interface MissedScheduleReport {
|
|
373
|
+
id: string;
|
|
374
|
+
name: string;
|
|
375
|
+
missedRun: MissedRun;
|
|
376
|
+
/** The outage this window falls inside, if any. */
|
|
377
|
+
downtime: DowntimeEntry | null;
|
|
378
|
+
/** What the USER can choose to do. Nothing here runs it. */
|
|
379
|
+
proposal: string;
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
export interface DowntimeReport {
|
|
383
|
+
now: number;
|
|
384
|
+
heartbeat: { at: number; ageMs: number } | null;
|
|
385
|
+
downtime: DowntimeEntry[];
|
|
386
|
+
lastDowntime: DowntimeEntry | null;
|
|
387
|
+
missedSchedules: MissedScheduleReport[];
|
|
388
|
+
slack?: SlackBackfill;
|
|
389
|
+
note: string;
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
/**
|
|
393
|
+
* Join phase 1's `missedRun` records to the outage that covers them.
|
|
394
|
+
*
|
|
395
|
+
* It does NOT recompute which windows were missed — the scheduler owns that decision (policy +
|
|
396
|
+
* staleness ceiling) and re-deriving it here would be a second, silently-diverging opinion.
|
|
397
|
+
*
|
|
398
|
+
* Read from schedules.json rather than the engine's in-memory list: the engine `saveSchedules()`
|
|
399
|
+
* on every mutation (incl. `noteMissed`), so disk is current, and reading it keeps this module
|
|
400
|
+
* independent of whether the engine has been started — which matters because a passive twin and
|
|
401
|
+
* this report both need the answer without owning the scheduler.
|
|
402
|
+
*/
|
|
403
|
+
export function missedSchedules(entries = listDowntime()): MissedScheduleReport[] {
|
|
404
|
+
return loadSchedules()
|
|
405
|
+
.filter((s): s is typeof s & { missedRun: MissedRun } => !!s.missedRun)
|
|
406
|
+
.map((s) => ({
|
|
407
|
+
id: s.id,
|
|
408
|
+
name: s.name,
|
|
409
|
+
missedRun: s.missedRun,
|
|
410
|
+
downtime: entries.find(e => s.missedRun.at >= e.from && s.missedRun.at <= e.to) ?? null,
|
|
411
|
+
proposal: `POST /api/schedules/${s.id}/run to run this window now (nothing has run it)`,
|
|
412
|
+
}));
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
/**
|
|
416
|
+
* Build the "what did I miss?" answer. Read-only apart from the Slack fetch, which is a read of
|
|
417
|
+
* Slack. Callers pass `slack: false` to skip the network entirely.
|
|
418
|
+
*/
|
|
419
|
+
export async function buildReport(opts: { slack?: boolean; backfill?: BackfillOptions } = {}): Promise<DowntimeReport> {
|
|
420
|
+
const now = Date.now();
|
|
421
|
+
const entries = listDowntime();
|
|
422
|
+
const last = entries[entries.length - 1] ?? null;
|
|
423
|
+
const hb = readHeartbeat();
|
|
424
|
+
const report: DowntimeReport = {
|
|
425
|
+
now,
|
|
426
|
+
heartbeat: hb === null ? null : { at: hb, ageMs: now - hb },
|
|
427
|
+
downtime: entries,
|
|
428
|
+
lastDowntime: last,
|
|
429
|
+
missedSchedules: missedSchedules(entries),
|
|
430
|
+
note: 'Report only — nothing here has been run, answered, or replayed. Act on an item explicitly.',
|
|
431
|
+
};
|
|
432
|
+
// A suspend gap froze the process instead of killing it, so the scheduler's boot-time catch-up
|
|
433
|
+
// scan never ran. The late timer fire on wake DOES record the windows it refuses, so the list is
|
|
434
|
+
// not empty — it is INCOMPLETE: it holds only what that fire judged. Say that, and say it for
|
|
435
|
+
// every recent suspend, not just the newest entry: a restart afterwards appends a `boot` entry
|
|
436
|
+
// but scans nothing retroactively, so gating on `last` would silently drop the warning.
|
|
437
|
+
const suspends = entries.filter(e => e.cause === 'suspend' && now - e.to <= SUSPEND_NOTE_MAX_AGE_MS);
|
|
438
|
+
if (suspends.length) {
|
|
439
|
+
const when = suspends.map(e => `${new Date(e.from).toISOString()}→${new Date(e.to).toISOString()}`).join(', ');
|
|
440
|
+
report.note += ` NOTE: a recent outage was a host SUSPEND (the process was frozen, not restarted: ${when}), so no scheduler catch-up scan ran. missedSchedules is INCOMPLETE for that gap — it lists only the windows the late timer fire itself judged on wake; any other window that elapsed inside the gap left no record at all. Check schedules whose window falls in the gap by hand.`;
|
|
441
|
+
}
|
|
442
|
+
if (opts.slack !== false) {
|
|
443
|
+
report.slack = last
|
|
444
|
+
// The snapshot frozen at boot, not the live cursors — see DowntimeEntry.slackCursors.
|
|
445
|
+
? await fetchSlackBackfill({ from: last.from }, { cursors: last.slackCursors, ...opts.backfill })
|
|
446
|
+
: { channels: [], messages: [], skipped: 'no recorded downtime to backfill' };
|
|
447
|
+
const cut = report.slack.channels.filter(c => c.truncated).map(c => c.channel);
|
|
448
|
+
if (cut.length) report.note += ` INCOMPLETE: hit the page cap on ${cut.join(', ')} — the OLDEST part of the outage is missing from this report.`;
|
|
449
|
+
}
|
|
450
|
+
return report;
|
|
451
|
+
}
|
package/src/server/mcp-server.ts
CHANGED
|
@@ -9,6 +9,7 @@ import { listWorkspaceTree, readWorkspaceFile, safeResolve, searchWorkspace } fr
|
|
|
9
9
|
import { listSkills, getSkill, saveSkill } from './skills.ts';
|
|
10
10
|
import { getAllSessions, loadConversation, isSessionLocked } from './sessions.ts';
|
|
11
11
|
import * as scheduler from './scheduler/index.ts';
|
|
12
|
+
import { buildReport } from './downtime.ts';
|
|
12
13
|
import { getAgentConfig } from './claude.ts';
|
|
13
14
|
import { validateApiKey } from './api-keys.ts';
|
|
14
15
|
import { verifyMcpToken } from './auth.ts';
|
|
@@ -247,8 +248,24 @@ export function createShragaMcp(deps: McpServerDeps) {
|
|
|
247
248
|
if (!id) return json({ error: 'id required' }, { status: 400 });
|
|
248
249
|
const schedule = scheduler.getSchedule(id);
|
|
249
250
|
if (!schedule) return json({ error: 'Schedule not found' }, { status: 404 });
|
|
250
|
-
scheduler.runNow(id);
|
|
251
|
-
return json({
|
|
251
|
+
const outcome = scheduler.runNow(id);
|
|
252
|
+
if (!outcome.ok) return json({ error: outcome.message, reason: outcome.reason }, { status: 409 });
|
|
253
|
+
return json({ ok: true, id, sessionId: outcome.sessionId, queued: outcome.queued ?? false });
|
|
254
|
+
} catch (e) { return json({ error: errMessage(e) }, { status: 500 }); }
|
|
255
|
+
});
|
|
256
|
+
|
|
257
|
+
// ── Downtime ─────────────────────────────────────────────────────────────
|
|
258
|
+
|
|
259
|
+
base.describeMCP('/downtime', 'GET', {
|
|
260
|
+
description: 'Answer "what did I miss while I was down?". Returns recorded downtime ranges, the schedule windows that were deliberately NOT replayed (missedRun), and the Slack messages that arrived during the last outage. Reports and proposes only — it never runs, replays, or answers anything; act on an item with the schedules/run tool.',
|
|
261
|
+
params: {
|
|
262
|
+
slack: { description: 'Set "0" to skip the Slack history read (no network).', type: 'string' },
|
|
263
|
+
},
|
|
264
|
+
annotations: { readOnlyHint: true },
|
|
265
|
+
});
|
|
266
|
+
router.get('/downtime', async (req: any) => {
|
|
267
|
+
try {
|
|
268
|
+
return json(await buildReport({ slack: str(req.query.slack) !== '0' }));
|
|
252
269
|
} catch (e) { return json({ error: errMessage(e) }, { status: 500 }); }
|
|
253
270
|
});
|
|
254
271
|
|