shraga 0.1.31 → 0.1.33

package/src/server/scheduler/engine.ts

@@ -1,10 +1,10 @@
-import { loadSchedules, saveSchedules, readCompletionMarker, writeCompletionMarker, readRunningMarker, isProcessAlive, loadThrottleState, saveThrottleState } from './storage.ts';
+import { loadSchedules, saveSchedules, readCompletionMarker, writeCompletionMarker, readRunningMarker, isProcessAlive, loadThrottleState, saveThrottleState, acquireRunLock, clearRunningMarker, markRunStarted } from './storage.ts';
 import { computeNextRun, computePrevRun, validateTrigger } from './timing.ts';
 import { runSchedule, type ResumeOptions, type EventContext } from './runner.ts';
 import { backfillScope, ensureBuiltinSchedules } from './builtins.ts';
 import { emitEvent } from '../events/bus.ts';
 import { getSessionUrl } from '../shraga-config.ts';
-import type { Schedule } from './types.ts';
+import type { Schedule, MissedPolicy, RunOutcome, RunRefusal } from './types.ts';
 
 type Broadcast = (data: object) => void;
 
@@ -26,6 +26,18 @@ interface RuntimeState {
 }
 
 const QUEUE_CAP = 5;
+/** Catch-up fires are delayed so MCP servers finish initializing first. Read per call so tests
+ *  (and an operator) can shorten it. */
+const catchupDelayMs = () => Number(process.env.SCHEDULER_CATCHUP_DELAY_MS ?? 10_000);
+/** Hard ceiling on how late a missed window may still be replayed, regardless of `onMissed`.
+ *  A missed window is never more than one period old, so a period-relative bound alone can never
+ *  suppress anything — yet replaying an 08:00 report at 22:00 is not the job the schedule
+ *  describes. This absolute cap is therefore the entire rule.
+ *
+ *  Deliberately NOT `min(period, cap)`: a cron missed window comes from `computePrevRun`, so its
+ *  age is always strictly less than one period — a period-relative term can never suppress
+ *  anything, and `min(period, cap)` is provably identical to `cap`. */
+const maxMissedAgeMs = () => Number(process.env.SCHEDULER_MAX_MISSED_AGE_MS ?? 6 * 60 * 60 * 1000);
 /** Max setTimeout delay (2^31-1 ms ≈ 24.8 days); longer delays overflow and fire immediately. */
 const MAX_TIMER_MS = 2_147_483_647;
 /** Only the designated instance fires schedules (DATA_SYNC_SCHEDULER_ACTIVE=true).
@@ -71,38 +83,55 @@ export function start(broadcast: Broadcast): void {
     }
   }
   // Catch up missed cron fires (e.g. process was down when cron should have fired)
-  const catchUps: string[] = [];
+  const catchUps: { id: string; window: number }[] = [];
   for (const s of state.schedules) {
     if (!s.enabled || s.trigger.kind !== 'cron') continue;
     const prev = computePrevRun(s.trigger);
     if (prev === null) continue;
     const lastAt = s.lastRun?.at;
     if (lastAt === undefined) continue; // never ran — nothing to catch up
-    if (lastAt < prev) {
-      const marker = readCompletionMarker(s.id);
-      if (marker && marker.completedAt >= prev) {
-        console.log(`[scheduler] skipping catch-up for ${s.id} — already completed at ${new Date(marker.completedAt).toISOString()} by ${marker.triggeredBy}`);
-        continue;
-      }
-      const running = readRunningMarker(s.id);
-      if (running && isProcessAlive(running.pid)) {
-        console.log(`[scheduler] skipping catch-up for ${s.id} — still running (pid ${running.pid}, started ${new Date(running.startedAt).toISOString()})`);
-        continue;
-      }
-      catchUps.push(s.id);
+    if (lastAt >= prev) continue;
+
+    const marker = readCompletionMarker(s.id);
+    if (marker && marker.completedAt >= prev) {
+      console.log(`[scheduler] skipping catch-up for ${s.id} — already completed at ${new Date(marker.completedAt).toISOString()} by ${marker.triggeredBy}`);
+      continue;
+    }
+    // An ATTEMPT on this window counts too: a run that errored or was killed already had its
+    // shot. Replaying it on the next boot is exactly the duplicate-fire this guards.
+    if (marker?.attemptWindow !== undefined && marker.attemptWindow >= prev) {
+      console.log(`[scheduler] skipping catch-up for ${s.id} — window ${new Date(prev).toISOString()} already attempted (${marker.status ?? 'started'})`);
+      continue;
+    }
+    // Someone is already on it — either another instance, or (far more common) the recovery
+    // path that resumed the interrupted run in-place moments ago.
+    const running = readRunningMarker(s.id);
+    if (running && isProcessAlive(running.pid)) {
+      console.log(`[scheduler] skipping catch-up for ${s.id} — still running (pid ${running.pid}, started ${new Date(running.startedAt).toISOString()})`);
+      continue;
     }
+
+    const verdict = judgeMissed(s, prev);
+    if (!verdict.replay) {
+      console.log(`[scheduler] not replaying ${s.id} — ${verdict.message}`);
+      noteMissed(s, prev, verdict.reason);
+      continue;
+    }
+    catchUps.push({ id: s.id, window: prev });
   }
   if (catchUps.length) {
-    console.log(`[scheduler] catch-up: ${catchUps.join(', ')} (delayed 10s for MCP init)`);
+    console.log(`[scheduler] catch-up: ${catchUps.map((c) => c.id).join(', ')} (delayed ${catchupDelayMs()}ms for MCP init)`);
     setTimeout(() => {
-      for (const id of catchUps) {
+      for (const { id, window } of catchUps) {
         const s = getSchedule(id);
-        if (s?.enabled) {
-          console.log(`[scheduler] catch-up: firing ${id}`);
-          runNow(id);
-        }
+        if (!s?.enabled) continue;
+        console.log(`[scheduler] catch-up: firing ${id}`);
+        // NOT runNow(): a catch-up is not a manual run. It must go through the same
+        // completion/attempt/lock guards as any timer fire, so that whatever claimed this
+        // window first (typically the in-place resume) makes this a no-op.
+        enqueueFire(s, window);
       }
-    }, 10_000);
+    }, catchupDelayMs());
   }
 
   saveSchedules(state.schedules);
@@ -125,6 +154,9 @@ export function getSchedule(id: string): Schedule | undefined {
 export function upsertSchedule(s: Schedule): { ok: true; schedule: Schedule } | { ok: false; error: string } {
   const err = validateTrigger(s.trigger);
   if (err) return { ok: false, error: err };
+  if (s.onMissed !== undefined && !MISSED_POLICIES.includes(s.onMissed)) {
+    return { ok: false, error: `Invalid onMissed "${s.onMissed}" (expected ${MISSED_POLICIES.join(' | ')})` };
+  }
 
   s.updatedAt = Date.now();
   if (s.enabled) {
@@ -180,9 +212,9 @@ export function toggleSchedule(id: string, enabled: boolean): Schedule | null {
   return s;
 }
 
-export function runNow(id: string, override?: string): string | null {
+export function runNow(id: string, override?: string): RunOutcome {
   const s = getSchedule(id);
-  if (!s) return null;
+  if (!s) return refuse('unknown-schedule', `No schedule ${id}`);
   return enqueueFire(s, Date.now(), override, true);
 }
 
@@ -275,22 +307,133 @@ export function cancelRun(id: string): boolean {
  * side-effects, e.g. a second Slack post). Mirrors the web/slack restart-resume path.
  * No-op if the schedule is unknown or already running.
  */
-export function resumeRun(scheduleId: string, sessionId: string, prompt: string): string | null {
+export function resumeRun(scheduleId: string, sessionId: string, prompt: string): RunOutcome {
   const s = getSchedule(scheduleId);
   if (!s) {
     console.warn(`[scheduler] resumeRun: unknown schedule ${scheduleId}`);
-    return null;
+    return refuse('unknown-schedule', `No schedule ${scheduleId}`);
   }
   if (state.running.has(scheduleId)) {
     console.log(`[scheduler] resumeRun: ${scheduleId} already running — skipping resume`);
-    return null;
+    return refuse('already-running', `${s.name} is already running`);
+  }
+  const window = interruptedWindow(s);
+  // A resume continues an existing conversation rather than starting a fresh one, so it dodges
+  // the duplicate-side-effect problem — but it does NOT dodge the "is this work still the work
+  // the schedule asked for" problem. Finishing the 08:00 report at 22:00 is the incident. So the
+  // resume path answers to the same onMissed policy and the same staleness ceiling as catch-up —
+  // for EVERY trigger kind, not just cron (interval/once/event have no catch-up path at all, so
+  // resume is their only gate).
+  if (window !== null) {
+    // Someone already finished this window — typically a catch-up that won the boot race and has
+    // since completed, so the run lock it held is gone. Resuming now would redo work that is
+    // already done: the duplicate fire, one step later. (An *attempt* on this window is NOT a
+    // refusal — the interrupted run we are resuming recorded one itself.)
+    const marker = readCompletionMarker(scheduleId);
+    if (marker && marker.completedAt >= window) {
+      const msg = `window ${new Date(window).toISOString()} already completed at ${new Date(marker.completedAt).toISOString()} by ${marker.triggeredBy}`;
+      console.log(`[scheduler] not resuming ${scheduleId} — ${msg}`);
+      return refuse('already-completed', msg);
+    }
+    const verdict = judgeMissed(s, window);
+    if (!verdict.replay) {
+      console.log(`[scheduler] not resuming ${scheduleId} — ${verdict.message}`);
+      noteMissed(s, window, verdict.reason);
+      return refuse(verdict.refusal, verdict.message);
+    }
   }
   console.log(`[scheduler] resuming ${scheduleId} in-place on session ${sessionId.slice(0, 30)}…`);
-  return startRun(s, Date.now(), undefined, { sessionId, prompt });
+  return startRun(s, window ?? Date.now(), undefined, { sessionId, prompt });
 }
 
 // ── Internals ───────────────────────────────────────────────────────────────
 
+const MISSED_POLICIES: MissedPolicy[] = ['run', 'skip', 'offer'];
+
+function refuse(reason: RunRefusal, message: string): RunOutcome {
+  return { ok: false, reason, message };
+}
+
+/**
+ * The window the interrupted run was covering — what "how late is this?" is measured against.
+ *
+ * For `cron` the window is derivable from the expression itself (`computePrevRun`), and that is
+ * preferred: it is correct even for legacy state written before markers recorded a window.
+ *
+ * `interval`/`once`/`event` have no schedule-derivable grid — but they are not therefore timeless.
+ * An interval run interrupted 14h ago is exactly as stale as a cron one, and it has no catch-up
+ * path to be caught by. So for those the window is the one the interrupted run ITSELF claimed,
+ * read back off the markers already on disk: the run lock's `window` (stamped by `acquireRunLock`
+ * at fire time), else its `startedAt`, else the attempt ledger, else the at-start `lastRun.at`.
+ * Only a schedule with no trace of ever having started has no window — and nothing to resume.
+ */
+function interruptedWindow(s: Schedule): number | null {
+  const prev = computePrevRun(s.trigger);
+  if (prev !== null) return prev;
+  const running = readRunningMarker(s.id);
+  if (running?.window !== undefined) return running.window;
+  if (running?.startedAt) return running.startedAt;
+  const marker = readCompletionMarker(s.id);
+  if (marker?.attemptWindow !== undefined) return marker.attemptWindow;
+  if (marker?.lastAttemptAt) return marker.lastAttemptAt;
+  return s.lastRun?.at ?? null;
+}
+
+/** Persist the terminal outcome of an attempt without touching the last SUCCESSFUL completion.
+ *  Every non-ok exit — reported failure or unexpected throw — must land here, else the marker
+ *  stays `started` forever and a crash is indistinguishable from a throw. */
+function recordAttemptOutcome(scheduleId: string, firedAt: number, at: number, status: 'error' | 'aborted' | 'started'): void {
+  const prev = readCompletionMarker(scheduleId);
+  writeCompletionMarker({
+    completedAt: prev?.completedAt ?? 0,
+    triggeredBy: 'scheduler',
+    scheduleId,
+    lastAttemptAt: prev?.lastAttemptAt ?? at,
+    attemptWindow: prev?.attemptWindow ?? firedAt,
+    status,
+  });
+}
+
+/** Default is `run`: it is what every existing schedule already does, and schedules.json has no
+ *  `onMissed` field on any of them — defaulting to anything else would silently change the
+ *  behaviour of live automations on upgrade. The unbounded-replay hazard that made `run`
+ *  dangerous is fixed by the staleness ceiling below, which applies to `run` too. */
+function missedPolicy(s: Schedule): MissedPolicy {
+  return s.onMissed ?? 'run';
+}
+
+type MissedVerdict =
+  | { replay: true }
+  | { replay: false; reason: 'skip' | 'offer' | 'stale'; refusal: 'policy-skip' | 'policy-offer' | 'stale'; message: string };
+
+/**
+ * The one decision behind "this window already elapsed — may it still run?", shared by all three
+ * replay paths: the boot catch-up scan, the in-place resume, and the late timer fire.
+ *
+ * `onMissed` first (an explicit skip/offer is a standing instruction, not an age question), then
+ * the absolute staleness ceiling, which overrides even `run`. Callers do the recording, because
+ * only they know how to refuse (a return value, or `continue`).
+ */
+function judgeMissed(s: Schedule, window: number, now: number = Date.now()): MissedVerdict {
+  const when = new Date(window).toISOString();
+  const policy = missedPolicy(s);
+  if (policy !== 'run') {
+    return { replay: false, reason: policy, refusal: policy === 'skip' ? 'policy-skip' : 'policy-offer', message: `window ${when} was missed and onMissed=${policy}` };
+  }
+  const age = now - window;
+  const grace = maxMissedAgeMs();
+  if (age > grace) {
+    return { replay: false, reason: 'stale', refusal: 'stale', message: `window ${when} is ${Math.round(age / 60_000)}m stale (ceiling ${Math.round(grace / 60_000)}m)` };
+  }
+  return { replay: true };
+}
+
+function noteMissed(s: Schedule, at: number, reason: 'skip' | 'offer' | 'stale'): void {
+  s.missedRun = { at, reason, noticedAt: Date.now() };
+  saveSchedules(state.schedules);
+  state.broadcast({ type: 'schedule:updated', schedule: s });
+}
+
 function replan(): void {
   if (!schedulerActive) return;
   if (state.timer) { clearTimeout(state.timer); state.timer = null; }
@@ -317,7 +460,24 @@ function fireDue(): void {
   const now = Date.now();
   for (const s of state.schedules) {
     if (!s.enabled || s.nextRun === undefined) continue;
-    if (s.nextRun <= now) enqueueFire(s, s.nextRun);
+    if (s.nextRun > now) continue;
+    // A timer fire is normally punctual, so it is NOT treated as a missed window — `onMissed`
+    // must never gate healthy operation. But a setTimeout does not survive a wall-clock jump:
+    // when the host sleeps (lid closed) or NTP steps the clock, the process is FROZEN, not
+    // killed — no restart, so the boot catch-up scan never runs — and the overdue timer fires
+    // the elapsed window the moment the machine wakes. Firing an 08:00 job at 21:36 is exactly
+    // what the catch-up ceiling exists to prevent, reached by the one path it didn't cover.
+    // Past the ceiling we hand the window to the same judgement the other two paths use.
+    // Past the ceiling `judgeMissed` can only refuse (it replays a `run` window only while it is
+    // within the ceiling), so there is no replay branch here — the shared decision is used for WHAT
+    // to record, not whether to fire.
+    if (now - s.nextRun > maxMissedAgeMs()) {
+      const verdict = judgeMissed(s, s.nextRun, now) as Extract<MissedVerdict, { replay: false }>;
+      console.log(`[scheduler] not firing ${s.id} — ${verdict.message} (timer fired late; host suspended or clock jumped)`);
+      noteMissed(s, s.nextRun, verdict.reason);
+      continue; // the advance loop below still re-arms / retires this schedule
+    }
+    enqueueFire(s, s.nextRun);
   }
   // Advance nextRun for recurring triggers; disable fired `once` triggers
   for (const s of state.schedules) {
@@ -336,21 +496,22 @@ function fireDue(): void {
   replan();
 }
 
-function enqueueFire(s: Schedule, firedAt: number, override?: string, manual = false, eventCtx?: EventContext): string | null {
-  // Skip if this cron period was already completed or still running from a prior server instance.
-  // Manual runs (runNow from UI/API) always proceed.
+function enqueueFire(s: Schedule, firedAt: number, override?: string, manual = false, eventCtx?: EventContext): RunOutcome {
+  // Skip if this cron period was already completed/attempted. Manual runs (runNow from UI/API)
+  // always proceed past the period guard — but never past the run lock in startRun().
   if (!manual && s.trigger.kind === 'cron') {
     const prev = computePrevRun(s.trigger, firedAt + 1);
     if (prev !== null) {
       const marker = readCompletionMarker(s.id);
       if (marker && marker.completedAt >= prev) {
-        console.log(`[scheduler] skipping fire for ${s.id} — already completed this period (at ${new Date(marker.completedAt).toISOString()})`);
-        return null;
+        const msg = `already completed this period (at ${new Date(marker.completedAt).toISOString()})`;
+        console.log(`[scheduler] skipping fire for ${s.id} — ${msg}`);
+        return refuse('already-completed', msg);
       }
-      const running = readRunningMarker(s.id);
-      if (running && isProcessAlive(running.pid)) {
-        console.log(`[scheduler] skipping fire for ${s.id} — still running (pid ${running.pid})`);
-        return null;
+      if (marker?.attemptWindow !== undefined && marker.attemptWindow >= prev) {
+        const msg = `this period was already attempted (${marker.status ?? 'started'})`;
+        console.log(`[scheduler] skipping fire for ${s.id} — ${msg}`);
+        return refuse('already-attempted', msg);
       }
     }
   }
@@ -369,10 +530,36 @@ function enqueueFire(s: Schedule, firedAt: number, override?: string, manual = f
     console.warn(`[scheduler] queue overflow for ${s.id} (cap=${QUEUE_CAP}), dropped fire @ ${dropped?.firedAt}`);
   }
   state.queues.set(s.id, q);
-  return null;
+  return { ok: true, sessionId: null, queued: true };
 }
 
-function startRun(s: Schedule, _firedAt: number, override?: string, resume?: ResumeOptions, eventCtx?: EventContext): string | null {
+/**
+ * Start one run, holding the cross-restart run lock for its whole life.
+ *
+ * Every path that can start a run funnels through here — timer fire, catch-up, event, manual
+ * runNow, and the in-place resume from crash recovery — so the lock is the single place that
+ * enforces "one live run per schedule". It is also what makes catch-up and recovery mutually
+ * exclusive: whichever reaches this first holds a live pid, and the other's acquire fails.
+ */
+function startRun(s: Schedule, firedAt: number, override?: string, resume?: ResumeOptions, eventCtx?: EventContext): RunOutcome {
+  const lock = acquireRunLock(s.id, firedAt);
+  if (!lock) {
+    const held = readRunningMarker(s.id);
+    const msg = `run lock held by pid ${held?.pid} since ${new Date(held?.startedAt ?? 0).toISOString()}`;
+    console.log(`[scheduler] not starting ${s.id} — ${msg}`);
+    return refuse('locked', msg);
+  }
+  // Record the attempt BEFORE running: a crash from here on must not look like "never tried".
+  markRunStarted(s.id, firedAt);
+  const pre = getSchedule(s.id);
+  if (pre) {
+    // Advance lastRun at start, not only on success. start() turns a leftover 'running' into
+    // 'error' on the next boot, so the distinction survives while the timestamp still blocks a
+    // replay of this window.
+    pre.lastRun = { at: Date.now(), sessionId: resume?.sessionId ?? '', status: 'running' };
+    pre.missedRun = undefined;
+    saveSchedules(state.schedules);
+  }
   // Deep-copy task so edits mid-run don't affect the in-flight execution
   const snapshot: Schedule = JSON.parse(JSON.stringify(s));
   let sessionId: string | null = null;
@@ -380,6 +567,14 @@ function startRun(s: Schedule, _firedAt: number, override?: string, resume?: Res
   const register = (sid: string, ac: AbortController) => {
     sessionId = sid;
     state.running.set(s.id, ac);
+    // Backfill the session link onto the at-start lastRun. Without this a crash mid-run persists
+    // an errored run with sessionId '' — no way back to the conversation that was interrupted,
+    // which is exactly what the recovery path needs.
+    const live = getSchedule(s.id);
+    if (live?.lastRun && live.lastRun.status === 'running' && !live.lastRun.sessionId) {
+      live.lastRun.sessionId = sid;
+      saveSchedules(state.schedules);
+    }
   };
 
   state.broadcast({ type: 'schedule:fired', scheduleId: s.id });
@@ -393,8 +588,13 @@ function startRun(s: Schedule, _firedAt: number, override?: string, resume?: Res
         saveSchedules(state.schedules);
         state.broadcast({ type: 'schedule:updated', schedule: live });
       }
+      if (summary.status !== 'ok') {
+        // Keep the attempt on record with its real outcome — the next boot must see that this
+        // window was tried and failed, not that it never ran.
+        recordAttemptOutcome(s.id, firedAt, summary.at, summary.status === 'running' ? 'started' : summary.status);
+      }
       if (summary.status === 'ok') {
-        writeCompletionMarker({ completedAt: summary.at, triggeredBy: 'scheduler', scheduleId: s.id });
+        writeCompletionMarker({ completedAt: summary.at, triggeredBy: 'scheduler', scheduleId: s.id, lastAttemptAt: summary.at, attemptWindow: firedAt, status: 'ok' });
         if (live && live.trigger.kind === 'once') {
           console.log(`[scheduler] auto-deleting completed once-schedule ${s.id}`);
           deleteSchedule(s.id);
@@ -422,9 +622,23 @@ function startRun(s: Schedule, _firedAt: number, override?: string, resume?: Res
     })
     .catch((err) => {
       console.error(`[scheduler] unexpected run failure for ${s.id}:`, err);
+      // A rejection never reaches the .then above, so without this the attempt marker stays
+      // 'started' forever and an unexpected throw is indistinguishable from a power cut.
+      const at = Date.now();
+      recordAttemptOutcome(s.id, firedAt, at, 'error');
+      const live = getSchedule(s.id);
+      if (live) {
+        live.lastRun = { at, sessionId: live.lastRun?.sessionId ?? '', status: 'error', error: `unexpected run failure: ${err?.message ?? String(err)}` };
+        saveSchedules(state.schedules);
+        state.broadcast({ type: 'schedule:updated', schedule: live });
+      }
     })
     .finally(() => {
       state.running.delete(s.id);
+      // Release the lock on EVERY exit path — ok, error, abort, or an unexpected throw. The
+      // runner already clears it on its own terminal states; this is idempotent and covers the
+      // paths that never reach the runner's finally.
+      clearRunningMarker(s.id);
       const q = state.queues.get(s.id);
       if (q && q.length > 0) {
         const next = q.shift()!;
@@ -434,5 +648,5 @@ function startRun(s: Schedule, _firedAt: number, override?: string, resume?: Res
       }
     });
 
-  return sessionId;
+  return { ok: true, sessionId };
 }

package/src/server/scheduler/runner.ts

@@ -6,7 +6,7 @@ import { streamChat, type PermissionHandler } from '../claude.ts';
 import { getMcpConfig } from '../mcp.ts';
 import { appendMessage, createScheduledSession, updateScheduledSessionStatus, setRunStatus, registerLivePartial, unregisterLivePartial, writePartial, clearPartial, acquireSessionLock, releaseSessionLock, type ConvBlock } from '../sessions.ts';
 import type { Schedule, ScheduleRunSummary } from './types.ts';
-import { writeRunningMarker, clearRunningMarker } from './storage.ts';
+import { updateRunLockPid, clearRunningMarker } from './storage.ts';
 import { addUnread } from '../unread.ts';
 
 export interface RunContext {
@@ -114,8 +114,8 @@ export async function runSchedule(
   acquireSessionLock(sessionId, 'scheduler', abortController);
   setRunStatus(sessionId, 'running', 'scheduler');
   onEvent({ type: 'session_busy', sessionId, busy: true });
-  // Mark this period as running with the live pid so startup catch-up won't double-fire it.
-  writeRunningMarker({ pid: process.pid, startedAt: now, scheduleId: schedule.id });
+  // The run lock (running marker) is acquired by the engine BEFORE this point — see
+  // engine.startRun. Writing it here too would let a direct runSchedule() call bypass the lock.
 
   const task = schedule.task;
   if (task.kind === 'job') {
@@ -414,9 +414,9 @@ function runCommandWithMarker(command: string, abortController: AbortController,
       stdio: ['ignore', 'pipe', 'pipe'],
     });
 
-    if (child.pid) {
-      writeRunningMarker({ pid: child.pid, startedAt: Date.now(), scheduleId });
-    }
+    // Re-point the held lock at the child: the job outlives nothing here, but if the server dies
+    // the child may still be alive, and a live pid must keep the schedule locked.
+    if (child.pid) updateRunLockPid(scheduleId, child.pid);
 
     let output = '';
     const handleData = (chunk: Buffer) => {

package/src/server/scheduler/storage.ts

@@ -96,3 +96,77 @@ export function clearRunningMarker(scheduleId: string): void {
 export function isProcessAlive(pid: number): boolean {
   try { process.kill(pid, 0); return true; } catch { return false; }
 }
+
+/** How long a held run lock is believed, before it is treated as abandoned regardless of whether
+ *  its pid answers. Bounds the blast radius of pid reuse: after a power cut the OS restarts pid
+ *  allocation low, so a persisted pid can plausibly be live again under the same uid — and a
+ *  liveness check alone would then wedge the schedule forever. Generous vs any real run (agent
+ *  runs are minutes, not hours) while capping the wedge at one window's worth of a daily job. */
+const runLockMaxAgeMs = () => Number(process.env.SCHEDULER_RUN_LOCK_MAX_AGE_MS ?? 6 * 60 * 60 * 1000);
+
+/**
+ * Claim the single live-run slot for a schedule.
+ *
+ * The lock IS the running marker — same file, same conventions — so it survives a process
+ * restart: after a crash the marker is still on disk but its pid is dead, and a dead pid is
+ * reclaimable (otherwise a power cut would wedge the schedule forever). A LIVE pid, ours or
+ * another instance's, means someone is already running this schedule: the caller must back off —
+ * UNLESS the claim is older than `runLockMaxAgeMs`, which is the escape hatch for a lock wedged
+ * by pid reuse. A live, in-ceiling claim is never stolen, not even by a manual run: on this
+ * single-active-instance design that pid is a real run (an agent session, or a spawned job the
+ * lock was re-pointed at), and starting a second one is the duplicate-fire we are preventing.
+ *
+ * Single-writer by design (only the DATA_SYNC_SCHEDULER_ACTIVE instance fires), so this is a
+ * read-then-write, not an atomic CAS.
+ */
+export function acquireRunLock(scheduleId: string, window: number): RunningMarker | null {
+  const existing = readRunningMarker(scheduleId);
+  if (existing) {
+    const alive = isProcessAlive(existing.pid);
+    const age = Date.now() - (Number.isFinite(existing.startedAt) ? existing.startedAt : 0);
+    const maxAge = runLockMaxAgeMs();
+    if (alive && age <= maxAge) return null;
+    const why = !alive
+      ? `dead pid ${existing.pid}`
+      : `held ${Math.round(age / 60_000)}m by live pid ${existing.pid}, past the ${Math.round(maxAge / 60_000)}m lock ceiling — assuming pid reuse`;
+    console.log(`[scheduler] reclaiming run lock for ${scheduleId} — ${why} (window ${new Date(existing.window ?? existing.startedAt).toISOString()})`);
+  }
+  const marker: RunningMarker = { pid: process.pid, startedAt: Date.now(), scheduleId, window };
+  writeRunningMarker(marker);
+  return marker;
+}
+
+/** Re-point a held lock at a spawned child process. Only ever touches a lock THIS process holds,
+ *  and preserves `startedAt` so re-pointing can't refresh the age ceiling above. */
+export function updateRunLockPid(scheduleId: string, pid: number): void {
+  const existing = readRunningMarker(scheduleId);
+  if (!existing) {
+    console.warn(`[scheduler] updateRunLockPid(${scheduleId}): no lock held — not creating one`);
+    return;
+  }
+  if (existing.pid !== process.pid) {
+    console.warn(`[scheduler] updateRunLockPid(${scheduleId}): lock is held by pid ${existing.pid}, not us — leaving it alone`);
+    return;
+  }
+  writeRunningMarker({ ...existing, scheduleId, pid });
+}
+
+/**
+ * Record that a run STARTED for `window`, before it can succeed or fail.
+ *
+ * Without this a run that errors or is killed leaves no trace on disk, so the next boot sees an
+ * un-completed window and replays it — the double-fire this ledger exists to prevent. The
+ * successful-completion timestamp is preserved untouched so "started" stays distinguishable
+ * from "completed ok".
+ */
+export function markRunStarted(scheduleId: string, window: number, triggeredBy: CompletionMarker['triggeredBy'] = 'scheduler'): void {
+  const prev = readCompletionMarker(scheduleId);
+  writeCompletionMarker({
+    completedAt: prev?.completedAt ?? 0,
+    triggeredBy,
+    scheduleId,
+    lastAttemptAt: Date.now(),
+    attemptWindow: window,
+    status: 'started',
+  });
+}

package/src/server/scheduler/types.ts

@@ -34,16 +34,77 @@ export interface ScheduleRunSummary {
   error?: string;
 }
 
+/** What the scheduler did about a window it found already elapsed at boot.
+ *  - `run`   replay it (today's behaviour, and the default — see engine.ts).
+ *  - `skip`  never replay; the window is simply lost.
+ *  - `offer` don't auto-fire, but record it on the schedule (`missedRun`) so a human can
+ *            see it was missed and run it on demand. */
+export type MissedPolicy = 'run' | 'skip' | 'offer';
+
+/** A window that elapsed while nothing was running and was NOT replayed.
+ *  Persisted on the schedule and returned verbatim by `GET /api/schedules[/:id]` — that read
+ *  path IS the affordance `offer` promises: a human (or the agent, via the scheduler skill) sees
+ *  the missed window and can `POST /api/schedules/:id/run` it on demand. */
+export interface MissedRun {
+  /** The window (fire time) that was missed. */
+  at: number;
+  reason: 'skip' | 'offer' | 'stale';
+  /** When the scheduler noticed — i.e. boot time. Read by whoever acts on the miss: `at` alone
+   *  can't tell "missed 20m ago, still worth running" from "found on a boot two days later". */
+  noticedAt: number;
+}
+
+/** Why a start request did not start a run. */
+export type RunRefusal =
+  | 'unknown-schedule'
+  | 'already-running'
+  | 'already-completed'
+  | 'already-attempted'
+  | 'policy-skip'
+  | 'policy-offer'
+  | 'stale'
+  | 'locked';
+
+/**
+ * Result of asking the engine to start a run (timer fire, catch-up, manual, event, resume).
+ * A refusal carries WHY, so the caller can record a truthful error and map a real HTTP status
+ * instead of collapsing every outcome into "null".
+ */
+export type RunOutcome =
+  | { ok: true; sessionId: string | null; queued?: boolean }
+  | { ok: false; reason: RunRefusal; message: string };
+
+/**
+ * Per-schedule attempt/completion ledger. `completedAt` records the last SUCCESSFUL run;
+ * `lastAttemptAt`/`attemptWindow` record the last run that STARTED, whether or not it finished.
+ * Both are needed: completion drives "this period is done", attempt drives "this period was
+ * already tried, don't replay it after a crash".
+ */
 export interface CompletionMarker {
+  /** Last successful completion; 0 when the schedule has never completed. */
   completedAt: number;
   triggeredBy: 'scheduler' | 'ssh' | 'api' | 'manual';
   scheduleId: string;
+  /** Wall-clock time the last attempt began. */
+  lastAttemptAt?: number;
+  /** The window (fire time) that attempt was covering. */
+  attemptWindow?: number;
+  /** Terminal state of the last attempt; `started` means it never reported back (crash). */
+  status?: 'started' | 'ok' | 'error' | 'aborted';
 }
 
+/**
+ * The run lock: at most one live run per scheduleId, across process restarts.
+ * Written by `acquireRunLock` before a run starts, removed when it reaches a terminal state.
+ * A lock whose `pid` is dead (crash / power loss) is reclaimable — see `acquireRunLock`.
+ */
 export interface RunningMarker {
   pid: number;
   startedAt: number;
   scheduleId: string;
+  /** The window (fire time) this run covers — lets a claimer see WHAT is held, not just that
+   *  something is. */
+  window?: number;
 }
 
 export interface Schedule {
@@ -59,6 +120,10 @@ export interface Schedule {
   nextRun?: number;
   lastRun?: ScheduleRunSummary;
   runCount: number;
+  /** What to do with a window that elapsed while the process was down. Absent ⇒ 'run'. */
+  onMissed?: MissedPolicy;
+  /** Last window that elapsed and was deliberately not replayed (policy or staleness). */
+  missedRun?: MissedRun;
   /** Set when a data-plane module owns this schedule (module name). Module reconcile
    *  updates trigger/task; enable/disable snapshots live in the module's state entry. */
   managedBy?: string;

package/src/server/slack/bot.ts

@@ -21,6 +21,7 @@ import { pipeAgentReply, type AgentEvent, type IngressMessage } from 'mcp-slack-
 import { makeSlackQuestionHandler } from './questions.ts';
 import * as contacts from '../contacts.ts';
 import { getChannelContext, invalidateChannelContext } from './context-cache.ts';
+import { noteSlackSeen } from '../downtime.ts';
 import { getOrCreateSession, registerThreadAlias, setLastMessageTs, setUseUserToken, findSlackSessionBySessionId, getProactiveOrigin, hasSessionForThread, isSlackBotPlaceholderEmail } from './sessions.ts';
 
 const MAX_DOWNLOAD_SIZE = 25 * 1024 * 1024;
@@ -45,6 +46,10 @@ const mdText = (b: ConvBlock): b is { type: 'text'; text: string } => b.type ===
 // half: agent-channel summon rules + threads the agent already owns. Referenced app session state
 // (proactive origins, known threads) is why it can't live in the package.
 export function shouldRespond(msg: IngressMessage): boolean {
+  // Advance the per-channel last-seen cursor for EVERY message we're handed, answered or not:
+  // "seen" is the honest baseline for downtime backfill, and it is also how we learn which
+  // channels exist at all. Purely a bookkeeping write — it never affects the gate below.
+  noteSlackSeen(msg.channel, msg.ts);
   const isAgentChannel = msg.channel === AGENT_CHANNEL;
   const isAgentOriginatedThread = msg.isThreadReply && !!(msg.rawThreadTs && getProactiveOrigin(msg.channel, msg.rawThreadTs));
   const isKnownAgentChannelThread = isAgentChannel && msg.isThreadReply && !!(msg.rawThreadTs && hasSessionForThread(msg.channel, msg.rawThreadTs));