shraga 0.1.37 → 0.1.39

package/defaults/skills/scheduler.md

@@ -150,6 +150,11 @@ with the process frozen), the scheduler decides whether to replay it. `onMissed`
 | `skip` | Never replay. The window is simply lost. |
 | `offer` | Don't auto-fire, but **record** it so a human/agent can run it on demand. |
 
+`skip`/`offer` suppress *unattended replay* generally — including the failed-run re-arm below, not
+just a window missed while the process was down. A schedule set to `offer` will not retry itself
+after an outage; that is the point of the setting, but it is also how a daily report goes quietly
+missing for a day.
+
 ```bash
 curl -s -X PUT -H "Content-Type: application/json" -H "x-internal-token: $INTERNAL_API_TOKEN" \
   http://localhost:$PORT/api/schedules/{id} -d '{ "onMissed": "offer" }' | jq .
@@ -158,8 +163,16 @@ curl -s -X PUT -H "Content-Type: application/json" -H "x-internal-token: $INTERN
 - **A staleness ceiling overrides every policy, `run` included**: a window more than
   `SCHEDULER_MAX_MISSED_AGE_MS` (default **6h**) late is never replayed. Finishing an 08:00 report
   at 22:00 is not the job the schedule describes.
-- One decision (`judgeMissed` in `scheduler/engine.ts`) governs **all three** paths that could
+- One decision (`judgeMissed` in `scheduler/engine.ts`) governs **all four** paths that could
   replay a window:
+  0. **Failed-run re-arm** — the run fired on time but errored *before producing anything* (no
+     `tool_use`, no text, no thinking: `sideEffectFree` on the run summary). The work provably
+     never started, so the window has not been spent. The engine retries the **same** window on a
+     5/15/30/60m backoff — the in-process ladder in `runner.ts` is sub-second, tuned for a flaky
+     engine, and an outage (a capped API key, a dead upstream) outlives it. A newer window
+     supersedes a pending retry; disable/delete cancels it; success clears it. The re-arm timer is
+     in-memory, so boot catch-up re-derives the same condition from `lastRun` and picks it up
+     across a restart.
   1. **Boot catch-up** (cron only) — the process was down when the window passed.
   2. **In-place resume** of a run interrupted mid-flight — every trigger kind
      (`interval`/`once`/`event` have no catch-up, so resume is their only gate; their window is

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "shraga",
-  "version": "0.1.37",
+  "version": "0.1.39",
   "description": "The teammate you delegate coding to — a self-hostable, multi-user AI coding agent web UI (Claude Code, with a pluggable engine seam).",
   "type": "module",
   "main": "./src/index.ts",

package/src/server/scheduler/engine.ts

@@ -1,4 +1,4 @@
-import { loadSchedules, saveSchedules, readCompletionMarker, writeCompletionMarker, readRunningMarker, isProcessAlive, loadThrottleState, saveThrottleState, acquireRunLock, clearRunningMarker, markRunStarted } from './storage.ts';
+import { loadSchedules, saveSchedules, readCompletionMarker, writeCompletionMarker, readRunningMarker, isProcessAlive, loadThrottleState, saveThrottleState, acquireRunLock, clearRunningMarker, markRunStarted, releaseAttemptWindow } from './storage.ts';
 import { computeNextRun, computePrevRun, validateTrigger } from './timing.ts';
 import { runSchedule, type ResumeOptions, type EventContext } from './runner.ts';
 import { backfillScope, ensureBuiltinSchedules } from './builtins.ts';
@@ -22,6 +22,9 @@ interface RuntimeState {
   queues: Map<string, QueuedFire[]>;
   /** Schedules currently running (id → AbortController for the live run). */
   running: Map<string, AbortController>;
+  /** Pending re-arms of a window whose run failed before producing anything (id → timer). One
+   *  per schedule: a schedule can only be retrying one window at a time. */
+  retries: Map<string, { timer: ReturnType<typeof setTimeout>; window: number; attempt: number }>;
   broadcast: Broadcast;
 }
 
@@ -50,6 +53,7 @@ const state: RuntimeState = {
   timer: null,
   queues: new Map(),
   running: new Map(),
+  retries: new Map(),
   broadcast: () => {},
 };
 
@@ -90,7 +94,13 @@ export function start(broadcast: Broadcast): void {
     if (prev === null) continue;
     const lastAt = s.lastRun?.at;
     if (lastAt === undefined) continue; // never ran — nothing to catch up
-    if (lastAt >= prev) continue;
+    // A run that errored WITHOUT producing anything did not spend its window (see rearmWindow):
+    // its work provably never started, so the window is still owed even though `lastRun.at` is
+    // newer than it. The in-memory re-arm timer does NOT survive a restart — and on a box whose
+    // watchdog restarts the service, that is the common case, not the exotic one — so boot must
+    // recognise the same condition or the retry silently dies with the process.
+    const owedByFailure = s.lastRun?.status === 'error' && s.lastRun.sideEffectFree === true;
+    if (lastAt >= prev && !owedByFailure) continue;
 
     const marker = readCompletionMarker(s.id);
     if (marker && marker.completedAt >= prev) {
@@ -182,6 +192,7 @@ export function deleteSchedule(id: string): boolean {
   if (idx < 0) return false;
   state.schedules.splice(idx, 1);
   state.queues.delete(id);
+  clearRearm(id);
   const ac = state.running.get(id);
   if (ac) ac.abort();
   saveSchedules(state.schedules);
@@ -205,6 +216,8 @@ export function toggleSchedule(id: string, enabled: boolean): Schedule | null {
     }
   } else {
     s.nextRun = undefined;
+    // A disabled schedule must not come back to life via a pending retry.
+    clearRearm(id);
   }
   saveSchedules(state.schedules);
   replan();
@@ -428,6 +441,75 @@ function judgeMissed(s: Schedule, window: number, now: number = Date.now()): Mis
   return { replay: true };
 }
 
+/** Backoff between re-arms of a side-effect-free failed window. Deliberately minutes, not the
+ *  sub-second ladder the in-process retry uses: that one exists for a flaky engine, this one for an
+ *  outage — a capped API key or a dead upstream is not coming back in 2 seconds. The last entry
+ *  repeats; the real bound is the staleness ceiling, which stops replay well before the ladder does.
+ *  Read per call (like `catchupDelayMs`) so tests — and an operator — can shorten it. */
+const rearmBackoffMs = (): number[] => {
+  const override = process.env.SCHEDULER_REARM_BACKOFF_MS;
+  if (!override) return [5 * 60_000, 15 * 60_000, 30 * 60_000, 60 * 60_000];
+  const parsed = override.split(',').map((v) => Number(v.trim())).filter((v) => Number.isFinite(v) && v >= 0);
+  return parsed.length ? parsed : [5 * 60_000];
+};
+
+/** Cancel any pending re-arm for a schedule (disabled, deleted, or superseded by a newer window). */
+function clearRearm(id: string): void {
+  const pending = state.retries.get(id);
+  if (!pending) return;
+  clearTimeout(pending.timer);
+  state.retries.delete(id);
+}
+
+/**
+ * Re-arm a window whose run failed before producing any output. Returns true when a retry is
+ * pending — the caller must then NOT record an attempt outcome, because an attempt marker is
+ * exactly what makes the window un-replayable.
+ *
+ * Refuses (returns false, caller falls through to the normal failure path) when the retry would
+ * land outside what `judgeMissed` permits: past the staleness ceiling, or against an explicit
+ * `onMissed` of `skip`/`offer`. Those are standing instructions about unattended replay and this
+ * is unattended replay, so it answers to them rather than routing around them.
+ */
+function rearmWindow(s: Schedule, window: number): boolean {
+  if (!schedulerActive) return false;
+  const live = getSchedule(s.id);
+  if (!live?.enabled) return false;
+
+  const pending = state.retries.get(s.id);
+  const attempt = pending?.window === window ? pending.attempt + 1 : 1;
+  clearRearm(s.id);
+
+  const ladder = rearmBackoffMs();
+  const delay = ladder[Math.min(attempt - 1, ladder.length - 1)]!;
+  // Judge the window as of WHEN THE RETRY WOULD FIRE, not now — arming a timer that is already
+  // doomed to be refused just burns the ceiling in silence.
+  const verdict = judgeMissed(live, window, Date.now() + delay);
+  if (!verdict.replay) {
+    console.log(`[scheduler] not re-arming ${s.id} — ${verdict.message}`);
+    if (verdict.reason !== 'skip') noteMissed(live, window, verdict.reason);
+    return false;
+  }
+
+  // The at-start marker already stamped this window as attempted; that stamp is what would make
+  // the replay a no-op, so drop it. Nothing else about the marker moves.
+  releaseAttemptWindow(s.id, window);
+  console.warn(`[scheduler] ${s.id} failed before producing any output — re-arming window ${new Date(window).toISOString()} in ${Math.round(delay / 60_000)}m (retry ${attempt})`);
+  const timer = setTimeout(() => {
+    // Deliberately NOT deleting the entry: it carries the attempt count, and the retry we are
+    // about to fire may fail again. Dropping it here would restart the ladder at 5m on every
+    // round — a fixed 5m poll wearing the shape of a backoff. It is cleared on success, on a
+    // superseding window, and on disable/delete.
+    const cur = getSchedule(s.id);
+    if (!cur?.enabled) return;
+    console.log(`[scheduler] retrying window ${new Date(window).toISOString()} for ${s.id}`);
+    enqueueFire(cur, window);
+  }, delay);
+  timer.unref?.();
+  state.retries.set(s.id, { timer, window, attempt });
+  return true;
+}
+
 function noteMissed(s: Schedule, at: number, reason: 'skip' | 'offer' | 'stale'): void {
   s.missedRun = { at, reason, noticedAt: Date.now() };
   saveSchedules(state.schedules);
@@ -497,6 +579,13 @@ function fireDue(): void {
 }
 
 function enqueueFire(s: Schedule, firedAt: number, override?: string, manual = false, eventCtx?: EventContext): RunOutcome {
+  // A newer window supersedes a pending retry of an older one: today's 08:00 report is the job,
+  // not yesterday's. Retrying both would double-post.
+  const pendingRetry = state.retries.get(s.id);
+  if (pendingRetry && firedAt > pendingRetry.window) {
+    console.log(`[scheduler] dropping pending retry of ${new Date(pendingRetry.window).toISOString()} for ${s.id} — superseded by a newer window`);
+    clearRearm(s.id);
+  }
   // Skip if this cron period was already completed/attempted. Manual runs (runNow from UI/API)
   // always proceed past the period guard — but never past the run lock in startRun().
   if (!manual && s.trigger.kind === 'cron') {
@@ -588,12 +677,21 @@ function startRun(s: Schedule, firedAt: number, override?: string, resume?: Resu
         saveSchedules(state.schedules);
         state.broadcast({ type: 'schedule:updated', schedule: live });
       }
+      // A run that failed WITHOUT producing anything has not spent its window: the work provably
+      // never started, so re-running cannot double-apply a side effect. Re-arm instead of burning
+      // the window — otherwise a multi-hour upstream outage (an API spend cap, a dead engine)
+      // silently costs the day's run, which is exactly how a morning report goes missing with
+      // every marker reading "attempted". Recording the attempt is deferred to the give-up path,
+      // because an attempt marker is precisely what blocks the replay we want.
+      if (summary.status === 'error' && summary.sideEffectFree && rearmWindow(s, firedAt)) return;
       if (summary.status !== 'ok') {
         // Keep the attempt on record with its real outcome — the next boot must see that this
         // window was tried and failed, not that it never ran.
         recordAttemptOutcome(s.id, firedAt, summary.at, summary.status === 'running' ? 'started' : summary.status);
       }
       if (summary.status === 'ok') {
+        // The window landed — drop any retry bookkeeping so the next failure starts a fresh ladder.
+        clearRearm(s.id);
         writeCompletionMarker({ completedAt: summary.at, triggeredBy: 'scheduler', scheduleId: s.id, lastAttemptAt: summary.at, attemptWindow: firedAt, status: 'ok' });
         if (live && live.trigger.kind === 'once') {
           console.log(`[scheduler] auto-deleting completed once-schedule ${s.id}`);

package/src/server/scheduler/runner.ts

@@ -193,6 +193,7 @@ export async function runSchedule(
 
   let status: ScheduleRunSummary['status'] = 'ok';
   let error: string | undefined;
+  let sideEffectFree = false;
 
   // A `bash` task is not exec'd — it's handed to the agent as a prompt, so the command's exit code
   // reaches nobody: the agent reports the failure in prose, its own turn succeeds, the run is stored
@@ -280,6 +281,10 @@ export async function runSchedule(
       // `tool_use_input`, `tool_result_image`, `permission_request` and `question_request` need no
       // separate flag: each is necessarily preceded by the `tool_use` that already set the boundary.
       const producedOutput = assistantBlocks.length > 0 || assistantText.length > 0 || producedThinking;
+      // Carry the boundary out to the summary. The in-process ladder is tuned for a flaky engine
+      // (sub-second backoff); an outage measured in hours outlives it, and the engine needs to know
+      // whether the window is still safely replayable once this run gives up.
+      sideEffectFree = !producedOutput;
       if (producedOutput || abortController.signal.aborted || attempt >= MAX_ATTEMPTS) {
         // Nobody watches stderr on a scheduled run — record the failure in the transcript.
         if (assistantText) { assistantBlocks.push({ type: 'text', text: assistantText }); assistantText = ''; }
@@ -317,7 +322,7 @@ export async function runSchedule(
   const preview = assistantText.slice(0, 120) || (status === 'ok' ? 'Schedule completed' : `Schedule ${status}`);
   addUnread(schedule.createdBy.uid, sessionId, preview, 'schedule', schedule.name);
 
-  const summary: ScheduleRunSummary = { at: now, sessionId, status, error };
+  const summary: ScheduleRunSummary = { at: now, sessionId, status, error, sideEffectFree: status === 'error' ? sideEffectFree : undefined };
   onEvent({ type: 'schedule:run_finished', scheduleId: schedule.id, sessionId, summary });
   return summary;
 }

package/src/server/scheduler/storage.ts

@@ -170,3 +170,21 @@ export function markRunStarted(scheduleId: string, window: number, triggeredBy:
     status: 'started',
   });
 }
+
+/** Undo the at-start attempt stamp for a window whose run turned out to have done nothing.
+ *  `markRunStarted` records `attemptWindow` BEFORE the work happens — right for a crash (the run
+ *  had its shot), wrong for a run that provably never started, because that stamp is precisely
+ *  what makes a window un-replayable. `completedAt` is untouched: a period that genuinely
+ *  completed must stay guarded no matter what a later attempt does. */
+export function releaseAttemptWindow(scheduleId: string, window: number): void {
+  const prev = readCompletionMarker(scheduleId);
+  if (!prev || prev.attemptWindow !== window) return;
+  writeCompletionMarker({
+    completedAt: prev.completedAt,
+    triggeredBy: prev.triggeredBy,
+    scheduleId,
+    lastAttemptAt: prev.lastAttemptAt,
+    attemptWindow: undefined,
+    status: prev.status,
+  });
+}

package/src/server/scheduler/types.ts

@@ -32,6 +32,11 @@ export interface ScheduleRunSummary {
   sessionId: string;
   status: 'running' | 'ok' | 'error' | 'aborted';
   error?: string;
+  /** Set on an errored run that produced NO output at all (no tool_use, no text, no thinking) —
+   *  the same side-effect boundary the in-process retry uses. It means the window's work provably
+   *  did not start, so the window has not really been spent and re-running cannot double-apply
+   *  anything. The engine uses it to re-arm the window instead of burning it. */
+  sideEffectFree?: boolean;
 }
 
 /** What the scheduler did about a window it found already elapsed at boot.