shraga 0.1.65 → 0.1.67

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "shraga",
-  "version": "0.1.65",
+  "version": "0.1.67",
   "description": "The teammate you delegate coding to — a self-hostable, multi-user AI coding agent web UI (Claude Code, with a pluggable engine seam).",
   "type": "module",
   "main": "./src/index.ts",

package/src/server/background-jobs.ts

@@ -0,0 +1,511 @@
+// Background shell jobs that OUTLIVE the turn that started them.
+//
+// The bug this exists for: an agent dispatches long work, promises "I'll report back", and the turn
+// is then cut at the engine's wall clock. The work finishes fine — and nobody is left to say so.
+// Any dispatch-and-poll pattern loses that race whenever the work outlasts the remaining budget, so
+// polling is not the fix; something has to WAKE the session when the process exits.
+//
+// Shape:
+//   • The registry is owned by the SERVER (module singleton), not by the turn. agentx builds a fresh
+//     `Agent` per turn, so a registry created there would die with the turn — the very failure above.
+//     Hosts hand each turn a thin per-session VIEW (`sessionJobRegistry`) over this one store; the
+//     view is duck-compatible with agentx's `ShellJobRegistry`, so `Shell({background:true})` and the
+//     ShellOutput/ShellStatus/ShellKill tools light up with no folklore `nohup`.
+//   • On exit we report through wake.ts — the same "close then report" path polls.ts already uses in
+//     production, so a job outcome lands wherever the session speaks (Slack thread / web UI).
+//   • No double-report: reading a finished job's OUTPUT from inside a turn marks it `observed` — the
+//     model had the result in hand and can speak for itself. Only that. `status`/`list` return no
+//     output, so seeing "exited" there is not holding the result; marking those observed suppressed
+//     the follow-up and lost the outcome entirely. We wake for every job whose output nobody read.
+//     (Known narrow gap: a turn that reads the output and is then cut before it speaks gets no
+//     follow-up. Much smaller window than the one being fixed.)
+//   • No collision with a live turn: the report defers while `isSessionLocked` — with a cap, past
+//     which it is delivered as plain text rather than never.
+//
+// Restart survival is PARTIAL and deliberately visible. Children are detached, so they keep running
+// across a server restart and their output keeps landing in the job's log file; boot re-adopts them
+// by pid and reports on exit as usual. What cannot survive is the exit CODE of a job that finished
+// while we were down — nothing was waiting on the process. Those are reported with an explicit
+// "exit code unknown (server restarted)" instead of a fabricated success.
+import { spawn, execFileSync } from 'node:child_process';
+import { mkdirSync, openSync, closeSync, readSync, readFileSync, writeFileSync, readdirSync, statSync, rmSync } from 'node:fs';
+import path from 'node:path';
+import { dataPath } from './paths.ts';
+import { isSessionLocked, getSession } from './sessions.ts';
+import { wakeSession, deliverToSession, wakeReady } from './wake.ts';
+
+const PREFIX = '[jobs]';
+
+// ── Bounds. Every one of these caps a blast radius the incident's `nohup` had none of. ──
+/** Hard cap on a job's log, enforced by `head -c` inside the job's own pipeline (see buildShell).
+ *  Polling a size and killing late cannot bound anything: at a 5s cadence an 8MB cap measured 830MB
+ *  on disk. `head` closes the pipe at exactly this many bytes instead — the writer then dies of
+ *  SIGPIPE, so the bound is the kernel's, not a timer's. */
+const MAX_LOG_BYTES = 8 * 1024 * 1024;
+/** How much of the tail we ever read back into memory (for a tool result or a report). */
+const TAIL_BYTES = 16 * 1024;
+/** How much of that tail goes into the wake prompt (the rest stays readable via ShellOutput). */
+const REPORT_CHARS = 4_000;
+const MAX_RUNNING_PER_SESSION = 4;
+const MAX_RUNNING_GLOBAL = 16;
+/** Grace before reporting: lets a still-live turn poll the just-finished job and own the telling. */
+const GRACE_MS = 8_000;
+/** Re-check cadence while the session is busy, or while an adopted (post-restart) job still runs. */
+const POLL_MS = 5_000;
+/** Give up deferring behind a busy session after this long and deliver the outcome as plain text. */
+const MAX_DEFER_MS = 30 * 60_000;
+/** Delete finished job records + logs after this long. */
+const PRUNE_AFTER_MS = 24 * 60 * 60_000;
+
+export type JobStatus = 'running' | 'exited' | 'killed' | 'error';
+
+export interface JobOwner {
+  sessionId: string;
+  uid: string;
+  userEmail?: string;
+  /** Working directory for the job's shell (the deployment/project root). */
+  cwd: string;
+  /** Extra env merged over the server's own for the child. */
+  env?: Record<string, string>;
+}
+
+interface JobRecord {
+  id: string;
+  sessionId: string;
+  uid: string;
+  userEmail?: string;
+  command: string;
+  cwd: string;
+  pid?: number;
+  startedAt: number;
+  status: JobStatus;
+  exitCode?: number;
+  endedAt?: number;
+  /** A turn read this job's terminal state — the model can report it itself, so we must not. */
+  observed?: boolean;
+  /** Terminal outcome of the follow-up delivery (absent = not attempted yet). */
+  reported?: 'woke' | 'raw' | 'skipped-observed' | 'no-session' | 'failed';
+  /** The exit code is unknown because the server restarted while this job ran. */
+  orphaned?: boolean;
+  /** The job hit MAX_LOG_BYTES: its output was cut off and the command was killed by SIGPIPE. */
+  truncated?: boolean;
+}
+
+// ── Storage ────────────────────────────────────────────────────────────────────
+const dir = (): string => { const d = dataPath('jobs'); mkdirSync(d, { recursive: true }); return d; };
+const recFile = (id: string): string => path.join(dir(), `${id}.json`);
+const logFile = (id: string): string => path.join(dir(), `${id}.log`);
+/** The command's REAL exit status, written by the job's own shell (see buildShell). */
+const statusFile = (id: string): string => path.join(dir(), `${id}.status`);
+
+/** The command's exit code as its shell recorded it, or null if it never got that far. */
+function readStatus(id: string): number | null {
+  try {
+    const n = Number(readFileSync(statusFile(id), 'utf-8').trim());
+    return Number.isInteger(n) ? n : null;
+  } catch { return null; }
+}
+
+/** Current size of a job's log, 0 if it does not exist yet. */
+function logSize(id: string): number {
+  try { return statSync(logFile(id)).size; } catch { return 0; }
+}
+
+/**
+ * Wrap the user's command so the log is bounded and the real exit code survives.
+ *
+ * `{ ( cmd ); echo $? > status; } 2>&1 | head -c N`, and every piece of that shape is load-bearing:
+ *   • `head -c N` — the actual disk bound. It closes the pipe at exactly N bytes; the writer then
+ *     takes SIGPIPE. Chosen over `ulimit -f`, which is also kernel-enforced but caps EVERY file the
+ *     job writes — that would kill an ordinary build the moment it emitted a >8MB artifact.
+ *   • the inner `( … )` subshell — a command ending in `exit 3` (agents write those) would otherwise
+ *     exit the group before the status line ever ran, and we'd lose the code. Verified.
+ *   • the status file — the pipeline's own exit code is `head`'s (always 0), so without this every
+ *     job would report success. The shell is not the process writing to the pipe, so it survives the
+ *     SIGPIPE that kills the command and still records the code.
+ * It also outlives us: the whole pipeline is in the job's detached process group, so a job adopted
+ * after a server restart can be given its true exit code instead of "unknown".
+ */
+function buildShell(id: string, cmd: string): string {
+  return `{\n(\n${cmd}\n)\n__st=$?\necho $__st > '${statusFile(id)}'\n} 2>&1 | head -c ${MAX_LOG_BYTES}`;
+}
+
+/** In-memory mirror of the on-disk records (disk is the restart-durable copy; this is the hot path). */
+const records = new Map<string, JobRecord>();
+/** Live timers per job, so nothing is left ticking after a job is reported or the record pruned. */
+const timers = new Map<string, ReturnType<typeof setTimeout>>();
+
+function save(j: JobRecord): void {
+  records.set(j.id, j);
+  try { writeFileSync(recFile(j.id), JSON.stringify(j, null, 2)); }
+  catch (e) { console.error(`${PREFIX} persist ${j.id} failed:`, (e as Error).message); }
+}
+
+function setTimer(id: string, ms: number, fn: () => void): void {
+  clearTimer(id);
+  timers.set(id, setTimeout(() => { timers.delete(id); fn(); }, ms));
+}
+function clearTimer(id: string): void {
+  const t = timers.get(id);
+  if (t) { clearTimeout(t); timers.delete(id); }
+}
+
+/** Last `TAIL_BYTES` of a job's log, decoded as utf-8. Never loads the whole file. */
+function readTail(id: string): string {
+  let fd: number | undefined;
+  try {
+    const size = statSync(logFile(id)).size;
+    const start = Math.max(0, size - TAIL_BYTES);
+    const len = size - start;
+    if (len <= 0) return '';
+    const buf = Buffer.allocUnsafe(len);
+    fd = openSync(logFile(id), 'r');
+    readSync(fd, buf, 0, len, start);
+    // A head-truncated read can split a multi-byte char — the replacement char is cosmetic, not corruption.
+    return (start > 0 ? `…(truncated, showing last ${TAIL_BYTES >> 10}KB)\n` : '') + buf.toString('utf-8');
+  } catch { return ''; }
+  finally { if (fd !== undefined) try { closeSync(fd); } catch { /* already closed */ } }
+}
+
+const runningCount = (sessionId?: string): number =>
+  [...records.values()].filter((j) => j.status === 'running' && (!sessionId || j.sessionId === sessionId)).length;
+
+/**
+ * Seconds this pid has been alive, or null if there is no such process.
+ *
+ * `etime` (formatted `[[DD-]HH:]MM:SS`), NOT `etimes` (raw seconds): etimes is a procps/Linux
+ * extension and BSD ps — macOS, which is what the live box runs — rejects it outright, so the whole
+ * adoption check silently degraded to "process gone" and every job that survived a restart was
+ * reported as `exit code unknown`. etime is in POSIX and works on both.
+ */
+function pidAgeSeconds(pid: number): number | null {
+  let out: string;
+  try {
+    // stderr piped: a missing pid is an expected outcome here, not something to print.
+    out = execFileSync('ps', ['-o', 'etime=', '-p', String(pid)], { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'pipe'] }).trim();
+  } catch { return null; /* no such process */ }
+  const m = out.match(/^(?:(?:(\d+)-)?(\d+):)?(\d+):(\d+)$/);
+  if (!m) return null;
+  const [, d, h, mi, sec] = m;
+  return Number(d ?? 0) * 86400 + Number(h ?? 0) * 3600 + Number(mi) * 60 + Number(sec);
+}
+
+/**
+ * Env for a job's shell: the server's own, minus anything that looks like a credential.
+ *
+ * agentx's foreground `Bash` redacts these by default (RealShellOptions.redactEnv). Backgrounding a
+ * command must not be a way around that — otherwise `Bash({background:true})` becomes a strictly
+ * weaker sandbox than `Bash({})`, and `echo $ANTHROPIC_API_KEY` lands in a job log we then feed back
+ * into the model. Mirrors agentx's SECRET_ENV_RE; a job that genuinely needs a credential gets it
+ * explicitly via `JobOwner.env`.
+ */
+const SECRET_ENV_RE = /(API_KEY|_TOKEN|_SECRET|_PASSWORD|_PRIVATE_KEY|^AWS_|^GITHUB_TOKEN$|^OPENAI_|^ANTHROPIC_|^GOOGLE_|^GEMINI_|^GROQ_|^NPM_TOKEN$|^SLACK_)/i;
+function childEnv(): Record<string, string | undefined> {
+  const out: Record<string, string | undefined> = {};
+  for (const [k, v] of Object.entries(process.env)) if (!SECRET_ENV_RE.test(k)) out[k] = v;
+  return out;
+}
+
+// ── Lifecycle ──────────────────────────────────────────────────────────────────
+
+/** Start `command` detached, in its own process group, with stdout+stderr appended to the job log.
+ *  Detached (setsid) for two reasons: the group is killable as a subtree, and the child is NOT torn
+ *  down with the server — which is what makes restart adoption possible at all. */
+export async function startJob(owner: JobOwner, command: string): Promise<string> {
+  const cmd = command.trim();
+  if (!cmd) throw new Error('empty command');
+  if (runningCount(owner.sessionId) >= MAX_RUNNING_PER_SESSION)
+    throw new Error(`too many background jobs in this session (max ${MAX_RUNNING_PER_SESSION}) — wait for one to finish or ShellKill it`);
+  if (runningCount() >= MAX_RUNNING_GLOBAL)
+    throw new Error(`too many background jobs on this server (max ${MAX_RUNNING_GLOBAL})`);
+
+  const id = `job-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
+  const j: JobRecord = {
+    id, sessionId: owner.sessionId, uid: owner.uid, userEmail: owner.userEmail,
+    command: cmd, cwd: owner.cwd, startedAt: Date.now(), status: 'running',
+  };
+  let fd: number;
+  try { fd = openSync(logFile(id), 'a'); }
+  catch (e) { throw new Error(`cannot open job log: ${(e as Error).message}`); }
+  try {
+    // stdin is /dev/null so a child can never block on (or steal) input; stdout+stderr go straight
+    // to the fd — nothing is buffered in this process, so a chatty job costs disk, not memory.
+    const proc = spawn('/bin/sh', ['-c', buildShell(id, cmd)], {
+      cwd: owner.cwd, env: { ...childEnv(), ...owner.env },
+      detached: true, stdio: ['ignore', fd, fd],
+    });
+    j.pid = proc.pid;
+    proc.on('error', (err) => finish(id, 'error', undefined, `spawn error: ${err.message}`));
+    // The pipeline's own code is `head`'s; the command's real one is in the status file.
+    proc.on('close', (code) => finish(id, 'exited', readStatus(id) ?? code ?? undefined));
+    proc.unref(); // never hold the event loop open for a background job
+  } catch (e) {
+    j.status = 'error';
+    save(j);
+    closeSync(fd);
+    throw new Error(`failed to spawn: ${(e as Error).message}`);
+  }
+  closeSync(fd); // the child holds its own dup of the fd
+  save(j);
+  console.log(`${PREFIX} started ${id} pid=${j.pid} session=${j.sessionId.slice(0, 8)} cmd=${cmd.slice(0, 120)}`);
+  return id;
+}
+
+/** Record a terminal state exactly once, then schedule the follow-up. */
+function finish(id: string, status: JobStatus, exitCode?: number, note?: string): void {
+  const j = records.get(id);
+  if (!j || j.status !== 'running') return; // already terminal (e.g. killed, then close fires)
+  j.status = status;
+  j.exitCode = exitCode;
+  j.endedAt = Date.now();
+  if (logSize(id) >= MAX_LOG_BYTES) {
+    // Say so IN the log: every path the model can read the result through goes via readTail, so the
+    // notice reaches it whether it polls ShellOutput or gets the wake report.
+    j.truncated = true;
+    try { writeFileSync(logFile(id), `\n[output limit: ${MAX_LOG_BYTES >> 20}MB reached — output was cut off here and the command was terminated]\n`, { flag: 'a' }); }
+    catch { /* nothing more we can do about the log */ }
+    console.warn(`${PREFIX} ${id} hit the ${MAX_LOG_BYTES >> 20}MB log cap — output truncated`);
+  }
+  save(j);
+  console.log(`${PREFIX} ${id} ${status}${exitCode != null ? ` exit=${exitCode}` : ''} in ${Math.round((j.endedAt - j.startedAt) / 1000)}s${note ? ` (${note})` : ''}`);
+  // Grace: a turn that is still alive gets first refusal on telling the user (see `observed`).
+  setTimer(id, GRACE_MS, () => { void report(id, Date.now()); });
+}
+
+export function killJob(id: string): boolean {
+  const j = records.get(id);
+  if (!j) return false;
+  if (j.status === 'running') {
+    // Group-kill: the child is its own group leader, so signalling `-pid` reaps the whole subtree —
+    // `kill(pid)` alone would hit /bin/sh and orphan whatever it forked.
+    if (j.pid) { try { process.kill(-j.pid, 'SIGTERM'); } catch { try { process.kill(j.pid, 'SIGTERM'); } catch { /* already gone */ } } }
+    finish(id, 'killed');
+  }
+  return true;
+}
+
+/** A job the model explicitly killed inside a turn needs no follow-up — it already knows. */
+function markObservedIfTerminal(j: JobRecord): void {
+  if (j.status !== 'running' && !j.observed) { j.observed = true; save(j); }
+}
+
+// ── Follow-up delivery ─────────────────────────────────────────────────────────
+
+function reportPrompt(j: JobRecord): string {
+  const dur = Math.round(((j.endedAt ?? Date.now()) - j.startedAt) / 1000);
+  const result = (j.orphaned
+    ? 'ended while the server was restarting — exit code unknown'
+    : j.status === 'killed' ? 'killed'
+    : j.status === 'error' ? 'failed to run'
+    : `exit ${j.exitCode ?? 0}`)
+    + (j.truncated ? `, cut off at the ${MAX_LOG_BYTES >> 20}MB output limit` : '');
+  const tail = readTail(j.id).slice(-REPORT_CHARS).trim();
+  return [
+    `[Background job finished] A command you started in an earlier turn has completed, after that turn ended.`,
+    `Command: \`${j.command}\``,
+    `Result: ${result} (after ${dur}s)`,
+    ``,
+    `Output (tail):`,
+    tail || '(no output)',
+    ``,
+    `Report this outcome to the user now — that turn promised a follow-up and this is it.`,
+    `Be brief and concrete. Do NOT re-run the command.`,
+  ].join('\n');
+}
+
+/** Plain-text form, for when we cannot run a turn (no runner, or the session never went idle). */
+function rawReport(j: JobRecord): string {
+  const dur = Math.round(((j.endedAt ?? Date.now()) - j.startedAt) / 1000);
+  const result = (j.orphaned ? 'exit code unknown (server restarted)' : j.status === 'exited' ? `exit ${j.exitCode ?? 0}` : j.status)
+    + (j.truncated ? `, cut off at the ${MAX_LOG_BYTES >> 20}MB output limit` : '');
+  const tail = readTail(j.id).slice(-REPORT_CHARS).trim();
+  return `Background job finished — \`${j.command}\`\nResult: ${result} (after ${dur}s)\n\n\`\`\`\n${tail || '(no output)'}\n\`\`\``;
+}
+
+/**
+ * Deliver a finished job's outcome, once.
+ *
+ * Order matters: `observed` is checked AFTER the grace window, so a turn that was still alive when
+ * the job exited has had its chance to poll and own the telling. `firstAttemptAt` bounds how long
+ * we defer behind a busy session.
+ */
+const report = (id: string, firstAttemptAt: number): Promise<void> =>
+  tryReport(id, firstAttemptAt).catch((e) => console.error(`${PREFIX} ${id} report threw:`, (e as Error).message));
+
+async function tryReport(id: string, firstAttemptAt: number): Promise<void> {
+  const j = records.get(id);
+  if (!j || j.reported) return;
+
+  if (j.observed) {
+    j.reported = 'skipped-observed';
+    save(j);
+    console.log(`${PREFIX} ${id} observed inside a turn — no follow-up needed`);
+    return;
+  }
+  if (!getSession(j.sessionId)) {
+    j.reported = 'no-session';
+    save(j);
+    console.warn(`${PREFIX} ${id} session ${j.sessionId} is gone — dropping follow-up`);
+    return;
+  }
+  // Never start a turn on top of a live one — they would interleave into one transcript.
+  if (isSessionLocked(j.sessionId)) {
+    if (Date.now() - firstAttemptAt < MAX_DEFER_MS) {
+      setTimer(id, POLL_MS, () => { void report(id, firstAttemptAt); });
+      return;
+    }
+    console.warn(`${PREFIX} ${id} session busy for ${Math.round(MAX_DEFER_MS / 60_000)}min — delivering raw`);
+    await deliverToSession({ sessionId: j.sessionId, uid: j.uid, text: rawReport(j), title: 'Background job' })
+      .catch((e) => console.error(`${PREFIX} raw deliver failed:`, (e as Error).message));
+    j.reported = 'raw';
+    save(j);
+    return;
+  }
+  try {
+    if (!wakeReady()) {
+      await deliverToSession({ sessionId: j.sessionId, uid: j.uid, text: rawReport(j), title: 'Background job' });
+      j.reported = 'raw';
+    } else {
+      const outcome = await wakeSession({
+        sessionId: j.sessionId, uid: j.uid, userEmail: j.userEmail,
+        prompt: reportPrompt(j), channel: 'job', title: 'Background job',
+      });
+      if (outcome === 'woke') j.reported = 'woke';
+      else {
+        // The wake ran but said nothing (or the session vanished mid-flight). Falling back to raw
+        // keeps the promise: the user sees the result rather than nothing at all.
+        await deliverToSession({ sessionId: j.sessionId, uid: j.uid, text: rawReport(j), title: 'Background job' });
+        j.reported = 'raw';
+        console.warn(`${PREFIX} ${id} wake returned '${outcome}' — delivered raw instead`);
+      }
+    }
+    console.log(`${PREFIX} ${id} follow-up delivered (${j.reported})`);
+  } catch (e) {
+    j.reported = 'failed';
+    console.error(`${PREFIX} ${id} follow-up FAILED:`, (e as Error).message);
+  }
+  save(j);
+}
+
+// ── Boot: adopt what survived, prune what is stale ─────────────────────────────
+
+/**
+ * Load persisted records and reconcile them with reality.
+ *
+ * `running` records are jobs we lost the `close` listener for when the process ended. Their children
+ * are detached, so many are genuinely still running — re-adopt those by polling the pid. The rest
+ * finished while we were down: we know THAT they ended but not with what code, which is reported
+ * honestly rather than guessed.
+ */
+export function initBackgroundJobs(): void {
+  let files: string[] = [];
+  try { files = readdirSync(dir()).filter((f) => f.endsWith('.json')); } catch { return; }
+  let adopted = 0, orphaned = 0;
+  for (const f of files) {
+    let j: JobRecord;
+    try { j = JSON.parse(readFileSync(path.join(dir(), f), 'utf-8')) as JobRecord; } catch { continue; }
+    if (!j?.id) continue;
+    records.set(j.id, j);
+
+    if (j.status !== 'running') {
+      // A terminal job whose follow-up never went out (we died between finish and report) still owes one.
+      if (!j.reported) setTimer(j.id, GRACE_MS, () => { void report(j.id, Date.now()); });
+      continue;
+    }
+    // pid liveness, guarded against pid REUSE. Our process started when the job did, so a pid that
+    // is still ours is AT LEAST as old as the job. A recycled pid belongs to a process the OS started
+    // later — i.e. YOUNGER than the job — which is exactly what this rejects. (Getting the comparison
+    // backwards adopts the stranger: we would report its exit as the job's, and ShellKill would
+    // `kill(-pid)` an unrelated process tree.) The 5s slack absorbs `etime`'s 1s granularity.
+    const age = j.pid ? pidAgeSeconds(j.pid) : null;
+    const jobAgeSec = (Date.now() - j.startedAt) / 1000;
+    if (age != null && age >= jobAgeSec - 5) { adoptRunning(j.id); adopted++; }
+    else {
+      if (age != null) console.warn(`${PREFIX} ${j.id} pid ${j.pid} is ${Math.round(age)}s old but the job is ${Math.round(jobAgeSec)}s old — pid was reused, not adopting`);
+      // Its own shell may still have recorded the real code before we went down.
+      const st = readStatus(j.id);
+      if (st != null) j.exitCode = st; else j.orphaned = true;
+      j.status = 'exited';
+      j.endedAt = Date.now();
+      save(j);
+      orphaned++;
+      setTimer(j.id, GRACE_MS, () => { void report(j.id, Date.now()); });
+    }
+  }
+  pruneOld();
+  setInterval(pruneOld, 60 * 60_000).unref?.();
+  if (adopted || orphaned) console.log(`${PREFIX} boot: adopted ${adopted} running job(s), ${orphaned} ended while down`);
+}
+
+/** Watch a job we no longer have a child handle for; report when its pid disappears. */
+function adoptRunning(id: string): void {
+  const j = records.get(id);
+  if (!j || j.status !== 'running' || !j.pid) return;
+  if (pidAgeSeconds(j.pid) == null) {
+    // `wait` only works for your own child, so the pid vanishing tells us THAT it ended, not how.
+    // Its own shell wrote the code down before exiting, though — prefer that over guessing.
+    const st = readStatus(id);
+    if (st == null) j.orphaned = true;
+    finish(id, 'exited', st ?? undefined);
+    return;
+  }
+  setTimer(id, POLL_MS, () => adoptRunning(id));
+}
+
+function pruneOld(): void {
+  const now = Date.now();
+  for (const j of [...records.values()]) {
+    if (j.status === 'running' || !j.endedAt || now - j.endedAt < PRUNE_AFTER_MS) continue;
+    clearTimer(j.id);
+    records.delete(j.id);
+    for (const p of [recFile(j.id), logFile(j.id), statusFile(j.id)]) try { rmSync(p); } catch { /* already gone */ }
+  }
+}
+
+// ── The per-turn view handed to the agent engine ───────────────────────────────
+
+/**
+ * A session-scoped facade over the process-wide store, duck-compatible with agentx's
+ * `ShellJobRegistry` — pass it as `makeRealShellTool({ registry })` and to `makeShellJobTools`.
+ *
+ * Cheap to build per turn (it holds no state); the jobs it starts belong to the server, so they
+ * outlive the turn that made it. Every lookup is session-scoped: one session's agent can neither
+ * read nor kill another's jobs.
+ */
+export function sessionJobRegistry(owner: JobOwner) {
+  const mine = (id: string): JobRecord | undefined => {
+    const j = records.get(id);
+    return j && j.sessionId === owner.sessionId ? j : undefined;
+  };
+  return {
+    start: (command: string) => startJob(owner, command),
+    output(id: string): string | null {
+      const j = mine(id);
+      if (!j) return null;
+      markObservedIfTerminal(j);
+      return readTail(id);
+    },
+    // NB: status() and list() deliberately do NOT mark the job observed — they return no output, so
+    // the model has not got the result and still needs the follow-up. Only output() may.
+    status(id: string): { status: JobStatus; exitCode?: number; bytes: number } | null {
+      const j = mine(id);
+      if (!j) return null;
+      return { status: j.status, exitCode: j.exitCode, bytes: logSize(id) };
+    },
+    list(): Array<{ id: string; command: string; status: JobStatus }> {
+      return [...records.values()]
+        .filter((j) => j.sessionId === owner.sessionId)
+        .map((j) => ({ id: j.id, command: j.command, status: j.status }));
+    },
+    kill(id: string): boolean {
+      const j = mine(id);
+      if (!j) return false;
+      const ok = killJob(id);
+      markObservedIfTerminal(j); // the model asked for this end — it does not need telling about it
+      return ok;
+    },
+  };
+}
+
+/** Test/introspection seam: the current record for a job (undefined once pruned). */
+export function getJob(id: string): Readonly<JobRecord> | undefined { return records.get(id); }

package/src/server/boot.ts

@@ -30,6 +30,7 @@ import { getAllSessions, getSession, getSessionHistory, upsertSession, appendMes
 import { setBroadcaster } from './session-bus.ts';
 import * as scheduler from './scheduler/index.ts';
 import { initPolls } from './polls.ts';
+import { initBackgroundJobs } from './background-jobs.ts';
 import { pushEnabled } from './push/push.ts';
 import { upsertToken, removeToken } from './push/store.ts';
 import { initPushTriggers, pushTurnDone, pushQuestion } from './push/triggers.ts';
@@ -1133,11 +1134,47 @@ if (!PASSIVE) {
 // runs in passive too. Otherwise a standby instance reports empty stats and /api/stats is a lie.
 statsSampler.start(broadcast);
 registerEventRoutes(app, requireAuth);
+/** How long an out-of-band wake turn waits for a busy session before giving up (see runTurn). */
+const WAKE_LOCK_WAIT_MS = 5 * 60_000;
 initPolls({
   broadcast,
-  runTurn: ({ prompt, sessionId, uid, userEmail }) =>
-    consumeStream(streamChat({ prompt, sessionId, uid, userEmail, mcpServers: getMcpConfig(uid), abortController: new AbortController(), onPermissionRequest: async () => ({ allow: true }) })),
+  // The out-of-band turn runner (polls + background-job follow-ups). It TAKES THE SESSION LOCK, and
+  // that is load-bearing: `streamChat` never acquires one itself, so before this every guard written
+  // against `isSessionLocked` — including background-jobs.ts's "never start a turn on top of a live
+  // one" — was reading a lock that this path never took. Two jobs finishing minutes apart then ran
+  // two concurrent wake turns in one session, each free to dispatch the next leg of the same
+  // workflow: two writers on one browser instance, which is the failure these workflows are built to
+  // prevent. Waiting (bounded) rather than failing fast, because the whole point of a wake is that
+  // the outcome gets told: a busy session usually means another wake is mid-turn and will be done in
+  // seconds.
+  runTurn: async ({ prompt, sessionId, uid, userEmail }) => {
+    const abortController = new AbortController();
+    const deadline = Date.now() + WAKE_LOCK_WAIT_MS;
+    while (!acquireSessionLock(sessionId, 'api', abortController)) {
+      if (Date.now() >= deadline) {
+        // Give up by returning NOTHING, never by throwing. A throw propagates out of wake.ts's
+        // unguarded `await runTurn` into background-jobs' catch, which records `reported: 'failed'`
+        // — the one delivery path with no raw fallback, so the job's outcome would reach the user in
+        // no form at all, after wake.ts had already appended the trigger prompt (a question with no
+        // answer in the transcript). Empty blocks are the 'no-output' contract callers already
+        // handle: the job store then delivers its raw report instead. Degraded, but never silent.
+        console.warn(`[wake] session ${sessionId} stayed busy for ${Math.round(WAKE_LOCK_WAIT_MS / 1000)}s — skipping the turn; the caller falls back to a raw report`);
+        return [];
+      }
+      await new Promise((r) => setTimeout(r, 2_000));
+    }
+    try {
+      return await consumeStream(streamChat({ prompt, sessionId, uid, userEmail, mcpServers: getMcpConfig(uid), abortController, onPermissionRequest: async () => ({ allow: true }) }));
+    } finally {
+      if (releaseSessionLock(sessionId, abortController)) setRunStatus(sessionId, 'idle');
+    }
+  },
 });
+// Background jobs outlive the turn that started them, so their follow-up must too. Must run AFTER
+// initPolls (which wires wake.ts's turn runner) — boot adoption can report a job that finished
+// while we were down, and that report runs a turn. Skipped when passive: a standby twin must not
+// adopt the active instance's children or double-report them.
+if (!PASSIVE) initBackgroundJobs();
 // Remote-push triggers: subscribe to schedule.finished and expose turn-done/question
 // hooks. isForeground reuses the existing presence tracking (see isUserViewingSession).
 initPushTriggers({

package/src/server/polls.ts

@@ -7,10 +7,9 @@
 import { mkdirSync, readFileSync, writeFileSync, readdirSync, rmSync } from 'node:fs';
 import path from 'node:path';
 import { dataPath } from './paths.ts';
-import { appendMessage, getSession, type ConvBlock } from './sessions.ts';
-import { addUnread } from './unread.ts';
-import { postMessage, slackPost, getUserName, buildPollBlocks, type PollSpec } from './slack/api.ts';
-import { findSlackSessionBySessionId } from './slack/sessions.ts';
+import { getSession } from './sessions.ts';
+import { slackPost, getUserName, buildPollBlocks, type PollSpec } from './slack/api.ts';
+import { initWake, wakeSession, deliverToSession, type TurnRunner } from './wake.ts';
 
 const PREFIX = '[polls]';
 
@@ -28,14 +27,10 @@ export interface PollRecord extends PollSpec {
   createdAt: number;
 }
 
-// ── IoC: injected by index.ts at startup to avoid a claude.ts <-> polls.ts cycle ──
-type TurnRunner = (args: { prompt: string; sessionId: string; uid: string; userEmail?: string }) => Promise<ConvBlock[]>;
-let runTurn: TurnRunner | null = null;
-let broadcastFn: ((ev: object) => void) | null = null;
-
+// ── IoC: injected by boot.ts at startup to avoid a claude.ts <-> polls.ts cycle. The runners live
+// in wake.ts now — every subsystem that reports back after its turn ended shares them. ──
 export function initPolls(deps: { runTurn: TurnRunner; broadcast: (ev: object) => void }): void {
-  runTurn = deps.runTurn;
-  broadcastFn = deps.broadcast;
+  initWake(deps);
   setInterval(sweep, 60_000);
   console.log(`${PREFIX} sweeper started`);
 }
@@ -140,7 +135,6 @@ async function closePoll(pollId: string, reason: 'deadline' | 'quorum' | 'manual
 
 // ── Close then report: wake the originating session once with the tally ─────────
 async function wakeAgent(p: PollRecord, reason: string): Promise<void> {
-  if (!runTurn) { console.warn(`${PREFIX} no turn runner; skipping wake for ${p.pollId}`); return; }
   if (!getSession(p.sessionId)) { console.warn(`${PREFIX} session ${p.sessionId} gone; skipping wake`); return; }
 
   // Per-option voters, with Slack user ids resolved to display names (cached).
@@ -161,15 +155,18 @@ async function wakeAgent(p: PollRecord, reason: string): Promise<void> {
   const headline = p.kind === 'question' ? 'Your question was answered' : `Your poll closed (${reason})`;
   const prompt = `[Poll result] ${headline}. Title: "${p.title}". ${voterCount(p)} participant(s).\n${lines}\n\nFollow up appropriately (summarize, take the next action, or notify the relevant people). Do not re-post the poll.`;
 
-  appendMessage(p.sessionId, { id: crypto.randomUUID(), role: 'user', blocks: [{ type: 'text', text: prompt }], channel: 'poll' });
-  broadcastFn?.({ type: 'session_messages_changed', sessionId: p.sessionId });
-
-  const blocks = await runTurn({ prompt, sessionId: p.sessionId, uid: p.uid, userEmail: p.userEmail });
-  if (!blocks.length) return;
-  appendMessage(p.sessionId, { id: crypto.randomUUID(), role: 'assistant', blocks });
-  broadcastFn?.({ type: 'session_messages_changed', sessionId: p.sessionId });
-  const text = blocks.filter((b): b is { type: 'text'; text: string } => b.type === 'text').map((b) => b.text).join('\n\n').trim();
-  addUnread(p.uid, p.sessionId, text.slice(0, 120) || 'Poll closed', 'proactive', p.title);
-  const slack = findSlackSessionBySessionId(p.sessionId);
-  if (slack && text) await postMessage(slack.channel, text, slack.threadTs, slack.useUserToken).catch((e) => console.error(`${PREFIX} slack deliver failed:`, (e as Error)?.message));
+  // The return value is NOT decoration: a wake that could not run a turn ('no-output' — no runner
+  // wired, or the session stayed busy past the wake lock's wait) leaves this poll CLOSED, already
+  // re-rendered as closed in Slack, and the transcript holding a `[Poll result]` prompt with no
+  // answer. The tally would then be lost for good (the record is pruned after 7 days). So when no
+  // turn ran, deliver the lines we already built — the same raw-report fallback the background-job
+  // caller makes. `unreadFallback` above cannot cover this: wake.ts returns before it consults it.
+  const outcome = await wakeSession({ sessionId: p.sessionId, uid: p.uid, userEmail: p.userEmail, prompt, channel: 'poll', title: p.title, unreadFallback: 'Poll closed' });
+  if (outcome !== 'woke') {
+    console.warn(`${PREFIX} wake for ${p.pollId} returned '${outcome}' — delivering the tally as plain text instead`);
+    await deliverToSession({
+      sessionId: p.sessionId, uid: p.uid, title: p.title,
+      text: `${headline} — "${p.title}" (${voterCount(p)} participant(s)).\n${lines}`,
+    }).catch((e) => console.error(`${PREFIX} raw tally deliver failed:`, (e as Error)?.message));
+  }
 }

package/src/server/scheduler/outcome.ts

@@ -0,0 +1,92 @@
+// A scheduled run's DECLARED outcome — the run saying what actually happened, instead of the
+// scheduler inferring success from "the agent's turn returned".
+//
+// The bug this exists for: a prompt run whose real work failed (or never started) still records
+// `ok`, because the only thing measured was that the turn came back. On 2026-08-28 the 15:30 social
+// run's scout died, nothing was delivered, the run stored `ok`, and the (enabled) failure notifier
+// stayed silent all day — it is event-driven on `status: 'error'` and was never given one.
+//
+// It gets worse with background jobs (server/background-jobs.ts): there, ending the turn early is
+// the CORRECT behaviour — the work outlives it and the job store wakes the session when it exits.
+// So "the turn returned" stops being even a weak proxy for the run's outcome.
+//
+// Shape: one JSON file per run session, written by the run itself (any tool that can write a file —
+// no new tool surface, nothing to plumb through the engine), read by runner.ts when the turn ends.
+//   { "status": "ok" }                                  → the run delivered
+//   { "status": "error", "error": "…" }                 → it did not; this fires the notifier
+//   { "status": "pending", "deadline": <epoch ms|ISO> }  → work is still in flight; the run stays
+//                                                          open until a terminal declaration lands,
+//                                                          and FAILS if the deadline passes first
+// Absent file ⇒ unchanged legacy behaviour (turn returned = ok), so no existing schedule changes.
+// Deliberately domain-free: it knows nothing about what the run was doing.
+import { mkdirSync, readFileSync, writeFileSync, rmSync } from 'node:fs';
+import path from 'node:path';
+import { dataPath } from '../paths.ts';
+
+/** Cap on how long a `pending` run may hold the window open, whatever deadline it asked for. */
+export const MAX_PENDING_MS = 6 * 60 * 60_000;
+/** Used when a `pending` declaration names no deadline. */
+export const DEFAULT_PENDING_MS = 60 * 60_000;
+
+export interface DeclaredOutcome {
+  status: 'ok' | 'error' | 'pending';
+  error?: string;
+  /** For `pending`: when the run gives up and is recorded as failed. Epoch ms or ISO-8601. */
+  deadline?: number | string;
+}
+
+const dir = (): string => { const d = dataPath('scheduler', 'outcomes'); mkdirSync(d, { recursive: true }); return d; };
+/** Session ids are server-minted (`sched-<id>-<ts>`), but this value reaches `path.join` and `rmSync`
+ *  — so it is validated rather than trusted. A `../` in there would delete outside the outcomes dir. */
+const safeId = (sessionId: string): string => {
+  if (!/^[A-Za-z0-9._-]+$/.test(sessionId) || sessionId.startsWith('.')) throw new Error(`unsafe session id for an outcome file: ${sessionId}`);
+  return sessionId;
+};
+export const outcomeFile = (sessionId: string): string => path.join(dir(), `${safeId(sessionId)}.json`);
+
+export function readOutcome(sessionId: string): DeclaredOutcome | null {
+  let raw: string;
+  try { raw = readFileSync(outcomeFile(sessionId), 'utf-8'); } catch { return null; }
+  let o: DeclaredOutcome;
+  // A malformed declaration is not "no declaration": the run tried to say something. Surfacing it
+  // as an error beats silently falling back to the optimistic default this module exists to remove.
+  // But a plain `Write` is not atomic, so a read can also land MID-write — that is a torn read, not
+  // a malformed declaration, and failing a run for it would be the same class of lie in reverse.
+  // Re-read once after a beat before believing it (the prompt also asks for write-temp-then-rename).
+  try { o = JSON.parse(raw) as DeclaredOutcome; }
+  catch {
+    try { raw = readFileSync(outcomeFile(sessionId), 'utf-8'); o = JSON.parse(raw) as DeclaredOutcome; }
+    catch { return { status: 'error', error: `run outcome file is not valid JSON: ${raw.slice(0, 200)}` }; }
+  }
+  if (o?.status !== 'ok' && o?.status !== 'error' && o?.status !== 'pending')
+    return { status: 'error', error: `run outcome file has an invalid status: ${JSON.stringify(o?.status)}` };
+  return o;
+}
+
+export function clearOutcome(sessionId: string): void {
+  try { rmSync(outcomeFile(sessionId)); } catch { /* nothing to clear */ }
+}
+
+/** Test/host seam — writes a declaration the way a run's own file write would. */
+export function writeOutcome(sessionId: string, o: DeclaredOutcome): void {
+  writeFileSync(outcomeFile(sessionId), JSON.stringify(o));
+}
+
+/** Absolute epoch ms a `pending` run expires at, clamped to MAX_PENDING_MS. */
+export function pendingDeadline(o: DeclaredOutcome, now: number): number {
+  const raw = typeof o.deadline === 'string' ? Date.parse(o.deadline) : o.deadline;
+  const asked = Number.isFinite(raw) ? (raw as number) : now + DEFAULT_PENDING_MS;
+  return Math.min(Math.max(asked, now), now + MAX_PENDING_MS);
+}
+
+/** The contract, appended to a scheduled prompt run so a run can report itself truthfully. */
+export function outcomePrompt(sessionId: string): string {
+  return `# Reporting this run's outcome
+This is a scheduled run. Unless you say otherwise, it is recorded as SUCCESSFUL the moment your turn returns — which is a lie whenever the work failed, was skipped, or is still in flight. Correct that by writing this file:
+\`${outcomeFile(sessionId)}\`
+- \`{"status":"ok"}\` — the run delivered what it was for.
+- \`{"status":"error","error":"<what went wrong>"}\` — it did not. This is what raises the failure alert; write it for a leg that never ran, a worker that died, or work you could not finish.
+- \`{"status":"pending","deadline":"<ISO-8601>"}\` — work you started outlives this turn (e.g. a background job). The run stays open and NOT successful until you write a terminal status from a later turn; if the deadline passes with no terminal status, the run is recorded as failed automatically.
+Write it atomically — write a temp file next to it and \`mv\` it into place — so a reader can never catch it half-written.
+Declare \`pending\` BEFORE you end a turn that leaves work running, and re-declare \`ok\`/\`error\` from the turn that sees it finish. Never declare \`ok\` for a run that did not deliver.`;
+}

package/src/server/scheduler/runner.ts

@@ -6,7 +6,8 @@ import { streamChat, type PermissionHandler } from '../claude.ts';
 import { getMcpConfig } from '../mcp.ts';
 import { appendMessage, createScheduledSession, updateScheduledSessionStatus, setRunStatus, registerLivePartial, unregisterLivePartial, writePartial, clearPartial, acquireSessionLock, releaseSessionLock, type ConvBlock } from '../sessions.ts';
 import type { Schedule, ScheduleRunSummary } from './types.ts';
-import { updateRunLockPid, clearRunningMarker } from './storage.ts';
+import { updateRunLockPid, clearRunningMarker, loadSchedules } from './storage.ts';
+import { readOutcome, clearOutcome, pendingDeadline, outcomePrompt, MAX_PENDING_MS } from './outcome.ts';
 import { addUnread } from '../unread.ts';
 
 export interface RunContext {
@@ -87,6 +88,70 @@ const sleep = (ms: number, signal?: AbortSignal) => new Promise<void>((resolve)
   signal?.addEventListener('abort', done, { once: true });
 });
 
+/** How often a `pending` run is re-checked while it waits for its terminal declaration. */
+const OUTCOME_POLL_MS = 10_000;
+
+/**
+ * Resolve what the run itself declared, once its turn has ended.
+ *
+ * Returns null when the run declared nothing — that is the legacy path and stays exactly as it was
+ * (turn returned ⇒ ok). A `pending` declaration keeps the run open until a terminal one lands; each
+ * fresh `pending` extends the wait (leg 2 re-declaring after leg 1 finished), bounded absolutely by
+ * MAX_PENDING_MS from the first one so an agent cannot extend forever. Silence past the deadline is
+ * a FAILURE — that is the whole point: a run that never came back must alert, not read as success.
+ */
+/**
+ * The schedule's CURRENT next fire time, read live from disk each time it is needed.
+ *
+ * Deliberately not the caller's snapshot: `fireDue()` starts the run in its first loop and only
+ * advances `nextRun` in its second, and `startRun` deep-copies the schedule BEFORE that advance —
+ * so the snapshot this run was handed still carries the window it is running FOR (already in the
+ * past), which would make the ceiling below `Infinity` and inert. The engine persists the advanced
+ * value (`saveSchedules` at the end of `fireDue`), so disk is the source of truth here. Reading it
+ * per poll also picks up an edit made while the run waits.
+ */
+function liveNextWindow(scheduleId: string): number | undefined {
+  try { return loadSchedules().find((s) => s.id === scheduleId)?.nextRun; }
+  catch { return undefined; }
+}
+
+async function resolveDeclaredOutcome(
+  sessionId: string,
+  ac: AbortController,
+  /** Schedule whose next window caps the wait — looked up live, never from the run's snapshot. */
+  scheduleId: string,
+): Promise<{ status: Exclude<ScheduleRunSummary['status'], 'running'>; error?: string } | null> {
+  let declared = readOutcome(sessionId);
+  if (!declared) return null;
+  // Waiting happens INSIDE the run promise, and engine.startRun only does `state.running.delete()`
+  // when that promise settles — so a pending run keeps its schedule marked running, and the next
+  // fire of the same schedule is QUEUED behind it rather than run on time. Verified in
+  // engine.ts (`state.running.delete` sits in `.finally`, and `fire()` queues when `state.running`
+  // has the id). For a 3×-daily schedule a multi-hour pending wait would therefore eat the next
+  // slot. So the wait ends at the next window at the latest: the run is then recorded as failed
+  // (the notifier fires) and the new window starts clean and on time.
+  const windowStop = (): number => {
+    const next = liveNextWindow(scheduleId);
+    return next && next > Date.now() ? next - 60_000 : Infinity;
+  };
+  const absoluteStop = Date.now() + MAX_PENDING_MS;
+  while (declared?.status === 'pending' && !ac.signal.aborted) {
+    const stop = windowStop();
+    const deadline = Math.min(pendingDeadline(declared, Date.now()), absoluteStop, stop);
+    if (Date.now() >= deadline) {
+      const why = deadline === stop ? 'its next scheduled window arrived first' : `deadline ${new Date(deadline).toISOString()}`;
+      return { status: 'error', error: `Run declared itself still in flight and never reported a terminal outcome (${why}).` };
+    }
+    await sleep(Math.min(OUTCOME_POLL_MS, deadline - Date.now()), ac.signal);
+    declared = readOutcome(sessionId);
+  }
+  if (ac.signal.aborted) return { status: 'aborted' };
+  if (!declared) return { status: 'error', error: 'Run outcome declaration disappeared before it reported a terminal state.' };
+  return declared.status === 'error'
+    ? { status: 'error', error: declared.error ? `Run reported failure: ${declared.error}` : 'Run reported failure with no detail.' }
+    : { status: 'ok' };
+}
+
 function formatEventBlock(e: EventContext): string {
   let body: string;
   try { body = JSON.stringify(e.payload, null, 2); } catch { body = String(e.payload); }
@@ -140,6 +205,17 @@ export async function runSchedule(
     if (eventCtx) base = `${base}\n\n---\n${formatEventBlock(eventCtx)}`;
     prompt = base;
   }
+  // Tell the run how to report its own truthful outcome (scheduler/outcome.ts). Prompt tasks only:
+  // a `bash` task's permission handler allows nothing but the task's own command, so such a run
+  // could not write the file even if it wanted to — its exit code is already the truth there.
+  if (task.kind === 'prompt') {
+    // Cleared on RESUME too: a resume reuses the interrupted run's session id, so a declaration left
+    // by the attempt that crashed would be adopted as this attempt's verdict. The contract is
+    // re-stated for the same reason — the resumed turn must be able to declare for itself.
+    clearOutcome(sessionId);
+    prompt = `${prompt}\n\n---\n${outcomePrompt(sessionId)}`;
+  }
+
   // task.engine/task.model ride the same prompt-directive channel users type by hand —
   // parseDirectives strips them and resolves aliases. Prepending (vs new plumbing) also persists the
   // choice into the saved prompt, so the session UI shows what the schedule actually requested.
@@ -322,6 +398,27 @@ export async function runSchedule(
     updateScheduledSessionStatus(sessionId, status);
   }
 
+  // The run's OWN verdict beats "the turn returned" — see scheduler/outcome.ts. Deliberately after
+  // the finally: the session lock is released by now, so a run that declared `pending` can be closed
+  // by a later turn in this session (a background job's wake, a follow-up message) while we wait.
+  if (status === 'ok' && task.kind === 'prompt') {
+    const declared = await resolveDeclaredOutcome(sessionId, abortController, schedule.id);
+    if (declared) {
+      status = declared.status;
+      error = declared.error;
+      updateScheduledSessionStatus(sessionId, status);
+      if (status !== 'ok') {
+        appendMessage(sessionId, {
+          id: crypto.randomUUID(),
+          role: 'assistant',
+          blocks: [{ type: 'error', text: error ?? `Run reported status ${status}` }],
+        });
+        onEvent({ type: 'session_messages_changed', sessionId });
+      }
+    }
+    clearOutcome(sessionId);
+  }
+
   const preview = assistantText.slice(0, 120) || (status === 'ok' ? 'Schedule completed' : `Schedule ${status}`);
   addUnread(schedule.createdBy.uid, sessionId, preview, 'schedule', schedule.name);
 

package/src/server/wake.ts

@@ -0,0 +1,95 @@
+// Waking a session from out-of-band work — "something finished while nobody was in the turn,
+// tell the user, wherever that session speaks".
+//
+// Every async subsystem that resolves after its turn is over needs the SAME tail: persist the
+// trigger as a user-channel message, run one agent turn on it, persist + broadcast the reply,
+// mark it unread, and relay to the Slack thread when the session has one. polls.ts grew it first
+// ("close then report"); background-jobs.ts needs it verbatim, so it lives here instead of twice.
+//
+// Decoupled from claude.ts via injected runners (see initWake) — same IoC polls.ts used, for the
+// same reason (claude.ts ← → subsystem import cycle).
+import { appendMessage, getSession, type ConvBlock } from './sessions.ts';
+import { addUnread } from './unread.ts';
+import { postMessage } from './slack/api.ts';
+import { findSlackSessionBySessionId } from './slack/sessions.ts';
+
+const PREFIX = '[wake]';
+
+export type TurnRunner = (args: { prompt: string; sessionId: string; uid: string; userEmail?: string }) => Promise<ConvBlock[]>;
+
+let runTurn: TurnRunner | null = null;
+let broadcastFn: ((ev: object) => void) | null = null;
+
+export function initWake(deps: { runTurn: TurnRunner; broadcast: (ev: object) => void }): void {
+  runTurn = deps.runTurn;
+  broadcastFn = deps.broadcast;
+}
+
+/** True once boot has wired a turn runner. Callers that can degrade (deliver raw text instead of
+ *  running a turn) check this rather than silently dropping their report. */
+export function wakeReady(): boolean { return !!runTurn; }
+
+export function broadcastSessionChanged(sessionId: string): void {
+  broadcastFn?.({ type: 'session_messages_changed', sessionId });
+}
+
+/**
+ * Put `text` into the session as an assistant message and push it to every surface that session
+ * speaks on (web stream + unread badge, and the Slack thread when one is mapped).
+ *
+ * This is the delivery half on its own: used directly when we must report WITHOUT running a turn
+ * (no runner wired, or the session stayed busy past the defer cap) so the outcome is still visible
+ * rather than silently dropped.
+ */
+export async function deliverToSession(i: { sessionId: string; uid: string; text: string; title?: string }): Promise<void> {
+  const text = i.text.trim();
+  if (!text) return;
+  appendMessage(i.sessionId, { id: crypto.randomUUID(), role: 'assistant', blocks: [{ type: 'text', text }] });
+  broadcastSessionChanged(i.sessionId);
+  addUnread(i.uid, i.sessionId, text.slice(0, 120), 'proactive', i.title);
+  const slack = findSlackSessionBySessionId(i.sessionId);
+  if (slack) await postMessage(slack.channel, text, slack.threadTs, slack.useUserToken)
+    .catch((e) => console.error(`${PREFIX} slack deliver failed:`, (e as Error)?.message));
+}
+
+export type WakeOutcome = 'woke' | 'no-session' | 'not-ready' | 'no-output';
+
+/**
+ * Run one agent turn on `prompt` in an existing session and deliver whatever it says.
+ *
+ * The caller is responsible for NOT calling this while a turn is already streaming in that
+ * session (see isSessionLocked) — two concurrent turns would interleave into one transcript.
+ */
+export async function wakeSession(i: {
+  sessionId: string; uid: string; userEmail?: string;
+  prompt: string;
+  /** Message channel tag for the injected trigger (e.g. 'poll', 'job') — shows provenance in the thread. */
+  channel: string;
+  /** Unread-badge title. */
+  title?: string;
+  /** Badge text to use when the turn ran but produced no TEXT block. Without it such a turn is
+   *  reported as 'no-output' and raises no badge — but a turn whose only block is `{type:'error'}`
+   *  is precisely when the user most needs telling. Callers that can degrade some other way (jobs
+   *  fall back to a raw report) leave this unset and handle 'no-output' themselves. */
+  unreadFallback?: string;
+}): Promise<WakeOutcome> {
+  if (!runTurn) { console.warn(`${PREFIX} no turn runner; cannot wake ${i.sessionId}`); return 'not-ready'; }
+  if (!getSession(i.sessionId)) { console.warn(`${PREFIX} session ${i.sessionId} gone; skipping wake`); return 'no-session'; }
+
+  appendMessage(i.sessionId, { id: crypto.randomUUID(), role: 'user', blocks: [{ type: 'text', text: i.prompt }], channel: i.channel });
+  broadcastSessionChanged(i.sessionId);
+
+  const blocks = await runTurn({ prompt: i.prompt, sessionId: i.sessionId, uid: i.uid, userEmail: i.userEmail });
+  if (!blocks.length) return 'no-output';
+  appendMessage(i.sessionId, { id: crypto.randomUUID(), role: 'assistant', blocks });
+  broadcastSessionChanged(i.sessionId);
+
+  const text = blocks.filter((b): b is { type: 'text'; text: string } => b.type === 'text').map((b) => b.text).join('\n\n').trim();
+  if (!text && !i.unreadFallback) return 'no-output';
+  addUnread(i.uid, i.sessionId, text.slice(0, 120) || i.unreadFallback!, 'proactive', i.title);
+  const slack = findSlackSessionBySessionId(i.sessionId);
+  // Nothing to say out loud when there is no text — but the badge above still went up.
+  if (slack && text) await postMessage(slack.channel, text, slack.threadTs, slack.useUserToken)
+    .catch((e) => console.error(`${PREFIX} slack deliver failed:`, (e as Error)?.message));
+  return text ? 'woke' : 'no-output';
+}