shraga 0.1.14 → 0.1.15

package/src/server/scheduler/builtins.ts

@@ -15,10 +15,12 @@ export const HOURLY_SUMMARIZER_SCHEDULE_ID = 'builtin-conversation-summarizer';
 export const FAILURE_NOTIFIER_SCHEDULE_ID = 'builtin-failure-notifier';
 
 /** Generic triage prompt for the failure notifier. Deployments override the prompt
- *  (recipients, runbooks, base URL, severity rules) — their edits survive reconcile. */
+ *  (recipients, runbooks, severity rules) — their edits survive reconcile. The session link
+ *  is NOT part of that: it comes from the event payload (see getSessionUrl), precisely so it
+ *  reaches deployments whose stored prompt reconcile will never touch. */
 const FAILURE_NOTIFIER_PROMPT = [
   'A scheduled job just FAILED. The failure event payload is included in this message',
-  '(fields: name, scheduleId, status, error, sessionId). Duplicate alerts for the same',
+  '(fields: name, scheduleId, status, error, sessionId, sessionUrl). Duplicate alerts for the same',
   'job+error are already suppressed by this trigger\'s throttle, so just handle this one.',
   '',
   'TRIAGE — classify the error:',
@@ -35,10 +37,25 @@ const FAILURE_NOTIFIER_PROMPT = [
   '  *What:* <one plain-language line>',
   '  *Fix:* <actionable next step from triage>',
   '  *Error:* <error, truncated to ~400 chars, in backticks>',
-  '  *Session:* <deployment URL>/?session=<sessionId>',
+  '  *Session:* <the payload\'s sessionUrl, verbatim>',
+  'The payload carries a ready-made absolute sessionUrl. Use it EXACTLY as given — never build a',
+  'link yourself. If sessionUrl is absent the deployment has no public origin configured: OMIT the',
+  'Session line entirely. Do NOT substitute localhost, $PORT or any host you infer from the box —',
+  'the alert is read off-box and such a link is always dead.',
   'Do NOT try to fix the job yourself.',
 ].join('\n');
 
+/** The pre-sessionUrl Session line: it asked the model to improvise "<deployment URL>", which
+ *  nothing ever supplied, so it fell back to the only host it could see ($PORT → localhost).
+ *  Reconcile deliberately preserves a builtin's stored `task.prompt` so deployment edits survive
+ *  upgrades — which also means a stored prompt keeps this broken line forever. Heal just the line
+ *  (not the whole prompt), so a deployment's other customisations are untouched.
+ *  Only the placeholder itself and the label before it are replaced: anything the deployment
+ *  appended after the placeholder is a hand-written annotation, so it is carried over verbatim,
+ *  and every occurrence is healed (a stored prompt may mention the link more than once). */
+const LEGACY_SESSION_LINE = /^.*<deployment URL>\/\?session=<sessionId>(.*)$/gm;
+const SESSION_LINE_FIX = "  *Session:* <the payload's sessionUrl, verbatim — omit this line if absent>";
+
 export function isSystemSchedule(schedule: Schedule): boolean {
   return schedule.scope === 'system';
 }
@@ -69,6 +86,13 @@ export function backfillScope(schedules: Schedule[]): void {
     if (task.kind === 'job' && task.command === 'bun run summarize:conversations') {
       task.command = SUMMARIZER_CMD;
     }
+    // Heal a persisted failure-notifier prompt still carrying the un-supplied "<deployment URL>"
+    // placeholder — reconcile won't touch task.prompt, so this is the only path that reaches it.
+    if (s.id === FAILURE_NOTIFIER_SCHEDULE_ID && typeof task.prompt === 'string') {
+      // `$1` keeps whatever the deployment wrote after the placeholder. No .test() guard:
+      // LEGACY_SESSION_LINE is global, and a global regex's .test() carries lastIndex between calls.
+      task.prompt = task.prompt.replace(LEGACY_SESSION_LINE, `${SESSION_LINE_FIX}$1`);
+    }
   }
 }
 

package/src/server/scheduler/engine.ts

@@ -3,6 +3,7 @@ import { computeNextRun, computePrevRun, validateTrigger } from './timing.ts';
 import { runSchedule, type ResumeOptions, type EventContext } from './runner.ts';
 import { backfillScope, ensureBuiltinSchedules } from './builtins.ts';
 import { emitEvent } from '../events/bus.ts';
+import { getSessionUrl } from '../shraga-config.ts';
 import type { Schedule } from './types.ts';
 
 type Broadcast = (data: object) => void;
@@ -410,6 +411,11 @@ function startRun(s: Schedule, _firedAt: number, override?: string, resume?: Res
           name: s.name,
           status: summary.status,
           sessionId: summary.sessionId,
+          // Ready-made absolute link. Supplied here rather than left to the consuming prompt:
+          // reconcile never syncs a builtin's stored `task.prompt`, so a prompt-only fix would
+          // miss every deployment that already persisted the schedule. Omitted when no public
+          // origin is configured — better no link than a localhost one.
+          sessionUrl: getSessionUrl(summary.sessionId),
           error: summary.error,
         }, { id: summary.sessionId });
       }

package/src/server/scheduler/runner.ts

@@ -66,6 +66,27 @@ export function resolvePromptFile(p: string): string {
   return existsSync(dataAnchored) ? dataAnchored : rootAnchored;
 }
 
+/**
+ * Bounded retry for transient engine failures on a prompt run.
+ *
+ * A cold-open race in the engine (observed: `database is locked` from @cursor/sdk, and an
+ * unreachable ANTHROPIC_BASE_URL) can kill a run in ~600ms having produced NOTHING — no first
+ * token, no tool call, no side effect. One blip then costs the whole day's run. The race window is
+ * milliseconds, so these delays are deliberately short: this is a blip retry, not an outage retry.
+ * Each delay is jittered (×0.5–1.5) so concurrent schedules don't retry in lockstep.
+ */
+const RETRY_BACKOFF_MS = [500, 2_000];
+const MAX_ATTEMPTS = RETRY_BACKOFF_MS.length + 1;
+
+/** Resolves after `ms`, or immediately on abort — a cancelled run must not sit out its backoff
+ *  holding the session lock and the running marker. */
+const sleep = (ms: number, signal?: AbortSignal) => new Promise<void>((resolve) => {
+  if (signal?.aborted) return resolve();
+  const done = () => { clearTimeout(timer); signal?.removeEventListener('abort', done); resolve(); };
+  const timer = setTimeout(done, ms);
+  signal?.addEventListener('abort', done, { once: true });
+});
+
 function formatEventBlock(e: EventContext): string {
   let body: string;
   try { body = JSON.stringify(e.payload, null, 2); } catch { body = String(e.payload); }
@@ -144,6 +165,9 @@ export async function runSchedule(
   const mcpServers = getMcpConfig(schedule.createdBy.uid);
 
   let assistantText = '';
+  // Thinking is real, billed model output but lands in no block (we don't persist it), so it needs
+  // its own flag to suppress retry — see the side-effect boundary below.
+  let producedThinking = false;
   const assistantBlocks: ConvBlock[] = [];
   const collectPartialBlocks = () => [
     ...assistantBlocks,
@@ -169,39 +193,81 @@ export async function runSchedule(
   onEvent({ type: 'schedule:run_started', scheduleId: schedule.id, sessionId, at: now });
 
   try {
-    for await (const ev of streamChat({
-      prompt,
-      sessionId,
-      uid: schedule.createdBy.uid,
-      userEmail: schedule.createdBy.email,
-      userName: schedule.createdBy.email.split('@')[0],
-      mcpServers,
-      abortController,
-      onPermissionRequest,
-    })) {
-      onEvent({ type: 'session_stream', sessionId, event: ev });
-      if (ev.type === 'text_delta') {
-        assistantText += ev.text;
-      } else if (ev.type === 'tool_use') {
+    for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
+      status = 'ok';
+      error = undefined;
+      try {
+        for await (const ev of streamChat({
+          prompt,
+          sessionId,
+          uid: schedule.createdBy.uid,
+          userEmail: schedule.createdBy.email,
+          userName: schedule.createdBy.email.split('@')[0],
+          mcpServers,
+          abortController,
+          onPermissionRequest,
+        })) {
+          onEvent({ type: 'session_stream', sessionId, event: ev });
+          if (ev.type === 'text_delta') {
+            assistantText += ev.text;
+          } else if (ev.type === 'tool_use') {
+            if (assistantText) { assistantBlocks.push({ type: 'text', text: assistantText }); assistantText = ''; }
+            assistantBlocks.push({ type: 'tool_use', tool: ev.tool, toolUseId: ev.toolUseId, input: ev.input });
+          } else if (ev.type === 'thinking_delta') {
+            producedThinking = true;
+          } else if (ev.type === 'tool_result') {
+            assistantBlocks.push({ type: 'tool_result', toolUseId: ev.toolUseId, output: ev.output });
+          } else if (ev.type === 'done') {
+            break;
+          } else if (ev.type === 'error') {
+            status = abortController.signal.aborted ? 'aborted' : 'error';
+            error = ev.message;
+            break;
+          }
+        }
+      } catch (err: any) {
+        if (abortController.signal.aborted) {
+          status = 'aborted';
+        } else {
+          status = 'error';
+          error = err?.message ?? String(err);
+        }
+      }
+
+      if (status !== 'error') break;
+
+      // The side-effect boundary. This is NOT an exact `ttft=-1` test — the engine emits events we
+      // don't track (model_resolved, stats) — it is the weaker but sufficient guarantee we actually
+      // need: if all three are empty, no side effect can have occurred, so re-running can't
+      // double-apply one (a posted DM, a written file). That holds because `tool_use` is yielded at
+      // content_block_start, BEFORE the tool executes — any tool that ran is always preceded by a
+      // `tool_use` already in `assistantBlocks`. `assistantText`/`producedThinking` additionally
+      // stop us re-billing a long model call that genuinely started producing before dying.
+      // Deliberately NOT counted: `model_resolved` and `stats` fire at spawn/on a timer before any
+      // generation — counting them would disable retry for the exact incident this exists for.
+      // `tool_use_input`, `tool_result_image`, `permission_request` and `question_request` need no
+      // separate flag: each is necessarily preceded by the `tool_use` that already set the boundary.
+      const producedOutput = assistantBlocks.length > 0 || assistantText.length > 0 || producedThinking;
+      if (producedOutput || abortController.signal.aborted || attempt >= MAX_ATTEMPTS) {
+        // Nobody watches stderr on a scheduled run — record the failure in the transcript.
         if (assistantText) { assistantBlocks.push({ type: 'text', text: assistantText }); assistantText = ''; }
-        assistantBlocks.push({ type: 'tool_use', tool: ev.tool, toolUseId: ev.toolUseId, input: ev.input });
-      } else if (ev.type === 'tool_result') {
-        assistantBlocks.push({ type: 'tool_result', toolUseId: ev.toolUseId, output: ev.output });
-      } else if (ev.type === 'done') {
-        break;
-      } else if (ev.type === 'error') {
-        status = abortController.signal.aborted ? 'aborted' : 'error';
-        error = ev.message;
+        assistantBlocks.push({ type: 'error', text: error ?? 'Unknown error' });
+        console.error(`[scheduler] run error for ${schedule.id} (attempt ${attempt}/${MAX_ATTEMPTS}):`, error);
         break;
       }
-    }
-  } catch (err: any) {
-    if (abortController.signal.aborted) {
-      status = 'aborted';
-    } else {
-      status = 'error';
-      error = err?.message ?? String(err);
-      console.error(`[scheduler] run error for ${schedule.id}:`, error);
+
+      // Retries stay visible: a silent one would hide that the upstream engine is flaky.
+      const delay = Math.round(RETRY_BACKOFF_MS[attempt - 1]! * (0.5 + Math.random()));
+      console.warn(`[scheduler] attempt ${attempt}/${MAX_ATTEMPTS} for ${schedule.id} failed before any output, retrying in ${delay}ms:`, error);
+      appendMessage(sessionId, {
+        id: crypto.randomUUID(),
+        role: 'assistant',
+        blocks: [{ type: 'text', text: `⚠️ Attempt ${attempt}/${MAX_ATTEMPTS} failed before producing any output — retrying in ${delay}ms.\n\n\`${error}\`` }],
+      });
+      onEvent({ type: 'session_messages_changed', sessionId });
+      await sleep(delay, abortController.signal);
+      // Cancelling during the backoff is a user cancel, not a failure — don't spawn attempt N+1.
+      if (abortController.signal.aborted) { status = 'aborted'; error = undefined; break; }
     }
   } finally {
     flush();

package/src/server/scheduler/types.ts

@@ -59,4 +59,7 @@ export interface Schedule {
   nextRun?: number;
   lastRun?: ScheduleRunSummary;
   runCount: number;
+  /** Set when a data-plane module owns this schedule (module name). Module reconcile
+   *  updates trigger/task; enable/disable snapshots live in the module's state entry. */
+  managedBy?: string;
 }

package/src/server/sessions.ts

@@ -356,6 +356,10 @@ export type ConvBlock =
   | { type: 'tool_use'; tool: string; toolUseId: string; input: unknown }
   | { type: 'tool_result'; toolUseId: string; output: string }
   | { type: 'thinking'; text: string }
+  // A run that failed at the engine/adapter level. Persisted so the transcript records the failure
+  // durably — otherwise a dead session (esp. an unattended scheduled one) is indistinguishable from
+  // one still thinking, with the cause only in stderr.
+  | { type: 'error'; text: string }
   // Persisted block written by an add-on engine's background worker (e.g. EE's duplex voice brain).
   // The core stores/renders it but owns none of its semantics; name kept for stored-history back-compat.
   | { type: 'duplex_result'; label?: string; tier?: string; text?: string }

package/src/server/shraga-config.ts

@@ -57,6 +57,10 @@ export interface ShragaConfig {
   /** @deprecated Use `mcps` instead */
   vendorMcps?: Record<string, McpShorthandEntry>;
   mcps?: Record<string, McpEntry>;
+  /** Public origin this deployment is reachable at, e.g. `https://agent.example.com`.
+   *  Used to build absolute session links for out-of-band notifications (push, alerts)
+   *  that have no incoming request to derive it from. Falls back to `$PUBLIC_ORIGIN`. */
+  publicOrigin?: string;
 }
 
 export interface HttpSidecarSpec {
@@ -114,6 +118,21 @@ export function getShragaConfigSync(): ShragaConfig {
   return _cached ?? {};
 }
 
+/** This deployment's public origin, without a trailing slash — data-dir config first, then
+ *  `$PUBLIC_ORIGIN`. Empty when unconfigured: callers MUST omit the link rather than fall back
+ *  to a locally-derived host, which is unreachable from wherever the notification is read. */
+export function getPublicOrigin(): string {
+  const origin = getShragaConfigSync().publicOrigin ?? process.env.PUBLIC_ORIGIN ?? '';
+  return origin.trim().replace(/\/+$/, '');
+}
+
+/** Absolute link to a session in the web UI, or `undefined` when no public origin is configured. */
+export function getSessionUrl(sessionId: string | undefined): string | undefined {
+  const origin = getPublicOrigin();
+  if (!origin || !sessionId) return undefined;
+  return `${origin}/?session=${encodeURIComponent(sessionId)}`;
+}
+
 /** Resolve global MCPs from the data-dir config (both shorthand vendor entries and full entries) */
 export function getGlobalMcpsFromConfig(): McpConfig {
   const ucConfig = getShragaConfigSync();

package/src/server/skills.ts

@@ -22,6 +22,8 @@ export interface SkillMeta {
   expires?: string;
   origin?: string;
   reviewed?: boolean;
+  /** `managed-by: <module>@<version>` — set on skills rendered by a data-plane module. */
+  managedBy?: string;
 }
 
 export function isExpired(meta: SkillMeta): boolean {
@@ -66,6 +68,7 @@ export function parseSkillFrontmatter(content: string): { meta: SkillMeta; body:
     }
     if (key === 'expires') meta.expires = val;
     if (key === 'origin') meta.origin = val;
+    if (key === 'managed-by') meta.managedBy = val;
     if (key === 'reviewed') meta.reviewed = val === 'true' ? true : val === 'false' ? false : undefined;
   }
   return { meta, body };

package/src/server/slack/bot.ts

@@ -98,8 +98,10 @@ async function* pumpStream(
         stopReason = ev.stopReason ?? 'end_turn';
         break;
       } else if (ev.type === 'error') {
+        // Slack gets it as text (it has no block renderer); the transcript gets a real error block.
         const t = `\n⚠️ ${ev.message}`;
-        assistantText += t;
+        if (assistantText) { assistantBlocks.push({ type: 'text', text: assistantText }); assistantText = ''; }
+        assistantBlocks.push({ type: 'error', text: ev.message });
         yield { type: 'text_delta', text: t };
         break;
       }