talon-agent 1.40.0 → 1.40.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "talon-agent",
-  "version": "1.40.0",
+  "version": "1.40.2",
   "description": "Multi-frontend AI agent with full tool access, streaming, cron jobs, and plugin system",
   "author": "Dylan Neve",
   "license": "MIT",

package/src/core/background/dream.ts

@@ -22,6 +22,7 @@ import { getDefaultModel } from "../models/catalog.js";
 import type { OneShotAgentParams } from "../types.js";
 import type { Backend } from "../agent-runtime/capabilities.js";
 import { getSoul } from "../soul/service.js";
+import { FailureBackoff } from "./failure-backoff.js";
 
 // ── Types ────────────────────────────────────────────────────────────────────
 
@@ -53,6 +54,13 @@ const DREAM_LOGS_DIR = resolve(dirs.logs, "dreams");
 // ── State ────────────────────────────────────────────────────────────────────
 
 let dreaming = false; // in-process guard (one dream at a time)
+/**
+ * A failed dream does not advance last_run, and maybeStartDream runs on
+ * every invocation — without a backoff a broken backend gets re-fired on
+ * every message (observed live: a model outage produced 403 identical
+ * consolidation failures). Exported for tests.
+ */
+export const dreamFailureBackoff = new FailureBackoff();
 let configRef: {
   model?: string;
   dreamModel?: string;
@@ -101,6 +109,9 @@ export function initDream(cfg: {
 export function maybeStartDream(): void {
   if (dreaming) return;
   if (configRef?.enabled === false) return;
+  // Inside a failure backoff window — stay quiet instead of re-firing the
+  // same error on every invocation. (Logged once, when the window was set.)
+  if (dreamFailureBackoff.active()) return;
 
   const state = readDreamState();
   const now = Date.now();
@@ -134,6 +145,7 @@ async function executeDream(trigger: "auto" | "forced"): Promise<void> {
   try {
     const dreamLogPath = await runDreamAgent(state?.last_run ?? 0);
     writeDreamState({ last_run: Date.now(), status: "idle" });
+    dreamFailureBackoff.succeed();
     log(
       "dream",
       `Memory consolidation complete (${trigger}), log: ${dreamLogPath}`,
@@ -145,6 +157,14 @@ async function executeDream(trigger: "auto" | "forced"): Promise<void> {
   } catch (err) {
     logError("dream", `Memory consolidation failed (${trigger})`, err);
     writeDreamState({ last_run: state?.last_run ?? 0, status: "idle" });
+    // A failed dream keeps the old last_run, so it would re-fire on the very
+    // next invocation — back off instead (forceDream bypasses the window).
+    const until = dreamFailureBackoff.fail(err);
+    logWarn(
+      "dream",
+      `Backing off until ${new Date(until).toISOString()} ` +
+        `after ${dreamFailureBackoff.failures} consecutive failure(s)`,
+    );
     if (trigger === "forced") throw err;
   } finally {
     dreaming = false;

package/src/core/background/failure-backoff.ts

@@ -0,0 +1,102 @@
+/**
+ * Failure backoff for background agents (heartbeat, dream, …).
+ *
+ * A failed background run typically does NOT advance its `last_run` marker,
+ * so whatever cadence check re-evaluates "is a run due?" re-fires the same
+ * failing run on every tick — observed live as hundreds of identical
+ * failures minutes apart (session-limit nights, model outages, auth
+ * breakage). This module centralizes the cure:
+ *
+ *   - generic failures back off exponentially (5 → 10 → 20 → 40 → 60min cap)
+ *   - session/rate-limit errors that state their own reset time
+ *     ("… resets 12:20am (UTC)") wait until then (+buffer) instead of
+ *     guessing
+ *   - a success clears the window
+ *
+ * Backoff state is in-memory only: a process restart forgets it, which is
+ * fine — the first post-restart attempt either works or re-arms the window.
+ */
+
+const FAILURE_BACKOFF_BASE_MS = 5 * 60 * 1000;
+const FAILURE_BACKOFF_MAX_MS = 60 * 60 * 1000;
+/** Safety margin added past a parsed limit-reset time (clock skew, rollout). */
+const LIMIT_RESET_BUFFER_MS = 2 * 60 * 1000;
+
+/**
+ * Parse the reset wall-clock time out of a session/rate-limit error message,
+ * e.g. "You've hit your session limit · resets 12:20am (UTC)" or
+ * "… resets 3pm (UTC)". Returns the epoch ms of the NEXT occurrence of that
+ * UTC time after `now`, or null when the message doesn't carry one.
+ */
+export function parseSessionLimitResetMs(
+  message: string,
+  now: number,
+): number | null {
+  const m = /resets\s+(\d{1,2})(?::(\d{2}))?\s*(am|pm)\s*\(UTC\)/i.exec(
+    message,
+  );
+  if (!m) return null;
+  const rawHour = Number(m[1]);
+  const minute = m[2] ? Number(m[2]) : 0;
+  if (rawHour < 1 || rawHour > 12 || minute > 59) return null;
+  let hour = rawHour % 12;
+  if (m[3].toLowerCase() === "pm") hour += 12;
+  const d = new Date(now);
+  d.setUTCHours(hour, minute, 0, 0);
+  let t = d.getTime();
+  if (t <= now) t += 24 * 60 * 60 * 1000;
+  return t;
+}
+
+/**
+ * When to next allow a run after the Nth consecutive failure (1-based).
+ * Limit errors with a stated reset time wait until then (+buffer);
+ * everything else doubles from 5min up to a 60min cap.
+ */
+export function failureBackoffUntil(
+  err: unknown,
+  consecutiveFailures: number,
+  now: number,
+): number {
+  const message = err instanceof Error ? err.message : String(err);
+  if (/session limit|rate limit/i.test(message)) {
+    const reset = parseSessionLimitResetMs(message, now);
+    if (reset !== null) return reset + LIMIT_RESET_BUFFER_MS;
+  }
+  const exp = Math.min(
+    FAILURE_BACKOFF_BASE_MS * 2 ** Math.max(0, consecutiveFailures - 1),
+    FAILURE_BACKOFF_MAX_MS,
+  );
+  return now + exp;
+}
+
+/**
+ * Per-component backoff holder. Cadence checks gate on `active()`; run
+ * completions call `succeed()` / `fail(err)`.
+ */
+export class FailureBackoff {
+  private consecutiveFailures = 0;
+  private until = 0;
+
+  /** True while inside the backoff window — the caller should skip firing. */
+  active(now = Date.now()): boolean {
+    return now < this.until;
+  }
+
+  /** Record a failure; returns the epoch ms the window is armed until. */
+  fail(err: unknown, now = Date.now()): number {
+    this.consecutiveFailures += 1;
+    this.until = failureBackoffUntil(err, this.consecutiveFailures, now);
+    return this.until;
+  }
+
+  /** Record a success — clears the window and the failure streak. */
+  succeed(): void {
+    this.consecutiveFailures = 0;
+    this.until = 0;
+  }
+
+  get failures(): number {
+    return this.consecutiveFailures;
+  }
+}

package/src/core/background/heartbeat/scheduler.ts

@@ -69,6 +69,9 @@ export function startHeartbeatTimer(intervalMinutes: number): void {
  */
 function runIfDue(intervalMs: number, startup: boolean): void {
   if (hb.running) return;
+  // Inside a failure backoff window — stay quiet instead of re-firing the
+  // same error every due check. (Logged once, when the window was set.)
+  if (hb.failureBackoff.active()) return;
   const lastRun = readHeartbeatState()?.last_run ?? 0;
   if (lastRun <= 0) {
     // Never ran on this install — fire now to establish the cadence.
@@ -178,6 +181,7 @@ async function executeHeartbeat(trigger: "auto" | "forced"): Promise<void> {
         status: "idle",
         run_count: previousRunCount + 1,
       });
+      hb.failureBackoff.succeed();
       log(
         "heartbeat",
         `Heartbeat #${previousRunCount + 1} complete (${trigger}), log: ${heartbeatLogPath}`,
@@ -199,6 +203,17 @@ async function executeHeartbeat(trigger: "auto" | "forced"): Promise<void> {
         status: "idle",
         run_count: isTimeout ? previousRunCount + 1 : previousRunCount,
       });
+      // Timeouts advance last_run (budget consumed), so the cadence itself
+      // spaces the next attempt. Every other failure retries against the same
+      // last_run — back off so the due check doesn't hammer it every minute.
+      if (!isTimeout) {
+        const until = hb.failureBackoff.fail(err);
+        logWarn(
+          "heartbeat",
+          `Backing off until ${new Date(until).toISOString()} ` +
+            `after ${hb.failureBackoff.failures} consecutive failure(s)`,
+        );
+      }
       if (trigger === "forced") throw err;
     } finally {
       hb.running = false;

package/src/core/background/heartbeat/state.ts

@@ -10,6 +10,7 @@ import { files as pathFiles } from "../../../util/paths.js";
 import { kvGet, kvSet } from "../../../storage/kv.js";
 import { importLegacyJson } from "../../../storage/legacy-import.js";
 import type { Backend } from "../../agent-runtime/capabilities.js";
+import { FailureBackoff } from "../failure-backoff.js";
 
 export type HeartbeatState = {
   /** Unix millisecond timestamp of the last successfully completed heartbeat run. */
@@ -71,6 +72,8 @@ export const hb: {
   intervalMinutesRef: number;
   config: HeartbeatConfig | null;
   logFileSequence: number;
+  /** Suppresses auto runs after failures (see failure-backoff.ts). */
+  failureBackoff: FailureBackoff;
 } = {
   running: false,
   currentRunPromise: null,
@@ -79,6 +82,7 @@ export const hb: {
   intervalMinutesRef: 60,
   config: null,
   logFileSequence: 0,
+  failureBackoff: new FailureBackoff(),
 };
 
 // ── State-file I/O ───────────────────────────────────────────────────────────