talon-agent 1.40.0 → 1.40.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "talon-agent",
-  "version": "1.40.0",
+  "version": "1.40.1",
   "description": "Multi-frontend AI agent with full tool access, streaming, cron jobs, and plugin system",
   "author": "Dylan Neve",
   "license": "MIT",

package/src/core/background/heartbeat/scheduler.ts

@@ -22,6 +22,65 @@ import { HeartbeatTimeoutError, runHeartbeatAgent } from "./agent.js";
 const STARTUP_DELAY_MS = 5 * 60 * 1000; // 5-minute delay before first run
 const DUE_CHECK_INTERVAL_MS = 60 * 1000;
 
+// Failure backoff. A failed run does NOT advance last_run, so without a
+// backoff the every-minute due check re-fires the heartbeat 60×/hour against
+// the same error (observed live: a session-limit night produced hundreds of
+// identical failures a minute apart). Generic failures back off
+// exponentially; session/rate-limit errors that state their own reset time
+// wait for it instead of guessing.
+const FAILURE_BACKOFF_BASE_MS = 5 * 60 * 1000;
+const FAILURE_BACKOFF_MAX_MS = 60 * 60 * 1000;
+/** Safety margin added past a parsed limit-reset time (clock skew, rollout). */
+const LIMIT_RESET_BUFFER_MS = 2 * 60 * 1000;
+
+/**
+ * Parse the reset wall-clock time out of a session/rate-limit error message,
+ * e.g. "You've hit your session limit · resets 12:20am (UTC)" or
+ * "… resets 3pm (UTC)". Returns the epoch ms of the NEXT occurrence of that
+ * UTC time after `now`, or null when the message doesn't carry one.
+ */
+export function parseSessionLimitResetMs(
+  message: string,
+  now: number,
+): number | null {
+  const m = /resets\s+(\d{1,2})(?::(\d{2}))?\s*(am|pm)\s*\(UTC\)/i.exec(
+    message,
+  );
+  if (!m) return null;
+  const rawHour = Number(m[1]);
+  const minute = m[2] ? Number(m[2]) : 0;
+  if (rawHour < 1 || rawHour > 12 || minute > 59) return null;
+  let hour = rawHour % 12;
+  if (m[3].toLowerCase() === "pm") hour += 12;
+  const d = new Date(now);
+  d.setUTCHours(hour, minute, 0, 0);
+  let t = d.getTime();
+  if (t <= now) t += 24 * 60 * 60 * 1000;
+  return t;
+}
+
+/**
+ * When to next allow an auto heartbeat after the Nth consecutive failure
+ * (1-based). Limit errors with a stated reset time wait until then (+buffer);
+ * everything else doubles from 5min up to a 60min cap.
+ */
+export function failureBackoffUntil(
+  err: unknown,
+  consecutiveFailures: number,
+  now: number,
+): number {
+  const message = err instanceof Error ? err.message : String(err);
+  if (/session limit|rate limit/i.test(message)) {
+    const reset = parseSessionLimitResetMs(message, now);
+    if (reset !== null) return reset + LIMIT_RESET_BUFFER_MS;
+  }
+  const exp = Math.min(
+    FAILURE_BACKOFF_BASE_MS * 2 ** Math.max(0, consecutiveFailures - 1),
+    FAILURE_BACKOFF_MAX_MS,
+  );
+  return now + exp;
+}
+
 export function initHeartbeat(cfg: HeartbeatConfig): void {
   hb.config = cfg;
 }
@@ -69,6 +128,9 @@ export function startHeartbeatTimer(intervalMinutes: number): void {
  */
 function runIfDue(intervalMs: number, startup: boolean): void {
   if (hb.running) return;
+  // Inside a failure backoff window — stay quiet instead of re-firing the
+  // same error every due check. (Logged once, when the window was set.)
+  if (Date.now() < hb.backoffUntil) return;
   const lastRun = readHeartbeatState()?.last_run ?? 0;
   if (lastRun <= 0) {
     // Never ran on this install — fire now to establish the cadence.
@@ -178,6 +240,8 @@ async function executeHeartbeat(trigger: "auto" | "forced"): Promise<void> {
         status: "idle",
         run_count: previousRunCount + 1,
       });
+      hb.consecutiveFailures = 0;
+      hb.backoffUntil = 0;
       log(
         "heartbeat",
         `Heartbeat #${previousRunCount + 1} complete (${trigger}), log: ${heartbeatLogPath}`,
@@ -199,6 +263,22 @@ async function executeHeartbeat(trigger: "auto" | "forced"): Promise<void> {
         status: "idle",
         run_count: isTimeout ? previousRunCount + 1 : previousRunCount,
       });
+      // Timeouts advance last_run (budget consumed), so the cadence itself
+      // spaces the next attempt. Every other failure retries against the same
+      // last_run — back off so the due check doesn't hammer it every minute.
+      if (!isTimeout) {
+        hb.consecutiveFailures += 1;
+        hb.backoffUntil = failureBackoffUntil(
+          err,
+          hb.consecutiveFailures,
+          Date.now(),
+        );
+        logWarn(
+          "heartbeat",
+          `Backing off until ${new Date(hb.backoffUntil).toISOString()} ` +
+            `after ${hb.consecutiveFailures} consecutive failure(s)`,
+        );
+      }
       if (trigger === "forced") throw err;
     } finally {
       hb.running = false;

package/src/core/background/heartbeat/state.ts

@@ -71,6 +71,10 @@ export const hb: {
   intervalMinutesRef: number;
   config: HeartbeatConfig | null;
   logFileSequence: number;
+  /** Consecutive failed runs — drives the failure backoff curve. */
+  consecutiveFailures: number;
+  /** Epoch ms before which auto runs are suppressed after a failure. */
+  backoffUntil: number;
 } = {
   running: false,
   currentRunPromise: null,
@@ -79,6 +83,8 @@ export const hb: {
   intervalMinutesRef: 60,
   config: null,
   logFileSequence: 0,
+  consecutiveFailures: 0,
+  backoffUntil: 0,
 };
 
 // ── State-file I/O ───────────────────────────────────────────────────────────