talon-agent 1.42.1 → 1.43.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "talon-agent",
-  "version": "1.42.1",
+  "version": "1.43.0",
   "description": "Multi-frontend AI agent with full tool access, streaming, cron jobs, and plugin system",
   "author": "Dylan Neve",
   "license": "MIT",

package/src/frontend/discord/handlers/queue.ts

@@ -5,7 +5,11 @@
  */
 
 import { classify, friendlyMessage } from "../../../core/errors.js";
-import { recordMessageProcessed, recordError } from "../../../util/watchdog.js";
+import {
+  recordMessageProcessed,
+  recordMessageReceived,
+  recordError,
+} from "../../../util/watchdog.js";
 import { appendDailyLog } from "../../../storage/daily-log.js";
 import { log, logError } from "../../../util/log.js";
 import { processAndReply } from "./delivery.js";
@@ -24,6 +28,9 @@ export function enqueueMessage(
   numericChatId: number,
   msg: QueuedMessage,
 ): void {
+  // Tell the watchdog work has arrived — stuck detection compares this
+  // against processing completions (idle chats must never look wedged).
+  recordMessageReceived();
   const existing = messageQueues.get(chatId);
   if (existing) {
     if (existing.messages.length >= MAX_QUEUED_PER_CHAT) return;

package/src/frontend/telegram/handlers/queue.ts

@@ -14,7 +14,11 @@ import {
   getRecentHistory,
   type HistoryMessage,
 } from "../../../storage/history.js";
-import { recordMessageProcessed, recordError } from "../../../util/watchdog.js";
+import {
+  recordMessageProcessed,
+  recordMessageReceived,
+  recordError,
+} from "../../../util/watchdog.js";
 import { log, logError, logWarn, logDebug } from "../../../util/log.js";
 import { appendDailyLog } from "../../../storage/daily-log.js";
 import { processAndReply, sendHtml } from "./delivery.js";
@@ -75,6 +79,9 @@ export function enqueueMessage(
   numericChatId: number,
   msg: QueuedMessage,
 ): void {
+  // Tell the watchdog work has arrived — stuck detection compares this
+  // against processing completions (idle chats must never look wedged).
+  recordMessageReceived();
   const existing = messageQueues.get(chatId);
   if (existing) {
     if (existing.messages.length >= MAX_QUEUED_PER_CHAT) return; // drop excess

package/src/util/watchdog.ts

@@ -9,13 +9,46 @@ import { logWarn } from "./log.js";
 // ── Message processing tracking ──────────────────────────────────────────────
 
 let lastProcessedAt = Date.now();
+let lastReceivedAt = 0;
 let totalMessagesProcessed = 0;
 const startTime = Date.now();
 
+/**
+ * Record that a message ARRIVED (enqueued for processing). Paired with
+ * `recordMessageProcessed`, this is what lets the watchdog tell a wedged
+ * message loop (work arrived, nothing finishing) apart from a bot that's
+ * simply idle because nobody is talking to it. Idle is not a fault.
+ */
+export function recordMessageReceived(): void {
+  lastReceivedAt = Date.now();
+}
+
 /** Record that a message was successfully processed. */
 export function recordMessageProcessed(): void {
   lastProcessedAt = Date.now();
   totalMessagesProcessed++;
+  resetStuckWarnBackoff();
+}
+
+/**
+ * Test-only: reset the activity clocks to "just processed, nothing pending"
+ * at the CURRENT Date.now(). Fake-timer suites need this because each test
+ * restarts its fake clock at the real now, while timestamps recorded by an
+ * earlier test may sit hours into that test's own fake future.
+ */
+export function resetWatchdogActivityForTests(): void {
+  lastProcessedAt = Date.now();
+  lastReceivedAt = 0;
+  resetStuckWarnBackoff();
+}
+
+/**
+ * How long the newest RECEIVED message has been waiting with no processing
+ * completed after it. Zero when idle or keeping up.
+ */
+function stuckMs(now: number): number {
+  if (lastReceivedAt === 0 || lastReceivedAt <= lastProcessedAt) return 0;
+  return now - lastReceivedAt;
 }
 
 /** Get total messages processed since startup. */
@@ -53,17 +86,38 @@ export function getRecentErrors(limit = 5): ErrorRecord[] {
 
 // ── Inactivity monitoring ────────────────────────────────────────────────────
 
-const INACTIVITY_WARN_MS = 10 * 60 * 1000; // 10 minutes
+const STUCK_WARN_MS = 10 * 60 * 1000; // 10 minutes with unprocessed work
+const STUCK_WARN_MAX_BACKOFF_MS = 60 * 60 * 1000;
 let watchdogTimer: ReturnType<typeof setInterval> | null = null;
+let stuckWarnBackoffMs = STUCK_WARN_MS;
+let nextStuckWarnAt = 0;
+
+function resetStuckWarnBackoff(): void {
+  stuckWarnBackoffMs = STUCK_WARN_MS;
+  nextStuckWarnAt = 0;
+}
 
 export function startWatchdog(workspaceDir?: string): void {
   if (watchdogTimer) return;
 
   watchdogTimer = setInterval(() => {
-    const elapsed = Date.now() - lastProcessedAt;
-    if (totalMessagesProcessed > 0 && elapsed > INACTIVITY_WARN_MS) {
-      const mins = Math.round(elapsed / 60000);
-      logWarn("watchdog", `No messages processed for ${mins} minutes`);
+    // Warn only when work is actually stuck — a message arrived and nothing
+    // has finished processing since. A quiet chat used to trip this every
+    // minute all night ("No messages processed for N minutes"), burying real
+    // warnings in noise. Repeat warnings back off exponentially.
+    const now = Date.now();
+    const stuck = stuckMs(now);
+    if (stuck > STUCK_WARN_MS && now >= nextStuckWarnAt) {
+      const mins = Math.round(stuck / 60000);
+      logWarn(
+        "watchdog",
+        `Message received ${mins} minutes ago is still unprocessed — the message loop may be wedged`,
+      );
+      nextStuckWarnAt = now + stuckWarnBackoffMs;
+      stuckWarnBackoffMs = Math.min(
+        stuckWarnBackoffMs * 2,
+        STUCK_WARN_MAX_BACKOFF_MS,
+      );
     }
 
     // Ensure workspace still exists (might have been deleted externally)
@@ -101,8 +155,11 @@ export function getHealthStatus(): HealthStatus {
   const now = Date.now();
   const msSinceLastMessage = now - lastProcessedAt;
   return {
-    // Unhealthy if no messages processed for 30+ minutes when we've seen at least one
-    healthy: totalMessagesProcessed === 0 || msSinceLastMessage < 30 * 60_000,
+    // Unhealthy only when the message loop is actually stuck: a received
+    // message has gone 30+ minutes with no processing completing after it.
+    // Mere idleness (nobody texting the bot) used to flip this false and
+    // show a red "unhealthy" row in the companion diagnostics overnight.
+    healthy: stuckMs(now) < 30 * 60_000,
     uptimeMs: now - startTime,
     totalMessagesProcessed,
     lastProcessedAt,