talon-agent 1.12.0 → 1.34.0

package/src/core/background/heartbeat/scheduler.ts

@@ -0,0 +1,211 @@
+/**
+ * Heartbeat scheduling + lifecycle — init, the cadence timer (Gleam
+ * scheduler-core driven, restart/suspend-safe), the one-at-a-time run guard,
+ * and the force/await/status public API.
+ */
+
+import { log, logError, logWarn } from "../../../util/log.js";
+import {
+  catchupRunCount,
+  missedRunCount,
+  nextDueMs,
+} from "../../../native/scheduler-core.js";
+import {
+  hb,
+  readHeartbeatState,
+  writeHeartbeatState,
+  type HeartbeatConfig,
+  type HeartbeatState,
+} from "./state.js";
+import { HeartbeatTimeoutError, runHeartbeatAgent } from "./agent.js";
+
+const STARTUP_DELAY_MS = 5 * 60 * 1000; // 5-minute delay before first run
+const DUE_CHECK_INTERVAL_MS = 60 * 1000;
+
+export function initHeartbeat(cfg: HeartbeatConfig): void {
+  hb.config = cfg;
+}
+
+/**
+ * Start the heartbeat timer. First run happens after a 5-minute startup delay,
+ * then repeats at the configured interval.
+ */
+export function startHeartbeatTimer(intervalMinutes: number): void {
+  if (hb.timer || hb.startupTimer) return; // already running or scheduled
+
+  if (!Number.isFinite(intervalMinutes) || intervalMinutes <= 0) {
+    logWarn(
+      "heartbeat",
+      `Refusing to start heartbeat timer with invalid intervalMinutes: ${intervalMinutes}`,
+    );
+    return;
+  }
+
+  hb.intervalMinutesRef = intervalMinutes;
+  const intervalMs = intervalMinutes * 60 * 1000;
+  log(
+    "heartbeat",
+    `Starting heartbeat timer (every ${intervalMinutes}min, first due check in 5min)`,
+  );
+
+  hb.startupTimer = setTimeout(() => {
+    hb.startupTimer = null;
+    runIfDue(intervalMs, true);
+    // Due checks every minute instead of one fixed setInterval(intervalMs):
+    // the cadence is computed from persisted last_run via the Gleam scheduler
+    // core, so it survives process restarts (a quick restart no longer resets
+    // the phase or double-fires) and system suspends (any number of missed
+    // fire times collapses into one catch-up run).
+    hb.timer = setInterval(() => {
+      runIfDue(intervalMs, false);
+    }, DUE_CHECK_INTERVAL_MS);
+  }, STARTUP_DELAY_MS);
+}
+
+/**
+ * Fire the heartbeat when its cadence says one (or more) runs are due. Decision
+ * logic is the Gleam scheduler core: `missedRunCount` counts fire times in
+ * (last_run, now], and the "once" catch-up policy collapses them into one run.
+ */
+function runIfDue(intervalMs: number, startup: boolean): void {
+  if (hb.running) return;
+  const lastRun = readHeartbeatState()?.last_run ?? 0;
+  if (lastRun <= 0) {
+    // Never ran on this install — fire now to establish the cadence.
+    executeHeartbeat("auto").catch(() => {});
+    return;
+  }
+  const now = Date.now();
+  const missed = missedRunCount(lastRun, intervalMs, now);
+  if (catchupRunCount(missed, "once", 1) > 0) {
+    executeHeartbeat("auto").catch(() => {});
+  } else if (startup) {
+    log(
+      "heartbeat",
+      `Cadence restored from state — next heartbeat due ${new Date(nextDueMs(lastRun, intervalMs, now)).toISOString()}`,
+    );
+  }
+}
+
+/**
+ * Stop the heartbeat timer. Does not wait for in-flight runs. Use
+ * awaitCurrentRun() after this to wait for a running heartbeat to finish.
+ */
+export function stopHeartbeatTimer(): void {
+  if (hb.startupTimer) {
+    clearTimeout(hb.startupTimer);
+    hb.startupTimer = null;
+  }
+  if (hb.timer) {
+    clearInterval(hb.timer);
+    hb.timer = null;
+    log("heartbeat", "Heartbeat timer stopped");
+  }
+}
+
+/**
+ * Wait for any in-flight heartbeat run to complete.
+ * Call after stopHeartbeatTimer() during graceful shutdown.
+ */
+export async function awaitCurrentRun(timeoutMs = 10_000): Promise<void> {
+  if (hb.currentRunPromise) {
+    log("heartbeat", "Waiting for in-flight heartbeat to complete...");
+    try {
+      await Promise.race([
+        hb.currentRunPromise,
+        new Promise<void>((resolve) => {
+          const t = setTimeout(() => {
+            logWarn(
+              "heartbeat",
+              "In-flight heartbeat did not finish within shutdown budget, proceeding",
+            );
+            resolve();
+          }, timeoutMs);
+          t.unref();
+        }),
+      ]);
+    } catch {
+      // Already logged in executeHeartbeat
+    }
+  }
+}
+
+/**
+ * Force a heartbeat run immediately. Resolves when the heartbeat completes.
+ * Throws if a heartbeat is already running.
+ */
+export async function forceHeartbeat(): Promise<void> {
+  if (hb.running) throw new Error("Heartbeat already running");
+  await executeHeartbeat("forced");
+}
+
+/** Get the current heartbeat status. */
+export function getHeartbeatStatus(): HeartbeatState | null {
+  return readHeartbeatState();
+}
+
+async function executeHeartbeat(trigger: "auto" | "forced"): Promise<void> {
+  if (hb.running) return;
+
+  const state = readHeartbeatState();
+  const now = Date.now();
+  const previousRunCount = state?.run_count ?? 0;
+  const previousLastRun = state?.last_run ?? 0;
+
+  hb.running = true;
+  // Mark as running with last_started, but preserve last_run from previous run
+  writeHeartbeatState({
+    last_run: previousLastRun,
+    last_started: now,
+    status: "running",
+    run_count: previousRunCount,
+  });
+  log(
+    "heartbeat",
+    `${trigger === "forced" ? "Force-triggering" : "Triggering"} heartbeat #${previousRunCount + 1} (last run: ${previousLastRun ? new Date(previousLastRun).toISOString() : "never"})`,
+  );
+
+  const run = (async () => {
+    try {
+      const heartbeatLogPath = await runHeartbeatAgent(
+        previousLastRun,
+        previousRunCount + 1,
+      );
+      // Only update last_run and increment run_count on success
+      writeHeartbeatState({
+        last_run: Date.now(),
+        last_started: now,
+        status: "idle",
+        run_count: previousRunCount + 1,
+      });
+      log(
+        "heartbeat",
+        `Heartbeat #${previousRunCount + 1} complete (${trigger}), log: ${heartbeatLogPath}`,
+      );
+    } catch (err) {
+      logError(
+        "heartbeat",
+        `Heartbeat #${previousRunCount + 1} failed (${trigger})`,
+        err,
+      );
+      // Timeouts consume the full hour budget — advance state so the next
+      // heartbeat sees a fresh window and a bumped run_count instead of
+      // re-triggering against the same `last_run` forever. Non-timeout errors
+      // retry from the previous successful run (no budget consumed).
+      const isTimeout = err instanceof HeartbeatTimeoutError;
+      writeHeartbeatState({
+        last_run: isTimeout ? Date.now() : previousLastRun,
+        last_started: now,
+        status: "idle",
+        run_count: isTimeout ? previousRunCount + 1 : previousRunCount,
+      });
+      if (trigger === "forced") throw err;
+    } finally {
+      hb.running = false;
+      hb.currentRunPromise = null;
+    }
+  })();
+
+  hb.currentRunPromise = run;
+  await run;
+}

package/src/core/background/heartbeat/state.ts

@@ -0,0 +1,153 @@
+/**
+ * Heartbeat shared state — the in-process run guard + timers + config ref,
+ * the persisted state-file shape, and its read/write helpers.
+ *
+ * The scheduler and agent submodules both read/write this state, so the
+ * mutable fields live on a single `hb` holder object imported by both.
+ */
+
+import { files as pathFiles } from "../../../util/paths.js";
+import { kvGet, kvSet } from "../../../storage/kv.js";
+import { importLegacyJson } from "../../../storage/legacy-import.js";
+import type { Backend } from "../../agent-runtime/capabilities.js";
+
+export type HeartbeatState = {
+  /** Unix millisecond timestamp of the last successfully completed heartbeat run. */
+  last_run: number;
+  /** Human-readable ISO timestamp of the last successfully completed heartbeat run. */
+  last_run_at?: string;
+  /** Unix millisecond timestamp of the last time a heartbeat was started (success or failure). */
+  last_started?: number;
+  /** "idle" when no heartbeat is running, "running" while one is active. */
+  status: "idle" | "running";
+  /** Total number of successfully completed heartbeat runs. */
+  run_count: number;
+};
+
+export type HeartbeatConfig = {
+  model?: string;
+  heartbeatModel?: string;
+  workspace?: string;
+  /**
+   * Accessor for the active backend — invoked each time a heartbeat fires so
+   * backend hot-swaps performed by the controller take effect on the next
+   * heartbeat without an `initHeartbeat` recall.
+   */
+  getBackend?: () => Backend | null;
+  /**
+   * Non-terminal frontends present at startup. Used to render the outbound
+   * messaging section of the heartbeat system prompt. Empty for terminal-only
+   * deployments.
+   */
+  frontends?: readonly string[];
+  /**
+   * MemPalace presence flag — when true, the prompt tells the agent to recall
+   * palace context before working a goal and store learnings afterwards.
+   */
+  mempalace?: boolean;
+};
+
+/**
+ * kv key owning the persisted heartbeat state. The blob used to live at
+ * `pathFiles.heartbeatState` as hand-rolled JSON; it now rides the shared
+ * `kv` table so it inherits transactional writes and TALON_DB_PATH test
+ * isolation like the rest of the unified storage layer.
+ */
+const HEARTBEAT_STATE_KEY = "heartbeat.state";
+
+/**
+ * Mutable heartbeat runtime state, shared across submodules:
+ *   - `running` / `currentRunPromise` — the one-at-a-time run guard
+ *   - `timer` / `startupTimer` — the cadence timers
+ *   - `intervalMinutesRef` — interval captured from startHeartbeatTimer
+ *   - `config` — injected via initHeartbeat
+ *   - `logFileSequence` — monotonic per-run log filename counter
+ */
+export const hb: {
+  running: boolean;
+  currentRunPromise: Promise<void> | null;
+  timer: ReturnType<typeof setInterval> | null;
+  startupTimer: ReturnType<typeof setTimeout> | null;
+  intervalMinutesRef: number;
+  config: HeartbeatConfig | null;
+  logFileSequence: number;
+} = {
+  running: false,
+  currentRunPromise: null,
+  timer: null,
+  startupTimer: null,
+  intervalMinutesRef: 60,
+  config: null,
+  logFileSequence: 0,
+};
+
+// ── State-file I/O ───────────────────────────────────────────────────────────
+
+function normalizeHeartbeatState(parsed: unknown): HeartbeatState | null {
+  if (!parsed || typeof parsed !== "object") return null;
+
+  const candidate = parsed as Record<string, unknown>;
+  const { last_run, last_run_at, last_started, status, run_count } = candidate;
+
+  if (typeof last_run !== "number" || !Number.isFinite(last_run)) return null;
+  if (typeof run_count !== "number" || !Number.isFinite(run_count)) return null;
+  if (status !== "idle" && status !== "running") return null;
+  if (last_run_at !== undefined && typeof last_run_at !== "string") return null;
+  if (
+    last_started !== undefined &&
+    (typeof last_started !== "number" || !Number.isFinite(last_started))
+  ) {
+    return null;
+  }
+
+  return {
+    last_run,
+    run_count,
+    status,
+    ...(last_run_at !== undefined ? { last_run_at } : {}),
+    ...(last_started !== undefined ? { last_started } : {}),
+  };
+}
+
+/**
+ * Fold the pre-SQLite JSON file into the kv store exactly once per
+ * process. Runs lazily on the first read rather than at boot so a
+ * heartbeat-free deployment never pays for it. Gated + idempotent via
+ * the module-level `imported` flag; the rename to `<file>.imported`
+ * inside importLegacyJson stops it re-running across restarts.
+ */
+let imported = false;
+function importLegacyStateOnce(): void {
+  if (imported) return;
+  imported = true;
+  importLegacyJson({
+    path: pathFiles.heartbeatState,
+    category: "heartbeat",
+    what: "heartbeat state",
+    ingest: (data) => {
+      const normalized = normalizeHeartbeatState(data);
+      if (!normalized) return 0;
+      kvSet(HEARTBEAT_STATE_KEY, normalized);
+      return 1;
+    },
+  });
+}
+
+export function readHeartbeatState(): HeartbeatState | null {
+  importLegacyStateOnce();
+  return normalizeHeartbeatState(kvGet(HEARTBEAT_STATE_KEY));
+}
+
+export function writeHeartbeatState(state: HeartbeatState): void {
+  // Re-derive last_run_at from last_run so the persisted ISO stamp can
+  // never drift from the millisecond field; omit it on the sentinel
+  // last_run === 0 (never-run) to match the pre-SQLite file format.
+  const { last_run_at: _lastRunAt, ...rest } = state;
+  const enriched: HeartbeatState = {
+    ...rest,
+    ...(state.last_run !== 0
+      ? { last_run_at: new Date(state.last_run).toISOString() }
+      : {}),
+  };
+  kvSet(HEARTBEAT_STATE_KEY, enriched);
+}

package/src/core/background/index.ts

@@ -0,0 +1,21 @@
+/**
+ * Background agents — everything that runs without a user message.
+ *
+ *   - `heartbeat` — periodic autonomous agent pass ("anything need
+ *     doing?"), prompted by prompts/heartbeat.md.
+ *   - `dream`    — memory-consolidation agent (daily notes → memory,
+ *     mempalace mining), prompted by prompts/dream.md.
+ *   - `pulse`    — lightweight liveness/typing-indicator ticker.
+ *   - `cron`     — persistent user-scheduled jobs (message / query).
+ *   - `triggers` — user-authored watcher scripts that fire wake-ups.
+ *
+ * All of these drive the backend through the same `BackgroundRunner`
+ * surface (core/agent-runtime/capabilities.ts) — they never talk to a
+ * concrete backend directly.
+ */
+
+export * from "./heartbeat/index.js";
+export * from "./dream.js";
+export * from "./pulse.js";
+export * from "./cron.js";
+export * from "./triggers/index.js";

package/src/core/background/isolated-agent.ts

@@ -0,0 +1,115 @@
+/**
+ * Generic isolated one-shot agent runner.
+ *
+ * Runs a `background.runOneShotAgent` under a hard timeout with the same
+ * abort → bounded-grace → (optional) orphan-eviction discipline heartbeat uses,
+ * so any unattended one-shot (heartbeat, dream, decoupled jobs) gets robust
+ * cancellation instead of hand-rolling it per caller.
+ *
+ * Flow: race the agent against a timeout. On timeout, abort the controller and
+ * give the backend a bounded grace window to honour it; if it doesn't, optionally
+ * ask the backend to evict orphan subprocesses (Linux /proc sweep). Eviction is
+ * opt-in via `evictLabel` because it matches subprocesses by an env tag — a
+ * caller that shares a tag with another context (e.g. heartbeat) should leave it
+ * unset to avoid sweeping the other context's subprocesses.
+ */
+
+import type { BackgroundRunner } from "../agent-runtime/capabilities.js";
+import type { OneShotAgentParams } from "../types.js";
+import { logWarn, logError } from "../../util/log.js";
+
+/** Default bounded grace after an abort before giving up on the backend. */
+export const DEFAULT_ABORT_GRACE_MS = 30 * 1000;
+
+export interface IsolatedRunOptions {
+  readonly background: BackgroundRunner;
+  /** Fully-built one-shot params (must carry an `abortController`). */
+  readonly params: OneShotAgentParams;
+  /** Hard timeout before the run is aborted. */
+  readonly timeoutMs: number;
+  /** Bounded grace for the backend to honour the abort (default 30s). */
+  readonly abortGraceMs?: number;
+  /**
+   * When set, evict orphan subprocesses carrying this context tag if the backend
+   * ignores the abort. Leave unset when the tag is shared with another context.
+   */
+  readonly evictLabel?: string;
+}
+
+/** Resolves to the value, or the string "timed_out" if `ms` elapses first. */
+async function raceWithTimeout<T>(
+  promise: Promise<T>,
+  ms: number,
+): Promise<T | "timed_out"> {
+  let handle: ReturnType<typeof setTimeout> | undefined;
+  const timeout = new Promise<"timed_out">((resolve) => {
+    handle = setTimeout(() => resolve("timed_out"), ms);
+    handle.unref();
+  });
+  try {
+    return await Promise.race([promise, timeout]);
+  } finally {
+    if (handle) clearTimeout(handle);
+  }
+}
+
+/**
+ * Run the one-shot under a hard timeout. Throws on timeout (after the grace
+ * window) and re-throws any agent error.
+ */
+export async function runIsolatedAgent(
+  opts: IsolatedRunOptions,
+): Promise<void> {
+  const { background, params, timeoutMs } = opts;
+  const graceMs = opts.abortGraceMs ?? DEFAULT_ABORT_GRACE_MS;
+
+  let timedOut = false;
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  const agentPromise = background.runOneShotAgent(params);
+
+  const timeoutPromise = new Promise<never>((_, reject) => {
+    timer = setTimeout(() => {
+      timedOut = true;
+      try {
+        params.abortController.abort();
+      } catch {
+        /* ignore */
+      }
+      reject(new Error(`isolated agent timed out after ${timeoutMs}ms`));
+    }, timeoutMs);
+    timer.unref();
+  });
+
+  try {
+    await Promise.race([agentPromise, timeoutPromise]);
+  } catch (err) {
+    // Snapshot + clear before any await so a late timer can't reclassify a
+    // non-timeout failure as a timeout (heartbeat learned this the hard way).
+    const wasTimeout = timedOut;
+    if (timer) {
+      clearTimeout(timer);
+      timer = undefined;
+    }
+    if (wasTimeout) {
+      const settled = await raceWithTimeout(
+        agentPromise.catch(() => "settled"),
+        graceMs,
+      );
+      if (settled === "timed_out" && opts.evictLabel) {
+        const evict = background.evictOrphanSubprocesses;
+        if (evict) {
+          evict(opts.evictLabel).catch((sweepErr: unknown) => {
+            logError("triggers", "orphan subprocess sweep failed", sweepErr);
+          });
+        } else {
+          logWarn("triggers", "backend ignored abort and has no eviction hook");
+        }
+      }
+    } else {
+      await agentPromise.catch(() => {});
+    }
+    throw err;
+  } finally {
+    if (timer) clearTimeout(timer);
+  }
+}

package/src/core/background/job-health.ts

@@ -0,0 +1,140 @@
+/**
+ * Per-job health tracking for the cron scheduler — circuit breakers
+ * with escalating cooldowns, built on the Gleam scheduler decision
+ * core (src/native/scheduler-core.ts).
+ *
+ * A job that keeps throwing stops being dispatched every tick: after
+ * `threshold` consecutive failures its breaker opens and runs are
+ * blocked while it cools down. Cooldown length comes from the core's
+ * deterministic backoff (escalating with each re-open, jittered per
+ * job so herds de-correlate). After cooling down, one probe run is
+ * allowed (half-open); success closes the breaker, failure re-opens it
+ * with a longer cooldown.
+ *
+ * State is in-memory by design — a restart gives every job a clean
+ * slate, which is the behavior you want after fixing whatever made the
+ * job fail.
+ */
+
+import {
+  backoffDelayMs,
+  breakerAllowsRun,
+  CLOSED_BREAKER,
+  stepBreaker,
+  type Breaker,
+} from "../../native/scheduler-core.js";
+
+export type JobHealthOptions = {
+  /** Consecutive failures before the breaker opens. */
+  threshold: number;
+  /** Cooldown after the first open. */
+  baseCooldownMs: number;
+  /** Cooldown ceiling for repeatedly re-opening jobs. */
+  maxCooldownMs: number;
+};
+
+type JobHealth = {
+  breaker: Breaker;
+  /** Times the breaker has opened since the last success — drives the
+   * cooldown escalation (the core's breaker resets its own failure
+   * count when it opens). */
+  opens: number;
+};
+
+const health = new Map<string, JobHealth>();
+
+function get(jobId: string): JobHealth {
+  let h = health.get(jobId);
+  if (!h) {
+    h = { breaker: CLOSED_BREAKER, opens: 0 };
+    health.set(jobId, h);
+  }
+  return h;
+}
+
+/** Deterministic per-job jitter seed (djb2 over the id). */
+function jobSeed(jobId: string): number {
+  let hash = 5381;
+  for (let i = 0; i < jobId.length; i++) {
+    hash = (hash * 33 + jobId.charCodeAt(i)) | 0;
+  }
+  return Math.abs(hash);
+}
+
+function cooldownMs(
+  jobId: string,
+  h: JobHealth,
+  opts: JobHealthOptions,
+): number {
+  return backoffDelayMs(Math.max(1, h.opens), {
+    baseMs: opts.baseCooldownMs,
+    capMs: opts.maxCooldownMs,
+    seed: jobSeed(jobId) + h.opens,
+  });
+}
+
+/**
+ * May this job run now? Also advances a cooled-down open breaker to
+ * half-open (the tick event), so the answer flips to true exactly when
+ * the cooldown elapses.
+ */
+export function jobAllowsRun(
+  jobId: string,
+  nowMs: number,
+  opts: JobHealthOptions,
+): boolean {
+  const h = get(jobId);
+  h.breaker = stepBreaker(h.breaker, "tick", nowMs, {
+    threshold: opts.threshold,
+    cooldownMs: cooldownMs(jobId, h, opts),
+  });
+  return breakerAllowsRun(h.breaker);
+}
+
+/** Record a successful run — closes the breaker, resets escalation. */
+export function recordJobSuccess(
+  jobId: string,
+  nowMs: number,
+  opts: JobHealthOptions,
+): void {
+  const h = get(jobId);
+  h.breaker = stepBreaker(h.breaker, "success", nowMs, {
+    threshold: opts.threshold,
+    cooldownMs: cooldownMs(jobId, h, opts),
+  });
+  h.opens = 0;
+}
+
+/**
+ * Record a failed run. Returns the cooldown in ms when this failure
+ * opened the breaker (caller logs it), else null.
+ */
+export function recordJobFailure(
+  jobId: string,
+  nowMs: number,
+  opts: JobHealthOptions,
+): number | null {
+  const h = get(jobId);
+  const wasOpen = h.breaker.state === "open";
+  h.breaker = stepBreaker(h.breaker, "failure", nowMs, {
+    threshold: opts.threshold,
+    cooldownMs: cooldownMs(jobId, h, opts),
+  });
+  if (h.breaker.state === "open" && !wasOpen) {
+    h.opens += 1;
+    return cooldownMs(jobId, h, opts);
+  }
+  return null;
+}
+
+/** Drop tracking for jobs that no longer exist (deleted cron jobs). */
+export function pruneJobHealth(liveIds: ReadonlySet<string>): void {
+  for (const id of health.keys()) {
+    if (!liveIds.has(id)) health.delete(id);
+  }
+}
+
+/** Test hook: forget everything. */
+export function resetJobHealth(): void {
+  health.clear();
+}