talon-agent 5.24.0 → 5.25.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/package.json +2 -1
  2. package/src/app.ts +30 -0
  3. package/src/backend/agy/process/orphans.ts +5 -1
  4. package/src/backend/claude-sdk/one-shot.ts +8 -1
  5. package/src/backend/codex/factory.ts +5 -1
  6. package/src/backend/codex/plan-usage.ts +12 -0
  7. package/src/backend/runtime/turn/turn-phases.ts +7 -1
  8. package/src/bootstrap.ts +1 -0
  9. package/src/cli.ts +18 -12
  10. package/src/core/agent-runtime/capabilities.ts +7 -0
  11. package/src/core/agents/index.ts +1 -1
  12. package/src/core/agents/prompt.ts +29 -2
  13. package/src/core/agents/runner.ts +7 -1
  14. package/src/core/agents/types.ts +6 -0
  15. package/src/core/background/cron/job-oneshot.ts +7 -1
  16. package/src/core/background/cron/scheduler.ts +28 -14
  17. package/src/core/background/cron/spec.ts +23 -3
  18. package/src/core/background/heartbeat/agent.ts +6 -0
  19. package/src/core/background/triggers/resume.ts +19 -4
  20. package/src/core/background/triggers/spawn.ts +1 -1
  21. package/src/core/config/index.ts +82 -0
  22. package/src/core/daemon/discovery.ts +160 -0
  23. package/src/core/daemon/pidfile.ts +107 -0
  24. package/src/core/daemon/respawn.ts +4 -1
  25. package/src/core/engine/backend-router/breaker.ts +158 -0
  26. package/src/core/engine/backend-router/headroom.ts +90 -22
  27. package/src/core/engine/backend-router/index.ts +11 -0
  28. package/src/core/engine/gateway-actions/agents/control.ts +9 -0
  29. package/src/core/engine/gateway-actions/agents/index.ts +4 -0
  30. package/src/core/engine/gateway-actions/agents/preflight.ts +215 -0
  31. package/src/core/engine/gateway-actions/cron.ts +5 -0
  32. package/src/core/engine/gateway-actions/fetch-url/guard.ts +13 -44
  33. package/src/core/engine/gateway-actions/fetch-url/index.ts +23 -43
  34. package/src/core/engine/gateway-actions/mesh.ts +4 -0
  35. package/src/core/engine/gateway-actions/models.ts +19 -4
  36. package/src/core/engine/gateway-routes.ts +31 -2
  37. package/src/core/engine/gateway.ts +11 -1
  38. package/src/core/fetch/classify.ts +73 -0
  39. package/src/core/fetch/curl-impersonate.ts +447 -0
  40. package/src/core/fetch/errors.ts +14 -0
  41. package/src/core/fetch/index.ts +133 -0
  42. package/src/core/fetch/ladder.ts +401 -0
  43. package/src/core/fetch/rungs.ts +319 -0
  44. package/src/core/fetch/types.ts +105 -0
  45. package/src/core/frontend-runtime/admin-notify.ts +100 -12
  46. package/src/core/mcp-hub/child-guard.ts +215 -0
  47. package/src/core/mcp-hub/child-transport.ts +89 -39
  48. package/src/core/mcp-hub/children.ts +65 -9
  49. package/src/core/mcp-hub/guest-scope.ts +3 -2
  50. package/src/core/mcp-hub/index.ts +36 -16
  51. package/src/core/mcp-hub/launcher.ts +81 -37
  52. package/src/core/mcp-hub/proxy-server.ts +12 -8
  53. package/src/core/mcp-hub/reaper.ts +120 -0
  54. package/src/core/mesh/credentials/index.ts +1 -1
  55. package/src/core/mesh/credentials/store.ts +1 -1
  56. package/src/core/mesh/devices/service.ts +8 -0
  57. package/src/core/mesh/links/bridge-links.ts +37 -0
  58. package/src/core/mesh/links/companion-pairing.ts +2 -2
  59. package/src/core/plugin/mcp.ts +8 -8
  60. package/src/core/tools/bridge.ts +5 -0
  61. package/src/core/tools/content/web.ts +1 -1
  62. package/src/core/tools/index.ts +2 -1
  63. package/src/core/tools/ops/agents.ts +28 -0
  64. package/src/core/tools/ops/mesh.ts +21 -0
  65. package/src/core/tools/ops/scheduling.ts +15 -0
  66. package/src/frontend/native/bridge/credentials/claims.ts +11 -1
  67. package/src/frontend/telegram/actions/media.ts +164 -12
  68. package/src/index.ts +22 -19
  69. package/src/plugins/playwright/version-coupling.ts +172 -62
  70. package/src/storage/cron.ts +34 -2
  71. package/src/storage/db.ts +1 -0
  72. package/src/storage/repositories/cron-repo.ts +9 -0
  73. package/src/storage/sql/cron.sql +5 -5
  74. package/src/storage/sql/db.sql +5 -0
  75. package/src/storage/sql/schema.sql +2 -1
  76. package/src/storage/sql/statements.generated.ts +10 -6
  77. package/src/util/log.ts +2 -1
  78. package/src/util/paths.ts +2 -0
  79. package/src/core/background/triggers/pid.ts +0 -28
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "talon-agent",
3
- "version": "5.24.0",
3
+ "version": "5.25.0",
4
4
  "description": "Multi-frontend AI agent with full tool access, streaming, cron jobs, and plugin system",
5
5
  "author": "The Falconry",
6
6
  "license": "Apache-2.0",
@@ -90,6 +90,7 @@
90
90
  "knip": "knip",
91
91
  "format": "prettier --write src/ prompts/",
92
92
  "format:check": "prettier --check src/ prompts/",
93
+ "preflight": "bash scripts/preflight.sh",
93
94
  "ci:protect": "node .github/scripts/enforce-ci-gate.mjs",
94
95
  "build:gleam": "cd native/scheduler-core && gleam build --target javascript && node embed.mjs",
95
96
  "build:zig": "node native/textops-wasm/build.mjs",
package/src/app.ts CHANGED
@@ -74,6 +74,7 @@ import {
74
74
  writePidRecord,
75
75
  removePidRecordIfOwnedBy,
76
76
  } from "./core/daemon/pidfile.js";
77
+ import { stampDaemonOwner } from "./core/daemon/pidfile.js";
77
78
  import {
78
79
  recordBootMetrics,
79
80
  startResourceSampler,
@@ -95,6 +96,24 @@ if (process.argv.includes(BOOT_SMOKE_FLAG)) {
95
96
  process.exit(0);
96
97
  }
97
98
 
99
+ // One daemon per install. This runs before anything with a side effect
100
+ // (restore, database, pidfile, trigger resume), because a second daemon
101
+ // does damage in each of them — see checkSingleInstance in core/daemon/discovery.ts.
102
+ {
103
+ const { checkSingleInstance, describeRefusal } =
104
+ await import("./core/daemon/discovery.js");
105
+ const verdict = await checkSingleInstance();
106
+ if (!verdict.ok) {
107
+ const message = describeRefusal(verdict.instance);
108
+ logError("bot", message);
109
+ console.error(`talon: ${message}`);
110
+ process.exit(1);
111
+ }
112
+ }
113
+ // Every child spawned from here on names this daemon as its owner, so an
114
+ // orphan sweep can tell a dead daemon's leftovers from a live one's runs.
115
+ stampDaemonOwner();
116
+
98
117
  /**
99
118
  * A `/backup restore <id>` from chat writes ~/.talon/restore-pending.json
100
119
  * and restarts. It is applied HERE, before anything else: every store
@@ -160,6 +179,17 @@ writePidRecord({ pid: process.pid, startedAt: bootedAt });
160
179
  // ── Create gateway + frontend ─────────────────────────────────────────────────
161
180
 
162
181
  const gateway = new Gateway("daemon");
182
+ // Fail rather than fall back when the gateway port belongs to another
183
+ // daemon: a fallback port is how a second daemon went unnoticed.
184
+ gateway.setPortInUseCheck(async (port) => {
185
+ const { probeHealth } = await import("./core/daemon/discovery.js");
186
+ const health = await probeHealth(port);
187
+ return health?.app === "talon" &&
188
+ health.mode === "daemon" &&
189
+ health.pid !== process.pid
190
+ ? `gateway port ${port} is held by another Talon daemon (pid ${String(health.pid)})`
191
+ : undefined;
192
+ });
163
193
  gateway.onStarted((port) =>
164
194
  writePidRecord({ pid: process.pid, port, startedAt: bootedAt }),
165
195
  );
@@ -12,6 +12,7 @@
12
12
 
13
13
  import { readdir, readFile } from "node:fs/promises";
14
14
  import { log } from "../../../util/log.js";
15
+ import { childBelongsToLiveDaemon } from "../../../core/daemon/pidfile.js";
15
16
  import { AGY_KILL_GRACE_MS } from "../constants.js";
16
17
  import { childChatIds, getChild } from "./child.js";
17
18
 
@@ -96,7 +97,10 @@ async function findOrphanPids(contextLabel: string): Promise<number[]> {
96
97
  // SIGKILL, so a chat id that happens to appear inside an
97
98
  // unrelated path must not select a victim.
98
99
  if (!argv.some((arg) => arg === "agy" || arg.endsWith("/agy"))) continue;
99
- if (argv.includes(contextLabel)) matched.push(pid);
100
+ if (!argv.includes(contextLabel)) continue;
101
+ // Another running daemon's live run, not an orphan of ours.
102
+ if (childBelongsToLiveDaemon(pid)) continue;
103
+ matched.push(pid);
100
104
  } catch {
101
105
  // Exited between readdir and readFile, or not ours. Skip.
102
106
  continue;
@@ -22,6 +22,10 @@ import { buildMcpServers, buildPluginMcpServers } from "./options.js";
22
22
  import { isBackgroundToolContext } from "../../core/agents/context.js";
23
23
  import { warnIfBelowCacheMinimum } from "../runtime/cache/cache-telemetry.js";
24
24
  import { emitAssistantText } from "../runtime/one-shot-hooks.js";
25
+ import {
26
+ isOtherLiveDaemon,
27
+ ownerFromEnviron,
28
+ } from "../../core/daemon/pidfile.js";
25
29
 
26
30
  const DEFAULT_SUBPROCESS_KILL_GRACE_MS = 5 * 1000;
27
31
 
@@ -290,8 +294,10 @@ async function formatAndAppendMessage(
290
294
  * triggers: the warden respawned them, the next sweep killed them again, and
291
295
  * the only visible symptom was a trigger stuck in "errored" with no output.
292
296
  *
293
- * Two independent guards, because this function issues SIGKILL:
297
+ * Three independent guards, because this function issues SIGKILL:
294
298
  * - refuse anything tagged `TALON_TRIGGER_ID` (a trigger, or its child);
299
+ * - refuse anything spawned by another daemon that is still running
300
+ * (see core/daemon/pidfile.ts). Its "orphans" are that daemon's live runs;
295
301
  * - require the argv to actually be the `claude` SDK binary, which is the
296
302
  * only thing this sweep was ever meant to reap.
297
303
  */
@@ -304,6 +310,7 @@ export function isEvictableOrphan(
304
310
  if (envEntries.some((entry) => entry.startsWith("TALON_TRIGGER_ID="))) {
305
311
  return false;
306
312
  }
313
+ if (isOtherLiveDaemon(ownerFromEnviron(envEntries))) return false;
307
314
  return argv.some((arg) => arg === "claude" || arg.endsWith("/claude"));
308
315
  }
309
316
 
@@ -19,7 +19,10 @@ import {
19
19
  type ModelCatalog,
20
20
  type UsageTelemetry,
21
21
  } from "../../core/agent-runtime/capabilities.js";
22
- import { getPlanUsage as getCodexPlanUsage } from "./plan-usage.js";
22
+ import {
23
+ getAuthFailure as getCodexAuthFailure,
24
+ getPlanUsage as getCodexPlanUsage,
25
+ } from "./plan-usage.js";
23
26
 
24
27
  import { initCodexAgent, getCodexAuthInfo } from "./init.js";
25
28
  import { handleMessage as codexHandleMessage } from "./handler/index.js";
@@ -88,6 +91,7 @@ const codexFactory: BackendFactory = {
88
91
  // report its rate-limit windows.
89
92
  const usage: UsageTelemetry = {
90
93
  getPlanUsage: () => getCodexPlanUsage(),
94
+ getAuthFailure: () => getCodexAuthFailure(),
91
95
  };
92
96
 
93
97
  const backend = composeBackend({
@@ -172,6 +172,18 @@ async function load(): Promise<PlanUsage | undefined> {
172
172
  }
173
173
  }
174
174
 
175
+ /**
176
+ * Why the Codex login is known to be rejected, or `undefined`. Set by a 401
177
+ * from the usage endpoint and cleared once `codex login` rewrites
178
+ * auth.json (seen on the next {@link getPlanUsage} refresh). The router
179
+ * reads it to stop sending work to a backend whose runs can only fail.
180
+ */
181
+ export function getAuthFailure(): string | undefined {
182
+ return rejectedAuthMtimeMs === undefined
183
+ ? undefined
184
+ : "Codex login expired (usage endpoint returned 401) — run `codex login`";
185
+ }
186
+
175
187
  /**
176
188
  * Plan windows for `/usage`, cached for a minute. A failed refresh keeps
177
189
  * serving the last known values — `fetchedAt` lets the caller age them.
@@ -26,7 +26,10 @@ import {
26
26
  } from "../../../storage/sessions.js";
27
27
  import { log } from "../../../util/log.js";
28
28
  import { extractSessionName } from "../../../core/weaver/session-name.js";
29
- import { recordBackendRunUsage } from "../../../core/engine/backend-router/index.js";
29
+ import {
30
+ recordBackendRunSuccess,
31
+ recordBackendRunUsage,
32
+ } from "../../../core/engine/backend-router/index.js";
30
33
  import { traceMessage } from "../../../util/trace.js";
31
34
  import {
32
35
  FLOW_VIOLATION_MAX_RETRIES,
@@ -120,6 +123,9 @@ export function accountTurn(inputs: AccountTurnInputs): void {
120
123
  // a provider with no account API still has a headroom signal. Backends
121
124
  // that DO report a plan simply outrank their own ledger.
122
125
  recordBackendRunUsage(inputs.backend, usage);
126
+ // A chat turn that completed is proof the backend works: close its
127
+ // breaker so background work may route there again.
128
+ if (!inputs.failed) recordBackendRunSuccess(inputs.backend);
123
129
  recordUsage(chatId, {
124
130
  ...usage,
125
131
  durationMs,
package/src/bootstrap.ts CHANGED
@@ -155,6 +155,7 @@ export async function bootstrap(
155
155
  ...(config.operatorIds ?? []),
156
156
  ...(config.discord?.adminUserIds ?? []).map((id) => `discord:${id}`),
157
157
  ],
158
+ guardChildren: true,
158
159
  });
159
160
 
160
161
  initWorkspace(config.workspace);
package/src/cli.ts CHANGED
@@ -16,22 +16,28 @@
16
16
  * `cli/index.ts`. Everything else lives under `cli/`.
17
17
  */
18
18
 
19
- import {
20
- MCP_LAUNCH_SUBCOMMAND,
21
- runSupervisor,
22
- } from "./core/mcp-hub/launcher.js";
23
- import { LUA_RUN_SUBCOMMAND, runLuaMain } from "./core/scripts/lua.js";
24
-
25
19
  // Hidden subcommand dispatch — must run before anything else. Talon
26
- // supervises MCP stdio children (`_mcp-launch`) and runs WASM-sandboxed
27
- // Lua trigger scripts (`_lua-run`) by re-invoking its own entrypoint
28
- // (see core/mcp-hub/launcher.ts). Neither call resolves; the helper process
29
- // exits from its own handlers.
30
- if (process.argv[2] === MCP_LAUNCH_SUBCOMMAND) {
20
+ // supervises MCP stdio children (`_mcp-launch`, `_mcp-reaper`) and runs
21
+ // WASM-sandboxed Lua trigger scripts (`_lua-run`) by re-invoking its own
22
+ // entrypoint (see core/mcp-hub/launcher.ts). None of these calls resolves;
23
+ // the helper process exits from its own handlers. Names are literals and
24
+ // modules load dynamically so a helper process only evaluates its own
25
+ // module (pinned to the exported constants by entry-dispatch.test.ts).
26
+ const subcommand = process.argv[2];
27
+ if (subcommand === "_mcp-launch") {
28
+ const { runSupervisor } = await import("./core/mcp-hub/launcher.js");
31
29
  await runSupervisor(process.argv.slice(3));
32
- } else if (process.argv[2] === LUA_RUN_SUBCOMMAND) {
30
+ } else if (subcommand === "_mcp-reaper") {
31
+ const { runReaper } = await import("./core/mcp-hub/reaper.js");
32
+ await runReaper();
33
+ } else if (subcommand === "_lua-run") {
34
+ const { runLuaMain } = await import("./core/scripts/lua.js");
33
35
  await runLuaMain(process.argv.slice(3));
34
36
  } else {
35
37
  const { runCli } = await import("./cli/index.js");
36
38
  await runCli();
37
39
  }
40
+
41
+ // No static imports (see above) — mark the file as an ES module so
42
+ // top-level await type-checks.
43
+ export {};
@@ -243,6 +243,13 @@ export interface UsageTelemetry {
243
243
  * plan concept; resolves `undefined` when the data can't be read.
244
244
  */
245
245
  getPlanUsage?(): Promise<PlanUsage | undefined>;
246
+ /**
247
+ * Why the backend's credential is known to be rejected (e.g. the usage
248
+ * endpoint returned 401), or `undefined` when nothing says it is.
249
+ * Synchronous and cheap: it reports what the last `getPlanUsage` saw.
250
+ * The router treats a backend that reports one as having no headroom.
251
+ */
252
+ getAuthFailure?(): string | undefined;
246
253
  /**
247
254
  * Banked one-shot limit resets, where the plan has them. Spending one is
248
255
  * irreversible, so this is reachable only from a human-pressed confirm
@@ -33,5 +33,5 @@ export {
33
33
  deliverToAgent,
34
34
  initAgentDelivery,
35
35
  } from "./delivery.js";
36
- export { describeParent } from "./prompt.js";
36
+ export { describeParent, wantsPreflight } from "./prompt.js";
37
37
  export type { AgentCaps, AgentParent, AgentRecord } from "./types.js";
@@ -50,11 +50,38 @@ export function buildAgentSystemPrompt(args: {
50
50
  });
51
51
  }
52
52
 
53
+ /**
54
+ * The standing pre-flight instruction appended to a PR-opening agent's
55
+ * brief: run the light CI suite locally, push only on green. GitHub then
56
+ * confirms a change instead of being the first compiler it meets.
57
+ */
58
+ const PREFLIGHT_INSTRUCTION =
59
+ "[Pre-flight lane] Before every `git push`, run `npm run preflight` in " +
60
+ "the repo (or call the run_preflight tool with cwd set to your checkout). " +
61
+ "Push only when it is green. If it is red, fix it — or, when a failure " +
62
+ "is genuinely out of scope, say in the PR body which step failed and why " +
63
+ "you pushed anyway.";
64
+
65
+ /** A brief that opens, updates or talks about a pull request. */
66
+ const PR_BRIEF = /\bPRs?\b|pull[ -]requests?/i;
67
+
68
+ /**
69
+ * Whether a spawn gets the pre-flight instruction: the caller's explicit
70
+ * choice when given, otherwise on for any brief that mentions a PR.
71
+ */
72
+ export function wantsPreflight(brief: string, explicit?: boolean): boolean {
73
+ return explicit ?? PR_BRIEF.test(brief);
74
+ }
75
+
53
76
  /** The activation prompt — the brief, framed as the job to start on. */
54
- export function buildAgentPrompt(brief: string): string {
77
+ export function buildAgentPrompt(
78
+ brief: string,
79
+ options: { preflight?: boolean } = {},
80
+ ): string {
81
+ const lane = options.preflight ? `\n\n${PREFLIGHT_INSTRUCTION}` : "";
55
82
  return (
56
83
  `[System: AGENT BRIEF. Work this to a conclusion, then call ` +
57
- `report_result exactly once.]\n\n${brief}`
84
+ `report_result exactly once.]\n\n${brief}${lane}`
58
85
  );
59
86
  }
60
87
 
@@ -35,6 +35,8 @@ import {
35
35
  } from "../engine/backend-controller/index.js";
36
36
  import {
37
37
  chooseBackend,
38
+ recordBackendRunFailure,
39
+ recordBackendRunSuccess,
38
40
  recordBackendRunUsage,
39
41
  taskClassForEffort,
40
42
  } from "../engine/backend-router/index.js";
@@ -306,7 +308,9 @@ async function buildRunParams(
306
308
  agentLogHeader(record, model),
307
309
  );
308
310
  return {
309
- prompt: buildAgentPrompt(record.brief),
311
+ prompt: buildAgentPrompt(record.brief, {
312
+ preflight: spec.preflight === true,
313
+ }),
310
314
  systemPrompt: buildAgentSystemPrompt({
311
315
  agentId: record.id,
312
316
  label: record.label,
@@ -437,9 +441,11 @@ async function runAgent(
437
441
  evictLabel: agentContextLabel(id),
438
442
  });
439
443
  recordBackendRunUsage(record.backendId, usage ?? undefined);
444
+ recordBackendRunSuccess(record.backendId);
440
445
  settled = settleSuccess(id, task, capture.last, usage ?? undefined);
441
446
  }
442
447
  } catch (err) {
448
+ recordBackendRunFailure(record.backendId, err);
443
449
  settled = settleFailure(id, task, err);
444
450
  } finally {
445
451
  await release().catch((err: unknown) =>
@@ -101,6 +101,12 @@ export interface AgentSpawnSpec {
101
101
  readonly reasoningEffort?: ReasoningEffortLevel;
102
102
  /** Hard wall-clock cap. Defaults to `agents.defaultTimeoutMs`. */
103
103
  readonly timeoutMs?: number;
104
+ /**
105
+ * Append the pre-flight lane instruction (run `npm run preflight` before
106
+ * every push) to the brief. Resolved by the spawn tool: explicit value,
107
+ * else on when the brief mentions a PR.
108
+ */
109
+ readonly preflight?: boolean;
104
110
  }
105
111
 
106
112
  /** `spawnAgent`'s answer — an error here is a tool error, never a throw. */
@@ -19,7 +19,11 @@ import {
19
19
  acquireBackendInstance,
20
20
  isModelValidForBackend,
21
21
  } from "../../engine/backend-controller/index.js";
22
- import { recordBackendRunUsage } from "../../engine/backend-router/index.js";
22
+ import {
23
+ recordBackendRunFailure,
24
+ recordBackendRunSuccess,
25
+ recordBackendRunUsage,
26
+ } from "../../engine/backend-router/index.js";
23
27
  import { taskTable } from "../../tasks/index.js";
24
28
  import type { OneShotAgentParams } from "../../types.js";
25
29
  import { runIsolatedAgent } from "../isolated-agent.js";
@@ -184,8 +188,10 @@ async function attemptJobOneShot(
184
188
  });
185
189
  task.succeed(usage ?? undefined);
186
190
  recordBackendRunUsage(backendId, usage ?? undefined);
191
+ recordBackendRunSuccess(backendId);
187
192
  } catch (err) {
188
193
  task.fail(err);
194
+ recordBackendRunFailure(backendId, err);
189
195
  throw err;
190
196
  }
191
197
  log(
@@ -223,15 +223,17 @@ async function runScheduled(job: CronJob): Promise<void> {
223
223
  }
224
224
  enforceRunCap(job.id);
225
225
  } catch (err) {
226
- // Advance lastRunAt on failure too (without bumping runCount — failed runs
227
- // don't count toward maxRuns). For interval jobs the anchor IS lastRunAt, so
228
- // skipping this would make a flaky job re-fire every 60s tick until the
229
- // breaker opens, instead of honoring its everyMs cadence between retries.
230
- updateCronJob(job.id, {
231
- lastRunAt: Date.now(),
232
- lastStatus: "error",
233
- lastError: err instanceof Error ? err.message : String(err),
234
- lastDurationMs: Date.now() - startedAt,
226
+ // A failed run is still a run: it advances lastRunAt and bumps runCount,
227
+ // so it counts toward maxRuns. For interval jobs the anchor IS lastRunAt,
228
+ // so not advancing it would make a flaky job re-fire every 60s tick until
229
+ // the breaker opens. And a capped job — a one-shot above all — whose
230
+ // failure didn't count would stay enabled and fire again at the next
231
+ // matching time: for a date-pinned cron expression that is a year later,
232
+ // long after anyone wanted it.
233
+ recordCronRun(job.id, {
234
+ status: "error",
235
+ error: err instanceof Error ? err.message : String(err),
236
+ durationMs: Date.now() - startedAt,
235
237
  });
236
238
  logError(
237
239
  "cron",
@@ -252,6 +254,14 @@ async function runScheduled(job: CronJob): Promise<void> {
252
254
  `Cron job "${job.name}" failed ${JOB_HEALTH.threshold} runs in a row: ${faultText(err)}. Paused for ~${mins} min.`,
253
255
  );
254
256
  }
257
+ if (enforceRunCap(job.id)) {
258
+ // The job's last permitted run failed and it is now retired — nothing
259
+ // will retry it, so the operator has to hear about it.
260
+ raiseAlert(
261
+ `cron.job.${job.id}`,
262
+ `Cron job "${job.name}" failed on its final run and was disabled: ${faultText(err)}. Re-enable or recreate it to try again.`,
263
+ );
264
+ }
255
265
  } finally {
256
266
  runningJobs.delete(job.id);
257
267
  }
@@ -261,11 +271,12 @@ async function runScheduled(job: CronJob): Promise<void> {
261
271
 
262
272
  /**
263
273
  * Disable a job that has reached its run cap (`maxRuns`; =1 means one-shot).
264
- * Call after a successful run, once runCount has been bumped.
274
+ * Call after every run — successful or failed — once runCount has been
275
+ * bumped. Returns true when this call retired the job.
265
276
  */
266
- function enforceRunCap(id: string): void {
277
+ function enforceRunCap(id: string): boolean {
267
278
  const job = getCronJob(id);
268
- if (!job || !job.enabled) return;
279
+ if (!job || !job.enabled) return false;
269
280
  if (job.maxRuns !== undefined && job.runCount >= job.maxRuns) {
270
281
  updateCronJob(id, { enabled: false });
271
282
  log(
@@ -276,7 +287,9 @@ function enforceRunCap(id: string): void {
276
287
  "Cron",
277
288
  `Job "${job.name}" finished after ${job.runCount} run(s)`,
278
289
  );
290
+ return true;
279
291
  }
292
+ return false;
280
293
  }
281
294
 
282
295
  /**
@@ -522,7 +535,8 @@ async function routeQueryJob(
522
535
  return { backendId: decision.backendId, model };
523
536
  }
524
537
 
525
- const CRON_JOB_TIMEOUT_MS = 10 * 60_000; // 10-minute max per job
538
+ /** Default hard limit per query run; a job's own `timeoutMs` overrides it. */
539
+ const CRON_JOB_TIMEOUT_MS = 10 * 60_000;
526
540
 
527
541
  export async function executeJob(job: CronJob): Promise<ExecuteJobResult> {
528
542
  if (!deps) return { status: "skipped", reason: "cron is not initialised" };
@@ -577,7 +591,7 @@ export async function executeJob(job: CronJob): Promise<ExecuteJobResult> {
577
591
  payload,
578
592
  label: job.name,
579
593
  kind: "cron",
580
- timeoutMs: CRON_JOB_TIMEOUT_MS,
594
+ timeoutMs: job.timeoutMs ?? CRON_JOB_TIMEOUT_MS,
581
595
  ...(fallback ? { fallback } : {}),
582
596
  });
583
597
  if (result.status === "skipped") {
@@ -24,6 +24,9 @@ import {
24
24
  const MIN_INTERVAL_SECONDS = 60;
25
25
  const MAX_CONTENT_LENGTH = 10_000;
26
26
  const CATCHUP_POLICIES = new Set<CatchupPolicy>(["skip", "once", "all"]);
27
+ /** Per-job run timeout bounds (seconds). The scheduler default is 10 minutes. */
28
+ const MIN_TIMEOUT_SECONDS = 60;
29
+ const MAX_TIMEOUT_SECONDS = 4 * 60 * 60;
27
30
 
28
31
  /** The job fields the spec covers; everything else is identity or telemetry. */
29
32
  type CronSpec = Pick<CronJob, "name" | "type" | "content" | "catchup"> &
@@ -40,6 +43,7 @@ type CronSpec = Pick<CronJob, "name" | "type" | "content" | "catchup"> &
40
43
  | "model"
41
44
  | "provider"
42
45
  | "instructions"
46
+ | "timeoutMs"
43
47
  >
44
48
  >;
45
49
 
@@ -220,8 +224,8 @@ const parseCatchup: Section = ({ body, editing, touched, updates }) => {
220
224
  };
221
225
 
222
226
  /**
223
- * Model / provider / instructions only make sense for "query" jobs (a
224
- * "message" job just sends text — no model runs), and a provider override
227
+ * Model / provider / instructions / timeout only make sense for "query" jobs
228
+ * (a "message" job just sends text — no model runs), and a provider override
225
229
  * needs a model to pick on it.
226
230
  */
227
231
  const parseOverrides: Section = (p) => {
@@ -230,9 +234,25 @@ const parseOverrides: Section = (p) => {
230
234
  if (touched(key))
231
235
  updates[key] = provided(body[key]) ? String(body[key]) : undefined;
232
236
  }
233
- const { type, model, provider, instructions } = merge(p);
237
+ if (touched("timeout_seconds")) {
238
+ if (!provided(body.timeout_seconds)) {
239
+ updates.timeoutMs = undefined;
240
+ } else {
241
+ const secs = Number(body.timeout_seconds);
242
+ if (
243
+ !Number.isFinite(secs) ||
244
+ secs < MIN_TIMEOUT_SECONDS ||
245
+ secs > MAX_TIMEOUT_SECONDS
246
+ )
247
+ return `'timeout_seconds' must be a number between ${MIN_TIMEOUT_SECONDS} and ${MAX_TIMEOUT_SECONDS}.`;
248
+ updates.timeoutMs = Math.round(secs * 1000);
249
+ }
250
+ }
251
+ const { type, model, provider, instructions, timeoutMs } = merge(p);
234
252
  if (type !== "query" && (model || provider || instructions))
235
253
  return "Model/provider/instructions only apply to 'query' jobs.";
254
+ if (type !== "query" && timeoutMs !== undefined)
255
+ return "'timeout_seconds' only applies to 'query' jobs.";
236
256
  if (provider && !model)
237
257
  return "A 'provider' override also requires a 'model'.";
238
258
  return null;
@@ -18,6 +18,8 @@ import type { OneShotAgentParams } from "../../types.js";
18
18
  import { acquireBackendInstance } from "../../engine/backend-controller/index.js";
19
19
  import {
20
20
  chooseBackend,
21
+ recordBackendRunFailure,
22
+ recordBackendRunSuccess,
21
23
  recordBackendRunUsage,
22
24
  resolveRoutedModel,
23
25
  } from "../../engine/backend-router/index.js";
@@ -471,6 +473,9 @@ export async function runHeartbeatAgent(
471
473
  runCount,
472
474
  heartbeatLogFile,
473
475
  );
476
+ } catch (err) {
477
+ recordBackendRunFailure(target.backendId, err);
478
+ throw err;
474
479
  } finally {
475
480
  // A routed run borrowed the instance from the pool; hand it back on
476
481
  // every path or the provider stays warm until the daemon restarts.
@@ -483,6 +488,7 @@ export async function runHeartbeatAgent(
483
488
  }
484
489
  }
485
490
  recordBackendRunUsage(target.backendId, usage ?? undefined);
491
+ recordBackendRunSuccess(target.backendId);
486
492
  task.succeed(usage ?? undefined);
487
493
  return heartbeatLogFile;
488
494
  }
@@ -1,8 +1,8 @@
1
1
  /**
2
2
  * Post-restart resume — respawn persistent triggers, fire late wakes for
3
3
  * recently-terminated ones, plus the orphan-kill used to avoid duplicate
4
- * spawns after an unclean crash. The /proc PID-starttime probe both this
5
- * and spawn use lives in ./pid.ts.
4
+ * spawns after an unclean crash. The /proc probes it relies on (pid start
5
+ * time, owning daemon) live in core/daemon/pidfile.ts.
6
6
  */
7
7
 
8
8
  import {
@@ -12,11 +12,14 @@ import {
12
12
  SHUTDOWN_KILL_ERROR,
13
13
  type Trigger,
14
14
  } from "../../../storage/triggers.js";
15
- import { log, logError } from "../../../util/log.js";
15
+ import { log, logError, logWarn } from "../../../util/log.js";
16
16
  import { depsHolder } from "./state.js";
17
17
  import { fireWake } from "./output.js";
18
18
  import { spawnTrigger } from "./spawn.js";
19
- import { readPidStarttimeSync } from "./pid.js";
19
+ import {
20
+ childBelongsToLiveDaemon,
21
+ readPidStarttimeSync,
22
+ } from "../../../core/daemon/pidfile.js";
20
23
 
21
24
  /**
22
25
  * After the dispatcher is wired, walk the store and clean up leftover state
@@ -85,6 +88,11 @@ export async function resumeAfterRestart(): Promise<void> {
85
88
  * boot and unchanged by exec(), so a match means the PID still belongs to our
86
89
  * process. On non-Linux (no /proc), pidStarttime is undefined and we fall
87
90
  * through to SIGKILL.
91
+ *
92
+ * Owner check: a process whose spawning daemon is still alive is not an
93
+ * orphan. It is another daemon's live trigger. That is what a second,
94
+ * concurrently started daemon found on 2026-09-27, and it SIGKILLed the
95
+ * running daemon's watchers.
88
96
  */
89
97
  function killOrphan(t: Trigger): void {
90
98
  if (t.pid === undefined) return;
@@ -93,6 +101,13 @@ function killOrphan(t: Trigger): void {
93
101
  } catch {
94
102
  return; // dead — nothing to do
95
103
  }
104
+ if (childBelongsToLiveDaemon(t.pid)) {
105
+ logWarn(
106
+ "triggers",
107
+ `Orphan probe: pid=${t.pid} ("${t.name}") belongs to another running daemon — leaving alone`,
108
+ );
109
+ return;
110
+ }
96
111
  if (t.pidStarttime !== undefined) {
97
112
  const current = readPidStarttimeSync(t.pid);
98
113
  if (current !== undefined && current !== t.pidStarttime) {
@@ -25,7 +25,7 @@ import {
25
25
  import { commandForLanguage } from "./command.js";
26
26
  import { handleStdoutLine, handleStderrLine } from "./output.js";
27
27
  import { handleTimeout, finalizeExit, failTrigger } from "./exit.js";
28
- import { readPidStarttimeSync } from "./pid.js";
28
+ import { readPidStarttimeSync } from "../../../core/daemon/pidfile.js";
29
29
 
30
30
  /**
31
31
  * Spawn a trigger's script as a supervised child process.