talon-agent 5.18.2 → 5.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/package.json +1 -1
- package/prompts/system/agent-brief.md +20 -3
- package/src/app.ts +13 -0
- package/src/backend/claude-sdk/handler.ts +4 -4
- package/src/backend/claude-sdk/mcp-ready.ts +16 -2
- package/src/backend/claude-sdk/stream.ts +2 -2
- package/src/backend/codex/auth.ts +1 -1
- package/src/backend/codex/handler/message.ts +11 -11
- package/src/backend/codex/init.ts +4 -9
- package/src/backend/codex/mcp-config.ts +1 -2
- package/src/backend/codex/oauth-incompat.ts +8 -4
- package/src/backend/codex/one-shot.ts +1 -1
- package/src/backend/openai-agents/builtins.ts +55 -27
- package/src/backend/openai-agents/factory.ts +3 -3
- package/src/backend/openai-agents/handler/message.ts +3 -5
- package/src/backend/openai-agents/mcp-pool.ts +10 -27
- package/src/backend/remote-server/chat-turn.ts +6 -6
- package/src/backend/remote-server/events.ts +1 -5
- package/src/backend/remote-server/index.ts +0 -1
- package/src/backend/remote-server/messages.ts +3 -7
- package/src/backend/remote-server/one-shot.ts +1 -3
- package/src/backend/remote-server/session-helpers.ts +1 -4
- package/src/backend/remote-server/sse-stream.ts +8 -9
- package/src/backend/runtime/metrics.ts +7 -13
- package/src/backend/runtime/sleep.ts +1 -2
- package/src/backend/runtime/turn/handle-retry.ts +48 -2
- package/src/backend/runtime/turn/handler-to-events.ts +3 -4
- package/src/bootstrap.ts +9 -1
- package/src/cli/doctor.ts +3 -0
- package/src/cli/index.ts +8 -10
- package/src/cli/logs.ts +148 -9
- package/src/cli/setup.ts +9 -11
- package/src/cli/status.ts +29 -0
- package/src/core/agent-runtime/README.md +5 -19
- package/src/core/agent-runtime/events.ts +3 -39
- package/src/core/agent-runtime/model-ref.ts +0 -8
- package/src/core/agents/registry.ts +49 -2
- package/src/core/auth/expiry-monitor.ts +9 -1
- package/src/core/auth/login-flow.ts +9 -1
- package/src/core/auth/status.ts +31 -3
- package/src/core/background/cron/scheduler.ts +25 -5
- package/src/core/background/dream/index.ts +29 -10
- package/src/core/background/failure-backoff.ts +30 -0
- package/src/core/background/heartbeat/agent.ts +2 -27
- package/src/core/background/heartbeat/index.ts +0 -2
- package/src/core/background/heartbeat/scheduler.ts +21 -13
- package/src/core/background/heartbeat/state.ts +10 -2
- package/src/core/background/isolated-agent.ts +6 -2
- package/src/core/background/pulse/pulse.ts +9 -0
- package/src/core/background/triggers/exit.ts +54 -0
- package/src/core/background/triggers/index.ts +1 -3
- package/src/core/background/triggers/resume.ts +2 -4
- package/src/core/backup/archive/tar.ts +14 -4
- package/src/core/backup/restore.ts +21 -13
- package/src/core/backup/scheduler.ts +28 -9
- package/src/core/backup/snapshot.ts +43 -10
- package/src/core/backup/store.ts +7 -19
- package/src/core/backup/targets.ts +69 -17
- package/src/core/config/index.ts +14 -0
- package/src/core/daemon/crash-marker.ts +141 -0
- package/src/core/daemon/crash.ts +9 -2
- package/src/core/daemon/handoff.ts +15 -0
- package/src/core/daemon/health-alerts.ts +297 -0
- package/src/core/daemon/log-reader.ts +289 -0
- package/src/core/doctor/index.ts +18 -2
- package/src/core/doctor/logs.ts +124 -0
- package/src/core/doctor/types.ts +1 -1
- package/src/core/engine/backend-controller/index.ts +1 -13
- package/src/core/engine/backend-router/router.ts +1 -1
- package/src/core/engine/dispatcher.ts +55 -2
- package/src/core/engine/fault-text.ts +40 -0
- package/src/core/engine/gateway-actions/agents/index.ts +3 -2
- package/src/core/engine/gateway-actions/agents/report.ts +62 -0
- package/src/core/engine/gateway-actions/history.ts +2 -4
- package/src/core/engine/gateway-actions/native/exec.ts +13 -16
- package/src/core/engine/gateway.ts +60 -1
- package/src/core/engine/turn-health.ts +222 -0
- package/src/core/errors.ts +2 -2
- package/src/core/frontend-runtime/admin-notify.ts +1 -1
- package/src/core/frontend-runtime/alerts.ts +130 -0
- package/src/core/mcp-hub/children.ts +78 -29
- package/src/core/mcp-hub/index.ts +21 -18
- package/src/core/mcp-hub/proxy-server.ts +8 -4
- package/src/core/mcp-hub/talon-server.ts +5 -12
- package/src/core/mesh/credentials/store.ts +16 -1
- package/src/core/mesh/devices/service.ts +26 -15
- package/src/core/mesh/devices/teleport.ts +14 -2
- package/src/core/mesh/links/node-binaries.ts +13 -6
- package/src/core/mesh/persist.ts +22 -10
- package/src/core/mesh/transfers/device-files.ts +5 -23
- package/src/core/mesh/transfers/transfers.ts +16 -3
- package/src/core/models/active-model.ts +2 -55
- package/src/core/plugin/actions.ts +19 -20
- package/src/core/plugin/builtins.ts +80 -90
- package/src/core/plugin/index.ts +1 -4
- package/src/core/plugin/loader.ts +25 -33
- package/src/core/plugin/mcp.ts +3 -5
- package/src/core/plugin/registry.ts +19 -35
- package/src/core/plugin/types.ts +2 -5
- package/src/core/prompt/assemble.ts +15 -3
- package/src/core/scripts/lua.ts +6 -2
- package/src/core/tasks/table.ts +8 -2
- package/src/core/tools/bridge.ts +2 -4
- package/src/core/tools/chat/cross-send.ts +1 -1
- package/src/core/tools/chat/messaging.ts +1 -1
- package/src/core/tools/index.ts +2 -2
- package/src/core/tools/mcp-env.ts +2 -59
- package/src/core/tools/ops/agents.ts +22 -1
- package/src/core/tools/schemas.ts +4 -9
- package/src/core/vfs/fusefs.ts +0 -5
- package/src/core/vfs/index.ts +9 -2
- package/src/core/vfs/mounts/diagnostics.ts +109 -0
- package/src/core/vfs/mounts/proc.ts +17 -1
- package/src/core/vfs/workspace.ts +7 -3
- package/src/core/weaver/shuttle.ts +8 -1
- package/src/core/weaver/turn-log.ts +320 -0
- package/src/core/weaver/weaver.ts +48 -5
- package/src/frontend/discord/actions/index.ts +8 -1
- package/src/frontend/discord/diagnostics.ts +82 -6
- package/src/frontend/discord/handlers/index.ts +0 -2
- package/src/frontend/discord/middleware.ts +7 -13
- package/src/frontend/discord/runtime.ts +1 -3
- package/src/frontend/health/delivery.ts +115 -0
- package/src/frontend/health/outage.ts +116 -0
- package/src/frontend/native/bridge/routes/chats.ts +3 -5
- package/src/frontend/native/bridge/server.ts +106 -21
- package/src/frontend/native/index.ts +1 -1
- package/src/frontend/native/media/media.ts +5 -1
- package/src/frontend/native/runtime.ts +12 -7
- package/src/frontend/native/surface/handlers.ts +1 -1
- package/src/frontend/native/surface/memory.ts +1 -1
- package/src/frontend/native/surface/models.ts +3 -3
- package/src/frontend/native/surface/settings.ts +20 -8
- package/src/frontend/native/turn/context.ts +6 -8
- package/src/frontend/native/turn/turn-meta.ts +2 -5
- package/src/frontend/native/turn/turn.ts +8 -10
- package/src/frontend/presentation/format.ts +2 -4
- package/src/frontend/presentation/session-status.ts +2 -6
- package/src/frontend/teams/actions.ts +8 -1
- package/src/frontend/teams/graph.ts +0 -1
- package/src/frontend/teams/index.ts +1 -4
- package/src/frontend/teams/poll.ts +40 -2
- package/src/frontend/teams/runtime.ts +14 -5
- package/src/frontend/telegram/actions/index.ts +4 -1
- package/src/frontend/telegram/actions/send.ts +8 -0
- package/src/frontend/telegram/handlers/context.ts +13 -2
- package/src/frontend/telegram/handlers/delivery.ts +12 -9
- package/src/frontend/telegram/handlers/index.ts +0 -2
- package/src/frontend/telegram/index.ts +35 -9
- package/src/frontend/telegram/polling/poll-health.ts +110 -0
- package/src/frontend/telegram/userbot.ts +100 -36
- package/src/frontend/terminal/builtins/session.ts +2 -2
- package/src/frontend/terminal/index.ts +1 -3
- package/src/frontend/terminal/renderer.ts +2 -18
- package/src/frontend/whatsapp/actions/index.ts +12 -1
- package/src/frontend/whatsapp/actions/messaging.ts +7 -2
- package/src/frontend/whatsapp/connection/connection.ts +29 -9
- package/src/frontend/whatsapp/connection/health.ts +89 -0
- package/src/frontend/whatsapp/connection/identity.ts +4 -4
- package/src/frontend/whatsapp/runtime.ts +11 -5
- package/src/native/blake3.ts +28 -2
- package/src/native/fusefs.ts +23 -5
- package/src/native/registry.ts +1 -1
- package/src/native/warden.ts +33 -5
- package/src/plugins/github/index.ts +0 -1
- package/src/plugins/mempalace/index.ts +9 -3
- package/src/plugins/playwright/index.ts +2 -4
- package/src/plugins/playwright/provision.ts +12 -4
- package/src/storage/chat-settings.ts +6 -1
- package/src/storage/cron.ts +29 -4
- package/src/storage/daily-log.ts +43 -47
- package/src/storage/db.ts +61 -33
- package/src/storage/history.ts +6 -1
- package/src/storage/journal.ts +9 -2
- package/src/storage/kv.ts +19 -6
- package/src/storage/media-index.ts +28 -6
- package/src/storage/repositories/chat-settings-repo.ts +8 -2
- package/src/storage/repositories/sessions-repo.ts +10 -3
- package/src/storage/scripts.ts +24 -13
- package/src/storage/sessions.ts +11 -2
- package/src/storage/skills.ts +21 -2
- package/src/storage/stickers.ts +17 -3
- package/src/storage/triggers.ts +8 -3
- package/src/storage/turn-meta.ts +25 -7
- package/src/util/log.ts +189 -6
- package/src/util/logging/turn-scope.ts +85 -0
- package/src/util/time.ts +3 -3
- package/src/util/watchdog.ts +30 -0
- package/src/core/engine/backend-controller/legacy.ts +0 -111
|
@@ -102,14 +102,6 @@ export interface ModelRef {
|
|
|
102
102
|
unavailableReason?: string;
|
|
103
103
|
}
|
|
104
104
|
|
|
105
|
-
/**
|
|
106
|
-
* Equality on identity only — two refs are the same run if they
|
|
107
|
-
* point at the same backend + id.
|
|
108
|
-
*/
|
|
109
|
-
export function sameModelRef(a: ModelRef, b: ModelRef): boolean {
|
|
110
|
-
return a.backend === b.backend && a.id === b.id;
|
|
111
|
-
}
|
|
112
|
-
|
|
113
105
|
/**
|
|
114
106
|
* Bare-minimum constructor for tests and adapters that don't yet
|
|
115
107
|
* carry rich metadata. Real catalog code should populate the
|
|
@@ -25,6 +25,7 @@ import type { TaskUsage } from "../tasks/types.js";
|
|
|
25
25
|
import type { AgentSettledEvent, AgentSpawnedEvent } from "../bus/events.js";
|
|
26
26
|
import type { ReasoningEffortLevel } from "../types.js";
|
|
27
27
|
import { bus } from "../bus/index.js";
|
|
28
|
+
import { logWarn } from "../../util/log.js";
|
|
28
29
|
|
|
29
30
|
/** Settled agents kept for status queries after they leave the live map. */
|
|
30
31
|
const DEFAULT_HISTORY_LIMIT = 100;
|
|
@@ -87,6 +88,23 @@ interface LiveAgent {
|
|
|
87
88
|
killRequested: boolean;
|
|
88
89
|
}
|
|
89
90
|
|
|
91
|
+
/**
|
|
92
|
+
* Whether two agents were spawned by the same parent.
|
|
93
|
+
*
|
|
94
|
+
* Compared structurally rather than by reference: records come from separate
|
|
95
|
+
* snapshots, so the parent objects are equal in value and never identical.
|
|
96
|
+
* The `kind` check is what stops a chat-parented agent matching an
|
|
97
|
+
* agent-parented one whose id happens to equal a chat key.
|
|
98
|
+
*/
|
|
99
|
+
function sameParent(a: AgentParent, b: AgentParent): boolean {
|
|
100
|
+
if (a.kind !== b.kind) return false;
|
|
101
|
+
return a.kind === "chat" && b.kind === "chat"
|
|
102
|
+
? a.chatId === b.chatId
|
|
103
|
+
: a.kind === "agent" && b.kind === "agent"
|
|
104
|
+
? a.agentId === b.agentId
|
|
105
|
+
: false;
|
|
106
|
+
}
|
|
107
|
+
|
|
90
108
|
function snapshot(entry: LiveAgent): AgentRecord {
|
|
91
109
|
return {
|
|
92
110
|
...entry.record,
|
|
@@ -279,8 +297,13 @@ export class AgentRegistry {
|
|
|
279
297
|
entry.killRequested = true;
|
|
280
298
|
try {
|
|
281
299
|
entry.abort?.abort();
|
|
282
|
-
} catch {
|
|
283
|
-
// An abort hook must not be able to break the kill path
|
|
300
|
+
} catch (err) {
|
|
301
|
+
// An abort hook must not be able to break the kill path — but a
|
|
302
|
+
// throwing one may leave the agent running, so say so.
|
|
303
|
+
logWarn(
|
|
304
|
+
"agents",
|
|
305
|
+
`Abort hook threw agent=${id}: ${err instanceof Error ? err.message : String(err)}`,
|
|
306
|
+
);
|
|
284
307
|
}
|
|
285
308
|
}
|
|
286
309
|
return true;
|
|
@@ -351,6 +374,30 @@ export class AgentRegistry {
|
|
|
351
374
|
return children.filter((child) => this.live.has(child));
|
|
352
375
|
}
|
|
353
376
|
|
|
377
|
+
/**
|
|
378
|
+
* An agent's live **peers** — the other agents sharing its parent.
|
|
379
|
+
*
|
|
380
|
+
* This is the addressing scope for agent-to-agent messaging, and it is
|
|
381
|
+
* deliberately narrower than "everything under the same chat". A swarm is
|
|
382
|
+
* a set of siblings spawned for one job, so siblings are the useful unit;
|
|
383
|
+
* widening to the whole chat tree would let an agent reach a cousin from an
|
|
384
|
+
* unrelated piece of work it knows nothing about.
|
|
385
|
+
*
|
|
386
|
+
* Live only: a settled agent has no mailbox to deliver into, and offering
|
|
387
|
+
* it as a peer would only produce a delivery failure one call later.
|
|
388
|
+
*/
|
|
389
|
+
peersOf(id: string): AgentRecord[] {
|
|
390
|
+
const self = this.get(id);
|
|
391
|
+
if (!self) return [];
|
|
392
|
+
const peers: AgentRecord[] = [];
|
|
393
|
+
for (const entry of this.live.values()) {
|
|
394
|
+
const record = snapshot(entry);
|
|
395
|
+
if (record.id === id) continue;
|
|
396
|
+
if (sameParent(record.parent, self.parent)) peers.push(record);
|
|
397
|
+
}
|
|
398
|
+
return peers.sort((a, b) => a.createdAt - b.createdAt);
|
|
399
|
+
}
|
|
400
|
+
|
|
354
401
|
/** Live agents plus the bounded settled ring, oldest first. */
|
|
355
402
|
list(): AgentRecord[] {
|
|
356
403
|
const records = [...this.history];
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
*/
|
|
11
11
|
|
|
12
12
|
import { notifyAdmin } from "../frontend-runtime/admin-notify.js";
|
|
13
|
+
import { logWarn } from "../../util/log.js";
|
|
13
14
|
import {
|
|
14
15
|
daysUntil,
|
|
15
16
|
PROVIDER_LABELS,
|
|
@@ -75,7 +76,14 @@ export function resetAuthExpiryAnnouncements(): void {
|
|
|
75
76
|
|
|
76
77
|
export function startAuthExpiryMonitor(): () => void {
|
|
77
78
|
const tick = (): void => {
|
|
78
|
-
void runAuthExpiryCheck().catch(() => {
|
|
79
|
+
void runAuthExpiryCheck().catch((err: unknown) => {
|
|
80
|
+
// The alert is already marked announced, so a failed send is not
|
|
81
|
+
// retried until the status changes — the log is its only trace.
|
|
82
|
+
logWarn(
|
|
83
|
+
"notify",
|
|
84
|
+
`auth expiry check failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
85
|
+
);
|
|
86
|
+
});
|
|
79
87
|
};
|
|
80
88
|
// First check shortly after boot so a lapsed login is surfaced right away.
|
|
81
89
|
const first = setTimeout(tick, 30_000);
|
|
@@ -160,7 +160,15 @@ export function startLogin(
|
|
|
160
160
|
new Error(outcome.ok ? "no prompt" : outcome.detail || outcome.reason),
|
|
161
161
|
);
|
|
162
162
|
if (tmpHome)
|
|
163
|
-
await rm(tmpHome, { recursive: true, force: true }).catch(
|
|
163
|
+
await rm(tmpHome, { recursive: true, force: true }).catch(
|
|
164
|
+
(err: unknown) =>
|
|
165
|
+
// The scratch HOME can hold a fresh credential copy — a leftover
|
|
166
|
+
// is worth knowing about.
|
|
167
|
+
logWarn(
|
|
168
|
+
"notify",
|
|
169
|
+
`${provider} login cleanup failed dir=${tmpHome}: ${err instanceof Error ? err.message : String(err)}`,
|
|
170
|
+
),
|
|
171
|
+
);
|
|
164
172
|
resolveDone(outcome);
|
|
165
173
|
};
|
|
166
174
|
|
package/src/core/auth/status.ts
CHANGED
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
import { readFile, stat } from "node:fs/promises";
|
|
16
16
|
import { homedir } from "node:os";
|
|
17
17
|
import { join } from "node:path";
|
|
18
|
+
import { logWarn } from "../../util/log.js";
|
|
18
19
|
|
|
19
20
|
export type AuthProvider = "claude" | "codex";
|
|
20
21
|
|
|
@@ -64,6 +65,19 @@ export function clearProviderExpired(provider: AuthProvider): void {
|
|
|
64
65
|
reportedExpired.delete(provider);
|
|
65
66
|
}
|
|
66
67
|
|
|
68
|
+
/**
|
|
69
|
+
* A credentials file that exists but isn't JSON reads as "not signed
|
|
70
|
+
* in" — indistinguishable from a missing login unless logged. Nothing
|
|
71
|
+
* from the file goes into the line: JSON.parse's message quotes the
|
|
72
|
+
* input, and the input is a token.
|
|
73
|
+
*/
|
|
74
|
+
function warnUnparseable(provider: AuthProvider): void {
|
|
75
|
+
logWarn(
|
|
76
|
+
"notify",
|
|
77
|
+
`${provider} credentials file is not valid JSON — reporting not signed in`,
|
|
78
|
+
);
|
|
79
|
+
}
|
|
80
|
+
|
|
67
81
|
export function parseClaudeCredentials(raw: string): ProviderAuthStatus {
|
|
68
82
|
const base: ProviderAuthStatus = {
|
|
69
83
|
provider: "claude",
|
|
@@ -81,6 +95,7 @@ export function parseClaudeCredentials(raw: string): ProviderAuthStatus {
|
|
|
81
95
|
try {
|
|
82
96
|
parsed = JSON.parse(raw);
|
|
83
97
|
} catch {
|
|
98
|
+
warnUnparseable("claude");
|
|
84
99
|
return base;
|
|
85
100
|
}
|
|
86
101
|
const oauth = parsed.claudeAiOauth;
|
|
@@ -115,6 +130,7 @@ export function parseCodexAuth(raw: string): ProviderAuthStatus {
|
|
|
115
130
|
try {
|
|
116
131
|
parsed = JSON.parse(raw);
|
|
117
132
|
} catch {
|
|
133
|
+
warnUnparseable("codex");
|
|
118
134
|
return base;
|
|
119
135
|
}
|
|
120
136
|
const apiKey =
|
|
@@ -136,11 +152,23 @@ export function parseCodexAuth(raw: string): ProviderAuthStatus {
|
|
|
136
152
|
};
|
|
137
153
|
}
|
|
138
154
|
|
|
139
|
-
async function readOrEmpty(
|
|
155
|
+
async function readOrEmpty(
|
|
156
|
+
provider: AuthProvider,
|
|
157
|
+
path: string,
|
|
158
|
+
): Promise<string | undefined> {
|
|
140
159
|
try {
|
|
141
160
|
await stat(path);
|
|
142
161
|
return await readFile(path, "utf8");
|
|
143
|
-
} catch {
|
|
162
|
+
} catch (err) {
|
|
163
|
+
// Missing is the normal signed-out case; anything else (EACCES,
|
|
164
|
+
// EISDIR) is a fault that would otherwise read as signed out too.
|
|
165
|
+
const code = (err as NodeJS.ErrnoException).code;
|
|
166
|
+
if (code !== "ENOENT") {
|
|
167
|
+
logWarn(
|
|
168
|
+
"notify",
|
|
169
|
+
`${provider} credentials unreadable code=${code ?? "?"} — reporting not signed in`,
|
|
170
|
+
);
|
|
171
|
+
}
|
|
144
172
|
return undefined;
|
|
145
173
|
}
|
|
146
174
|
}
|
|
@@ -151,7 +179,7 @@ export async function readProviderStatus(
|
|
|
151
179
|
): Promise<ProviderAuthStatus> {
|
|
152
180
|
const path =
|
|
153
181
|
provider === "claude" ? claudeCredentialsPath(env) : codexAuthPath(env);
|
|
154
|
-
const raw = await readOrEmpty(path);
|
|
182
|
+
const raw = await readOrEmpty(provider, path);
|
|
155
183
|
const status =
|
|
156
184
|
raw === undefined
|
|
157
185
|
? { provider, loggedIn: false, expired: true }
|
|
@@ -34,6 +34,8 @@ import {
|
|
|
34
34
|
} from "../../../storage/cron.js";
|
|
35
35
|
import { appendDailyLog } from "../../../storage/daily-log.js";
|
|
36
36
|
import { log, logError, logWarn } from "../../../util/log.js";
|
|
37
|
+
import { raiseAlert, resolveAlert } from "../../frontend-runtime/alerts.js";
|
|
38
|
+
import { faultText } from "../../engine/fault-text.js";
|
|
37
39
|
import { numericChatIdFor } from "../../frontend-runtime/chat-id.js";
|
|
38
40
|
import {
|
|
39
41
|
chooseBackend,
|
|
@@ -156,8 +158,12 @@ async function runCronTick(): Promise<void> {
|
|
|
156
158
|
pruneJobHealth(new Set(jobs.map((j) => j.id)));
|
|
157
159
|
|
|
158
160
|
let loadShed = false;
|
|
159
|
-
for (const
|
|
160
|
-
|
|
161
|
+
for (const listed of jobs) {
|
|
162
|
+
// Re-read: each run is awaited, so a slow job lets the next tick start
|
|
163
|
+
// and run later jobs before this loop reaches them. The listing's copy
|
|
164
|
+
// would carry the pre-run lastRunAt and fire such a job a second time.
|
|
165
|
+
const job = getCronJob(listed.id);
|
|
166
|
+
if (!job?.enabled) continue;
|
|
161
167
|
// Expiry takes priority over dueness: a job past its end time is disabled
|
|
162
168
|
// and skipped even if this minute would otherwise match.
|
|
163
169
|
if (expireIfPast(job, nowMs)) continue;
|
|
@@ -201,6 +207,10 @@ async function runScheduled(job: CronJob): Promise<void> {
|
|
|
201
207
|
durationMs: Date.now() - startedAt,
|
|
202
208
|
};
|
|
203
209
|
recordJobSuccess(job.id, Date.now(), JOB_HEALTH);
|
|
210
|
+
resolveAlert(
|
|
211
|
+
`cron.job.${job.id}`,
|
|
212
|
+
`Cron job "${job.name}" is running again.`,
|
|
213
|
+
);
|
|
204
214
|
recordCronRun(job.id, outcome);
|
|
205
215
|
appendDailyLog(
|
|
206
216
|
"Cron",
|
|
@@ -223,12 +233,23 @@ async function runScheduled(job: CronJob): Promise<void> {
|
|
|
223
233
|
lastError: err instanceof Error ? err.message : String(err),
|
|
224
234
|
lastDurationMs: Date.now() - startedAt,
|
|
225
235
|
});
|
|
226
|
-
logError(
|
|
236
|
+
logError(
|
|
237
|
+
"cron",
|
|
238
|
+
`Job "${job.name}" [${job.id}] failed chat=${job.chatId} type=${job.type} ms=${Date.now() - startedAt}`,
|
|
239
|
+
err,
|
|
240
|
+
);
|
|
227
241
|
const cooldown = recordJobFailure(job.id, Date.now(), JOB_HEALTH);
|
|
228
242
|
if (cooldown !== null) {
|
|
243
|
+
const mins = Math.round(cooldown / 60_000);
|
|
229
244
|
logWarn(
|
|
230
245
|
"cron",
|
|
231
|
-
`Breaker opened for "${job.name}" [${job.id}] — cooling down ~${
|
|
246
|
+
`Breaker opened for "${job.name}" [${job.id}] — cooling down ~${mins}min`,
|
|
247
|
+
);
|
|
248
|
+
// The breaker opens at JOB_HEALTH.threshold consecutive failures —
|
|
249
|
+
// the job is now paused, which the operator should hear about.
|
|
250
|
+
raiseAlert(
|
|
251
|
+
`cron.job.${job.id}`,
|
|
252
|
+
`Cron job "${job.name}" failed ${JOB_HEALTH.threshold} runs in a row: ${faultText(err)}. Paused for ~${mins} min.`,
|
|
232
253
|
);
|
|
233
254
|
}
|
|
234
255
|
} finally {
|
|
@@ -461,7 +482,6 @@ function isCronDue(job: CronJob, now: Date, windowStartMs: number): boolean {
|
|
|
461
482
|
export const _cronInternals = {
|
|
462
483
|
isDue,
|
|
463
484
|
isCronDue,
|
|
464
|
-
MAX_TICK_LOOKBACK_MS,
|
|
465
485
|
};
|
|
466
486
|
|
|
467
487
|
/**
|
|
@@ -24,6 +24,7 @@ import type { Backend } from "../../agent-runtime/capabilities.js";
|
|
|
24
24
|
import { taskTable } from "../../tasks/index.js";
|
|
25
25
|
import { resolveBackgroundEffort } from "../effort.js";
|
|
26
26
|
import { FailureBackoff } from "../failure-backoff.js";
|
|
27
|
+
import { faultText } from "../../engine/fault-text.js";
|
|
27
28
|
|
|
28
29
|
// ── Types ────────────────────────────────────────────────────────────────────
|
|
29
30
|
|
|
@@ -61,7 +62,10 @@ let dreaming = false; // in-process guard (one dream at a time)
|
|
|
61
62
|
* every message (observed live: a model outage produced 403 identical
|
|
62
63
|
* consolidation failures). Exported for tests.
|
|
63
64
|
*/
|
|
64
|
-
export const dreamFailureBackoff = new FailureBackoff(
|
|
65
|
+
export const dreamFailureBackoff = new FailureBackoff({
|
|
66
|
+
key: "dream.failing",
|
|
67
|
+
label: "Dream (memory consolidation)",
|
|
68
|
+
});
|
|
65
69
|
let configRef: {
|
|
66
70
|
model?: string;
|
|
67
71
|
dreamModel?: string;
|
|
@@ -141,6 +145,16 @@ export async function forceDream(): Promise<void> {
|
|
|
141
145
|
await executeDream("forced");
|
|
142
146
|
}
|
|
143
147
|
|
|
148
|
+
/** Arm the failure backoff so maybeStartDream stays quiet for a while. */
|
|
149
|
+
function backOff(err: unknown): void {
|
|
150
|
+
const until = dreamFailureBackoff.fail(err);
|
|
151
|
+
logWarn(
|
|
152
|
+
"dream",
|
|
153
|
+
`dream.backoff failures=${dreamFailureBackoff.failures} ` +
|
|
154
|
+
`until=${new Date(until).toISOString()} error="${faultText(err)}"`,
|
|
155
|
+
);
|
|
156
|
+
}
|
|
157
|
+
|
|
144
158
|
/** Shared dream execution — claims lock, runs agent, releases lock. */
|
|
145
159
|
async function executeDream(trigger: "auto" | "forced"): Promise<void> {
|
|
146
160
|
const state = readDreamState();
|
|
@@ -153,8 +167,13 @@ async function executeDream(trigger: "auto" | "forced"): Promise<void> {
|
|
|
153
167
|
|
|
154
168
|
try {
|
|
155
169
|
const dreamLogPath = await runDreamAgent(state?.last_run ?? 0);
|
|
156
|
-
|
|
157
|
-
|
|
170
|
+
// A run whose last_run didn't stick still looks overdue, and
|
|
171
|
+
// maybeStartDream would run it all again on the very next message.
|
|
172
|
+
if (writeDreamState({ last_run: Date.now(), status: "idle" })) {
|
|
173
|
+
dreamFailureBackoff.succeed();
|
|
174
|
+
} else {
|
|
175
|
+
backOff(new Error("dream state was not persisted"));
|
|
176
|
+
}
|
|
158
177
|
log(
|
|
159
178
|
"dream",
|
|
160
179
|
`Memory consolidation complete (${trigger}), log: ${dreamLogPath}`,
|
|
@@ -164,12 +183,7 @@ async function executeDream(trigger: "auto" | "forced"): Promise<void> {
|
|
|
164
183
|
writeDreamState({ last_run: state?.last_run ?? 0, status: "idle" });
|
|
165
184
|
// A failed dream keeps the old last_run, so it would re-fire on the very
|
|
166
185
|
// next invocation — back off instead (forceDream bypasses the window).
|
|
167
|
-
|
|
168
|
-
logWarn(
|
|
169
|
-
"dream",
|
|
170
|
-
`Backing off until ${new Date(until).toISOString()} ` +
|
|
171
|
-
`after ${dreamFailureBackoff.failures} consecutive failure(s)`,
|
|
172
|
-
);
|
|
186
|
+
backOff(err);
|
|
173
187
|
if (trigger === "forced") throw err;
|
|
174
188
|
} finally {
|
|
175
189
|
dreaming = false;
|
|
@@ -427,10 +441,15 @@ function readDreamState(): DreamState | null {
|
|
|
427
441
|
return parsed;
|
|
428
442
|
}
|
|
429
443
|
|
|
430
|
-
|
|
444
|
+
/**
|
|
445
|
+
* Persist the state. Returns whether last_run actually landed: kvSet logs
|
|
446
|
+
* and swallows a failed write (full disk), so the store is read back.
|
|
447
|
+
*/
|
|
448
|
+
function writeDreamState(state: DreamState): boolean {
|
|
431
449
|
const enriched: DreamState = {
|
|
432
450
|
...state,
|
|
433
451
|
last_run_at: new Date(state.last_run).toISOString(),
|
|
434
452
|
};
|
|
435
453
|
kvSet(DREAM_STATE_KEY, enriched);
|
|
454
|
+
return readDreamState()?.last_run === state.last_run;
|
|
436
455
|
}
|
|
@@ -15,8 +15,16 @@
|
|
|
15
15
|
*
|
|
16
16
|
* Backoff state is in-memory only: a process restart forgets it, which is
|
|
17
17
|
* fine — the first post-restart attempt either works or re-arms the window.
|
|
18
|
+
*
|
|
19
|
+
* A holder built with an `alert` also tells the operator: once the streak
|
|
20
|
+
* reaches its threshold (2 by default — one failure is a blip, a second in
|
|
21
|
+
* a row after the backoff is a broken component) it raises `alert.key`
|
|
22
|
+
* with the error, and the next success resolves it.
|
|
18
23
|
*/
|
|
19
24
|
|
|
25
|
+
import { faultText } from "../engine/fault-text.js";
|
|
26
|
+
import { raiseAlert, resolveAlert } from "../frontend-runtime/alerts.js";
|
|
27
|
+
|
|
20
28
|
const FAILURE_BACKOFF_BASE_MS = 5 * 60 * 1000;
|
|
21
29
|
const FAILURE_BACKOFF_MAX_MS = 60 * 60 * 1000;
|
|
22
30
|
/** Safety margin added past a parsed limit-reset time (clock skew, rollout). */
|
|
@@ -70,6 +78,16 @@ export function failureBackoffUntil(
|
|
|
70
78
|
return now + exp;
|
|
71
79
|
}
|
|
72
80
|
|
|
81
|
+
/** Operator alert a backoff holder raises while its component keeps failing. */
|
|
82
|
+
type BackoffAlert = {
|
|
83
|
+
/** Stable alert key, e.g. "heartbeat.failing". */
|
|
84
|
+
key: string;
|
|
85
|
+
/** What failed, for the message: "The heartbeat", "Dream (memory consolidation)". */
|
|
86
|
+
label: string;
|
|
87
|
+
/** Consecutive failures before raising. Default 2. */
|
|
88
|
+
threshold?: number;
|
|
89
|
+
};
|
|
90
|
+
|
|
73
91
|
/**
|
|
74
92
|
* Per-component backoff holder. Cadence checks gate on `active()`; run
|
|
75
93
|
* completions call `succeed()` / `fail(err)`.
|
|
@@ -78,6 +96,8 @@ export class FailureBackoff {
|
|
|
78
96
|
private consecutiveFailures = 0;
|
|
79
97
|
private until = 0;
|
|
80
98
|
|
|
99
|
+
constructor(private readonly alert?: BackoffAlert) {}
|
|
100
|
+
|
|
81
101
|
/** True while inside the backoff window — the caller should skip firing. */
|
|
82
102
|
active(now = Date.now()): boolean {
|
|
83
103
|
return now < this.until;
|
|
@@ -87,6 +107,14 @@ export class FailureBackoff {
|
|
|
87
107
|
fail(err: unknown, now = Date.now()): number {
|
|
88
108
|
this.consecutiveFailures += 1;
|
|
89
109
|
this.until = failureBackoffUntil(err, this.consecutiveFailures, now);
|
|
110
|
+
const alert = this.alert;
|
|
111
|
+
if (alert && this.consecutiveFailures >= (alert.threshold ?? 2)) {
|
|
112
|
+
const retryAt = new Date(this.until).toISOString().slice(11, 16);
|
|
113
|
+
raiseAlert(
|
|
114
|
+
alert.key,
|
|
115
|
+
`${alert.label} has failed ${this.consecutiveFailures} times in a row: ${faultText(err)}. Next attempt after ${retryAt} UTC.`,
|
|
116
|
+
);
|
|
117
|
+
}
|
|
90
118
|
return this.until;
|
|
91
119
|
}
|
|
92
120
|
|
|
@@ -94,6 +122,8 @@ export class FailureBackoff {
|
|
|
94
122
|
succeed(): void {
|
|
95
123
|
this.consecutiveFailures = 0;
|
|
96
124
|
this.until = 0;
|
|
125
|
+
if (this.alert)
|
|
126
|
+
resolveAlert(this.alert.key, `${this.alert.label} is running again.`);
|
|
97
127
|
}
|
|
98
128
|
|
|
99
129
|
get failures(): number {
|
|
@@ -22,6 +22,7 @@ import {
|
|
|
22
22
|
resolveRoutedModel,
|
|
23
23
|
} from "../../engine/backend-router/index.js";
|
|
24
24
|
import { resolveBackgroundEffort } from "../effort.js";
|
|
25
|
+
import { raceWithTimeout } from "../isolated-agent.js";
|
|
25
26
|
import { hb } from "./state.js";
|
|
26
27
|
|
|
27
28
|
const DEFAULT_HEARTBEAT_TIMEOUT_MS = 10 * 60 * 1000; // 10-minute soft cap
|
|
@@ -88,7 +89,7 @@ export function buildHeartbeatSystemPrompt(): string {
|
|
|
88
89
|
/**
|
|
89
90
|
* Render the open-goal listing for the heartbeat prompt. Cross-chat by design:
|
|
90
91
|
* the heartbeat is a global agent, so it sees every chat's open goals (with
|
|
91
|
-
* chat ids for routing updates back).
|
|
92
|
+
* chat ids for routing updates back).
|
|
92
93
|
*/
|
|
93
94
|
function renderGoalsBlock(): { text: string; count: number } {
|
|
94
95
|
let text = "(no open goals)";
|
|
@@ -486,32 +487,6 @@ export async function runHeartbeatAgent(
|
|
|
486
487
|
return heartbeatLogFile;
|
|
487
488
|
}
|
|
488
489
|
|
|
489
|
-
/**
|
|
490
|
-
* Race a promise against a timeout. Returns the promise's resolved value, or
|
|
491
|
-
* the sentinel `"timed_out"` if the timeout fires first.
|
|
492
|
-
*
|
|
493
|
-
* NOTE: if `p` rejects before the timeout fires, that rejection propagates —
|
|
494
|
-
* callers that need a never-throwing race should `.catch()` the input promise
|
|
495
|
-
* themselves.
|
|
496
|
-
*/
|
|
497
|
-
async function raceWithTimeout<T>(
|
|
498
|
-
p: Promise<T>,
|
|
499
|
-
ms: number,
|
|
500
|
-
): Promise<T | "timed_out"> {
|
|
501
|
-
let t: ReturnType<typeof setTimeout> | null = null;
|
|
502
|
-
try {
|
|
503
|
-
return await Promise.race([
|
|
504
|
-
p,
|
|
505
|
-
new Promise<"timed_out">((resolve) => {
|
|
506
|
-
t = setTimeout(() => resolve("timed_out"), ms);
|
|
507
|
-
t.unref();
|
|
508
|
-
}),
|
|
509
|
-
]);
|
|
510
|
-
} finally {
|
|
511
|
-
if (t) clearTimeout(t);
|
|
512
|
-
}
|
|
513
|
-
}
|
|
514
|
-
|
|
515
490
|
// ── Logging helpers ─────────────────────────────────────────────────────────
|
|
516
491
|
|
|
517
492
|
async function createHeartbeatLogFile(): Promise<string> {
|
|
@@ -9,8 +9,6 @@
|
|
|
9
9
|
* - `state` — shared run-guard/timers/config + persisted-state I/O
|
|
10
10
|
* - `agent` — prompt building + the one-shot agent run + log helpers
|
|
11
11
|
* - `scheduler` — init, the cadence timer, run guard, force/await/status API
|
|
12
|
-
*
|
|
13
|
-
* Re-exports the same public surface the old single-file module exposed.
|
|
14
12
|
*/
|
|
15
13
|
|
|
16
14
|
export {
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
7
|
import { log, logError, logWarn } from "../../../util/log.js";
|
|
8
|
+
import { faultText } from "../../engine/fault-text.js";
|
|
8
9
|
import {
|
|
9
10
|
catchupRunCount,
|
|
10
11
|
missedRunCount,
|
|
@@ -147,6 +148,16 @@ export function getHeartbeatStatus(): HeartbeatState | null {
|
|
|
147
148
|
return readHeartbeatState();
|
|
148
149
|
}
|
|
149
150
|
|
|
151
|
+
/** Arm the failure backoff so the due check stops re-firing for a while. */
|
|
152
|
+
function backOff(err: unknown): void {
|
|
153
|
+
const until = hb.failureBackoff.fail(err);
|
|
154
|
+
logWarn(
|
|
155
|
+
"heartbeat",
|
|
156
|
+
`heartbeat.backoff failures=${hb.failureBackoff.failures} ` +
|
|
157
|
+
`until=${new Date(until).toISOString()} error="${faultText(err)}"`,
|
|
158
|
+
);
|
|
159
|
+
}
|
|
160
|
+
|
|
150
161
|
async function executeHeartbeat(trigger: "auto" | "forced"): Promise<void> {
|
|
151
162
|
if (hb.running) return;
|
|
152
163
|
|
|
@@ -175,13 +186,16 @@ async function executeHeartbeat(trigger: "auto" | "forced"): Promise<void> {
|
|
|
175
186
|
previousRunCount + 1,
|
|
176
187
|
);
|
|
177
188
|
// Only update last_run and increment run_count on success
|
|
178
|
-
writeHeartbeatState({
|
|
189
|
+
const persisted = writeHeartbeatState({
|
|
179
190
|
last_run: Date.now(),
|
|
180
191
|
last_started: now,
|
|
181
192
|
status: "idle",
|
|
182
193
|
run_count: previousRunCount + 1,
|
|
183
194
|
});
|
|
184
|
-
|
|
195
|
+
// A run whose last_run didn't stick looks overdue to the next due
|
|
196
|
+
// check, which would run the whole agent again every minute.
|
|
197
|
+
if (persisted) hb.failureBackoff.succeed();
|
|
198
|
+
else backOff(new Error("heartbeat state was not persisted"));
|
|
185
199
|
log(
|
|
186
200
|
"heartbeat",
|
|
187
201
|
`Heartbeat #${previousRunCount + 1} complete (${trigger}), log: ${heartbeatLogPath}`,
|
|
@@ -197,23 +211,17 @@ async function executeHeartbeat(trigger: "auto" | "forced"): Promise<void> {
|
|
|
197
211
|
// re-triggering against the same `last_run` forever. Non-timeout errors
|
|
198
212
|
// retry from the previous successful run (no budget consumed).
|
|
199
213
|
const isTimeout = err instanceof HeartbeatTimeoutError;
|
|
200
|
-
writeHeartbeatState({
|
|
214
|
+
const persisted = writeHeartbeatState({
|
|
201
215
|
last_run: isTimeout ? Date.now() : previousLastRun,
|
|
202
216
|
last_started: now,
|
|
203
217
|
status: "idle",
|
|
204
218
|
run_count: isTimeout ? previousRunCount + 1 : previousRunCount,
|
|
205
219
|
});
|
|
206
220
|
// Timeouts advance last_run (budget consumed), so the cadence itself
|
|
207
|
-
// spaces the next attempt. Every other failure
|
|
208
|
-
// last_run — back off so the due check
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
logWarn(
|
|
212
|
-
"heartbeat",
|
|
213
|
-
`Backing off until ${new Date(until).toISOString()} ` +
|
|
214
|
-
`after ${hb.failureBackoff.failures} consecutive failure(s)`,
|
|
215
|
-
);
|
|
216
|
-
}
|
|
221
|
+
// spaces the next attempt — if that write landed. Every other failure
|
|
222
|
+
// retries against the same last_run — back off so the due check
|
|
223
|
+
// doesn't hammer it every minute.
|
|
224
|
+
if (!isTimeout || !persisted) backOff(err);
|
|
217
225
|
if (trigger === "forced") throw err;
|
|
218
226
|
} finally {
|
|
219
227
|
hb.running = false;
|
|
@@ -100,7 +100,10 @@ export const hb: {
|
|
|
100
100
|
intervalMinutesRef: 60,
|
|
101
101
|
config: null,
|
|
102
102
|
logFileSequence: 0,
|
|
103
|
-
failureBackoff: new FailureBackoff(
|
|
103
|
+
failureBackoff: new FailureBackoff({
|
|
104
|
+
key: "heartbeat.failing",
|
|
105
|
+
label: "The heartbeat",
|
|
106
|
+
}),
|
|
104
107
|
};
|
|
105
108
|
|
|
106
109
|
// ── State-file I/O ───────────────────────────────────────────────────────────
|
|
@@ -160,7 +163,11 @@ export function readHeartbeatState(): HeartbeatState | null {
|
|
|
160
163
|
return normalizeHeartbeatState(kvGet(HEARTBEAT_STATE_KEY));
|
|
161
164
|
}
|
|
162
165
|
|
|
163
|
-
|
|
166
|
+
/**
|
|
167
|
+
* Persist the state. Returns whether last_run actually landed: kvSet logs
|
|
168
|
+
* and swallows a failed write (full disk), so the store is read back.
|
|
169
|
+
*/
|
|
170
|
+
export function writeHeartbeatState(state: HeartbeatState): boolean {
|
|
164
171
|
// Re-derive last_run_at from last_run so the persisted ISO stamp can
|
|
165
172
|
// never drift from the millisecond field; omit it on the sentinel
|
|
166
173
|
// last_run === 0 (never-run) to match the pre-SQLite file format.
|
|
@@ -172,4 +179,5 @@ export function writeHeartbeatState(state: HeartbeatState): void {
|
|
|
172
179
|
: {}),
|
|
173
180
|
};
|
|
174
181
|
kvSet(HEARTBEAT_STATE_KEY, enriched);
|
|
182
|
+
return readHeartbeatState()?.last_run === state.last_run;
|
|
175
183
|
}
|
|
@@ -50,8 +50,12 @@ export interface IsolatedRunOptions {
|
|
|
50
50
|
readonly logCategory?: LogComponent;
|
|
51
51
|
}
|
|
52
52
|
|
|
53
|
-
/**
|
|
54
|
-
|
|
53
|
+
/**
|
|
54
|
+
* Resolves to the value, or the string "timed_out" if `ms` elapses first.
|
|
55
|
+
* A rejection of `promise` inside the window propagates — `.catch()` it
|
|
56
|
+
* first for a never-throwing race.
|
|
57
|
+
*/
|
|
58
|
+
export async function raceWithTimeout<T>(
|
|
55
59
|
promise: Promise<T>,
|
|
56
60
|
ms: number,
|
|
57
61
|
): Promise<T | "timed_out"> {
|
|
@@ -27,6 +27,8 @@ import { formatSmartTimestamp } from "../../../util/time.js";
|
|
|
27
27
|
let timer: ReturnType<typeof setInterval> | null = null;
|
|
28
28
|
const registeredChats = new Set<string>();
|
|
29
29
|
const lastCheckMessageId = new Map<string, number>();
|
|
30
|
+
/** Chats whose pulse turn is still queued or running. */
|
|
31
|
+
const inFlight = new Set<string>();
|
|
30
32
|
|
|
31
33
|
const DEFAULT_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes
|
|
32
34
|
let activeIntervalMs = DEFAULT_INTERVAL_MS;
|
|
@@ -126,6 +128,10 @@ async function runPulse(): Promise<void> {
|
|
|
126
128
|
}
|
|
127
129
|
|
|
128
130
|
async function pulseChat(chatId: string): Promise<void> {
|
|
131
|
+
// The pulse turn queues behind the chat's other turns, so it can outlast
|
|
132
|
+
// the interval. Until it settles the checkpoint hasn't moved, and the next
|
|
133
|
+
// tick would dispatch the same unread messages a second time.
|
|
134
|
+
if (inFlight.has(chatId)) return;
|
|
129
135
|
const numericChatId = parseInt(chatId, 10);
|
|
130
136
|
if (isNaN(numericChatId)) {
|
|
131
137
|
logError("pulse", `Invalid chatId: ${chatId}`);
|
|
@@ -157,6 +163,7 @@ async function pulseChat(chatId: string): Promise<void> {
|
|
|
157
163
|
})
|
|
158
164
|
.join("\n");
|
|
159
165
|
|
|
166
|
+
inFlight.add(chatId);
|
|
160
167
|
try {
|
|
161
168
|
const prompt =
|
|
162
169
|
`[System: Pulse check — ${unread.length} new message(s) since last check. ` +
|
|
@@ -179,5 +186,7 @@ async function pulseChat(chatId: string): Promise<void> {
|
|
|
179
186
|
} catch (err) {
|
|
180
187
|
logError("pulse", `Chat ${chatId} failed`, err);
|
|
181
188
|
// Don't update lastCheckMessageId — messages will be retried next pulse
|
|
189
|
+
} finally {
|
|
190
|
+
inFlight.delete(chatId);
|
|
182
191
|
}
|
|
183
192
|
}
|