talon-agent 5.18.1 → 5.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -21
- package/LICENSE-MIT +21 -0
- package/NOTICE +16 -0
- package/README.md +8 -3
- package/package.json +4 -2
- package/prompts/system/agent-brief.md +20 -3
- package/src/app.ts +13 -0
- package/src/backend/claude-sdk/handler.ts +4 -4
- package/src/backend/claude-sdk/mcp-ready.ts +16 -2
- package/src/backend/claude-sdk/stream.ts +2 -2
- package/src/backend/codex/auth.ts +1 -1
- package/src/backend/codex/handler/message.ts +11 -11
- package/src/backend/codex/init.ts +4 -9
- package/src/backend/codex/mcp-config.ts +1 -2
- package/src/backend/codex/oauth-incompat.ts +8 -4
- package/src/backend/codex/one-shot.ts +1 -1
- package/src/backend/openai-agents/builtins.ts +55 -27
- package/src/backend/openai-agents/factory.ts +3 -3
- package/src/backend/openai-agents/handler/message.ts +3 -5
- package/src/backend/openai-agents/mcp-pool.ts +10 -27
- package/src/backend/remote-server/chat-turn.ts +6 -6
- package/src/backend/remote-server/events.ts +1 -5
- package/src/backend/remote-server/index.ts +0 -1
- package/src/backend/remote-server/messages.ts +3 -7
- package/src/backend/remote-server/one-shot.ts +1 -3
- package/src/backend/remote-server/session-helpers.ts +1 -4
- package/src/backend/remote-server/sse-stream.ts +8 -9
- package/src/backend/runtime/metrics.ts +7 -13
- package/src/backend/runtime/sleep.ts +1 -2
- package/src/backend/runtime/turn/handle-retry.ts +48 -2
- package/src/backend/runtime/turn/handler-to-events.ts +3 -4
- package/src/bootstrap.ts +9 -1
- package/src/cli/doctor.ts +3 -0
- package/src/cli/index.ts +8 -10
- package/src/cli/logs.ts +148 -9
- package/src/cli/setup.ts +9 -11
- package/src/cli/status.ts +29 -0
- package/src/core/agent-runtime/README.md +5 -19
- package/src/core/agent-runtime/events.ts +3 -39
- package/src/core/agent-runtime/model-ref.ts +0 -8
- package/src/core/agents/registry.ts +49 -2
- package/src/core/auth/expiry-monitor.ts +9 -1
- package/src/core/auth/login-flow.ts +9 -1
- package/src/core/auth/status.ts +31 -3
- package/src/core/background/cron/scheduler.ts +25 -5
- package/src/core/background/dream/index.ts +29 -10
- package/src/core/background/failure-backoff.ts +30 -0
- package/src/core/background/heartbeat/agent.ts +2 -27
- package/src/core/background/heartbeat/index.ts +0 -2
- package/src/core/background/heartbeat/scheduler.ts +21 -13
- package/src/core/background/heartbeat/state.ts +10 -2
- package/src/core/background/isolated-agent.ts +6 -2
- package/src/core/background/pulse/pulse.ts +9 -0
- package/src/core/background/triggers/exit.ts +54 -0
- package/src/core/background/triggers/index.ts +1 -3
- package/src/core/background/triggers/resume.ts +2 -4
- package/src/core/backup/archive/tar.ts +14 -4
- package/src/core/backup/restore.ts +21 -13
- package/src/core/backup/scheduler.ts +28 -9
- package/src/core/backup/snapshot.ts +43 -10
- package/src/core/backup/store.ts +7 -19
- package/src/core/backup/targets.ts +69 -17
- package/src/core/config/index.ts +14 -0
- package/src/core/daemon/crash-marker.ts +141 -0
- package/src/core/daemon/crash.ts +9 -2
- package/src/core/daemon/handoff.ts +15 -0
- package/src/core/daemon/health-alerts.ts +297 -0
- package/src/core/daemon/log-reader.ts +289 -0
- package/src/core/doctor/index.ts +18 -2
- package/src/core/doctor/logs.ts +124 -0
- package/src/core/doctor/types.ts +1 -1
- package/src/core/engine/backend-controller/index.ts +1 -13
- package/src/core/engine/backend-router/router.ts +1 -1
- package/src/core/engine/dispatcher.ts +55 -2
- package/src/core/engine/fault-text.ts +40 -0
- package/src/core/engine/gateway-actions/agents/index.ts +3 -2
- package/src/core/engine/gateway-actions/agents/report.ts +62 -0
- package/src/core/engine/gateway-actions/history.ts +2 -4
- package/src/core/engine/gateway-actions/native/exec.ts +13 -16
- package/src/core/engine/gateway.ts +60 -1
- package/src/core/engine/turn-health.ts +222 -0
- package/src/core/errors.ts +2 -2
- package/src/core/frontend-runtime/admin-notify.ts +1 -1
- package/src/core/frontend-runtime/alerts.ts +130 -0
- package/src/core/mcp-hub/children.ts +78 -29
- package/src/core/mcp-hub/index.ts +21 -18
- package/src/core/mcp-hub/proxy-server.ts +8 -4
- package/src/core/mcp-hub/talon-server.ts +5 -12
- package/src/core/mesh/credentials/store.ts +16 -1
- package/src/core/mesh/devices/service.ts +26 -15
- package/src/core/mesh/devices/teleport.ts +14 -2
- package/src/core/mesh/links/node-binaries.ts +13 -6
- package/src/core/mesh/persist.ts +22 -10
- package/src/core/mesh/transfers/device-files.ts +5 -23
- package/src/core/mesh/transfers/transfers.ts +16 -3
- package/src/core/models/active-model.ts +2 -55
- package/src/core/plugin/actions.ts +19 -20
- package/src/core/plugin/builtins.ts +80 -90
- package/src/core/plugin/index.ts +1 -4
- package/src/core/plugin/loader.ts +25 -33
- package/src/core/plugin/mcp.ts +3 -5
- package/src/core/plugin/registry.ts +19 -35
- package/src/core/plugin/types.ts +2 -5
- package/src/core/prompt/assemble.ts +15 -3
- package/src/core/scripts/lua.ts +6 -2
- package/src/core/tasks/table.ts +8 -2
- package/src/core/tools/bridge.ts +2 -4
- package/src/core/tools/chat/cross-send.ts +1 -1
- package/src/core/tools/chat/messaging.ts +1 -1
- package/src/core/tools/index.ts +2 -2
- package/src/core/tools/mcp-env.ts +2 -59
- package/src/core/tools/ops/agents.ts +22 -1
- package/src/core/tools/schemas.ts +4 -9
- package/src/core/vfs/fusefs.ts +0 -5
- package/src/core/vfs/index.ts +9 -2
- package/src/core/vfs/mounts/diagnostics.ts +109 -0
- package/src/core/vfs/mounts/proc.ts +17 -1
- package/src/core/vfs/workspace.ts +7 -3
- package/src/core/weaver/shuttle.ts +8 -1
- package/src/core/weaver/turn-log.ts +320 -0
- package/src/core/weaver/weaver.ts +48 -5
- package/src/frontend/discord/actions/index.ts +8 -1
- package/src/frontend/discord/diagnostics.ts +82 -6
- package/src/frontend/discord/handlers/index.ts +0 -2
- package/src/frontend/discord/middleware.ts +7 -13
- package/src/frontend/discord/runtime.ts +1 -3
- package/src/frontend/health/delivery.ts +115 -0
- package/src/frontend/health/outage.ts +116 -0
- package/src/frontend/native/bridge/routes/chats.ts +3 -5
- package/src/frontend/native/bridge/server.ts +106 -21
- package/src/frontend/native/index.ts +1 -1
- package/src/frontend/native/media/media.ts +5 -1
- package/src/frontend/native/runtime.ts +12 -7
- package/src/frontend/native/surface/handlers.ts +1 -1
- package/src/frontend/native/surface/memory.ts +1 -1
- package/src/frontend/native/surface/models.ts +3 -3
- package/src/frontend/native/surface/settings.ts +20 -8
- package/src/frontend/native/turn/context.ts +6 -8
- package/src/frontend/native/turn/turn-meta.ts +2 -5
- package/src/frontend/native/turn/turn.ts +8 -10
- package/src/frontend/presentation/format.ts +2 -4
- package/src/frontend/presentation/session-status.ts +2 -6
- package/src/frontend/teams/actions.ts +8 -1
- package/src/frontend/teams/graph.ts +0 -1
- package/src/frontend/teams/index.ts +1 -4
- package/src/frontend/teams/poll.ts +40 -2
- package/src/frontend/teams/runtime.ts +14 -5
- package/src/frontend/telegram/actions/index.ts +4 -1
- package/src/frontend/telegram/actions/outgoing-log.ts +70 -0
- package/src/frontend/telegram/actions/send.ts +12 -0
- package/src/frontend/telegram/handlers/context.ts +13 -2
- package/src/frontend/telegram/handlers/delivery.ts +12 -9
- package/src/frontend/telegram/handlers/index.ts +0 -2
- package/src/frontend/telegram/index.ts +35 -9
- package/src/frontend/telegram/polling/poll-health.ts +110 -0
- package/src/frontend/telegram/userbot.ts +147 -41
- package/src/frontend/terminal/builtins/session.ts +2 -2
- package/src/frontend/terminal/index.ts +1 -3
- package/src/frontend/terminal/renderer.ts +2 -18
- package/src/frontend/whatsapp/actions/index.ts +12 -1
- package/src/frontend/whatsapp/actions/messaging.ts +7 -2
- package/src/frontend/whatsapp/connection/connection.ts +29 -9
- package/src/frontend/whatsapp/connection/health.ts +89 -0
- package/src/frontend/whatsapp/connection/identity.ts +4 -4
- package/src/frontend/whatsapp/runtime.ts +11 -5
- package/src/native/blake3.ts +28 -2
- package/src/native/fusefs.ts +23 -5
- package/src/native/registry.ts +1 -1
- package/src/native/warden.ts +33 -5
- package/src/plugins/github/index.ts +0 -1
- package/src/plugins/mempalace/index.ts +9 -3
- package/src/plugins/playwright/index.ts +2 -4
- package/src/plugins/playwright/provision.ts +12 -4
- package/src/storage/chat-settings.ts +6 -1
- package/src/storage/cron.ts +29 -4
- package/src/storage/daily-log.ts +43 -47
- package/src/storage/db.ts +61 -33
- package/src/storage/history.ts +6 -1
- package/src/storage/journal.ts +9 -2
- package/src/storage/kv.ts +19 -6
- package/src/storage/media-index.ts +28 -6
- package/src/storage/repositories/chat-settings-repo.ts +8 -2
- package/src/storage/repositories/sessions-repo.ts +10 -3
- package/src/storage/scripts.ts +24 -13
- package/src/storage/sessions.ts +11 -2
- package/src/storage/skills.ts +21 -2
- package/src/storage/stickers.ts +17 -3
- package/src/storage/triggers.ts +8 -3
- package/src/storage/turn-meta.ts +25 -7
- package/src/util/log.ts +189 -6
- package/src/util/logging/turn-scope.ts +85 -0
- package/src/util/time.ts +3 -3
- package/src/util/watchdog.ts +30 -0
- package/src/core/engine/backend-controller/legacy.ts +0 -111
package/src/cli/logs.ts
CHANGED
|
@@ -1,13 +1,72 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* `talon logs` — pretty-print the last lines of the JSON log file and then
|
|
3
3
|
* tail it live.
|
|
4
|
+
*
|
|
5
|
+
* Filters (`--errors`, `--component`, `--since`, `--grep`, `--turn`)
|
|
6
|
+
* switch to the shared log reader (core/daemon/log-reader.ts — the same
|
|
7
|
+
* parser behind proc/log, proc/errors and doctor): the backlog is read
|
|
8
|
+
* across rotated generations, and the live tail applies the same filter.
|
|
9
|
+
* `--no-follow` prints the backlog and exits, for scripts and agents.
|
|
4
10
|
*/
|
|
5
11
|
|
|
6
12
|
import pc from "picocolors";
|
|
7
13
|
import { existsSync, readFileSync, watchFile } from "node:fs";
|
|
14
|
+
import type { LogFilter, LogRecord } from "../core/daemon/log-reader.js";
|
|
8
15
|
import { printBanner } from "./config.js";
|
|
9
16
|
import { LOG_FILE } from "./context.js";
|
|
10
17
|
|
|
18
|
+
export type LogsOptions = {
|
|
19
|
+
filter: LogFilter;
|
|
20
|
+
/** Keep tailing after the backlog (default true). */
|
|
21
|
+
follow: boolean;
|
|
22
|
+
/** Backlog size; undefined = 30, or 1000 under `--since`. */
|
|
23
|
+
lines?: number;
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
const LOGS_USAGE =
|
|
27
|
+
"talon logs [--errors] [--component <c>] [--since <30m|2h|1d>] " +
|
|
28
|
+
"[--grep <text>] [--turn <id>] [-n <lines>] [--no-follow]";
|
|
29
|
+
|
|
30
|
+
/** Parse `talon logs` argv; a string is a usage error to print. */
|
|
31
|
+
export async function parseLogsArgs(
|
|
32
|
+
args: string[],
|
|
33
|
+
now: number = Date.now(),
|
|
34
|
+
): Promise<LogsOptions | string> {
|
|
35
|
+
const { parseLogDuration } = await import("../core/daemon/log-reader.js");
|
|
36
|
+
const opts: LogsOptions = { filter: {}, follow: true };
|
|
37
|
+
for (let i = 0; i < args.length; i++) {
|
|
38
|
+
const flag = args[i];
|
|
39
|
+
const value = (): string | undefined => args[++i];
|
|
40
|
+
if (flag === "--errors") opts.filter.minLevel = "warn";
|
|
41
|
+
else if (flag === "--no-follow") opts.follow = false;
|
|
42
|
+
else if (flag === "--component") opts.filter.component = value();
|
|
43
|
+
else if (flag === "--grep") opts.filter.grep = value();
|
|
44
|
+
else if (flag === "--turn") opts.filter.turn = value();
|
|
45
|
+
else if (flag === "--since") {
|
|
46
|
+
const ms = parseLogDuration(value() ?? "");
|
|
47
|
+
if (ms === null) return `--since needs a duration like 30m, 2h, 1d`;
|
|
48
|
+
opts.filter.since = now - ms;
|
|
49
|
+
} else if (flag === "-n" || flag === "--lines") {
|
|
50
|
+
const n = Number(value());
|
|
51
|
+
if (!Number.isInteger(n) || n <= 0) return `${flag} needs a count`;
|
|
52
|
+
opts.lines = n;
|
|
53
|
+
} else return `unknown option: ${flag}\n usage: ${LOGS_USAGE}`;
|
|
54
|
+
if (i >= args.length) return `${flag} needs a value`;
|
|
55
|
+
}
|
|
56
|
+
return opts;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** `parseLogsArgs` then run; usage errors exit non-zero. */
|
|
60
|
+
export async function runLogsCommand(args: string[]): Promise<void> {
|
|
61
|
+
const opts = await parseLogsArgs(args);
|
|
62
|
+
if (typeof opts === "string") {
|
|
63
|
+
console.error(` ${pc.red("✖")} ${opts}`);
|
|
64
|
+
process.exitCode = 1;
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
await tailLogs(opts);
|
|
68
|
+
}
|
|
69
|
+
|
|
11
70
|
const LEVEL_LABELS: Record<number, string> = {
|
|
12
71
|
10: pc.dim("TRC"),
|
|
13
72
|
20: pc.dim("DBG"),
|
|
@@ -17,6 +76,25 @@ const LEVEL_LABELS: Record<number, string> = {
|
|
|
17
76
|
60: pc.bgRed(pc.white("FTL")),
|
|
18
77
|
};
|
|
19
78
|
|
|
79
|
+
const LEVEL_NAME_LABELS: Record<LogRecord["level"], string> = {
|
|
80
|
+
trace: LEVEL_LABELS[10],
|
|
81
|
+
debug: LEVEL_LABELS[20],
|
|
82
|
+
info: LEVEL_LABELS[30],
|
|
83
|
+
warn: LEVEL_LABELS[40],
|
|
84
|
+
error: LEVEL_LABELS[50],
|
|
85
|
+
fatal: LEVEL_LABELS[60],
|
|
86
|
+
};
|
|
87
|
+
|
|
88
|
+
/** A parsed record in the same shape as {@link formatLogLine}. */
|
|
89
|
+
function formatRecord(rec: LogRecord): string {
|
|
90
|
+
const time = pc.dim(new Date(rec.ts).toTimeString().slice(0, 8));
|
|
91
|
+
const comp = pc.cyan((rec.component ?? "?").padEnd(10));
|
|
92
|
+
const turn =
|
|
93
|
+
rec.turn && !rec.msg.includes("turn=") ? ` turn=${rec.turn}` : "";
|
|
94
|
+
const err = rec.err ? pc.red(` (${rec.err})`) : "";
|
|
95
|
+
return ` ${time} ${LEVEL_NAME_LABELS[rec.level]} ${comp} ${rec.msg}${turn}${err}`;
|
|
96
|
+
}
|
|
97
|
+
|
|
20
98
|
function formatLogLine(line: string): string {
|
|
21
99
|
try {
|
|
22
100
|
const obj = JSON.parse(line);
|
|
@@ -31,7 +109,13 @@ function formatLogLine(line: string): string {
|
|
|
31
109
|
}
|
|
32
110
|
}
|
|
33
111
|
|
|
34
|
-
|
|
112
|
+
function hasFilter(filter: LogFilter): boolean {
|
|
113
|
+
return Object.values(filter).some((v) => v !== undefined);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
export async function tailLogs(
|
|
117
|
+
opts: LogsOptions = { filter: {}, follow: true },
|
|
118
|
+
): Promise<void> {
|
|
35
119
|
printBanner();
|
|
36
120
|
if (!existsSync(LOG_FILE)) {
|
|
37
121
|
console.log(
|
|
@@ -39,18 +123,73 @@ export async function tailLogs(): Promise<void> {
|
|
|
39
123
|
);
|
|
40
124
|
return;
|
|
41
125
|
}
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
126
|
+
if (hasFilter(opts.filter)) return tailFiltered(opts);
|
|
127
|
+
if (opts.follow) {
|
|
128
|
+
console.log(
|
|
129
|
+
` ${pc.dim("Tailing")} ${pc.dim(LOG_FILE)}\n ${pc.dim("Press Ctrl+C to stop")}\n`,
|
|
130
|
+
);
|
|
131
|
+
}
|
|
45
132
|
const content = readFileSync(LOG_FILE, "utf-8");
|
|
46
133
|
const lines = content.trim().split("\n");
|
|
47
|
-
for (const line of lines.slice(-30))
|
|
48
|
-
|
|
49
|
-
|
|
134
|
+
for (const line of lines.slice(-(opts.lines ?? 30)))
|
|
135
|
+
console.log(formatLogLine(line));
|
|
136
|
+
if (!opts.follow) return;
|
|
137
|
+
await followLog(lines.length, (line) => console.log(formatLogLine(line)));
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/** The filtered path: backlog across generations, then a filtered tail. */
|
|
141
|
+
async function tailFiltered(opts: LogsOptions): Promise<void> {
|
|
142
|
+
const { readLogRecords, parseLogLine, matchesLogFilter } =
|
|
143
|
+
await import("../core/daemon/log-reader.js");
|
|
144
|
+
const limit = opts.lines ?? (opts.filter.since !== undefined ? 1000 : 30);
|
|
145
|
+
const backlog = readLogRecords(LOG_FILE, {
|
|
146
|
+
limit,
|
|
147
|
+
filter: opts.filter,
|
|
148
|
+
// The CLI owns its process: read whole generations, not a window.
|
|
149
|
+
maxBytesPerFile: 64 * 1024 * 1024,
|
|
150
|
+
});
|
|
151
|
+
const desc = describeFilter(opts.filter);
|
|
152
|
+
console.log(
|
|
153
|
+
` ${pc.dim(`${backlog.length} matching entr${backlog.length === 1 ? "y" : "ies"} (${desc})`)}` +
|
|
154
|
+
(opts.follow ? `\n ${pc.dim("Following — Ctrl+C to stop")}` : "") +
|
|
155
|
+
"\n",
|
|
156
|
+
);
|
|
157
|
+
for (const rec of backlog) console.log(formatRecord(rec));
|
|
158
|
+
if (!opts.follow) return;
|
|
159
|
+
const start = readFileSync(LOG_FILE, "utf-8").trim().split("\n").length;
|
|
160
|
+
await followLog(start, (line) => {
|
|
161
|
+
const rec = parseLogLine(line);
|
|
162
|
+
if (rec && matchesLogFilter(rec, opts.filter))
|
|
163
|
+
console.log(formatRecord(rec));
|
|
164
|
+
});
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
function describeFilter(filter: LogFilter): string {
|
|
168
|
+
const parts: string[] = [];
|
|
169
|
+
if (filter.minLevel) parts.push(`>=${filter.minLevel}`);
|
|
170
|
+
if (filter.component) parts.push(`component=${filter.component}`);
|
|
171
|
+
if (filter.since !== undefined)
|
|
172
|
+
parts.push(`since ${new Date(filter.since).toTimeString().slice(0, 8)}`);
|
|
173
|
+
if (filter.grep) parts.push(`grep "${filter.grep}"`);
|
|
174
|
+
if (filter.turn) parts.push(`turn=${filter.turn}`);
|
|
175
|
+
return parts.join(", ");
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/** Print each line appended to the log from line `from` on, forever. */
|
|
179
|
+
async function followLog(
|
|
180
|
+
from: number,
|
|
181
|
+
onLine: (line: string) => void,
|
|
182
|
+
): Promise<void> {
|
|
183
|
+
let lastSize = from;
|
|
184
|
+
watchFile(LOG_FILE, { interval: 500 }, (curr, prev) => {
|
|
185
|
+
// A rotation (daemon start → talon.log.old, or the sink's runtime
|
|
186
|
+
// shift to talon.log.1) leaves a new file under the name, and counting
|
|
187
|
+
// its lines against the old one's would stay silent until it outgrew
|
|
188
|
+
// the file it replaced.
|
|
189
|
+
if (curr.ino !== prev.ino || curr.size < prev.size) lastSize = 0;
|
|
50
190
|
try {
|
|
51
191
|
const nl = readFileSync(LOG_FILE, "utf-8").trim().split("\n");
|
|
52
|
-
for (let i = lastSize; i < nl.length; i++)
|
|
53
|
-
console.log(formatLogLine(nl[i]));
|
|
192
|
+
for (let i = lastSize; i < nl.length; i++) onLine(nl[i]);
|
|
54
193
|
lastSize = nl.length;
|
|
55
194
|
} catch {
|
|
56
195
|
/* ignore */
|
package/src/cli/setup.ts
CHANGED
|
@@ -26,11 +26,9 @@ const trimmedOrUndefined = (raw: string) => raw.trim() || undefined;
|
|
|
26
26
|
/**
|
|
27
27
|
* Await a clack prompt; on Esc/Ctrl-C say so and leave the wizard.
|
|
28
28
|
*
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
* ever resolves a symbol to mean "cancelled", so narrowing on `typeof`
|
|
33
|
-
* here removes both.
|
|
29
|
+
* clack only ever resolves a symbol to mean "cancelled", so narrowing on
|
|
30
|
+
* `typeof` replaces a per-prompt `isCancel` guard and the `as` cast that
|
|
31
|
+
* `@clack/core`'s own unique-symbol narrowing would still need.
|
|
34
32
|
*
|
|
35
33
|
* The answer type is subtracted with `Exclude` rather than inferred from a
|
|
36
34
|
* `Promise<T | symbol>` parameter: clack 1.8.1 retyped `CANCEL_SYMBOL` as a
|
|
@@ -667,12 +665,12 @@ function telegramAccess(
|
|
|
667
665
|
*
|
|
668
666
|
* Extracted from `runSetup` so the merge is testable without driving the
|
|
669
667
|
* prompts — the behaviour that matters here is what it *doesn't* touch.
|
|
670
|
-
* The wizard models roughly half of ~/.talon/config.json
|
|
671
|
-
*
|
|
672
|
-
*
|
|
673
|
-
*
|
|
674
|
-
*
|
|
675
|
-
* `
|
|
668
|
+
* The wizard models roughly half of ~/.talon/config.json. Spreading
|
|
669
|
+
* `existing` first keeps every key it does not model (whatsapp, native,
|
|
670
|
+
* memory, github, heartbeat/dream, allowlists, plugin blocks…) — building
|
|
671
|
+
* from the named fields alone silently deletes them. Fields below still
|
|
672
|
+
* override, and an explicit `undefined` still deletes, because
|
|
673
|
+
* `saveConfig` strips undefined.
|
|
676
674
|
*/
|
|
677
675
|
export function buildSetupConfig(
|
|
678
676
|
existing: Config,
|
package/src/cli/status.ts
CHANGED
|
@@ -15,6 +15,29 @@ function formatUptime(seconds: number): string {
|
|
|
15
15
|
return `${Math.floor(seconds / 3600)}h ${Math.floor((seconds % 3600) / 60)}m`;
|
|
16
16
|
}
|
|
17
17
|
|
|
18
|
+
type HealthAlert = { key: string; severity: string; message: string };
|
|
19
|
+
|
|
20
|
+
/** The daemon's active alerts from its /health body (older daemons: none). */
|
|
21
|
+
function healthAlerts(health: Record<string, unknown>): HealthAlert[] {
|
|
22
|
+
const raw = health.alerts;
|
|
23
|
+
if (!Array.isArray(raw)) return [];
|
|
24
|
+
return raw.filter(
|
|
25
|
+
(a): a is HealthAlert =>
|
|
26
|
+
typeof a === "object" &&
|
|
27
|
+
a !== null &&
|
|
28
|
+
typeof a.key === "string" &&
|
|
29
|
+
typeof a.message === "string",
|
|
30
|
+
);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** One line per active alert, coloured by severity. Shared with doctor. */
|
|
34
|
+
export function formatAlertLines(health: Record<string, unknown>): string[] {
|
|
35
|
+
return healthAlerts(health).map((a) => {
|
|
36
|
+
const dot = a.severity === "warn" ? pc.yellow("●") : pc.red("●");
|
|
37
|
+
return ` ${dot} ${pc.bold(a.key)} ${a.message}`;
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
|
|
18
41
|
export async function showStatus(): Promise<void> {
|
|
19
42
|
printBanner();
|
|
20
43
|
const instance = await findRunningInstance();
|
|
@@ -38,6 +61,12 @@ export async function showStatus(): Promise<void> {
|
|
|
38
61
|
console.log(` ${pc.dim("Queue")} ${h.queue} pending`);
|
|
39
62
|
console.log(` ${pc.dim("Errors")} ${h.errors}`);
|
|
40
63
|
console.log(` ${pc.dim("Last active")} ${h.lastActivity}\n`);
|
|
64
|
+
const alerts = formatAlertLines(h);
|
|
65
|
+
if (alerts.length > 0) {
|
|
66
|
+
console.log(` ${pc.bold("Active alerts")}\n`);
|
|
67
|
+
for (const line of alerts) console.log(line);
|
|
68
|
+
console.log();
|
|
69
|
+
}
|
|
41
70
|
return;
|
|
42
71
|
}
|
|
43
72
|
|
|
@@ -41,15 +41,16 @@ error | completed
|
|
|
41
41
|
```
|
|
42
42
|
|
|
43
43
|
Plus `UsageSnapshot`, `AgentError` (with `AgentErrorKind`),
|
|
44
|
-
`AgentResult
|
|
45
|
-
`
|
|
44
|
+
`AgentResult`, and `AgentRunError` — what the dispatcher rethrows an
|
|
45
|
+
`error` terminator as. Helpers: `emptyUsage`, `toolInputToRecord`,
|
|
46
|
+
`classifiedToAgentError`.
|
|
46
47
|
|
|
47
48
|
### `model-ref.ts`
|
|
48
49
|
|
|
49
50
|
Typed model identity. `ModelRef = { backend: BackendId, id,
|
|
50
51
|
displayName, ... }`. Owns the `BACKEND_IDS` literal — single source of
|
|
51
52
|
truth for which backends the typed union can route to. Helpers:
|
|
52
|
-
`isBackendId`, `
|
|
53
|
+
`isBackendId`, `makeBareModelRef`.
|
|
53
54
|
|
|
54
55
|
### `capabilities.ts`
|
|
55
56
|
|
|
@@ -101,16 +102,6 @@ Backend contract assertions any conforming `Backend` must pass:
|
|
|
101
102
|
|
|
102
103
|
Each throws `ContractViolation` with a descriptive message.
|
|
103
104
|
|
|
104
|
-
### `event-bridge.ts`
|
|
105
|
-
|
|
106
|
-
The bridge between the canonical `AgentEvent` stream and the
|
|
107
|
-
callback-shaped consumer contract the dispatcher uses upstream of the
|
|
108
|
-
backend. `pipeEventsToCallbacks(stream, callbacks)` consumes an
|
|
109
|
-
`AgentEvent` stream and invokes the supplied callbacks (`onStreamDelta`
|
|
110
|
-
/ `onTextBlock` / `onToolUse`), returns the final `AgentResult`, and
|
|
111
|
-
throws `BridgedAgentError` carrying the original `AgentError` if the
|
|
112
|
-
stream terminates with an error event.
|
|
113
|
-
|
|
114
105
|
## Migration cookbook
|
|
115
106
|
|
|
116
107
|
### Adding a new backend
|
|
@@ -164,11 +155,6 @@ returns `{ model: string | null, ref: ModelRef | null, source }`:
|
|
|
164
155
|
- `source` carries the chain step that produced the model, useful
|
|
165
156
|
for toast wording and stale-slot cleanup.
|
|
166
157
|
|
|
167
|
-
Convenience wrappers:
|
|
168
|
-
|
|
169
|
-
- `getActiveModelForChat(...)` → `model`
|
|
170
|
-
- `getActiveModelRefForChat(...)` → `ref`
|
|
171
|
-
|
|
172
158
|
### Adding a new store
|
|
173
159
|
|
|
174
160
|
New structured state goes in the SQLite layer — see the layering doc
|
|
@@ -178,7 +164,7 @@ in `src/storage/db.ts` (sql/<store>.sql → repositories/<store>-repo.ts
|
|
|
178
164
|
## Invariants
|
|
179
165
|
|
|
180
166
|
- `BACKEND_IDS` in `model-ref.ts` is the source of truth for the typed
|
|
181
|
-
union. `
|
|
167
|
+
union. `core/config/index.ts` zod enums are wired to the same literal.
|
|
182
168
|
- `AgentEvent.type` is the ONLY discrimination mechanism. No class
|
|
183
169
|
hierarchy, no `instanceof` checks.
|
|
184
170
|
- Every `ChatBackend.runChatTurn` stream terminates with `completed`
|
|
@@ -5,12 +5,12 @@
|
|
|
5
5
|
* Every backend (Claude SDK, Codex, Kilo, OpenCode, OpenAI Agents)
|
|
6
6
|
* translates its SDK's native event stream into `AgentEvent`s. Core
|
|
7
7
|
* renderers (Telegram dispatch, terminal output, heartbeat log,
|
|
8
|
-
* dream log, `/status`, tests) consume `AgentEvent`s. Backends
|
|
9
|
-
*
|
|
8
|
+
* dream log, `/status`, tests) consume `AgentEvent`s. Backends don't
|
|
9
|
+
* render markdown logs themselves and core never parses
|
|
10
10
|
* backend-specific output.
|
|
11
11
|
*
|
|
12
12
|
* The shared wrapper `backend/runtime/turn/handler-to-events.ts` converts
|
|
13
|
-
* each backend's
|
|
13
|
+
* each backend's callback-driven `handleMessage` into the
|
|
14
14
|
* canonical sequence: `run_started → text_delta* →
|
|
15
15
|
* assistant_message* → tool_call* → usage → completed`. Backends
|
|
16
16
|
* with richer SDKs can emit events directly without the wrapper.
|
|
@@ -146,27 +146,6 @@ export type AgentEvent =
|
|
|
146
146
|
| { type: "error"; error: AgentError }
|
|
147
147
|
| { type: "completed"; result?: AgentResult };
|
|
148
148
|
|
|
149
|
-
/**
|
|
150
|
-
* Type-narrowing helper. Saves callers from writing
|
|
151
|
-
* `event.type === "completed"` in two places when they need both the
|
|
152
|
-
* narrowing and a boolean expression.
|
|
153
|
-
*/
|
|
154
|
-
export function isAgentEventOf<K extends AgentEvent["type"]>(
|
|
155
|
-
event: AgentEvent,
|
|
156
|
-
kind: K,
|
|
157
|
-
): event is Extract<AgentEvent, { type: K }> {
|
|
158
|
-
return event.type === kind;
|
|
159
|
-
}
|
|
160
|
-
|
|
161
|
-
/**
|
|
162
|
-
* Whether this event is a stream terminator — `completed` (success)
|
|
163
|
-
* or `error` (failure). Useful for stream consumers that want to
|
|
164
|
-
* release a typing indicator or close a log section on either.
|
|
165
|
-
*/
|
|
166
|
-
export function isAgentRunTerminator(event: AgentEvent): boolean {
|
|
167
|
-
return event.type === "completed" || event.type === "error";
|
|
168
|
-
}
|
|
169
|
-
|
|
170
149
|
/**
|
|
171
150
|
* Error thrown when an `AgentEvent` stream terminates with an `error`
|
|
172
151
|
* event. The dispatcher consumes the canonical event stream directly
|
|
@@ -234,21 +213,6 @@ export function emptyUsage(): UsageSnapshot {
|
|
|
234
213
|
};
|
|
235
214
|
}
|
|
236
215
|
|
|
237
|
-
/**
|
|
238
|
-
* Accumulate two usage snapshots. Pure — caller passes both, gets a
|
|
239
|
-
* new object back. Used by stream consumers that aggregate per-event
|
|
240
|
-
* usage into a final figure for `/status`.
|
|
241
|
-
*/
|
|
242
|
-
export function addUsage(a: UsageSnapshot, b: UsageSnapshot): UsageSnapshot {
|
|
243
|
-
return {
|
|
244
|
-
inputTokens: a.inputTokens + b.inputTokens,
|
|
245
|
-
outputTokens: a.outputTokens + b.outputTokens,
|
|
246
|
-
cacheRead: a.cacheRead + b.cacheRead,
|
|
247
|
-
cacheWrite: a.cacheWrite + b.cacheWrite,
|
|
248
|
-
modelId: b.modelId ?? a.modelId,
|
|
249
|
-
};
|
|
250
|
-
}
|
|
251
|
-
|
|
252
216
|
/**
|
|
253
217
|
* The `core/errors.ts` reasons that map to a specific `AgentErrorKind`.
|
|
254
218
|
* Anything not listed collapses to `unknown`. `ErrorReason` is
|
|
@@ -102,14 +102,6 @@ export interface ModelRef {
|
|
|
102
102
|
unavailableReason?: string;
|
|
103
103
|
}
|
|
104
104
|
|
|
105
|
-
/**
|
|
106
|
-
* Equality on identity only — two refs are the same run if they
|
|
107
|
-
* point at the same backend + id.
|
|
108
|
-
*/
|
|
109
|
-
export function sameModelRef(a: ModelRef, b: ModelRef): boolean {
|
|
110
|
-
return a.backend === b.backend && a.id === b.id;
|
|
111
|
-
}
|
|
112
|
-
|
|
113
105
|
/**
|
|
114
106
|
* Bare-minimum constructor for tests and adapters that don't yet
|
|
115
107
|
* carry rich metadata. Real catalog code should populate the
|
|
@@ -25,6 +25,7 @@ import type { TaskUsage } from "../tasks/types.js";
|
|
|
25
25
|
import type { AgentSettledEvent, AgentSpawnedEvent } from "../bus/events.js";
|
|
26
26
|
import type { ReasoningEffortLevel } from "../types.js";
|
|
27
27
|
import { bus } from "../bus/index.js";
|
|
28
|
+
import { logWarn } from "../../util/log.js";
|
|
28
29
|
|
|
29
30
|
/** Settled agents kept for status queries after they leave the live map. */
|
|
30
31
|
const DEFAULT_HISTORY_LIMIT = 100;
|
|
@@ -87,6 +88,23 @@ interface LiveAgent {
|
|
|
87
88
|
killRequested: boolean;
|
|
88
89
|
}
|
|
89
90
|
|
|
91
|
+
/**
|
|
92
|
+
* Whether two agents were spawned by the same parent.
|
|
93
|
+
*
|
|
94
|
+
* Compared structurally rather than by reference: records come from separate
|
|
95
|
+
* snapshots, so the parent objects are equal in value and never identical.
|
|
96
|
+
* The `kind` check is what stops a chat-parented agent matching an
|
|
97
|
+
* agent-parented one whose id happens to equal a chat key.
|
|
98
|
+
*/
|
|
99
|
+
function sameParent(a: AgentParent, b: AgentParent): boolean {
|
|
100
|
+
if (a.kind !== b.kind) return false;
|
|
101
|
+
return a.kind === "chat" && b.kind === "chat"
|
|
102
|
+
? a.chatId === b.chatId
|
|
103
|
+
: a.kind === "agent" && b.kind === "agent"
|
|
104
|
+
? a.agentId === b.agentId
|
|
105
|
+
: false;
|
|
106
|
+
}
|
|
107
|
+
|
|
90
108
|
function snapshot(entry: LiveAgent): AgentRecord {
|
|
91
109
|
return {
|
|
92
110
|
...entry.record,
|
|
@@ -279,8 +297,13 @@ export class AgentRegistry {
|
|
|
279
297
|
entry.killRequested = true;
|
|
280
298
|
try {
|
|
281
299
|
entry.abort?.abort();
|
|
282
|
-
} catch {
|
|
283
|
-
// An abort hook must not be able to break the kill path
|
|
300
|
+
} catch (err) {
|
|
301
|
+
// An abort hook must not be able to break the kill path — but a
|
|
302
|
+
// throwing one may leave the agent running, so say so.
|
|
303
|
+
logWarn(
|
|
304
|
+
"agents",
|
|
305
|
+
`Abort hook threw agent=${id}: ${err instanceof Error ? err.message : String(err)}`,
|
|
306
|
+
);
|
|
284
307
|
}
|
|
285
308
|
}
|
|
286
309
|
return true;
|
|
@@ -351,6 +374,30 @@ export class AgentRegistry {
|
|
|
351
374
|
return children.filter((child) => this.live.has(child));
|
|
352
375
|
}
|
|
353
376
|
|
|
377
|
+
/**
|
|
378
|
+
* An agent's live **peers** — the other agents sharing its parent.
|
|
379
|
+
*
|
|
380
|
+
* This is the addressing scope for agent-to-agent messaging, and it is
|
|
381
|
+
* deliberately narrower than "everything under the same chat". A swarm is
|
|
382
|
+
* a set of siblings spawned for one job, so siblings are the useful unit;
|
|
383
|
+
* widening to the whole chat tree would let an agent reach a cousin from an
|
|
384
|
+
* unrelated piece of work it knows nothing about.
|
|
385
|
+
*
|
|
386
|
+
* Live only: a settled agent has no mailbox to deliver into, and offering
|
|
387
|
+
* it as a peer would only produce a delivery failure one call later.
|
|
388
|
+
*/
|
|
389
|
+
peersOf(id: string): AgentRecord[] {
|
|
390
|
+
const self = this.get(id);
|
|
391
|
+
if (!self) return [];
|
|
392
|
+
const peers: AgentRecord[] = [];
|
|
393
|
+
for (const entry of this.live.values()) {
|
|
394
|
+
const record = snapshot(entry);
|
|
395
|
+
if (record.id === id) continue;
|
|
396
|
+
if (sameParent(record.parent, self.parent)) peers.push(record);
|
|
397
|
+
}
|
|
398
|
+
return peers.sort((a, b) => a.createdAt - b.createdAt);
|
|
399
|
+
}
|
|
400
|
+
|
|
354
401
|
/** Live agents plus the bounded settled ring, oldest first. */
|
|
355
402
|
list(): AgentRecord[] {
|
|
356
403
|
const records = [...this.history];
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
*/
|
|
11
11
|
|
|
12
12
|
import { notifyAdmin } from "../frontend-runtime/admin-notify.js";
|
|
13
|
+
import { logWarn } from "../../util/log.js";
|
|
13
14
|
import {
|
|
14
15
|
daysUntil,
|
|
15
16
|
PROVIDER_LABELS,
|
|
@@ -75,7 +76,14 @@ export function resetAuthExpiryAnnouncements(): void {
|
|
|
75
76
|
|
|
76
77
|
export function startAuthExpiryMonitor(): () => void {
|
|
77
78
|
const tick = (): void => {
|
|
78
|
-
void runAuthExpiryCheck().catch(() => {
|
|
79
|
+
void runAuthExpiryCheck().catch((err: unknown) => {
|
|
80
|
+
// The alert is already marked announced, so a failed send is not
|
|
81
|
+
// retried until the status changes — the log is its only trace.
|
|
82
|
+
logWarn(
|
|
83
|
+
"notify",
|
|
84
|
+
`auth expiry check failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
85
|
+
);
|
|
86
|
+
});
|
|
79
87
|
};
|
|
80
88
|
// First check shortly after boot so a lapsed login is surfaced right away.
|
|
81
89
|
const first = setTimeout(tick, 30_000);
|
|
@@ -160,7 +160,15 @@ export function startLogin(
|
|
|
160
160
|
new Error(outcome.ok ? "no prompt" : outcome.detail || outcome.reason),
|
|
161
161
|
);
|
|
162
162
|
if (tmpHome)
|
|
163
|
-
await rm(tmpHome, { recursive: true, force: true }).catch(
|
|
163
|
+
await rm(tmpHome, { recursive: true, force: true }).catch(
|
|
164
|
+
(err: unknown) =>
|
|
165
|
+
// The scratch HOME can hold a fresh credential copy — a leftover
|
|
166
|
+
// is worth knowing about.
|
|
167
|
+
logWarn(
|
|
168
|
+
"notify",
|
|
169
|
+
`${provider} login cleanup failed dir=${tmpHome}: ${err instanceof Error ? err.message : String(err)}`,
|
|
170
|
+
),
|
|
171
|
+
);
|
|
164
172
|
resolveDone(outcome);
|
|
165
173
|
};
|
|
166
174
|
|
package/src/core/auth/status.ts
CHANGED
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
import { readFile, stat } from "node:fs/promises";
|
|
16
16
|
import { homedir } from "node:os";
|
|
17
17
|
import { join } from "node:path";
|
|
18
|
+
import { logWarn } from "../../util/log.js";
|
|
18
19
|
|
|
19
20
|
export type AuthProvider = "claude" | "codex";
|
|
20
21
|
|
|
@@ -64,6 +65,19 @@ export function clearProviderExpired(provider: AuthProvider): void {
|
|
|
64
65
|
reportedExpired.delete(provider);
|
|
65
66
|
}
|
|
66
67
|
|
|
68
|
+
/**
|
|
69
|
+
* A credentials file that exists but isn't JSON reads as "not signed
|
|
70
|
+
* in" — indistinguishable from a missing login unless logged. Nothing
|
|
71
|
+
* from the file goes into the line: JSON.parse's message quotes the
|
|
72
|
+
* input, and the input is a token.
|
|
73
|
+
*/
|
|
74
|
+
function warnUnparseable(provider: AuthProvider): void {
|
|
75
|
+
logWarn(
|
|
76
|
+
"notify",
|
|
77
|
+
`${provider} credentials file is not valid JSON — reporting not signed in`,
|
|
78
|
+
);
|
|
79
|
+
}
|
|
80
|
+
|
|
67
81
|
export function parseClaudeCredentials(raw: string): ProviderAuthStatus {
|
|
68
82
|
const base: ProviderAuthStatus = {
|
|
69
83
|
provider: "claude",
|
|
@@ -81,6 +95,7 @@ export function parseClaudeCredentials(raw: string): ProviderAuthStatus {
|
|
|
81
95
|
try {
|
|
82
96
|
parsed = JSON.parse(raw);
|
|
83
97
|
} catch {
|
|
98
|
+
warnUnparseable("claude");
|
|
84
99
|
return base;
|
|
85
100
|
}
|
|
86
101
|
const oauth = parsed.claudeAiOauth;
|
|
@@ -115,6 +130,7 @@ export function parseCodexAuth(raw: string): ProviderAuthStatus {
|
|
|
115
130
|
try {
|
|
116
131
|
parsed = JSON.parse(raw);
|
|
117
132
|
} catch {
|
|
133
|
+
warnUnparseable("codex");
|
|
118
134
|
return base;
|
|
119
135
|
}
|
|
120
136
|
const apiKey =
|
|
@@ -136,11 +152,23 @@ export function parseCodexAuth(raw: string): ProviderAuthStatus {
|
|
|
136
152
|
};
|
|
137
153
|
}
|
|
138
154
|
|
|
139
|
-
async function readOrEmpty(
|
|
155
|
+
async function readOrEmpty(
|
|
156
|
+
provider: AuthProvider,
|
|
157
|
+
path: string,
|
|
158
|
+
): Promise<string | undefined> {
|
|
140
159
|
try {
|
|
141
160
|
await stat(path);
|
|
142
161
|
return await readFile(path, "utf8");
|
|
143
|
-
} catch {
|
|
162
|
+
} catch (err) {
|
|
163
|
+
// Missing is the normal signed-out case; anything else (EACCES,
|
|
164
|
+
// EISDIR) is a fault that would otherwise read as signed out too.
|
|
165
|
+
const code = (err as NodeJS.ErrnoException).code;
|
|
166
|
+
if (code !== "ENOENT") {
|
|
167
|
+
logWarn(
|
|
168
|
+
"notify",
|
|
169
|
+
`${provider} credentials unreadable code=${code ?? "?"} — reporting not signed in`,
|
|
170
|
+
);
|
|
171
|
+
}
|
|
144
172
|
return undefined;
|
|
145
173
|
}
|
|
146
174
|
}
|
|
@@ -151,7 +179,7 @@ export async function readProviderStatus(
|
|
|
151
179
|
): Promise<ProviderAuthStatus> {
|
|
152
180
|
const path =
|
|
153
181
|
provider === "claude" ? claudeCredentialsPath(env) : codexAuthPath(env);
|
|
154
|
-
const raw = await readOrEmpty(path);
|
|
182
|
+
const raw = await readOrEmpty(provider, path);
|
|
155
183
|
const status =
|
|
156
184
|
raw === undefined
|
|
157
185
|
? { provider, loggedIn: false, expired: true }
|
|
@@ -34,6 +34,8 @@ import {
|
|
|
34
34
|
} from "../../../storage/cron.js";
|
|
35
35
|
import { appendDailyLog } from "../../../storage/daily-log.js";
|
|
36
36
|
import { log, logError, logWarn } from "../../../util/log.js";
|
|
37
|
+
import { raiseAlert, resolveAlert } from "../../frontend-runtime/alerts.js";
|
|
38
|
+
import { faultText } from "../../engine/fault-text.js";
|
|
37
39
|
import { numericChatIdFor } from "../../frontend-runtime/chat-id.js";
|
|
38
40
|
import {
|
|
39
41
|
chooseBackend,
|
|
@@ -156,8 +158,12 @@ async function runCronTick(): Promise<void> {
|
|
|
156
158
|
pruneJobHealth(new Set(jobs.map((j) => j.id)));
|
|
157
159
|
|
|
158
160
|
let loadShed = false;
|
|
159
|
-
for (const
|
|
160
|
-
|
|
161
|
+
for (const listed of jobs) {
|
|
162
|
+
// Re-read: each run is awaited, so a slow job lets the next tick start
|
|
163
|
+
// and run later jobs before this loop reaches them. The listing's copy
|
|
164
|
+
// would carry the pre-run lastRunAt and fire such a job a second time.
|
|
165
|
+
const job = getCronJob(listed.id);
|
|
166
|
+
if (!job?.enabled) continue;
|
|
161
167
|
// Expiry takes priority over dueness: a job past its end time is disabled
|
|
162
168
|
// and skipped even if this minute would otherwise match.
|
|
163
169
|
if (expireIfPast(job, nowMs)) continue;
|
|
@@ -201,6 +207,10 @@ async function runScheduled(job: CronJob): Promise<void> {
|
|
|
201
207
|
durationMs: Date.now() - startedAt,
|
|
202
208
|
};
|
|
203
209
|
recordJobSuccess(job.id, Date.now(), JOB_HEALTH);
|
|
210
|
+
resolveAlert(
|
|
211
|
+
`cron.job.${job.id}`,
|
|
212
|
+
`Cron job "${job.name}" is running again.`,
|
|
213
|
+
);
|
|
204
214
|
recordCronRun(job.id, outcome);
|
|
205
215
|
appendDailyLog(
|
|
206
216
|
"Cron",
|
|
@@ -223,12 +233,23 @@ async function runScheduled(job: CronJob): Promise<void> {
|
|
|
223
233
|
lastError: err instanceof Error ? err.message : String(err),
|
|
224
234
|
lastDurationMs: Date.now() - startedAt,
|
|
225
235
|
});
|
|
226
|
-
logError(
|
|
236
|
+
logError(
|
|
237
|
+
"cron",
|
|
238
|
+
`Job "${job.name}" [${job.id}] failed chat=${job.chatId} type=${job.type} ms=${Date.now() - startedAt}`,
|
|
239
|
+
err,
|
|
240
|
+
);
|
|
227
241
|
const cooldown = recordJobFailure(job.id, Date.now(), JOB_HEALTH);
|
|
228
242
|
if (cooldown !== null) {
|
|
243
|
+
const mins = Math.round(cooldown / 60_000);
|
|
229
244
|
logWarn(
|
|
230
245
|
"cron",
|
|
231
|
-
`Breaker opened for "${job.name}" [${job.id}] — cooling down ~${
|
|
246
|
+
`Breaker opened for "${job.name}" [${job.id}] — cooling down ~${mins}min`,
|
|
247
|
+
);
|
|
248
|
+
// The breaker opens at JOB_HEALTH.threshold consecutive failures —
|
|
249
|
+
// the job is now paused, which the operator should hear about.
|
|
250
|
+
raiseAlert(
|
|
251
|
+
`cron.job.${job.id}`,
|
|
252
|
+
`Cron job "${job.name}" failed ${JOB_HEALTH.threshold} runs in a row: ${faultText(err)}. Paused for ~${mins} min.`,
|
|
232
253
|
);
|
|
233
254
|
}
|
|
234
255
|
} finally {
|
|
@@ -461,7 +482,6 @@ function isCronDue(job: CronJob, now: Date, windowStartMs: number): boolean {
|
|
|
461
482
|
export const _cronInternals = {
|
|
462
483
|
isDue,
|
|
463
484
|
isCronDue,
|
|
464
|
-
MAX_TICK_LOOKBACK_MS,
|
|
465
485
|
};
|
|
466
486
|
|
|
467
487
|
/**
|