talon-agent 5.18.2 → 5.19.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/package.json +2 -2
- package/prompts/system/agent-brief.md +20 -3
- package/src/app.ts +13 -0
- package/src/backend/claude-sdk/handler.ts +4 -4
- package/src/backend/claude-sdk/mcp-ready.ts +16 -2
- package/src/backend/claude-sdk/one-shot.ts +32 -2
- package/src/backend/claude-sdk/stream.ts +2 -2
- package/src/backend/codex/auth.ts +1 -1
- package/src/backend/codex/handler/message.ts +11 -11
- package/src/backend/codex/init.ts +4 -9
- package/src/backend/codex/mcp-config.ts +1 -2
- package/src/backend/codex/oauth-incompat.ts +8 -4
- package/src/backend/codex/one-shot.ts +1 -1
- package/src/backend/openai-agents/builtins.ts +55 -27
- package/src/backend/openai-agents/factory.ts +3 -3
- package/src/backend/openai-agents/handler/message.ts +3 -5
- package/src/backend/openai-agents/mcp-pool.ts +10 -27
- package/src/backend/remote-server/chat-turn.ts +6 -6
- package/src/backend/remote-server/events.ts +1 -5
- package/src/backend/remote-server/index.ts +0 -1
- package/src/backend/remote-server/messages.ts +3 -7
- package/src/backend/remote-server/one-shot.ts +1 -3
- package/src/backend/remote-server/session-helpers.ts +1 -4
- package/src/backend/remote-server/sse-stream.ts +8 -9
- package/src/backend/runtime/metrics.ts +7 -13
- package/src/backend/runtime/sleep.ts +1 -2
- package/src/backend/runtime/turn/handle-retry.ts +48 -2
- package/src/backend/runtime/turn/handler-to-events.ts +3 -4
- package/src/bootstrap.ts +9 -1
- package/src/cli/doctor.ts +3 -0
- package/src/cli/index.ts +8 -10
- package/src/cli/logs.ts +148 -9
- package/src/cli/setup.ts +9 -11
- package/src/cli/status.ts +29 -0
- package/src/core/agent-runtime/README.md +5 -19
- package/src/core/agent-runtime/events.ts +3 -39
- package/src/core/agent-runtime/model-ref.ts +0 -8
- package/src/core/agents/registry.ts +65 -2
- package/src/core/auth/expiry-monitor.ts +9 -1
- package/src/core/auth/login-flow.ts +9 -1
- package/src/core/auth/status.ts +31 -3
- package/src/core/background/cron/scheduler.ts +25 -5
- package/src/core/background/dream/index.ts +29 -10
- package/src/core/background/failure-backoff.ts +30 -0
- package/src/core/background/heartbeat/agent.ts +2 -27
- package/src/core/background/heartbeat/index.ts +0 -2
- package/src/core/background/heartbeat/scheduler.ts +21 -13
- package/src/core/background/heartbeat/state.ts +10 -2
- package/src/core/background/isolated-agent.ts +6 -2
- package/src/core/background/pulse/pulse.ts +9 -0
- package/src/core/background/triggers/exit.ts +54 -0
- package/src/core/background/triggers/index.ts +1 -3
- package/src/core/background/triggers/resume.ts +2 -4
- package/src/core/backup/archive/tar.ts +14 -4
- package/src/core/backup/plan.ts +1 -0
- package/src/core/backup/restore.ts +21 -13
- package/src/core/backup/scheduler.ts +28 -9
- package/src/core/backup/snapshot.ts +43 -10
- package/src/core/backup/store.ts +7 -19
- package/src/core/backup/targets.ts +69 -17
- package/src/core/config/index.ts +14 -0
- package/src/core/daemon/crash-marker.ts +141 -0
- package/src/core/daemon/crash.ts +9 -2
- package/src/core/daemon/handoff.ts +15 -0
- package/src/core/daemon/health-alerts.ts +297 -0
- package/src/core/daemon/log-reader.ts +289 -0
- package/src/core/doctor/index.ts +18 -2
- package/src/core/doctor/logs.ts +124 -0
- package/src/core/doctor/types.ts +1 -1
- package/src/core/engine/backend-controller/index.ts +1 -13
- package/src/core/engine/backend-router/router.ts +1 -1
- package/src/core/engine/dispatcher.ts +55 -2
- package/src/core/engine/fault-text.ts +40 -0
- package/src/core/engine/gateway-actions/agents/index.ts +3 -2
- package/src/core/engine/gateway-actions/agents/report.ts +62 -0
- package/src/core/engine/gateway-actions/history.ts +2 -4
- package/src/core/engine/gateway-actions/mesh.ts +26 -14
- package/src/core/engine/gateway-actions/native/exec.ts +13 -16
- package/src/core/engine/gateway-actions/native/teleport.ts +1 -1
- package/src/core/engine/gateway.ts +60 -1
- package/src/core/engine/turn-health.ts +222 -0
- package/src/core/errors.ts +2 -2
- package/src/core/frontend-runtime/admin-notify.ts +1 -1
- package/src/core/frontend-runtime/alerts.ts +130 -0
- package/src/core/mcp-hub/children.ts +78 -29
- package/src/core/mcp-hub/index.ts +21 -18
- package/src/core/mcp-hub/proxy-server.ts +8 -4
- package/src/core/mcp-hub/talon-server.ts +5 -12
- package/src/core/mesh/credentials/store.ts +16 -1
- package/src/core/mesh/devices/registry.ts +1 -17
- package/src/core/mesh/devices/service.ts +49 -17
- package/src/core/mesh/devices/teleport.ts +14 -2
- package/src/core/mesh/links/node-binaries.ts +13 -6
- package/src/core/mesh/persist.ts +22 -10
- package/src/core/mesh/transfers/device-files.ts +5 -23
- package/src/core/mesh/transfers/transfers.ts +16 -3
- package/src/core/models/active-model.ts +2 -55
- package/src/core/plugin/actions.ts +19 -20
- package/src/core/plugin/builtins.ts +80 -90
- package/src/core/plugin/index.ts +1 -4
- package/src/core/plugin/loader.ts +25 -33
- package/src/core/plugin/mcp.ts +3 -5
- package/src/core/plugin/registry.ts +19 -35
- package/src/core/plugin/types.ts +2 -5
- package/src/core/prompt/assemble.ts +15 -3
- package/src/core/scripts/lua.ts +6 -2
- package/src/core/tasks/table.ts +8 -2
- package/src/core/tools/bridge.ts +2 -4
- package/src/core/tools/chat/cross-send.ts +1 -1
- package/src/core/tools/chat/messaging.ts +1 -1
- package/src/core/tools/index.ts +2 -2
- package/src/core/tools/mcp-env.ts +2 -59
- package/src/core/tools/ops/agents.ts +22 -1
- package/src/core/tools/schemas.ts +4 -9
- package/src/core/vfs/fusefs.ts +0 -5
- package/src/core/vfs/index.ts +9 -2
- package/src/core/vfs/mounts/diagnostics.ts +109 -0
- package/src/core/vfs/mounts/proc.ts +17 -1
- package/src/core/vfs/workspace.ts +7 -3
- package/src/core/weaver/shuttle.ts +8 -1
- package/src/core/weaver/turn-log.ts +320 -0
- package/src/core/weaver/weaver.ts +48 -5
- package/src/frontend/discord/actions/index.ts +8 -1
- package/src/frontend/discord/diagnostics.ts +82 -6
- package/src/frontend/discord/handlers/index.ts +0 -2
- package/src/frontend/discord/middleware.ts +7 -13
- package/src/frontend/discord/runtime.ts +1 -3
- package/src/frontend/health/delivery.ts +115 -0
- package/src/frontend/health/outage.ts +116 -0
- package/src/frontend/native/bridge/routes/chats.ts +3 -5
- package/src/frontend/native/bridge/server.ts +106 -21
- package/src/frontend/native/index.ts +1 -1
- package/src/frontend/native/media/media.ts +5 -1
- package/src/frontend/native/runtime.ts +12 -7
- package/src/frontend/native/surface/handlers.ts +1 -1
- package/src/frontend/native/surface/memory.ts +1 -1
- package/src/frontend/native/surface/models.ts +3 -3
- package/src/frontend/native/surface/settings.ts +20 -8
- package/src/frontend/native/turn/context.ts +6 -8
- package/src/frontend/native/turn/turn-meta.ts +2 -5
- package/src/frontend/native/turn/turn.ts +8 -10
- package/src/frontend/presentation/format.ts +2 -4
- package/src/frontend/presentation/session-status.ts +2 -6
- package/src/frontend/teams/actions.ts +8 -1
- package/src/frontend/teams/graph.ts +0 -1
- package/src/frontend/teams/index.ts +1 -4
- package/src/frontend/teams/poll.ts +40 -2
- package/src/frontend/teams/runtime.ts +14 -5
- package/src/frontend/telegram/actions/index.ts +4 -1
- package/src/frontend/telegram/actions/send.ts +8 -0
- package/src/frontend/telegram/handlers/context.ts +13 -2
- package/src/frontend/telegram/handlers/delivery.ts +12 -9
- package/src/frontend/telegram/handlers/index.ts +0 -2
- package/src/frontend/telegram/index.ts +35 -9
- package/src/frontend/telegram/polling/poll-health.ts +110 -0
- package/src/frontend/telegram/userbot.ts +100 -36
- package/src/frontend/terminal/builtins/session.ts +2 -2
- package/src/frontend/terminal/index.ts +1 -3
- package/src/frontend/terminal/renderer.ts +2 -18
- package/src/frontend/whatsapp/actions/index.ts +12 -1
- package/src/frontend/whatsapp/actions/messaging.ts +7 -2
- package/src/frontend/whatsapp/connection/connection.ts +29 -9
- package/src/frontend/whatsapp/connection/health.ts +89 -0
- package/src/frontend/whatsapp/connection/identity.ts +4 -4
- package/src/frontend/whatsapp/runtime.ts +11 -5
- package/src/native/blake3.ts +28 -2
- package/src/native/fusefs.ts +23 -5
- package/src/native/registry.ts +1 -1
- package/src/native/warden.ts +33 -5
- package/src/plugins/github/index.ts +0 -1
- package/src/plugins/mempalace/index.ts +9 -3
- package/src/plugins/playwright/index.ts +2 -4
- package/src/plugins/playwright/provision.ts +12 -4
- package/src/storage/chat-settings.ts +6 -1
- package/src/storage/cron.ts +29 -4
- package/src/storage/daily-log.ts +43 -47
- package/src/storage/db.ts +61 -33
- package/src/storage/history.ts +6 -1
- package/src/storage/journal.ts +9 -2
- package/src/storage/kv.ts +19 -6
- package/src/storage/media-index.ts +28 -6
- package/src/storage/repositories/chat-settings-repo.ts +8 -2
- package/src/storage/repositories/sessions-repo.ts +10 -3
- package/src/storage/scripts.ts +24 -13
- package/src/storage/sessions.ts +11 -2
- package/src/storage/skills.ts +21 -2
- package/src/storage/stickers.ts +17 -3
- package/src/storage/triggers.ts +8 -3
- package/src/storage/turn-meta.ts +25 -7
- package/src/util/log.ts +189 -6
- package/src/util/logging/turn-scope.ts +85 -0
- package/src/util/time.ts +3 -3
- package/src/util/watchdog.ts +30 -0
- package/src/core/engine/backend-controller/legacy.ts +0 -111
|
@@ -0,0 +1,289 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Daemon log reader — one parser for talon.log shared by every surface
|
|
3
|
+
* that shows it: `talon logs` filters, the proc/log + proc/errors views
|
|
4
|
+
* in the namespace, and doctor's recent-errors probe.
|
|
5
|
+
*
|
|
6
|
+
* talon.log is pino JSON Lines (see util/log.ts). The live file rotates
|
|
7
|
+
* at runtime into numbered generations (`talon.log.1` newest …) and, at
|
|
8
|
+
* daemon start, into `talon.log.old`; readers walk them newest-first so
|
|
9
|
+
* a question like "errors in the last hour" survives a rotation.
|
|
10
|
+
*
|
|
11
|
+
* Everything here is synchronous and bounded by a byte window per file:
|
|
12
|
+
* callers are a CLI process, a doctor pass, or a FUSE view read — none
|
|
13
|
+
* may slurp an unbounded history.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { existsSync, readdirSync, statSync } from "node:fs";
|
|
17
|
+
import { basename, dirname, join } from "node:path";
|
|
18
|
+
import { tailFile } from "../../util/tail-file.js";
|
|
19
|
+
|
|
20
|
+
type LogLevelName = "trace" | "debug" | "info" | "warn" | "error" | "fatal";
|
|
21
|
+
|
|
22
|
+
const LEVEL_BY_NUM: Record<number, LogLevelName> = {
|
|
23
|
+
10: "trace",
|
|
24
|
+
20: "debug",
|
|
25
|
+
30: "info",
|
|
26
|
+
40: "warn",
|
|
27
|
+
50: "error",
|
|
28
|
+
60: "fatal",
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
const LEVEL_RANK: Record<LogLevelName, number> = {
|
|
32
|
+
trace: 10,
|
|
33
|
+
debug: 20,
|
|
34
|
+
info: 30,
|
|
35
|
+
warn: 40,
|
|
36
|
+
error: 50,
|
|
37
|
+
fatal: 60,
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
/** One parsed talon.log line. */
|
|
41
|
+
export type LogRecord = {
|
|
42
|
+
/** Epoch ms (0 when the line carried none). */
|
|
43
|
+
ts: number;
|
|
44
|
+
level: LogLevelName;
|
|
45
|
+
component?: string;
|
|
46
|
+
msg: string;
|
|
47
|
+
err?: string;
|
|
48
|
+
stack?: string;
|
|
49
|
+
/** Turn id — a `turn` field, or a `turn=<id>` pair in the message. */
|
|
50
|
+
turn?: string;
|
|
51
|
+
};
|
|
52
|
+
|
|
53
|
+
export type LogFilter = {
|
|
54
|
+
/** Drop records below this severity. */
|
|
55
|
+
minLevel?: LogLevelName;
|
|
56
|
+
/** Exact component match. */
|
|
57
|
+
component?: string;
|
|
58
|
+
/** Drop records older than this epoch ms. */
|
|
59
|
+
since?: number;
|
|
60
|
+
/** Case-insensitive substring of the message or error text. */
|
|
61
|
+
grep?: string;
|
|
62
|
+
/** Exact turn id match. */
|
|
63
|
+
turn?: string;
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
const TURN_IN_MSG = /\bturn=([^\s,;)]+)/;
|
|
67
|
+
|
|
68
|
+
function turnOf(obj: Record<string, unknown>, msg: string): string | undefined {
|
|
69
|
+
if (typeof obj.turn === "string" || typeof obj.turn === "number") {
|
|
70
|
+
return String(obj.turn);
|
|
71
|
+
}
|
|
72
|
+
return TURN_IN_MSG.exec(msg)?.[1];
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** Parse one line; null for anything that is not a pino record. */
|
|
76
|
+
export function parseLogLine(line: string): LogRecord | null {
|
|
77
|
+
const trimmed = line.trim();
|
|
78
|
+
if (!trimmed.startsWith("{")) return null;
|
|
79
|
+
let obj: Record<string, unknown>;
|
|
80
|
+
try {
|
|
81
|
+
obj = JSON.parse(trimmed) as Record<string, unknown>;
|
|
82
|
+
} catch {
|
|
83
|
+
return null; // partial first line of a byte window, torn write
|
|
84
|
+
}
|
|
85
|
+
const msg = typeof obj.msg === "string" ? obj.msg : "";
|
|
86
|
+
const turn = turnOf(obj, msg);
|
|
87
|
+
return {
|
|
88
|
+
ts: typeof obj.time === "number" ? obj.time : 0,
|
|
89
|
+
level: LEVEL_BY_NUM[obj.level as number] ?? "info",
|
|
90
|
+
...(typeof obj.component === "string" ? { component: obj.component } : {}),
|
|
91
|
+
msg,
|
|
92
|
+
...(typeof obj.err === "string" ? { err: obj.err } : {}),
|
|
93
|
+
...(typeof obj.stack === "string" ? { stack: obj.stack } : {}),
|
|
94
|
+
...(turn !== undefined ? { turn } : {}),
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
export function matchesLogFilter(rec: LogRecord, filter: LogFilter): boolean {
|
|
99
|
+
if (filter.minLevel && LEVEL_RANK[rec.level] < LEVEL_RANK[filter.minLevel])
|
|
100
|
+
return false;
|
|
101
|
+
if (filter.component && rec.component !== filter.component) return false;
|
|
102
|
+
if (filter.since !== undefined && rec.ts < filter.since) return false;
|
|
103
|
+
if (filter.turn && rec.turn !== filter.turn) return false;
|
|
104
|
+
if (filter.grep) {
|
|
105
|
+
const needle = filter.grep.toLowerCase();
|
|
106
|
+
const hay = `${rec.msg}\n${rec.err ?? ""}`.toLowerCase();
|
|
107
|
+
if (!hay.includes(needle)) return false;
|
|
108
|
+
}
|
|
109
|
+
return true;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/** Parse + filter a raw chunk of the log, oldest first. */
|
|
113
|
+
function parseLogChunk(raw: string, filter: LogFilter): LogRecord[] {
|
|
114
|
+
const out: LogRecord[] = [];
|
|
115
|
+
// pino writes `"component":"<c>"` verbatim — a substring test skips the
|
|
116
|
+
// JSON.parse for every other component's line (most of a trace log).
|
|
117
|
+
const tag = filter.component
|
|
118
|
+
? `"component":${JSON.stringify(filter.component)}`
|
|
119
|
+
: null;
|
|
120
|
+
for (const line of raw.split("\n")) {
|
|
121
|
+
if (tag !== null && !line.includes(tag)) continue;
|
|
122
|
+
const rec = parseLogLine(line);
|
|
123
|
+
if (rec && matchesLogFilter(rec, filter)) out.push(rec);
|
|
124
|
+
}
|
|
125
|
+
return out;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* The live file and its rotated generations that exist, newest first:
|
|
130
|
+
* the live file, then every `<name>.<n>` (util/log.ts's runtime shift)
|
|
131
|
+
* and the start-time `<name>.old`, ordered by mtime — the two schemes
|
|
132
|
+
* interleave in time. Found by listing the directory, so the reader
|
|
133
|
+
* needs no knowledge of how many generations the writer keeps.
|
|
134
|
+
*/
|
|
135
|
+
export function logGenerations(path: string): string[] {
|
|
136
|
+
const dir = dirname(path);
|
|
137
|
+
const prefix = `${basename(path)}.`;
|
|
138
|
+
const isGeneration = (name: string): boolean =>
|
|
139
|
+
name.startsWith(prefix) && /^(\d+|old)$/.test(name.slice(prefix.length));
|
|
140
|
+
let names: string[];
|
|
141
|
+
try {
|
|
142
|
+
names = readdirSync(dir).filter(isGeneration);
|
|
143
|
+
} catch {
|
|
144
|
+
names = [];
|
|
145
|
+
}
|
|
146
|
+
const rotated: Array<{ path: string; mtime: number }> = [];
|
|
147
|
+
for (const name of names) {
|
|
148
|
+
try {
|
|
149
|
+
const full = join(dir, name);
|
|
150
|
+
rotated.push({ path: full, mtime: statSync(full).mtimeMs });
|
|
151
|
+
} catch {
|
|
152
|
+
/* rotated away between the listing and the stat */
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
rotated.sort((a, b) => b.mtime - a.mtime);
|
|
156
|
+
return [...(existsSync(path) ? [path] : []), ...rotated.map((r) => r.path)];
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
export type ReadLogOptions = {
|
|
160
|
+
/** Max records returned (the newest ones, oldest first). */
|
|
161
|
+
limit: number;
|
|
162
|
+
filter?: LogFilter;
|
|
163
|
+
/** Trailing byte window read from each file (default 2 MB). */
|
|
164
|
+
maxBytesPerFile?: number;
|
|
165
|
+
/** Generations consulted at most (default: all). */
|
|
166
|
+
maxFiles?: number;
|
|
167
|
+
};
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* The newest `limit` records matching `filter`, oldest first, walking
|
|
171
|
+
* back through rotated generations until the limit is met, the `since`
|
|
172
|
+
* cutoff is passed, or `maxFiles` is spent. Missing files read as empty.
|
|
173
|
+
*/
|
|
174
|
+
export function readLogRecords(
|
|
175
|
+
path: string,
|
|
176
|
+
opts: ReadLogOptions,
|
|
177
|
+
): LogRecord[] {
|
|
178
|
+
const filter = opts.filter ?? {};
|
|
179
|
+
const maxBytes = opts.maxBytesPerFile ?? 2 * 1024 * 1024;
|
|
180
|
+
const files = logGenerations(path).slice(0, opts.maxFiles);
|
|
181
|
+
let collected: LogRecord[] = [];
|
|
182
|
+
for (const file of files) {
|
|
183
|
+
let raw: string;
|
|
184
|
+
let mtime: number;
|
|
185
|
+
try {
|
|
186
|
+
mtime = statSync(file).mtimeMs;
|
|
187
|
+
if (filter.since !== undefined && mtime < filter.since) break;
|
|
188
|
+
raw = tailFile(file, Number.MAX_SAFE_INTEGER, maxBytes);
|
|
189
|
+
} catch {
|
|
190
|
+
continue;
|
|
191
|
+
}
|
|
192
|
+
collected = [...parseLogChunk(raw, filter), ...collected];
|
|
193
|
+
if (collected.length >= opts.limit) break;
|
|
194
|
+
}
|
|
195
|
+
return collected.slice(-opts.limit);
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
const LEVEL_TAG: Record<LogLevelName, string> = {
|
|
199
|
+
trace: "TRC",
|
|
200
|
+
debug: "DBG",
|
|
201
|
+
info: "INF",
|
|
202
|
+
warn: "WRN",
|
|
203
|
+
error: "ERR",
|
|
204
|
+
fatal: "FTL",
|
|
205
|
+
};
|
|
206
|
+
|
|
207
|
+
/**
|
|
208
|
+
* One human line: ISO time, level, component, message, turn id (when
|
|
209
|
+
* it came as a field rather than in the message), error — the
|
|
210
|
+
* shape an agent or operator reads without a JSON parser. `stackLines`
|
|
211
|
+
* appends that many stack frames, indented, under error records.
|
|
212
|
+
*/
|
|
213
|
+
export function formatLogRecord(
|
|
214
|
+
rec: LogRecord,
|
|
215
|
+
opts: { stackLines?: number; maxMsgChars?: number } = {},
|
|
216
|
+
): string {
|
|
217
|
+
const time = rec.ts > 0 ? new Date(rec.ts).toISOString() : "-";
|
|
218
|
+
const comp = (rec.component ?? "?").padEnd(10);
|
|
219
|
+
const max = opts.maxMsgChars ?? 1000;
|
|
220
|
+
const msg = rec.msg.length > max ? `${rec.msg.slice(0, max)}…` : rec.msg;
|
|
221
|
+
let line = `${time} ${LEVEL_TAG[rec.level]} ${comp} ${msg}`;
|
|
222
|
+
if (rec.turn && !TURN_IN_MSG.test(rec.msg)) line += ` turn=${rec.turn}`;
|
|
223
|
+
if (rec.err) line += ` (${rec.err})`;
|
|
224
|
+
const frames = opts.stackLines ?? 0;
|
|
225
|
+
if (frames > 0 && rec.stack) {
|
|
226
|
+
const stack = rec.stack
|
|
227
|
+
.split("\n")
|
|
228
|
+
.slice(1, frames + 1)
|
|
229
|
+
.map((f) => ` ${f.trim()}`);
|
|
230
|
+
if (stack.length > 0) line += `\n${stack.join("\n")}`;
|
|
231
|
+
}
|
|
232
|
+
return line;
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
const DURATION_UNITS: Record<string, number> = {
|
|
236
|
+
s: 1_000,
|
|
237
|
+
m: 60_000,
|
|
238
|
+
h: 3_600_000,
|
|
239
|
+
d: 86_400_000,
|
|
240
|
+
};
|
|
241
|
+
|
|
242
|
+
/** "90s" / "15m" / "2h" / "1d" → ms; a bare number is minutes. */
|
|
243
|
+
export function parseLogDuration(input: string): number | null {
|
|
244
|
+
const match = /^(\d+(?:\.\d+)?)([smhd]?)$/.exec(input.trim().toLowerCase());
|
|
245
|
+
if (!match) return null;
|
|
246
|
+
const ms = Number(match[1]) * DURATION_UNITS[match[2] || "m"];
|
|
247
|
+
return ms > 0 ? ms : null;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
export type LoggedAlert = {
|
|
251
|
+
key: string;
|
|
252
|
+
severity: string;
|
|
253
|
+
message: string;
|
|
254
|
+
since: number;
|
|
255
|
+
};
|
|
256
|
+
|
|
257
|
+
// The shapes alerts.ts logs: raiseAlert → `[<severity>] <key>: <message>`,
|
|
258
|
+
// resolveAlert → `resolved <key> after <n> min`, both as component "alert".
|
|
259
|
+
const ALERT_RAISED = /^\[(\w+)\] (\S+): ([\s\S]*)$/;
|
|
260
|
+
const ALERT_RESOLVED = /^resolved (\S+) after /;
|
|
261
|
+
/** The daemon's first boot line — in-memory alert state starts empty. */
|
|
262
|
+
const DAEMON_START = "Starting Talon...";
|
|
263
|
+
|
|
264
|
+
/**
|
|
265
|
+
* Replay alert raise/resolve lines to the set still open at the end of
|
|
266
|
+
* `records` (oldest first). For a process that cannot see the daemon's
|
|
267
|
+
* in-memory alert table — `talon doctor` runs standalone. A daemon
|
|
268
|
+
* restart clears the table, so a boot line clears the replay too.
|
|
269
|
+
*/
|
|
270
|
+
export function replayAlerts(records: readonly LogRecord[]): LoggedAlert[] {
|
|
271
|
+
const open = new Map<string, LoggedAlert>();
|
|
272
|
+
for (const rec of records) {
|
|
273
|
+
if (rec.component === "bot" && rec.msg === DAEMON_START) {
|
|
274
|
+
open.clear();
|
|
275
|
+
continue;
|
|
276
|
+
}
|
|
277
|
+
if (rec.component !== "alert") continue;
|
|
278
|
+
const raised = ALERT_RAISED.exec(rec.msg);
|
|
279
|
+
if (raised) {
|
|
280
|
+
const [, severity, key, message] = raised;
|
|
281
|
+
const prior = open.get(key);
|
|
282
|
+
open.set(key, { key, severity, message, since: prior?.since ?? rec.ts });
|
|
283
|
+
continue;
|
|
284
|
+
}
|
|
285
|
+
const resolved = ALERT_RESOLVED.exec(rec.msg);
|
|
286
|
+
if (resolved) open.delete(resolved[1]);
|
|
287
|
+
}
|
|
288
|
+
return [...open.values()];
|
|
289
|
+
}
|
package/src/core/doctor/index.ts
CHANGED
|
@@ -16,6 +16,7 @@ import { stat } from "node:fs/promises";
|
|
|
16
16
|
import { NATIVE_MODULES } from "../../native/registry.js";
|
|
17
17
|
import { dirs } from "../../util/paths.js";
|
|
18
18
|
import { getBackend, listBackends } from "../agent-runtime/backend-registry.js";
|
|
19
|
+
import { checkOpenAlerts, checkRecentErrors } from "./logs.js";
|
|
19
20
|
import type {
|
|
20
21
|
DoctorCheck,
|
|
21
22
|
DoctorConfigSlice,
|
|
@@ -270,6 +271,22 @@ async function checkOneBackend(
|
|
|
270
271
|
}
|
|
271
272
|
}
|
|
272
273
|
|
|
274
|
+
/**
|
|
275
|
+
* What the deployment is doing rather than what it is built from: the
|
|
276
|
+
* provisioned plugin runtimes, the backends, and what the daemon's log
|
|
277
|
+
* and alert table say went wrong lately (./logs.ts).
|
|
278
|
+
*/
|
|
279
|
+
async function checkRunningState(
|
|
280
|
+
config: DoctorConfigSlice | undefined,
|
|
281
|
+
): Promise<DoctorCheck[]> {
|
|
282
|
+
return [
|
|
283
|
+
...(await checkPluginRuntimes(config)),
|
|
284
|
+
...(await checkBackend(config)),
|
|
285
|
+
checkRecentErrors(),
|
|
286
|
+
...checkOpenAlerts(),
|
|
287
|
+
];
|
|
288
|
+
}
|
|
289
|
+
|
|
273
290
|
/**
|
|
274
291
|
* Run every check and return the structured report. `config` is
|
|
275
292
|
* undefined when no config file exists yet (first run).
|
|
@@ -440,8 +457,7 @@ export async function collectDoctorReport(opts: {
|
|
|
440
457
|
}
|
|
441
458
|
|
|
442
459
|
const native = await checkNativeModules();
|
|
443
|
-
checks.push(...(await
|
|
444
|
-
checks.push(...(await checkBackend(opts.config)));
|
|
460
|
+
checks.push(...(await checkRunningState(opts.config)));
|
|
445
461
|
|
|
446
462
|
const issues =
|
|
447
463
|
checks.filter((c) => c.status === "fail" || c.issue).length +
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Doctor's "what went wrong lately" probes: the last hour of talon.log
|
|
3
|
+
* summarised by component, and the operator alerts still open — so
|
|
4
|
+
* `talon doctor` points at the failing subsystem instead of only at the
|
|
5
|
+
* environment.
|
|
6
|
+
*
|
|
7
|
+
* Alerts come from the daemon's in-memory table when doctor runs inside
|
|
8
|
+
* the daemon (the chat /doctor command). `talon doctor` is its own
|
|
9
|
+
* process and sees an empty table, so it replays the alert lines in the
|
|
10
|
+
* log instead; those are reported as warnings without counting toward
|
|
11
|
+
* the issue total — the log can only say what was last announced.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import {
|
|
15
|
+
readLogRecords,
|
|
16
|
+
replayAlerts,
|
|
17
|
+
type LogRecord,
|
|
18
|
+
} from "../daemon/log-reader.js";
|
|
19
|
+
import { activeAlerts } from "../frontend-runtime/alerts.js";
|
|
20
|
+
import { files } from "../../util/paths.js";
|
|
21
|
+
import type { DoctorCheck } from "./types.js";
|
|
22
|
+
|
|
23
|
+
const HOUR_MS = 3_600_000;
|
|
24
|
+
/** How far back the alert replay looks for a raise still unresolved. */
|
|
25
|
+
const ALERT_WINDOW_MS = 24 * HOUR_MS;
|
|
26
|
+
const READ = { limit: Number.MAX_SAFE_INTEGER, maxBytesPerFile: 16 << 20 };
|
|
27
|
+
const DETAIL_CHARS = 160;
|
|
28
|
+
|
|
29
|
+
export type RecentLogOptions = {
|
|
30
|
+
logPath?: string;
|
|
31
|
+
now?: number;
|
|
32
|
+
/** The live alert table; defaults to this process's `activeAlerts()`. */
|
|
33
|
+
liveAlerts?: ReturnType<typeof activeAlerts>;
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
function clip(text: string): string {
|
|
37
|
+
return text.length > DETAIL_CHARS ? `${text.slice(0, DETAIL_CHARS)}…` : text;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function byComponent(records: readonly LogRecord[]): string {
|
|
41
|
+
const counts = new Map<string, number>();
|
|
42
|
+
for (const r of records) {
|
|
43
|
+
const c = r.component ?? "?";
|
|
44
|
+
counts.set(c, (counts.get(c) ?? 0) + 1);
|
|
45
|
+
}
|
|
46
|
+
return [...counts]
|
|
47
|
+
.sort((a, b) => b[1] - a[1])
|
|
48
|
+
.map(([c, n]) => `${c} ${n}`)
|
|
49
|
+
.join(", ");
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** The last hour's warn/error lines, as one doctor check. */
|
|
53
|
+
export function checkRecentErrors(opts: RecentLogOptions = {}): DoctorCheck {
|
|
54
|
+
const logPath = opts.logPath ?? files.log;
|
|
55
|
+
const now = opts.now ?? Date.now();
|
|
56
|
+
const recent = readLogRecords(logPath, {
|
|
57
|
+
...READ,
|
|
58
|
+
filter: { since: now - HOUR_MS, minLevel: "warn" },
|
|
59
|
+
});
|
|
60
|
+
const errors = recent.filter(
|
|
61
|
+
(r) => r.level === "error" || r.level === "fatal",
|
|
62
|
+
);
|
|
63
|
+
const warnings = recent.length - errors.length;
|
|
64
|
+
if (errors.length === 0) {
|
|
65
|
+
return warnings === 0
|
|
66
|
+
? { label: "Log: no errors or warnings in the last hour", status: "ok" }
|
|
67
|
+
: {
|
|
68
|
+
label: `Log: ${warnings} warning(s) in the last hour`,
|
|
69
|
+
status: "info",
|
|
70
|
+
detail: byComponent(recent),
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
const last = errors.at(-1)!;
|
|
74
|
+
const at = new Date(last.ts).toTimeString().slice(0, 5);
|
|
75
|
+
const what = clip(`${last.msg}${last.err ? ` (${last.err})` : ""}`);
|
|
76
|
+
return {
|
|
77
|
+
label: `Log: ${errors.length} error(s) in the last hour — ${byComponent(errors)}`,
|
|
78
|
+
status: "warn",
|
|
79
|
+
detail:
|
|
80
|
+
`latest ${at} ${last.component ?? "?"}: ${what} — ` +
|
|
81
|
+
`talon logs --errors --since 1h, or ~/.talon/ns/proc/errors`,
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/** Open alerts, one check each (or one "none" line). */
|
|
86
|
+
export function checkOpenAlerts(opts: RecentLogOptions = {}): DoctorCheck[] {
|
|
87
|
+
const now = opts.now ?? Date.now();
|
|
88
|
+
const live = opts.liveAlerts ?? activeAlerts();
|
|
89
|
+
const fromLog = live.length === 0;
|
|
90
|
+
const alerts = fromLog ? loggedAlerts(opts.logPath ?? files.log, now) : live;
|
|
91
|
+
if (alerts.length === 0)
|
|
92
|
+
return [{ label: "Alerts: none open", status: "ok" }];
|
|
93
|
+
return alerts.map((a) => {
|
|
94
|
+
const mins = Math.max(0, Math.round((now - a.since) / 60_000));
|
|
95
|
+
const label = `Alert ${a.key} (${a.severity}, ${mins} min)`;
|
|
96
|
+
if (fromLog) {
|
|
97
|
+
return {
|
|
98
|
+
label,
|
|
99
|
+
status: "warn",
|
|
100
|
+
detail: `last logged: ${clip(a.message)}`,
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
return {
|
|
104
|
+
label,
|
|
105
|
+
status: a.severity === "warn" ? "warn" : "fail",
|
|
106
|
+
detail: clip(a.message),
|
|
107
|
+
issue: true,
|
|
108
|
+
};
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/** Replay the last day's alert lines (plus boot lines, which reset them). */
|
|
113
|
+
function loggedAlerts(logPath: string, now: number) {
|
|
114
|
+
const since = now - ALERT_WINDOW_MS;
|
|
115
|
+
const alertLines = readLogRecords(logPath, {
|
|
116
|
+
...READ,
|
|
117
|
+
filter: { since, component: "alert" },
|
|
118
|
+
});
|
|
119
|
+
const boots = readLogRecords(logPath, {
|
|
120
|
+
...READ,
|
|
121
|
+
filter: { since, component: "bot", grep: "Starting Talon" },
|
|
122
|
+
});
|
|
123
|
+
return replayAlerts([...alertLines, ...boots].sort((a, b) => a.ts - b.ts));
|
|
124
|
+
}
|
package/src/core/doctor/types.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* The doctor's data shapes, on their own so a backend factory can declare
|
|
3
3
|
* its checks (`BackendFactory.doctor`) without importing the collector —
|
|
4
|
-
* `core/doctor.ts` reads the registry, so the types living there would
|
|
4
|
+
* `core/doctor/index.ts` reads the registry, so the types living there would
|
|
5
5
|
* close a cycle. Checks are pure data (label / status / detail); renderers
|
|
6
6
|
* decide presentation.
|
|
7
7
|
*/
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
* Different roles, and different chats, have different cost/latency/
|
|
11
11
|
* quality needs. Typical post-Anthropic-metering setup: chat default
|
|
12
12
|
* on free-tier OpenAI Agents, heartbeats on Claude Sonnet, dream
|
|
13
|
-
* shared with chat — plus
|
|
13
|
+
* shared with chat — plus one group chat stays on Claude while DMs use the
|
|
14
14
|
* cheap default. A single-active model can't express that. The pool
|
|
15
15
|
* lets each holder bind independently while deduplicating instances
|
|
16
16
|
* when ids overlap.
|
|
@@ -24,7 +24,6 @@
|
|
|
24
24
|
* - `pool` — init/teardown, role accessors, availability, snapshot,
|
|
25
25
|
* listeners.
|
|
26
26
|
* - `rebind` — rebind/release holders + per-chat accessors.
|
|
27
|
-
* - `legacy` — single-active aliases routed to the chat role.
|
|
28
27
|
*/
|
|
29
28
|
|
|
30
29
|
export * from "./types.js";
|
|
@@ -57,14 +56,3 @@ export {
|
|
|
57
56
|
hasChatBackendOverride,
|
|
58
57
|
resolveChatBackend,
|
|
59
58
|
} from "./rebind.js";
|
|
60
|
-
export {
|
|
61
|
-
initBackendController,
|
|
62
|
-
getActiveBackend,
|
|
63
|
-
hasActiveBackend,
|
|
64
|
-
getActiveBackendOrNull,
|
|
65
|
-
getActiveBackendId,
|
|
66
|
-
getActiveBackendLabel,
|
|
67
|
-
switchBackend,
|
|
68
|
-
cleanupBackendController,
|
|
69
|
-
resetBackendControllerForTest,
|
|
70
|
-
} from "./legacy.js";
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Plan-aware backend routing for background work.
|
|
3
3
|
*
|
|
4
4
|
* Background work — `spawn_agent` sub-agents, cron `query` jobs, the
|
|
5
|
-
* heartbeat —
|
|
5
|
+
* heartbeat — would otherwise inherit whichever backend the chat is on.
|
|
6
6
|
* On a multi-subscription install that is a good way to burn one plan to its
|
|
7
7
|
* ceiling while another sits idle. When nothing is pinned, this picks the
|
|
8
8
|
* backend with the most headroom instead.
|
|
@@ -12,8 +12,15 @@ import type { ExecuteParams, ExecuteResult } from "../types.js";
|
|
|
12
12
|
import { formatRelayBlock, takePendingRelay } from "./cross-chat-relay.js";
|
|
13
13
|
import { recordMessageSignal } from "../memory/taps.js";
|
|
14
14
|
import { log } from "../../util/log.js";
|
|
15
|
+
import { setStuckLoopListener } from "../../util/watchdog.js";
|
|
15
16
|
import { taskTable, type KillOutcome } from "../tasks/index.js";
|
|
16
17
|
import { initWeaver, type Weaver, type WeaverDeps } from "../weaver/index.js";
|
|
18
|
+
import {
|
|
19
|
+
noteTurnFailed,
|
|
20
|
+
noteTurnSucceeded,
|
|
21
|
+
stuckLoopAlert,
|
|
22
|
+
type TurnBinding,
|
|
23
|
+
} from "./turn-health.js";
|
|
17
24
|
|
|
18
25
|
// ── Dependencies (injected at startup) ──────────────────────────────────────
|
|
19
26
|
|
|
@@ -21,7 +28,7 @@ import { initWeaver, type Weaver, type WeaverDeps } from "../weaver/index.js";
|
|
|
21
28
|
* `getBackend` takes the string chat id so it can route per-chat —
|
|
22
29
|
* a chat with a backend override returns its override backend, others
|
|
23
30
|
* fall through to the global chat-role backend. Tests can pass a
|
|
24
|
-
* stub that ignores the chat id. See `core/backend-controller
|
|
31
|
+
* stub that ignores the chat id. See `core/engine/backend-controller/`.
|
|
25
32
|
*
|
|
26
33
|
* `resolveActiveModel` walks the 5-step active-model resolution
|
|
27
34
|
* chain for the chat and returns both the resolved `ModelRef` and
|
|
@@ -33,9 +40,18 @@ import { initWeaver, type Weaver, type WeaverDeps } from "../weaver/index.js";
|
|
|
33
40
|
type DispatcherDeps = WeaverDeps;
|
|
34
41
|
|
|
35
42
|
let weaver: Weaver | null = null;
|
|
43
|
+
let backendIdFor: (chatId: string) => string = () => "unknown";
|
|
36
44
|
|
|
37
45
|
export function initDispatcher(d: DispatcherDeps): void {
|
|
38
46
|
weaver = initWeaver(d);
|
|
47
|
+
backendIdFor = (chatId) => {
|
|
48
|
+
try {
|
|
49
|
+
return d.getBackend(chatId).id;
|
|
50
|
+
} catch {
|
|
51
|
+
return "unknown";
|
|
52
|
+
}
|
|
53
|
+
};
|
|
54
|
+
setStuckLoopListener(stuckLoopAlert);
|
|
39
55
|
log("dispatcher", "Initialized (per-chat serial, cross-chat parallel)");
|
|
40
56
|
}
|
|
41
57
|
|
|
@@ -94,5 +110,42 @@ export async function execute(params: ExecuteParams): Promise<ExecuteResult> {
|
|
|
94
110
|
const prompt = relayed.length
|
|
95
111
|
? formatRelayBlock(relayed) + params.prompt
|
|
96
112
|
: params.prompt;
|
|
97
|
-
|
|
113
|
+
const startedAt = Date.now();
|
|
114
|
+
try {
|
|
115
|
+
const result = await weaver.runTurn(
|
|
116
|
+
relayed.length ? { ...params, prompt } : params,
|
|
117
|
+
);
|
|
118
|
+
// Only a turn that reached a backend vouches for it — a refusal or a
|
|
119
|
+
// turn killed in the queue binds no warp this turn.
|
|
120
|
+
const bound = turnBinding(weaver, params, startedAt, false);
|
|
121
|
+
if (bound) noteTurnSucceeded(bound);
|
|
122
|
+
return result;
|
|
123
|
+
} catch (err) {
|
|
124
|
+
const bound = turnBinding(weaver, params, startedAt, true);
|
|
125
|
+
if (bound) noteTurnFailed(bound, err);
|
|
126
|
+
throw err;
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* The backend/model this turn ran on, for turn-health alerts: the warp
|
|
132
|
+
* the Weaver bound for it, or — when the turn failed before binding one
|
|
133
|
+
* — the backend currently serving the chat.
|
|
134
|
+
*/
|
|
135
|
+
function turnBinding(
|
|
136
|
+
w: Weaver,
|
|
137
|
+
params: ExecuteParams,
|
|
138
|
+
startedAt: number,
|
|
139
|
+
failed: boolean,
|
|
140
|
+
): TurnBinding | null {
|
|
141
|
+
const common = {
|
|
142
|
+
chatId: params.chatId,
|
|
143
|
+
source: params.source,
|
|
144
|
+
durationMs: Date.now() - startedAt,
|
|
145
|
+
};
|
|
146
|
+
const warp = w.loom.get(params.chatId)?.warp;
|
|
147
|
+
if (warp && warp.boundAt >= startedAt) {
|
|
148
|
+
return { ...common, backendId: warp.backendId, model: warp.model };
|
|
149
|
+
}
|
|
150
|
+
return failed ? { ...common, backendId: backendIdFor(params.chatId) } : null;
|
|
98
151
|
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The error text an operator alert or a key=value log line may carry.
|
|
3
|
+
*
|
|
4
|
+
* Alerts land on a phone and log lines get pasted into issues, so the
|
|
5
|
+
* raw message is flattened to one line, clipped, and stripped of the
|
|
6
|
+
* credential shapes a provider error can echo back (bearer tokens, API
|
|
7
|
+
* keys, `token=` query parameters).
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
const SECRET_PATTERNS: readonly RegExp[] = [
|
|
11
|
+
/\bBearer\s+[\w.~+/=-]+/gi,
|
|
12
|
+
/\b(?:sk|pk|rk)-[\w-]{8,}/g,
|
|
13
|
+
/\b(?:gh[pousr]|github_pat)_\w{8,}/g,
|
|
14
|
+
/\bxox[abprs]-[\w-]{8,}/g,
|
|
15
|
+
/\b(access_token|refresh_token|api_key|apikey|token|key|secret|password)=[^\s&"']+/gi,
|
|
16
|
+
];
|
|
17
|
+
|
|
18
|
+
/** One-line, clipped, secret-free text for `err`. */
|
|
19
|
+
export function faultText(err: unknown, max = 200): string {
|
|
20
|
+
let raw: string;
|
|
21
|
+
if (err instanceof Error) raw = err.message || err.name;
|
|
22
|
+
else if (typeof err === "string") raw = err;
|
|
23
|
+
else {
|
|
24
|
+
try {
|
|
25
|
+
raw = String(err);
|
|
26
|
+
} catch {
|
|
27
|
+
raw = "[unprintable error]";
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
let text = raw.replace(/\s+/g, " ").trim();
|
|
31
|
+
for (const pattern of SECRET_PATTERNS) {
|
|
32
|
+
text = text.replace(pattern, (match, name?: string) =>
|
|
33
|
+
typeof name === "string" && match.includes("=")
|
|
34
|
+
? `${name}=[redacted]`
|
|
35
|
+
: "[redacted]",
|
|
36
|
+
);
|
|
37
|
+
}
|
|
38
|
+
if (!text) return "unknown error";
|
|
39
|
+
return text.length > max ? `${text.slice(0, max - 1)}…` : text;
|
|
40
|
+
}
|
|
@@ -3,8 +3,9 @@
|
|
|
3
3
|
*
|
|
4
4
|
* - `control` — what a parent calls: spawn / list / status / wait / send /
|
|
5
5
|
* kill. Works from a chat and from inside another agent.
|
|
6
|
-
* - `report` — what a sub-agent calls about itself
|
|
7
|
-
* message_parent / check_inbox
|
|
6
|
+
* - `report` — what a sub-agent calls about itself and its siblings:
|
|
7
|
+
* report_result / message_parent / check_inbox / list_peers /
|
|
8
|
+
* message_peer.
|
|
8
9
|
*
|
|
9
10
|
* Both sets are reachable from an `agent:<id>` context: the gateway routes
|
|
10
11
|
* those chat keys straight here (see `Gateway.handleAction`), because a
|