toolroll 0.9.46 → 0.9.48

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/CHANGELOG.md +27 -0
  2. package/dist/activity-line.d.ts +26 -0
  3. package/dist/activity-line.js +47 -0
  4. package/dist/browser/workspace.js +25 -25
  5. package/dist/browser-workspace.d.ts +18 -1
  6. package/dist/browser-workspace.js +9 -3
  7. package/dist/builder.js +4 -2
  8. package/dist/cli.d.ts +1 -1
  9. package/dist/cli.js +4 -0
  10. package/dist/contracts/cli.d.ts +87 -0
  11. package/dist/contracts/cli.js +2 -0
  12. package/dist/contracts/health.d.ts +84 -0
  13. package/dist/contracts/health.js +11 -0
  14. package/dist/flow-card-task.d.ts +2 -0
  15. package/dist/flow-card-task.js +57 -0
  16. package/dist/flow-live.d.ts +7 -15
  17. package/dist/flow-live.js +77 -115
  18. package/dist/flows-ui.js +15 -1
  19. package/dist/health-cli.d.ts +3 -0
  20. package/dist/health-cli.js +79 -0
  21. package/dist/live-bus.d.ts +89 -0
  22. package/dist/live-bus.js +321 -0
  23. package/dist/live.d.ts +14 -0
  24. package/dist/live.js +71 -22
  25. package/dist/metrics.d.ts +3 -0
  26. package/dist/metrics.js +53 -1
  27. package/dist/operate.js +43 -25
  28. package/dist/planner.js +4 -3
  29. package/dist/read-executor-worker.d.ts +1 -0
  30. package/dist/read-executor-worker.js +26 -0
  31. package/dist/read-executor.d.ts +49 -0
  32. package/dist/read-executor.js +119 -0
  33. package/dist/request-budget.d.ts +1 -0
  34. package/dist/request-budget.js +6 -0
  35. package/dist/serve.d.ts +7 -0
  36. package/dist/serve.js +118 -42
  37. package/dist/server-telemetry.d.ts +86 -0
  38. package/dist/server-telemetry.js +263 -0
  39. package/dist/setup-guide.d.ts +2 -2
  40. package/dist/sqlite-telemetry.d.ts +9 -0
  41. package/dist/sqlite-telemetry.js +98 -0
  42. package/dist/store.d.ts +22 -0
  43. package/dist/store.js +59 -3
  44. package/dist/surface.js +2 -1
  45. package/dist/task-activity.d.ts +27 -0
  46. package/dist/task-activity.js +53 -0
  47. package/dist/task-live.d.ts +8 -0
  48. package/dist/task-live.js +47 -0
  49. package/dist/work-index.js +12 -1
  50. package/dist/workspace-revision.d.ts +2 -0
  51. package/dist/workspace-revision.js +3 -1
  52. package/package.json +1 -1
@@ -0,0 +1,321 @@
1
+ /**
2
+ * Live views push on write. Nothing here polls once a second.
3
+ *
4
+ * - The bus. The database's own revision triggers (workspace-revision.ts)
5
+ * move one counter on every meaningful write. After each in-process
6
+ * transact() COMMIT, and whenever another process (the worker, the CLI)
7
+ * writes the database's write-ahead log, the revision is read once; when
8
+ * it moved, every subscriber hears `{ revision }`. A rolled-back or quiet
9
+ * write moves nothing and says nothing. A page left open with nothing
10
+ * being written costs no database work at all.
11
+ * - The rooms. Each open task or flow is a room keyed by what it shows.
12
+ * When the bus speaks, a room rechecks every viewer's access (a signed-out
13
+ * or narrowed account is dropped), takes its fingerprint once, and only if
14
+ * that moved tells its pages `change` with the fingerprint and revision —
15
+ * an invalidation, never content. The page reads itself again under its
16
+ * own checks, so redaction stays the page's.
17
+ * - The safety net. Every room also checks itself about every 30 s, so a
18
+ * missed signal heals, and sends a keep-alive between.
19
+ * - Back-pressure. Each page's stream is bounded: a page that falls behind
20
+ * gets one `reload` and ends so its EventSource can reconnect.
21
+ *
22
+ * Transport seam: a future WebSocket (live cursors) subscribes to the same
23
+ * LiveBus beside these SSE rooms; nothing in the Store or the triggers changes.
24
+ */
25
+ import { watch } from "node:fs";
26
+ import { WORKSPACE_REVISION_KEY } from "./workspace-revision.js";
27
+ export function createLiveBus() {
28
+ const listeners = new Set();
29
+ let last = null;
30
+ return {
31
+ subscribe(listener) { listeners.add(listener); return () => { listeners.delete(listener); }; },
32
+ publish(change) {
33
+ last = change.revision;
34
+ for (const listener of [...listeners]) {
35
+ try {
36
+ listener(change);
37
+ }
38
+ catch { /* one room never stops another */ }
39
+ }
40
+ },
41
+ revision: () => last,
42
+ listeners: () => listeners.size,
43
+ };
44
+ }
45
+ const SIGNAL = "toolroll_live_written";
46
+ /**
47
+ * Publish the workspace revision when it moves: after this process commits,
48
+ * and when the database's write-ahead log changes under another process.
49
+ * Signals in one turn of the event loop coalesce into one read.
50
+ */
51
+ export function followWorkspace(store, current, bus, options = {}) {
52
+ let last = null;
53
+ try {
54
+ last = current();
55
+ }
56
+ catch { /* the first signal reads it */ }
57
+ let scheduled = false, closed = false;
58
+ let timer = null;
59
+ const check = () => {
60
+ scheduled = false;
61
+ timer = null;
62
+ if (closed)
63
+ return;
64
+ // Never read inside an open transaction: what it wrote isn't committed yet.
65
+ if (store.handle.isTransaction === true) {
66
+ schedule(10);
67
+ return;
68
+ }
69
+ let now;
70
+ try {
71
+ now = current();
72
+ }
73
+ catch {
74
+ schedule(250);
75
+ return; /* busy: read again shortly */
76
+ }
77
+ if (now === last)
78
+ return;
79
+ last = now;
80
+ bus.publish({ revision: now });
81
+ };
82
+ const schedule = (delay = 0) => {
83
+ // With no page open, nobody needs the read.
84
+ if (scheduled || closed || bus.listeners() === 0)
85
+ return;
86
+ scheduled = true;
87
+ timer = setTimeout(check, delay);
88
+ timer.unref?.();
89
+ };
90
+ const unhook = store.onCommit(() => schedule());
91
+ // A write this process makes outside transact() (an autocommit statement)
92
+ // moves the revision too: a connection-local TEMP trigger on the revision
93
+ // row calls back here. Other processes never see it. The read waits for
94
+ // the statement's own commit, and a rollback leaves the revision unmoved.
95
+ let signalled = false;
96
+ const db = store.handle;
97
+ if (typeof db.function === "function") {
98
+ try {
99
+ db.function(SIGNAL, { deterministic: false }, () => { if (!closed)
100
+ schedule(); return null; });
101
+ db.exec(`CREATE TEMP TRIGGER IF NOT EXISTS ${SIGNAL}_update AFTER UPDATE ON main.service_cursor WHEN NEW.key = '${WORKSPACE_REVISION_KEY}' BEGIN SELECT ${SIGNAL}(); END;
102
+ CREATE TEMP TRIGGER IF NOT EXISTS ${SIGNAL}_insert AFTER INSERT ON main.service_cursor WHEN NEW.key = '${WORKSPACE_REVISION_KEY}' BEGIN SELECT ${SIGNAL}(); END;`);
103
+ signalled = true;
104
+ }
105
+ catch { /* commits and the safety net still reach the rooms */ }
106
+ }
107
+ const file = options.file ?? null;
108
+ let watcher = null;
109
+ if (file !== null) {
110
+ const start = options.watchFile ?? ((path, listener) => { try {
111
+ return watch(path, { persistent: false }, listener);
112
+ }
113
+ catch {
114
+ return null;
115
+ } });
116
+ // The worker and CLI write from their own processes; their commits land in the WAL.
117
+ watcher = start(`${file}-wal`, () => schedule());
118
+ watcher?.on?.("error", () => { watcher?.close(); watcher = null; });
119
+ }
120
+ return {
121
+ check,
122
+ close() {
123
+ closed = true;
124
+ if (timer !== null)
125
+ clearTimeout(timer);
126
+ unhook();
127
+ watcher?.close();
128
+ watcher = null;
129
+ if (signalled) {
130
+ try {
131
+ db.exec(`DROP TRIGGER IF EXISTS temp.${SIGNAL}_update; DROP TRIGGER IF EXISTS temp.${SIGNAL}_insert;`);
132
+ }
133
+ catch { /* the database is closing */ }
134
+ }
135
+ },
136
+ };
137
+ }
138
+ /** About every 30 s a room checks itself; a keep-alive goes between. */
139
+ export const SAFETY_NET_MS = 30_000;
140
+ export const KEEP_ALIVE_MS = 15_000;
141
+ /** What a page's stream may hold unsent before it is "behind". */
142
+ export const STREAM_LIMIT_BYTES = 32 * 1024;
143
+ /** One page's stream, bounded. Overflow is terminal, even below Node's drain threshold. */
144
+ export class LiveStream {
145
+ response;
146
+ limit;
147
+ onOverflow;
148
+ ended = false;
149
+ constructor(response, limit = STREAM_LIMIT_BYTES, onOverflow = () => { }) {
150
+ this.response = response;
151
+ this.limit = limit;
152
+ this.onOverflow = onOverflow;
153
+ }
154
+ get open() { return !this.ended && !this.response.writableEnded && !this.response.destroyed; }
155
+ overflow() {
156
+ if (!this.open)
157
+ return;
158
+ this.ended = true;
159
+ // end(data) queues at most this final frame; it never waits for a drain
160
+ // that Node may not emit when our limit is below its high-water mark.
161
+ try {
162
+ this.response.end("event: reload\ndata: {}\n\n");
163
+ }
164
+ finally {
165
+ this.onOverflow();
166
+ }
167
+ }
168
+ write(frame) {
169
+ if (!this.open)
170
+ return;
171
+ if (this.response.writableLength > this.limit) {
172
+ this.overflow();
173
+ return;
174
+ }
175
+ this.response.write(frame);
176
+ if (this.response.writableLength > this.limit)
177
+ this.overflow();
178
+ }
179
+ send(event, data) {
180
+ this.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
181
+ }
182
+ keepAlive() {
183
+ this.write(": keep-alive\n\n");
184
+ }
185
+ }
186
+ const PRESENCE_SETTLE_MS = 150;
187
+ /** Rooms of open pages that hear the bus. `presence` says who else is here, as each page should see it. */
188
+ export function createLiveRooms(fingerprint, presence, options = {}) {
189
+ const bus = options.bus ?? null;
190
+ const keepAliveMs = options.keepAliveMs ?? KEEP_ALIVE_MS;
191
+ const checkEvery = Math.max(1, Math.round((options.safetyNetMs ?? SAFETY_NET_MS) / keepAliveMs));
192
+ const rooms = new Map();
193
+ let unsubscribe = null;
194
+ const here = (room, viewer) => presence([...room.viewers.keys()], viewer.name);
195
+ const announce = (room) => {
196
+ // A page that moves from one card to the next leaves and joins at once: say it once, after it settles.
197
+ if (room.settling !== null)
198
+ clearTimeout(room.settling);
199
+ room.settling = setTimeout(() => {
200
+ room.settling = null;
201
+ for (const [viewer, stream] of room.viewers)
202
+ stream.send("here", { people: here(room, viewer) });
203
+ }, PRESENCE_SETTLE_MS);
204
+ room.settling.unref();
205
+ };
206
+ const leave = (key, viewer) => {
207
+ const room = rooms.get(key);
208
+ if (room === undefined || !room.viewers.delete(viewer))
209
+ return;
210
+ if (!viewer.response.writableEnded)
211
+ viewer.response.end();
212
+ if (room.viewers.size > 0) {
213
+ announce(room);
214
+ return;
215
+ }
216
+ clearInterval(room.timer);
217
+ if (room.settling !== null)
218
+ clearTimeout(room.settling);
219
+ if (room.retry !== null)
220
+ clearTimeout(room.retry);
221
+ rooms.delete(key);
222
+ if (rooms.size === 0 && unsubscribe !== null) {
223
+ unsubscribe();
224
+ unsubscribe = null;
225
+ }
226
+ };
227
+ const drop = (key, viewer) => {
228
+ rooms.get(key)?.viewers.get(viewer)?.send("gone", {});
229
+ leave(key, viewer);
230
+ };
231
+ /** Recheck who may still see the room, then whether what it shows moved. */
232
+ const refresh = (key, room, revision) => {
233
+ // A signed-out or narrowed account stops hearing about it, at the first write after.
234
+ for (const viewer of [...room.viewers.keys()]) {
235
+ let ok = false;
236
+ try {
237
+ ok = viewer.valid();
238
+ }
239
+ catch {
240
+ ok = false;
241
+ }
242
+ if (!ok)
243
+ drop(key, viewer);
244
+ }
245
+ if (rooms.get(key) !== room)
246
+ return;
247
+ let now;
248
+ try {
249
+ now = fingerprint(key);
250
+ }
251
+ catch {
252
+ // A busy database: read again shortly rather than wait for the safety net.
253
+ if (room.retry === null) {
254
+ room.retry = setTimeout(() => { room.retry = null; if (rooms.get(key) === room)
255
+ refresh(key, room, revision); }, 1_000);
256
+ room.retry.unref();
257
+ }
258
+ return;
259
+ }
260
+ if (now === null) {
261
+ for (const viewer of [...room.viewers.keys()])
262
+ drop(key, viewer);
263
+ return;
264
+ }
265
+ if (now === room.last)
266
+ return;
267
+ room.last = now;
268
+ for (const stream of room.viewers.values())
269
+ stream.send("change", { at: now, revision: revision ?? bus?.revision() ?? null });
270
+ };
271
+ const tick = (key, room) => {
272
+ room.ticks += 1;
273
+ if (room.ticks % checkEvery === 0) {
274
+ const before = room.last;
275
+ refresh(key, room, null);
276
+ if (rooms.get(key) === room && room.last !== before)
277
+ return;
278
+ }
279
+ for (const stream of room.viewers.values())
280
+ stream.keepAlive();
281
+ };
282
+ const heard = (change) => {
283
+ for (const [key, room] of [...rooms])
284
+ refresh(key, room, change.revision);
285
+ };
286
+ return {
287
+ join(key, viewer) {
288
+ let room = rooms.get(key);
289
+ if (room === undefined) {
290
+ let last = null;
291
+ try {
292
+ last = fingerprint(key);
293
+ }
294
+ catch { /* the next signal or the safety net reads it */ }
295
+ const created = { viewers: new Map(), last, timer: setInterval(() => tick(key, created), keepAliveMs), ticks: 0, settling: null, retry: null };
296
+ created.timer.unref();
297
+ room = created;
298
+ rooms.set(key, room);
299
+ if (bus !== null && unsubscribe === null)
300
+ unsubscribe = bus.subscribe(heard);
301
+ }
302
+ // Remove an overflowed page immediately, even while its final bytes
303
+ // wait for the client. Peers must not keep seeing stale presence.
304
+ const stream = new LiveStream(viewer.response, options.limitBytes, () => leave(key, viewer));
305
+ room.viewers.set(viewer, stream);
306
+ viewer.response.once("close", () => leave(key, viewer));
307
+ stream.send("change", { at: room.last, revision: bus?.revision() ?? null });
308
+ if (!stream.open)
309
+ return;
310
+ stream.send("here", { people: here(room, viewer) });
311
+ if (stream.open)
312
+ announce(room);
313
+ },
314
+ close() {
315
+ for (const [key, room] of [...rooms])
316
+ for (const viewer of [...room.viewers.keys()])
317
+ leave(key, viewer);
318
+ },
319
+ size: () => [...rooms.values()].reduce((sum, room) => sum + room.viewers.size, 0),
320
+ };
321
+ }
package/dist/live.d.ts CHANGED
@@ -25,6 +25,20 @@ export declare function liveDir(root: string): string;
25
25
  /** The exact filename, from the numeric id and nothing else — no user
26
26
  * input is ever parsed into this path. */
27
27
  export declare function liveLogPath(root: string, runId: number): string;
28
+ /**
29
+ * The fixed tool vocabulary. Known harness tools map to OUR kinds and
30
+ * phrases; anything unrecognized gets the generic one. The tool's own
31
+ * name, arguments, and metadata never reach the file: a tool named after
32
+ * a secret stays unspoken.
33
+ */
34
+ export type LiveActivityKind = "session" | "message" | "edit" | "command" | "read" | "search" | "lookup" | "plan" | "delegate" | "tool";
35
+ export declare const LIVE_ACTIVITY_KINDS: readonly LiveActivityKind[];
36
+ /**
37
+ * What one stream event says the agent just did, as one fixed kind: the
38
+ * last-activity line's only input. Never text, names or arguments; an
39
+ * event that says nothing (results, deltas, subagent chatter) is null.
40
+ */
41
+ export declare function liveActivityKind(event: Record<string, unknown>): LiveActivityKind | null;
28
42
  /**
29
43
  * One stream event → zero or more display lines, in the fixed vocabulary.
30
44
  * Assistant TEXT is the agent's own words (sanitized below); tool use is a
package/dist/live.js CHANGED
@@ -32,28 +32,77 @@ export function liveDir(root) {
32
32
  export function liveLogPath(root, runId) {
33
33
  return join(liveDir(root), `${Math.trunc(runId)}.log`);
34
34
  }
35
+ export const LIVE_ACTIVITY_KINDS = ["session", "message", "edit", "command", "read", "search", "lookup", "plan", "delegate", "tool"];
36
+ const TOOL_KINDS = new Map([
37
+ ["Edit", "edit"],
38
+ ["Write", "edit"],
39
+ ["MultiEdit", "edit"],
40
+ ["NotebookEdit", "edit"],
41
+ ["Bash", "command"],
42
+ ["Read", "read"],
43
+ ["Glob", "search"],
44
+ ["Grep", "search"],
45
+ ["LS", "read"],
46
+ ["WebFetch", "lookup"],
47
+ ["WebSearch", "lookup"],
48
+ ["TodoWrite", "plan"],
49
+ ["Task", "delegate"],
50
+ ]);
51
+ const KIND_HEADLINES = {
52
+ session: "the agent session started",
53
+ message: "",
54
+ edit: "→ editing files",
55
+ command: "→ running a command",
56
+ read: "→ reading the code",
57
+ search: "→ searching the code",
58
+ lookup: "→ looking something up",
59
+ plan: "→ organizing its plan",
60
+ delegate: "→ delegating to a helper",
61
+ tool: "→ using a tool",
62
+ };
63
+ const toolKind = (name) => TOOL_KINDS.get(typeof name === "string" ? name : "") ?? "tool";
35
64
  /**
36
- * The fixed tool-headline vocabulary. Known harness tools map to OUR
37
- * phrases; anything unrecognized gets the generic line. The tool's own
38
- * name, arguments, and metadata never reach the file: a tool named after
39
- * a secret stays unspoken.
65
+ * What one stream event says the agent just did, as one fixed kind: the
66
+ * last-activity line's only input. Never text, names or arguments; an
67
+ * event that says nothing (results, deltas, subagent chatter) is null.
40
68
  */
41
- const TOOL_HEADLINES = new Map([
42
- ["Edit", "→ editing files"],
43
- ["Write", "→ editing files"],
44
- ["MultiEdit", "→ editing files"],
45
- ["NotebookEdit", "→ editing files"],
46
- ["Bash", "→ running a command"],
47
- ["Read", "→ reading the code"],
48
- ["Glob", "→ searching the code"],
49
- ["Grep", "→ searching the code"],
50
- ["LS", "→ reading the code"],
51
- ["WebFetch", "→ looking something up"],
52
- ["WebSearch", "→ looking something up"],
53
- ["TodoWrite", "→ organizing its plan"],
54
- ["Task", "→ delegating to a helper"],
55
- ]);
56
- const TOOL_HEADLINE_GENERIC = "→ using a tool";
69
+ export function liveActivityKind(event) {
70
+ try {
71
+ const type = String(event["type"] ?? "");
72
+ if ((type === "system" && String(event["subtype"] ?? "") === "init") || type === "thread.started" || type === "init")
73
+ return "session";
74
+ if (type === "item.completed") {
75
+ const item = event["item"];
76
+ const kind = item === undefined || item === null ? "" : String(item["type"] ?? "");
77
+ return kind === "agent_message" ? "message" : kind === "command_execution" ? "command" : kind === "file_change" ? "edit" : kind === "web_search" ? "lookup" : null;
78
+ }
79
+ if (type === "message")
80
+ return String(event["role"] ?? "") === "assistant" && event["delta"] !== true ? "message" : null;
81
+ if (type === "tool_call" || type === "tool_use")
82
+ return toolKind(event["name"] ?? event["tool"]?.["name"]);
83
+ if (type !== "assistant")
84
+ return null;
85
+ const parent = event["parent_tool_use_id"];
86
+ if (parent !== undefined && parent !== null)
87
+ return null;
88
+ const content = event["message"]?.["content"];
89
+ if (!Array.isArray(content))
90
+ return null;
91
+ let last = null;
92
+ for (const block of content) {
93
+ if (block === null || typeof block !== "object")
94
+ continue;
95
+ if (block["type"] === "tool_use")
96
+ last = toolKind(block["name"]);
97
+ else if (block["type"] === "text" && typeof block["text"] === "string" && block["text"].trim() !== "")
98
+ last = "message";
99
+ }
100
+ return last;
101
+ }
102
+ catch {
103
+ return null;
104
+ }
105
+ }
57
106
  /** Strip ANSI escapes and every control character; tabs become spaces. */
58
107
  function stripControls(text) {
59
108
  return text
@@ -108,7 +157,7 @@ export function renderTranscriptLines(event) {
108
157
  }
109
158
  if (type === "tool_call" || type === "tool_use") {
110
159
  const name = event["name"] ?? event["tool"]?.["name"];
111
- return [TOOL_HEADLINES.get(typeof name === "string" ? name : "") ?? TOOL_HEADLINE_GENERIC];
160
+ return [KIND_HEADLINES[toolKind(name)]];
112
161
  }
113
162
  if (type !== "assistant")
114
163
  return [];
@@ -136,7 +185,7 @@ export function renderTranscriptLines(event) {
136
185
  }
137
186
  else if (kind === "tool_use") {
138
187
  const name = block["name"];
139
- lines.push(TOOL_HEADLINES.get(typeof name === "string" ? name : "") ?? TOOL_HEADLINE_GENERIC);
188
+ lines.push(KIND_HEADLINES[toolKind(name)]);
140
189
  }
141
190
  }
142
191
  return lines;
package/dist/metrics.d.ts CHANGED
@@ -1,3 +1,6 @@
1
1
  import type { Store } from "./store.js";
2
+ import { type ServerTelemetry } from "./server-telemetry.js";
2
3
  /** `repos`: the projects this console serves (null: all); task, run and worker figures count only theirs. */
3
4
  export declare function prometheusMetrics(store: Store, now: Date, repos?: readonly string[] | null, destinations?: ReadonlyMap<string, string>): string;
5
+ /** Performance labels are fixed vocabularies only, independent of the store's project labels. */
6
+ export declare function performanceMetrics(telemetry: ServerTelemetry): string;
package/dist/metrics.js CHANGED
@@ -6,6 +6,7 @@
6
6
  */
7
7
  import { createHash } from "node:crypto";
8
8
  import { basename } from "node:path";
9
+ import { LATENCY_BUCKETS, QUEUE_BUCKETS, STREAM_FAMILIES } from "./server-telemetry.js";
9
10
  const label = (value) => value.replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/\n/g, "\\n");
10
11
  const line = (name, sample) => {
11
12
  const labels = Object.entries(sample.labels ?? {});
@@ -52,5 +53,56 @@ export function prometheusMetrics(store, now, repos = null, destinations) {
52
53
  add("toolroll_monitoring_last_ok_timestamp_seconds", "When a destination last took a delivery.", "gauge", status.filter(one => one.lastOkAt !== null).map(one => ({ labels: { destination: one.sink }, value: Math.floor(Date.parse(one.lastOkAt) / 1000) })));
53
54
  add("toolroll_worker_heartbeat_age_seconds", "Seconds since each project's worker last checked in.", "gauge", rows(`SELECT repo, MAX(heartbeat_at) AS at FROM watch_lease WHERE ${inProjects("repo")} GROUP BY repo`, ...scoped).map(row => ({ labels: { project: project(String(row["repo"])) }, value: Math.max(0, Math.round((now.getTime() - Date.parse(String(row["at"]))) / 1000)) })));
54
55
  add("toolroll_checkouts", "Build checkouts on disk, by state.", "gauge", rows(`SELECT CASE WHEN runner IS NOT NULL AND released_at IS NULL THEN 'leased' ELSE 'released' END AS state, COUNT(*) AS n FROM worktree WHERE ${inProjects("repo")} GROUP BY 1`, ...scoped).map(row => ({ labels: { state: String(row["state"]) }, value: Number(row["n"]) })));
55
- return `${metrics.map(metric => [`# HELP ${metric.name} ${metric.help}`, `# TYPE ${metric.name} ${metric.type}`, ...metric.samples.map(sample => line(metric.name, sample))].join("\n")).join("\n")}\n`;
56
+ return `${metrics.map(metric => [`# HELP ${metric.name} ${metric.help}`, `# TYPE ${metric.name} ${metric.type}`, ...metric.samples.map(sample => line(metric.name, sample))].join("\n")).join("\n")}\n${performanceMetrics(store.telemetry)}`;
57
+ }
58
+ /** Performance labels are fixed vocabularies only, independent of the store's project labels. */
59
+ export function performanceMetrics(telemetry) {
60
+ const out = [];
61
+ const header = (name, help, type) => out.push(`# HELP ${name} ${help}`, `# TYPE ${name} ${type}`);
62
+ const histogram = (name, value, labels = {}) => {
63
+ let cumulative = 0;
64
+ for (const [i, n] of value.total.bins.entries()) {
65
+ cumulative += n;
66
+ out.push(line(`${name}_bucket`, { labels: { ...labels, le: String(LATENCY_BUCKETS[i] ?? '+Inf') }, value: cumulative }));
67
+ }
68
+ out.push(line(`${name}_sum`, { labels, value: value.total.sum }), line(`${name}_count`, { labels, value: value.total.count }));
69
+ };
70
+ header('toolroll_http_request_duration_seconds', 'Request duration; SSE measures opening headers, not stream lifetime.', 'histogram');
71
+ for (const [route, value] of telemetry.routes)
72
+ histogram('toolroll_http_request_duration_seconds', value, { route });
73
+ for (const [name, help, value] of [
74
+ ['toolroll_sqlite_write_wait_seconds', 'Elapsed acquisition of an explicit write lock, including failed waits and all retry slices.', telemetry.writeWait],
75
+ ['toolroll_sqlite_write_hold_seconds', 'Time holding an explicit write transaction, through commit or rollback.', telemetry.writeHold],
76
+ ['toolroll_sqlite_write_statement_seconds', 'Standalone write duration, including native busy waits that SQLite cannot separate.', telemetry.writeStatement],
77
+ ]) {
78
+ header(name, help, 'histogram');
79
+ histogram(name, value);
80
+ }
81
+ const snapshot = telemetry.snapshot();
82
+ header('toolroll_event_loop_delay_seconds', 'Approximate event-loop delay over the last five minutes; 20 ms sampling interval.', 'gauge');
83
+ for (const [quantile, value] of [['0.5', snapshot.eventLoop.p50Ms], ['0.99', snapshot.eventLoop.p99Ms]]) {
84
+ if (value !== null)
85
+ out.push(line('toolroll_event_loop_delay_seconds', { labels: { quantile }, value: value / 1000 }));
86
+ }
87
+ header('toolroll_event_loop_delay_samples', 'Event-loop samples in the recent window; zero means no measurement yet.', 'gauge');
88
+ out.push(line('toolroll_event_loop_delay_samples', { value: snapshot.eventLoop.count }));
89
+ header('toolroll_sse_connections', 'Currently open live streams.', 'gauge');
90
+ for (const route of STREAM_FAMILIES)
91
+ out.push(line('toolroll_sse_connections', { labels: { route }, value: telemetry.streamQueues(route).length }));
92
+ header('toolroll_sse_queue_bytes', 'Bytes currently waiting to send across open streams.', 'gauge');
93
+ header('toolroll_sse_queue_max_bytes', 'Largest current per-stream send queue.', 'gauge');
94
+ header('toolroll_sse_queue_bytes_bucket', 'Current per-stream queue depth distribution, not a lifetime histogram; each stream contributes once.', 'gauge');
95
+ for (const route of STREAM_FAMILIES) {
96
+ const queues = telemetry.streamQueues(route);
97
+ out.push(line('toolroll_sse_queue_bytes', { labels: { route }, value: queues.reduce((sum, n) => sum + n, 0) }));
98
+ out.push(line('toolroll_sse_queue_max_bytes', { labels: { route }, value: queues.reduce((max, n) => Math.max(max, n), 0) }));
99
+ for (const bound of [...QUEUE_BUCKETS, Infinity])
100
+ out.push(line('toolroll_sse_queue_bytes_bucket', { labels: { route, le: Number.isFinite(bound) ? String(bound) : '+Inf' }, value: queues.filter(n => n <= bound).length }));
101
+ }
102
+ header('toolroll_request_budget_refusals_total', 'Requests refused by the shared CLI and MCP request budget, including unavailable budget checks.', 'counter');
103
+ for (const route of ['api', 'mcp'])
104
+ for (const reason of ['read-per-minute', 'act-per-minute', 'per-day', 'unavailable']) {
105
+ out.push(line('toolroll_request_budget_refusals_total', { labels: { route, reason }, value: telemetry.refusals.get(`${route}:${reason}`) ?? 0 }));
106
+ }
107
+ return `${out.join('\n')}\n`;
56
108
  }
package/dist/operate.js CHANGED
@@ -90,7 +90,7 @@ import { openStore, openStoreNoMigrate, CONCURRENT_WRITER_WAIT_MS, isDatabaseBus
90
90
  import { createHash, randomBytes, randomInt, randomUUID } from "node:crypto";
91
91
  import { authorizePlanUnderMode } from "./plan-auto.js";
92
92
  import { ghDispatchAdapter, mirrorTaskId, syncPass } from "./sync.js";
93
- import { sweepLiveLogs } from "./live.js";
93
+ import { LIVE_FILE_BOUND, LIVE_RETAIN_MS, sweepLiveLogs } from "./live.js";
94
94
  import { configPath, addRepos, removeRepos, updateRepos, loadRepos, loadProjectRegistry, updateProjectRegistry } from "./repos.js";
95
95
  import { deleteProject, holdingsWords, projectHoldings, projectRunning } from "./project-delete.js";
96
96
  import { pushPass } from "./push.js";
@@ -181,6 +181,7 @@ import { WorktreePool } from "./worktree.js";
181
181
  import { requestTaskStop, resumeTaskStop, taskControlOf } from "./task-control.js";
182
182
  import { worktreeAdoptionNotice } from "./worktree-notices.js";
183
183
  import { workIndexPage, WorkIndexCursorError } from "./work-index.js";
184
+ import { readExecutorOf } from "./read-executor.js";
184
185
  import { parseWorkView } from "./workspace-ui.js";
185
186
  import { taskWorkSummaryOf } from "./work-summary.js";
186
187
  import { assignmentOf, assignmentBrief, syncAssignmentHandoffs } from "./assignment.js";
@@ -4671,6 +4672,11 @@ async function reconcileCommand(flags, context) {
4671
4672
  // day, orphans on mtime, active runs untouchable. Never evidence, never
4672
4673
  // load-bearing — a sweep that finds nothing is the common case.
4673
4674
  const liveSwept = sweepLiveLogs(store, context.evidenceRoot, clock());
4675
+ // What each agent did last goes by the same rules, from its own row.
4676
+ try {
4677
+ store.sweepRunActivity(clock(), LIVE_RETAIN_MS, LIVE_FILE_BOUND);
4678
+ }
4679
+ catch { /* display state: the next pass sweeps */ }
4674
4680
  const worktrees = new WorktreePool(store, {
4675
4681
  root: pool,
4676
4682
  ...(context.gitRunner === undefined ? {} : { runner: context.gitRunner }),
@@ -10751,35 +10757,47 @@ function listTasks(flags, context) {
10751
10757
  !Number.isSafeInteger(limit) || limit < 1 || limit > 100) {
10752
10758
  return fail(write, json, 'task list', 'usage', '--view takes all, needs-you, running or completed; --limit takes 1–100', EXIT.usage);
10753
10759
  }
10754
- let page;
10755
- try {
10756
- // A remote person limited to some projects (remoteLensOf) pages and counts only theirs.
10757
- const lens = context.principal?.lens ?? null;
10758
- page = workIndexPage(store, now, { principal: 'operator', repos: lens, includeUnplaced: lens === null }, {
10759
- view: parseWorkView(rawView ?? null), limit, cursor: text(flags, 'cursor') ?? null,
10760
- ...(wanted === undefined ? {} : { state: wanted }),
10761
- ...(text(flags, 'repo') === undefined ? {} : { project: text(flags, 'repo') }),
10762
- });
10763
- }
10764
- catch (error) {
10760
+ // A remote person limited to some projects (remoteLensOf) pages and counts only theirs.
10761
+ const lens = context.principal?.lens ?? null;
10762
+ const access = { principal: 'operator', repos: lens, includeUnplaced: lens === null };
10763
+ const options = {
10764
+ view: parseWorkView(rawView ?? null), limit, cursor: text(flags, 'cursor') ?? null,
10765
+ ...(wanted === undefined ? {} : { state: wanted }),
10766
+ ...(text(flags, 'repo') === undefined ? {} : { project: text(flags, 'repo') }),
10767
+ };
10768
+ const refused = (error) => {
10765
10769
  if (!(error instanceof WorkIndexCursorError))
10766
10770
  throw error;
10767
10771
  return fail(write, json, 'task list', 'usage', 'Invalid page cursor. Start again without --cursor.', EXIT.usage);
10768
- }
10769
- const tasks = page.items.map(item => ({ ...item, id: item.rootId }));
10770
- if (json) {
10771
- write(envelopeJson({ ok: true, command: 'task list', count: tasks.length, tasks,
10772
- totals: page.totals, nextCursor: page.nextCursor, limit: page.limit, view: page.view, evidence: 'recorded' }));
10772
+ };
10773
+ const show = (page) => {
10774
+ const tasks = page.items.map(item => ({ ...item, id: item.rootId }));
10775
+ if (json) {
10776
+ write(envelopeJson({ ok: true, command: 'task list', count: tasks.length, tasks,
10777
+ totals: page.totals, nextCursor: page.nextCursor, limit: page.limit, view: page.view, evidence: 'recorded' }));
10778
+ return EXIT.ok;
10779
+ }
10780
+ if (tasks.length === 0)
10781
+ write(wanted === undefined ? 'The task list is empty.' : `Nothing is ${wanted}.`);
10782
+ const width = Math.max(0, ...tasks.map(task => task.id.length));
10783
+ for (const task of tasks)
10784
+ write(` ${task.id.padEnd(width)} ${task.status.label.padEnd(16)} ${task.title}`);
10785
+ if (page.nextCursor !== null)
10786
+ write(`Next page: --cursor ${page.nextCursor}`);
10773
10787
  return EXIT.ok;
10788
+ };
10789
+ // On the server, the same page is read on a worker's read-only connection so the request loop stays free.
10790
+ const executor = readExecutorOf(store);
10791
+ if (executor !== null)
10792
+ return executor.workIndexPage(now, access, options).then(show, refused);
10793
+ let page;
10794
+ try {
10795
+ page = workIndexPage(store, now, access, options);
10774
10796
  }
10775
- if (tasks.length === 0)
10776
- write(wanted === undefined ? 'The task list is empty.' : `Nothing is ${wanted}.`);
10777
- const width = Math.max(0, ...tasks.map(task => task.id.length));
10778
- for (const task of tasks)
10779
- write(` ${task.id.padEnd(width)} ${task.status.label.padEnd(16)} ${task.title}`);
10780
- if (page.nextCursor !== null)
10781
- write(`Next page: --cursor ${page.nextCursor}`);
10782
- return EXIT.ok;
10797
+ catch (error) {
10798
+ return refused(error);
10799
+ }
10800
+ return show(page);
10783
10801
  }
10784
10802
  function showTask(positional, context) {
10785
10803
  const { store, write, json, now } = context;